@arke-institute/sdk 2.3.6 → 2.3.8

package/dist/index.js

@@ -160,563 +160,28 @@ function createArkeClient(config) {
 }
 
 // src/operations/upload/engine.ts
-var PHASE_COUNT = 3;
-var PHASE_INDEX = {
-  creating: 0,
-  backlinking: 1,
-  uploading: 2,
-  complete: 3,
-  error: -1
-};
-async function parallelLimit(items, concurrency, fn) {
-  const results = [];
-  let index = 0;
-  async function worker() {
-    while (index < items.length) {
-      const currentIndex = index++;
-      const item = items[currentIndex];
-      results[currentIndex] = await fn(item, currentIndex);
-    }
-  }
-  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => worker());
-  await Promise.all(workers);
-  return results;
-}
 var TARGET_BYTES_IN_FLIGHT = 200 * 1024 * 1024;
-var BytePool = class {
-  constructor(targetBytes = TARGET_BYTES_IN_FLIGHT) {
-    this.targetBytes = targetBytes;
-    this.bytesInFlight = 0;
-    this.waitQueue = [];
-  }
-  async run(size, fn) {
-    while (this.bytesInFlight > 0 && this.bytesInFlight + size > this.targetBytes) {
-      await new Promise((resolve) => this.waitQueue.push(resolve));
-    }
-    this.bytesInFlight += size;
-    try {
-      return await fn();
-    } finally {
-      this.bytesInFlight -= size;
-      const next = this.waitQueue.shift();
-      if (next) next();
-    }
-  }
-};
-function getParentPath(relativePath) {
-  const lastSlash = relativePath.lastIndexOf("/");
-  if (lastSlash === -1) return null;
-  return relativePath.slice(0, lastSlash);
-}
-function groupFoldersByDepth(folders) {
-  const byDepth = /* @__PURE__ */ new Map();
-  for (const folder of folders) {
-    const depth = folder.relativePath.split("/").length - 1;
-    if (!byDepth.has(depth)) byDepth.set(depth, []);
-    byDepth.get(depth).push(folder);
-  }
-  return byDepth;
-}
-async function uploadTree(client, tree, options) {
-  const { target, onProgress, concurrency = 10, continueOnError = false, note } = options;
-  const errors = [];
-  const createdFolders = [];
-  const createdFiles = [];
-  const foldersByPath = /* @__PURE__ */ new Map();
-  const totalEntities = tree.files.length + tree.folders.length;
-  const totalBytes = tree.files.reduce((sum, f) => sum + f.size, 0);
-  let completedEntities = 0;
-  let bytesUploaded = 0;
-  const reportProgress = (progress) => {
-    if (onProgress) {
-      const phase = progress.phase || "creating";
-      const phaseIndex = PHASE_INDEX[phase] ?? -1;
-      let phasePercent = 0;
-      if (phase === "creating") {
-        const done = progress.completedEntities ?? completedEntities;
-        phasePercent = totalEntities > 0 ? Math.round(done / totalEntities * 100) : 100;
-      } else if (phase === "backlinking") {
-        const done = progress.completedParents ?? 0;
-        const total = progress.totalParents ?? 0;
-        phasePercent = total > 0 ? Math.round(done / total * 100) : 100;
-      } else if (phase === "uploading") {
-        const done = progress.bytesUploaded ?? bytesUploaded;
-        phasePercent = totalBytes > 0 ? Math.round(done / totalBytes * 100) : 100;
-      } else if (phase === "complete") {
-        phasePercent = 100;
-      }
-      onProgress({
-        phase,
-        phaseIndex,
-        phaseCount: PHASE_COUNT,
-        phasePercent,
-        totalEntities,
-        completedEntities,
-        totalParents: 0,
-        completedParents: 0,
-        totalBytes,
-        bytesUploaded,
-        ...progress
-      });
-    }
-  };
-  try {
-    let collectionId;
-    let collectionCid;
-    let collectionCreated = false;
-    if (target.createCollection) {
-      const collectionBody = {
-        label: target.createCollection.label,
-        description: target.createCollection.description,
-        roles: target.createCollection.roles,
-        note
-      };
-      const { data, error } = await client.api.POST("/collections", {
-        body: collectionBody
-      });
-      if (error || !data) {
-        throw new Error(`Failed to create collection: ${JSON.stringify(error)}`);
-      }
-      collectionId = data.id;
-      collectionCid = data.cid;
-      collectionCreated = true;
-    } else if (target.collectionId) {
-      collectionId = target.collectionId;
-      const { data, error } = await client.api.GET("/collections/{id}", {
-        params: { path: { id: collectionId } }
-      });
-      if (error || !data) {
-        throw new Error(`Failed to fetch collection: ${JSON.stringify(error)}`);
-      }
-      collectionCid = data.cid;
-    } else {
-      throw new Error("Must provide either collectionId or createCollection in target");
-    }
-    const rootParentId = target.parentId ?? collectionId;
-    reportProgress({ phase: "creating", completedEntities: 0 });
-    const foldersByDepth = groupFoldersByDepth(tree.folders);
-    const sortedDepths = [...foldersByDepth.keys()].sort((a, b) => a - b);
-    for (const depth of sortedDepths) {
-      const foldersAtDepth = foldersByDepth.get(depth);
-      await Promise.all(
-        foldersAtDepth.map(async (folder) => {
-          try {
-            const parentPath = getParentPath(folder.relativePath);
-            const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
-            const parentType = parentPath ? "folder" : parentId === collectionId ? "collection" : "folder";
-            const folderBody = {
-              label: folder.name,
-              collection: collectionId,
-              note,
-              relationships: [{ predicate: "in", peer: parentId, peer_type: parentType }]
-            };
-            const { data, error } = await client.api.POST("/folders", {
-              body: folderBody
-            });
-            if (error || !data) {
-              throw new Error(JSON.stringify(error));
-            }
-            foldersByPath.set(folder.relativePath, { id: data.id, cid: data.cid });
-            createdFolders.push({
-              name: folder.name,
-              relativePath: folder.relativePath,
-              id: data.id,
-              entityCid: data.cid
-            });
-            completedEntities++;
-            reportProgress({
-              phase: "creating",
-              completedEntities,
-              currentItem: folder.relativePath
-            });
-          } catch (err) {
-            const errorMsg = err instanceof Error ? err.message : String(err);
-            if (continueOnError) {
-              errors.push({ path: folder.relativePath, error: `Folder creation failed: ${errorMsg}` });
-              completedEntities++;
-            } else {
-              throw new Error(`Failed to create folder ${folder.relativePath}: ${errorMsg}`);
-            }
-          }
-        })
-      );
-    }
-    const FILE_CREATION_CONCURRENCY = 50;
-    await parallelLimit(tree.files, FILE_CREATION_CONCURRENCY, async (file) => {
-      try {
-        const parentPath = getParentPath(file.relativePath);
-        const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
-        const parentType = parentPath ? "folder" : parentId === collectionId ? "collection" : "folder";
-        const fileBody = {
-          key: crypto.randomUUID(),
-          // Generate unique storage key
-          filename: file.name,
-          content_type: file.mimeType,
-          size: file.size,
-          collection: collectionId,
-          relationships: [{ predicate: "in", peer: parentId, peer_type: parentType }]
-        };
-        const { data, error } = await client.api.POST("/files", {
-          body: fileBody
-        });
-        if (error || !data) {
-          throw new Error(`Entity creation failed: ${JSON.stringify(error)}`);
-        }
-        createdFiles.push({
-          ...file,
-          id: data.id,
-          entityCid: data.cid
-        });
-        completedEntities++;
-        reportProgress({
-          phase: "creating",
-          completedEntities,
-          currentItem: file.relativePath
-        });
-      } catch (err) {
-        const errorMsg = err instanceof Error ? err.message : String(err);
-        if (continueOnError) {
-          errors.push({ path: file.relativePath, error: errorMsg });
-          completedEntities++;
-        } else {
-          throw new Error(`Failed to create file ${file.relativePath}: ${errorMsg}`);
-        }
-      }
-    });
-    const childrenByParent = /* @__PURE__ */ new Map();
-    for (const folder of createdFolders) {
-      const parentPath = getParentPath(folder.relativePath);
-      const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
-      if (!childrenByParent.has(parentId)) childrenByParent.set(parentId, []);
-      childrenByParent.get(parentId).push({ id: folder.id, type: "folder" });
-    }
-    for (const file of createdFiles) {
-      const parentPath = getParentPath(file.relativePath);
-      const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
-      if (!childrenByParent.has(parentId)) childrenByParent.set(parentId, []);
-      childrenByParent.get(parentId).push({ id: file.id, type: "file" });
-    }
-    const totalParents = childrenByParent.size;
-    let completedParents = 0;
-    reportProgress({ phase: "backlinking", totalParents, completedParents: 0 });
-    const parentEntries = [...childrenByParent.entries()];
-    await parallelLimit(parentEntries, concurrency, async ([parentId, children]) => {
-      try {
-        const isCollection = parentId === collectionId;
-        const relationshipsAdd = children.map((child) => ({
-          predicate: "contains",
-          peer: child.id,
-          peer_type: child.type
-        }));
-        if (isCollection) {
-          const { data: collData, error: getError } = await client.api.GET("/collections/{id}", {
-            params: { path: { id: parentId } }
-          });
-          if (getError || !collData) {
-            throw new Error(`Failed to fetch collection: ${JSON.stringify(getError)}`);
-          }
-          const updateBody = {
-            expect_tip: collData.cid,
-            relationships_add: relationshipsAdd,
-            note: note ? `${note} (backlink)` : "Upload backlink"
-          };
-          const { error } = await client.api.PUT("/collections/{id}", {
-            params: { path: { id: parentId } },
-            body: updateBody
-          });
-          if (error) {
-            throw new Error(JSON.stringify(error));
-          }
-        } else {
-          const { data: folderData, error: getError } = await client.api.GET("/folders/{id}", {
-            params: { path: { id: parentId } }
-          });
-          if (getError || !folderData) {
-            throw new Error(`Failed to fetch folder: ${JSON.stringify(getError)}`);
-          }
-          const updateBody = {
-            expect_tip: folderData.cid,
-            relationships_add: relationshipsAdd,
-            note: note ? `${note} (backlink)` : "Upload backlink"
-          };
-          const { error } = await client.api.PUT("/folders/{id}", {
-            params: { path: { id: parentId } },
-            body: updateBody
-          });
-          if (error) {
-            throw new Error(JSON.stringify(error));
-          }
-        }
-        completedParents++;
-        reportProgress({
-          phase: "backlinking",
-          totalParents,
-          completedParents,
-          currentItem: `parent:${parentId}`
-        });
-      } catch (err) {
-        const errorMsg = err instanceof Error ? err.message : String(err);
-        if (continueOnError) {
-          errors.push({ path: `parent:${parentId}`, error: `Backlink failed: ${errorMsg}` });
-          completedParents++;
-        } else {
-          throw new Error(`Failed to backlink parent ${parentId}: ${errorMsg}`);
-        }
-      }
-    });
-    reportProgress({ phase: "uploading", bytesUploaded: 0 });
-    const pool = new BytePool();
-    await Promise.all(
-      createdFiles.map(async (file) => {
-        await pool.run(file.size, async () => {
-          try {
-            const fileData = await file.getData();
-            let body;
-            if (fileData instanceof Blob) {
-              body = fileData;
-            } else if (fileData instanceof Uint8Array) {
-              const arrayBuffer = new ArrayBuffer(fileData.byteLength);
-              new Uint8Array(arrayBuffer).set(fileData);
-              body = new Blob([arrayBuffer], { type: file.mimeType });
-            } else {
-              body = new Blob([fileData], { type: file.mimeType });
-            }
-            const { error: uploadError } = await client.api.POST("/files/{id}/content", {
-              params: { path: { id: file.id } },
-              body,
-              bodySerializer: (b) => b,
-              headers: { "Content-Type": file.mimeType }
-            });
-            if (uploadError) {
-              throw new Error(`Upload failed: ${JSON.stringify(uploadError)}`);
-            }
-            bytesUploaded += file.size;
-            reportProgress({
-              phase: "uploading",
-              bytesUploaded,
-              currentItem: file.relativePath
-            });
-          } catch (err) {
-            const errorMsg = err instanceof Error ? err.message : String(err);
-            if (continueOnError) {
-              errors.push({ path: file.relativePath, error: `Upload failed: ${errorMsg}` });
-            } else {
-              throw new Error(`Failed to upload ${file.relativePath}: ${errorMsg}`);
-            }
-          }
-        });
-      })
-    );
-    reportProgress({ phase: "complete", totalParents, completedParents, bytesUploaded });
-    const resultFolders = createdFolders.map((f) => ({
-      id: f.id,
-      cid: f.entityCid,
-      type: "folder",
-      relativePath: f.relativePath
-    }));
-    const resultFiles = createdFiles.map((f) => ({
-      id: f.id,
-      cid: f.entityCid,
-      type: "file",
-      relativePath: f.relativePath
-    }));
-    return {
-      success: errors.length === 0,
-      collection: {
-        id: collectionId,
-        cid: collectionCid,
-        created: collectionCreated
-      },
-      folders: resultFolders,
-      files: resultFiles,
-      errors
-    };
-  } catch (err) {
-    const errorMsg = err instanceof Error ? err.message : String(err);
-    reportProgress({
-      phase: "error",
-      error: errorMsg
-    });
-    return {
-      success: false,
-      collection: {
-        id: target.collectionId ?? "",
-        cid: "",
-        created: false
-      },
-      folders: createdFolders.map((f) => ({
-        id: f.id,
-        cid: f.entityCid,
-        type: "folder",
-        relativePath: f.relativePath
-      })),
-      files: createdFiles.map((f) => ({
-        id: f.id,
-        cid: f.entityCid,
-        type: "file",
-        relativePath: f.relativePath
-      })),
-      errors: [...errors, { path: "", error: errorMsg }]
-    };
-  }
-}
 
 // src/operations/upload/cid.ts
 import { CID } from "multiformats/cid";
 import { sha256 } from "multiformats/hashes/sha2";
 import * as raw from "multiformats/codecs/raw";
 
-// src/operations/upload/scanners.ts
-function getMimeType(filename) {
-  const ext = filename.toLowerCase().split(".").pop() || "";
-  const mimeTypes = {
-    // Images
-    jpg: "image/jpeg",
-    jpeg: "image/jpeg",
-    png: "image/png",
-    gif: "image/gif",
-    webp: "image/webp",
-    svg: "image/svg+xml",
-    ico: "image/x-icon",
-    bmp: "image/bmp",
-    tiff: "image/tiff",
-    tif: "image/tiff",
-    // Documents
-    pdf: "application/pdf",
-    doc: "application/msword",
-    docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-    xls: "application/vnd.ms-excel",
-    xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    ppt: "application/vnd.ms-powerpoint",
-    pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-    odt: "application/vnd.oasis.opendocument.text",
-    ods: "application/vnd.oasis.opendocument.spreadsheet",
-    odp: "application/vnd.oasis.opendocument.presentation",
-    // Text
-    txt: "text/plain",
-    md: "text/markdown",
-    csv: "text/csv",
-    html: "text/html",
-    htm: "text/html",
-    css: "text/css",
-    xml: "text/xml",
-    rtf: "application/rtf",
-    // Code
-    js: "text/javascript",
-    mjs: "text/javascript",
-    ts: "text/typescript",
-    jsx: "text/javascript",
-    tsx: "text/typescript",
-    json: "application/json",
-    yaml: "text/yaml",
-    yml: "text/yaml",
-    // Archives
-    zip: "application/zip",
-    tar: "application/x-tar",
-    gz: "application/gzip",
-    rar: "application/vnd.rar",
-    "7z": "application/x-7z-compressed",
-    // Audio
-    mp3: "audio/mpeg",
-    wav: "audio/wav",
-    ogg: "audio/ogg",
-    m4a: "audio/mp4",
-    flac: "audio/flac",
-    // Video
-    mp4: "video/mp4",
-    webm: "video/webm",
-    avi: "video/x-msvideo",
-    mov: "video/quicktime",
-    mkv: "video/x-matroska",
-    // Fonts
-    woff: "font/woff",
-    woff2: "font/woff2",
-    ttf: "font/ttf",
-    otf: "font/otf",
-    // Other
-    wasm: "application/wasm"
-  };
-  return mimeTypes[ext] || "application/octet-stream";
-}
-async function scanDirectory(directoryPath, options = {}) {
-  const fs = await import("fs/promises");
-  const path = await import("path");
-  const { ignore = ["node_modules", ".git", ".DS_Store"], includeHidden = false } = options;
-  const files = [];
-  const folders = [];
-  const rootName = path.basename(directoryPath);
-  async function scanDir(dirPath, relativePath) {
-    const entries = await fs.readdir(dirPath, { withFileTypes: true });
-    for (const entry of entries) {
-      const name = entry.name;
-      if (!includeHidden && name.startsWith(".")) {
-        continue;
-      }
-      if (ignore.some((pattern) => name === pattern || name.match(pattern))) {
-        continue;
-      }
-      const fullPath = path.join(dirPath, name);
-      const entryRelativePath = relativePath ? `${relativePath}/${name}` : name;
-      if (entry.isDirectory()) {
-        folders.push({
-          name,
-          relativePath: entryRelativePath
-        });
-        await scanDir(fullPath, entryRelativePath);
-      } else if (entry.isFile()) {
-        const stat = await fs.stat(fullPath);
-        files.push({
-          name,
-          relativePath: entryRelativePath,
-          size: stat.size,
-          mimeType: getMimeType(name),
-          getData: async () => {
-            const buffer = await fs.readFile(fullPath);
-            return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
-          }
-        });
-      }
-    }
-  }
-  await scanDir(directoryPath, "");
-  folders.sort((a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length);
-  return { files, folders };
-}
-
 // src/operations/folders.ts
 var FolderOperations = class {
   constructor(client) {
     this.client = client;
+    void client;
   }
   /**
    * Upload a local directory to Arke
    *
-   * @deprecated Use uploadTree and scanDirectory instead
+   * @deprecated This method has been removed. Use uploadTree and buildUploadTree instead.
    */
-  async uploadDirectory(localPath, options) {
-    const tree = await scanDirectory(localPath);
-    const result = await uploadTree(this.client, tree, {
-      target: {
-        collectionId: options.collectionId,
-        parentId: options.parentFolderId
-      },
-      concurrency: options.concurrency,
-      onProgress: options.onProgress ? (p) => {
-        options.onProgress({
-          phase: p.phase === "creating" ? "creating-folders" : p.phase === "uploading" ? "uploading-files" : p.phase === "backlinking" ? "linking" : p.phase === "complete" ? "complete" : "scanning",
-          totalFiles: p.totalEntities,
-          completedFiles: p.completedEntities,
-          totalFolders: p.totalParents,
-          completedFolders: p.completedParents,
-          currentFile: p.currentItem
-        });
-      } : void 0
-    });
-    return {
-      rootFolder: result.folders[0] || null,
-      folders: result.folders,
-      files: result.files
-    };
+  async uploadDirectory(_localPath, _options) {
+    throw new Error(
+      "FolderOperations.uploadDirectory has been removed. Use uploadTree() with buildUploadTree() instead. See: https://github.com/arke-institute/arke-sdk#upload-module"
+    );
   }
 };