@arke-institute/sdk 2.0.0 → 2.1.0

package/dist/operations/index.js

@@ -1,3 +1,653 @@
+// src/operations/upload/cid.ts
+import { CID } from "multiformats/cid";
+import { sha256 } from "multiformats/hashes/sha2";
+import * as raw from "multiformats/codecs/raw";
+async function computeCid(data) {
+  let bytes;
+  if (data instanceof Blob) {
+    const buffer = await data.arrayBuffer();
+    bytes = new Uint8Array(buffer);
+  } else if (data instanceof ArrayBuffer) {
+    bytes = new Uint8Array(data);
+  } else {
+    bytes = data;
+  }
+  const hash = await sha256.digest(bytes);
+  const cid = CID.create(1, raw.code, hash);
+  return cid.toString();
+}
+async function verifyCid(data, expectedCid) {
+  const computed = await computeCid(data);
+  return computed === expectedCid;
+}
+
+// src/operations/upload/engine.ts
+async function parallelLimit(items, concurrency, fn) {
+  const results = [];
+  let index = 0;
+  async function worker() {
+    while (index < items.length) {
+      const currentIndex = index++;
+      const item = items[currentIndex];
+      results[currentIndex] = await fn(item, currentIndex);
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
+function getParentPath(relativePath) {
+  const lastSlash = relativePath.lastIndexOf("/");
+  if (lastSlash === -1) return null;
+  return relativePath.slice(0, lastSlash);
+}
+async function uploadTree(client, tree, options) {
+  const { target, onProgress, concurrency = 5, continueOnError = false, note } = options;
+  const errors = [];
+  const createdFolders = [];
+  const createdFiles = [];
+  const reportProgress = (progress) => {
+    if (onProgress) {
+      onProgress({
+        phase: "scanning",
+        totalFiles: tree.files.length,
+        completedFiles: 0,
+        totalFolders: tree.folders.length,
+        completedFolders: 0,
+        ...progress
+      });
+    }
+  };
+  try {
+    let collectionId;
+    let collectionCid;
+    let collectionCreated = false;
+    if (target.createCollection) {
+      reportProgress({ phase: "scanning", currentFolder: "Creating collection..." });
+      const collectionBody = {
+        label: target.createCollection.label,
+        description: target.createCollection.description,
+        roles: target.createCollection.roles,
+        note
+      };
+      const { data, error } = await client.api.POST("/collections", {
+        body: collectionBody
+      });
+      if (error || !data) {
+        throw new Error(`Failed to create collection: ${JSON.stringify(error)}`);
+      }
+      collectionId = data.id;
+      collectionCid = data.cid;
+      collectionCreated = true;
+    } else if (target.collectionId) {
+      collectionId = target.collectionId;
+      const { data, error } = await client.api.GET("/collections/{id}", {
+        params: { path: { id: collectionId } }
+      });
+      if (error || !data) {
+        throw new Error(`Failed to fetch collection: ${JSON.stringify(error)}`);
+      }
+      collectionCid = data.cid;
+    } else {
+      throw new Error("Must provide either collectionId or createCollection in target");
+    }
+    const rootParentId = target.parentId ?? collectionId;
+    reportProgress({
+      phase: "computing-cids",
+      totalFiles: tree.files.length,
+      completedFiles: 0
+    });
+    const preparedFiles = [];
+    let cidProgress = 0;
+    await parallelLimit(tree.files, concurrency, async (file) => {
+      try {
+        const data = await file.getData();
+        const cid = await computeCid(data);
+        preparedFiles.push({
+          ...file,
+          cid
+        });
+        cidProgress++;
+        reportProgress({
+          phase: "computing-cids",
+          completedFiles: cidProgress,
+          currentFile: file.relativePath
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({ path: file.relativePath, error: `CID computation failed: ${errorMsg}` });
+        } else {
+          throw new Error(`Failed to compute CID for ${file.relativePath}: ${errorMsg}`);
+        }
+      }
+    });
+    reportProgress({
+      phase: "creating-folders",
+      totalFolders: tree.folders.length,
+      completedFolders: 0
+    });
+    const sortedFolders = [...tree.folders].sort(
+      (a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length
+    );
+    for (let i = 0; i < sortedFolders.length; i++) {
+      const folder = sortedFolders[i];
+      try {
+        const folderBody = {
+          label: folder.name,
+          collection: collectionId,
+          note
+        };
+        const { data, error } = await client.api.POST("/folders", {
+          body: folderBody
+        });
+        if (error || !data) {
+          throw new Error(JSON.stringify(error));
+        }
+        createdFolders.push({
+          name: folder.name,
+          relativePath: folder.relativePath,
+          id: data.id,
+          entityCid: data.cid
+        });
+        reportProgress({
+          phase: "creating-folders",
+          completedFolders: i + 1,
+          currentFolder: folder.relativePath
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({ path: folder.relativePath, error: `Folder creation failed: ${errorMsg}` });
+        } else {
+          throw new Error(`Failed to create folder ${folder.relativePath}: ${errorMsg}`);
+        }
+      }
+    }
+    const folderPathToEntity = /* @__PURE__ */ new Map();
+    for (const folder of createdFolders) {
+      folderPathToEntity.set(folder.relativePath, folder);
+    }
+    reportProgress({
+      phase: "creating-files",
+      totalFiles: preparedFiles.length,
+      completedFiles: 0
+    });
+    let fileCreateProgress = 0;
+    await parallelLimit(preparedFiles, concurrency, async (file) => {
+      try {
+        const fileBody = {
+          key: file.cid,
+          // Use CID as storage key (best practice)
+          filename: file.name,
+          content_type: file.mimeType,
+          size: file.size,
+          cid: file.cid,
+          collection: collectionId
+        };
+        const { data, error } = await client.api.POST("/files", {
+          body: fileBody
+        });
+        if (error || !data) {
+          throw new Error(JSON.stringify(error));
+        }
+        createdFiles.push({
+          ...file,
+          id: data.id,
+          entityCid: data.cid,
+          uploadUrl: data.upload_url,
+          uploadExpiresAt: data.upload_expires_at
+        });
+        fileCreateProgress++;
+        reportProgress({
+          phase: "creating-files",
+          completedFiles: fileCreateProgress,
+          currentFile: file.relativePath
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({ path: file.relativePath, error: `File creation failed: ${errorMsg}` });
+        } else {
+          throw new Error(`Failed to create file ${file.relativePath}: ${errorMsg}`);
+        }
+      }
+    });
+    const totalBytes = createdFiles.reduce((sum, f) => sum + f.size, 0);
+    let bytesUploaded = 0;
+    reportProgress({
+      phase: "uploading-content",
+      totalFiles: createdFiles.length,
+      completedFiles: 0,
+      totalBytes,
+      bytesUploaded: 0
+    });
+    let uploadProgress = 0;
+    await parallelLimit(createdFiles, concurrency, async (file) => {
+      try {
+        const data = await file.getData();
+        let body;
+        if (data instanceof Blob) {
+          body = data;
+        } else if (data instanceof Uint8Array) {
+          const arrayBuffer = new ArrayBuffer(data.byteLength);
+          new Uint8Array(arrayBuffer).set(data);
+          body = new Blob([arrayBuffer], { type: file.mimeType });
+        } else {
+          body = new Blob([data], { type: file.mimeType });
+        }
+        const response = await fetch(file.uploadUrl, {
+          method: "PUT",
+          body,
+          headers: {
+            "Content-Type": file.mimeType
+          }
+        });
+        if (!response.ok) {
+          throw new Error(`Upload failed with status ${response.status}`);
+        }
+        bytesUploaded += file.size;
+        uploadProgress++;
+        reportProgress({
+          phase: "uploading-content",
+          completedFiles: uploadProgress,
+          currentFile: file.relativePath,
+          bytesUploaded,
+          totalBytes
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({ path: file.relativePath, error: `Upload failed: ${errorMsg}` });
+        } else {
+          throw new Error(`Failed to upload ${file.relativePath}: ${errorMsg}`);
+        }
+      }
+    });
+    reportProgress({ phase: "linking" });
+    const filePathToEntity = /* @__PURE__ */ new Map();
+    for (const file of createdFiles) {
+      filePathToEntity.set(file.relativePath, file);
+    }
+    const parentGroups = /* @__PURE__ */ new Map();
+    for (const folder of createdFolders) {
+      const parentPath = getParentPath(folder.relativePath);
+      let parentId;
+      if (parentPath === null) {
+        parentId = rootParentId;
+      } else {
+        const parentFolder = folderPathToEntity.get(parentPath);
+        if (!parentFolder) {
+          errors.push({
+            path: folder.relativePath,
+            error: `Parent folder not found: ${parentPath}`
+          });
+          continue;
+        }
+        parentId = parentFolder.id;
+      }
+      if (!parentGroups.has(parentId)) {
+        parentGroups.set(parentId, { folderId: parentId, children: [] });
+      }
+      parentGroups.get(parentId).children.push({ id: folder.id });
+    }
+    for (const file of createdFiles) {
+      const parentPath = getParentPath(file.relativePath);
+      let parentId;
+      if (parentPath === null) {
+        parentId = rootParentId;
+      } else {
+        const parentFolder = folderPathToEntity.get(parentPath);
+        if (!parentFolder) {
+          errors.push({
+            path: file.relativePath,
+            error: `Parent folder not found: ${parentPath}`
+          });
+          continue;
+        }
+        parentId = parentFolder.id;
+      }
+      if (!parentGroups.has(parentId)) {
+        parentGroups.set(parentId, { folderId: parentId, children: [] });
+      }
+      parentGroups.get(parentId).children.push({ id: file.id });
+    }
+    for (const [parentId, group] of parentGroups) {
+      if (group.children.length === 0) continue;
+      try {
+        let expectTip;
+        if (parentId === collectionId) {
+          const { data, error: error2 } = await client.api.GET("/collections/{id}", {
+            params: { path: { id: collectionId } }
+          });
+          if (error2 || !data) {
+            throw new Error(`Failed to fetch collection CID: ${JSON.stringify(error2)}`);
+          }
+          expectTip = data.cid;
+        } else {
+          const { data, error: error2 } = await client.api.GET("/folders/{id}", {
+            params: { path: { id: parentId } }
+          });
+          if (error2 || !data) {
+            throw new Error(`Failed to fetch folder CID: ${JSON.stringify(error2)}`);
+          }
+          expectTip = data.cid;
+        }
+        const bulkBody = {
+          expect_tip: expectTip,
+          children: group.children,
+          note
+        };
+        const { error } = await client.api.POST("/folders/{id}/children/bulk", {
+          params: { path: { id: parentId } },
+          body: bulkBody
+        });
+        if (error) {
+          throw new Error(JSON.stringify(error));
+        }
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({
+            path: `parent:${parentId}`,
+            error: `Bulk linking failed: ${errorMsg}`
+          });
+        } else {
+          throw new Error(`Failed to link children to ${parentId}: ${errorMsg}`);
+        }
+      }
+    }
+    reportProgress({ phase: "complete" });
+    const resultFolders = createdFolders.map((f) => ({
+      id: f.id,
+      cid: f.entityCid,
+      type: "folder",
+      relativePath: f.relativePath
+    }));
+    const resultFiles = createdFiles.map((f) => ({
+      id: f.id,
+      cid: f.entityCid,
+      type: "file",
+      relativePath: f.relativePath
+    }));
+    return {
+      success: errors.length === 0,
+      collection: {
+        id: collectionId,
+        cid: collectionCid,
+        created: collectionCreated
+      },
+      folders: resultFolders,
+      files: resultFiles,
+      errors
+    };
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    reportProgress({
+      phase: "error",
+      error: errorMsg
+    });
+    return {
+      success: false,
+      collection: {
+        id: target.collectionId ?? "",
+        cid: "",
+        created: false
+      },
+      folders: createdFolders.map((f) => ({
+        id: f.id,
+        cid: f.entityCid,
+        type: "folder",
+        relativePath: f.relativePath
+      })),
+      files: createdFiles.map((f) => ({
+        id: f.id,
+        cid: f.entityCid,
+        type: "file",
+        relativePath: f.relativePath
+      })),
+      errors: [...errors, { path: "", error: errorMsg }]
+    };
+  }
+}
+
+// src/operations/upload/scanners.ts
+function getMimeType(filename) {
+  const ext = filename.toLowerCase().split(".").pop() || "";
+  const mimeTypes = {
+    // Images
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    png: "image/png",
+    gif: "image/gif",
+    webp: "image/webp",
+    svg: "image/svg+xml",
+    ico: "image/x-icon",
+    bmp: "image/bmp",
+    tiff: "image/tiff",
+    tif: "image/tiff",
+    // Documents
+    pdf: "application/pdf",
+    doc: "application/msword",
+    docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    xls: "application/vnd.ms-excel",
+    xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ppt: "application/vnd.ms-powerpoint",
+    pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    odt: "application/vnd.oasis.opendocument.text",
+    ods: "application/vnd.oasis.opendocument.spreadsheet",
+    odp: "application/vnd.oasis.opendocument.presentation",
+    // Text
+    txt: "text/plain",
+    md: "text/markdown",
+    csv: "text/csv",
+    html: "text/html",
+    htm: "text/html",
+    css: "text/css",
+    xml: "text/xml",
+    rtf: "application/rtf",
+    // Code
+    js: "text/javascript",
+    mjs: "text/javascript",
+    ts: "text/typescript",
+    jsx: "text/javascript",
+    tsx: "text/typescript",
+    json: "application/json",
+    yaml: "text/yaml",
+    yml: "text/yaml",
+    // Archives
+    zip: "application/zip",
+    tar: "application/x-tar",
+    gz: "application/gzip",
+    rar: "application/vnd.rar",
+    "7z": "application/x-7z-compressed",
+    // Audio
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    ogg: "audio/ogg",
+    m4a: "audio/mp4",
+    flac: "audio/flac",
+    // Video
+    mp4: "video/mp4",
+    webm: "video/webm",
+    avi: "video/x-msvideo",
+    mov: "video/quicktime",
+    mkv: "video/x-matroska",
+    // Fonts
+    woff: "font/woff",
+    woff2: "font/woff2",
+    ttf: "font/ttf",
+    otf: "font/otf",
+    // Other
+    wasm: "application/wasm"
+  };
+  return mimeTypes[ext] || "application/octet-stream";
+}
+async function scanDirectory(directoryPath, options = {}) {
+  const fs = await import("fs/promises");
+  const path = await import("path");
+  const { ignore = ["node_modules", ".git", ".DS_Store"], includeHidden = false } = options;
+  const files = [];
+  const folders = [];
+  const rootName = path.basename(directoryPath);
+  async function scanDir(dirPath, relativePath) {
+    const entries = await fs.readdir(dirPath, { withFileTypes: true });
+    for (const entry of entries) {
+      const name = entry.name;
+      if (!includeHidden && name.startsWith(".")) {
+        continue;
+      }
+      if (ignore.some((pattern) => name === pattern || name.match(pattern))) {
+        continue;
+      }
+      const fullPath = path.join(dirPath, name);
+      const entryRelativePath = relativePath ? `${relativePath}/${name}` : name;
+      if (entry.isDirectory()) {
+        folders.push({
+          name,
+          relativePath: entryRelativePath
+        });
+        await scanDir(fullPath, entryRelativePath);
+      } else if (entry.isFile()) {
+        const stat = await fs.stat(fullPath);
+        files.push({
+          name,
+          relativePath: entryRelativePath,
+          size: stat.size,
+          mimeType: getMimeType(name),
+          getData: async () => {
+            const buffer = await fs.readFile(fullPath);
+            return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+          }
+        });
+      }
+    }
+  }
+  await scanDir(directoryPath, "");
+  folders.sort((a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length);
+  return { files, folders };
+}
+async function scanFileSystemEntries(entries, options = {}) {
+  const { ignore = ["node_modules", ".git", ".DS_Store"] } = options;
+  const files = [];
+  const folders = [];
+  async function processEntry(entry, parentPath) {
+    const name = entry.name;
+    if (ignore.some((pattern) => name === pattern)) {
+      return;
+    }
+    const relativePath = parentPath ? `${parentPath}/${name}` : name;
+    if (entry.isFile) {
+      const fileEntry = entry;
+      const file = await new Promise((resolve, reject) => {
+        fileEntry.file(resolve, reject);
+      });
+      files.push({
+        name,
+        relativePath,
+        size: file.size,
+        mimeType: file.type || getMimeType(name),
+        getData: async () => file.arrayBuffer()
+      });
+    } else if (entry.isDirectory) {
+      const dirEntry = entry;
+      folders.push({
+        name,
+        relativePath
+      });
+      const reader = dirEntry.createReader();
+      const childEntries = await new Promise((resolve, reject) => {
+        const allEntries = [];
+        function readEntries() {
+          reader.readEntries((entries2) => {
+            if (entries2.length === 0) {
+              resolve(allEntries);
+            } else {
+              allEntries.push(...entries2);
+              readEntries();
+            }
+          }, reject);
+        }
+        readEntries();
+      });
+      for (const childEntry of childEntries) {
+        await processEntry(childEntry, relativePath);
+      }
+    }
+  }
+  for (const entry of entries) {
+    if (entry) {
+      await processEntry(entry, "");
+    }
+  }
+  folders.sort((a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length);
+  return { files, folders };
+}
+async function scanFileList(fileList, options = {}) {
+  const { ignore = ["node_modules", ".git", ".DS_Store"] } = options;
+  const files = [];
+  const folderPaths = /* @__PURE__ */ new Set();
+  for (let i = 0; i < fileList.length; i++) {
+    const file = fileList[i];
+    if (!file) continue;
+    const relativePath = file.webkitRelativePath || file.name;
+    const name = file.name;
+    const pathSegments = relativePath.split("/");
+    if (pathSegments.some((segment) => ignore.includes(segment))) {
+      continue;
+    }
+    const pathParts = relativePath.split("/");
+    for (let j = 1; j < pathParts.length; j++) {
+      const folderPath = pathParts.slice(0, j).join("/");
+      folderPaths.add(folderPath);
+    }
+    const fileRef = file;
+    files.push({
+      name,
+      relativePath,
+      size: fileRef.size,
+      mimeType: fileRef.type || getMimeType(name),
+      getData: async () => fileRef.arrayBuffer()
+    });
+  }
+  const folders = Array.from(folderPaths).map((path) => ({
+    name: path.split("/").pop(),
+    relativePath: path
+  })).sort((a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length);
+  return { files, folders };
+}
+function buildUploadTree(items) {
+  const files = [];
+  const folderPaths = /* @__PURE__ */ new Set();
+  for (const item of items) {
+    const pathParts = item.path.split("/");
+    const name = pathParts.pop();
+    for (let i = 1; i <= pathParts.length; i++) {
+      folderPaths.add(pathParts.slice(0, i).join("/"));
+    }
+    let size;
+    if (item.data instanceof Blob) {
+      size = item.data.size;
+    } else if (item.data instanceof ArrayBuffer) {
+      size = item.data.byteLength;
+    } else {
+      size = item.data.length;
+    }
+    files.push({
+      name,
+      relativePath: item.path,
+      size,
+      mimeType: item.mimeType || getMimeType(name),
+      getData: async () => item.data
+    });
+  }
+  const folders = Array.from(folderPaths).map((path) => ({
+    name: path.split("/").pop(),
+    relativePath: path
+  })).sort((a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length);
+  return { files, folders };
+}
+
 // src/operations/folders.ts
 var FolderOperations = class {
   constructor(client) {
@@ -6,15 +656,32 @@ var FolderOperations = class {
   /**
    * Upload a local directory to Arke
    *
-   * TODO: Implement this method
-   * Steps:
-   * 1. Scan directory structure
-   * 2. Create folder hierarchy (depth-first)
-   * 3. Upload files in parallel (with concurrency limit)
-   * 4. Create bidirectional relationships (folder contains file)
+   * @deprecated Use uploadTree and scanDirectory instead
    */
-  async uploadDirectory(_localPath, _options) {
-    throw new Error("FolderOperations.uploadDirectory is not yet implemented");
+  async uploadDirectory(localPath, options) {
+    const tree = await scanDirectory(localPath);
+    const result = await uploadTree(this.client, tree, {
+      target: {
+        collectionId: options.collectionId,
+        parentId: options.parentFolderId
+      },
+      concurrency: options.concurrency,
+      onProgress: options.onProgress ? (p) => {
+        options.onProgress({
+          phase: p.phase === "computing-cids" || p.phase === "creating-folders" ? "creating-folders" : p.phase === "creating-files" || p.phase === "uploading-content" ? "uploading-files" : p.phase === "linking" ? "linking" : p.phase === "complete" ? "complete" : "scanning",
+          totalFiles: p.totalFiles,
+          completedFiles: p.completedFiles,
+          totalFolders: p.totalFolders,
+          completedFolders: p.completedFolders,
+          currentFile: p.currentFile
+        });
+      } : void 0
+    });
+    return {
+      rootFolder: result.folders[0] || null,
+      folders: result.folders,
+      files: result.files
+    };
   }
 };
 
@@ -79,6 +746,14 @@ var CryptoOperations = class {
 export {
   BatchOperations,
   CryptoOperations,
-  FolderOperations
+  FolderOperations,
+  buildUploadTree,
+  computeCid,
+  getMimeType,
+  scanDirectory,
+  scanFileList,
+  scanFileSystemEntries,
+  uploadTree,
+  verifyCid
 };
 //# sourceMappingURL=index.js.map