@arke-institute/sdk 2.0.0 → 2.2.0

package/dist/index.js

@@ -9,9 +9,9 @@ var DEFAULT_CONFIG = {
 
 // src/client/errors.ts
 var ArkeError = class extends Error {
-  constructor(message, code, status, details) {
+  constructor(message, code2, status, details) {
     super(message);
-    this.code = code;
+    this.code = code2;
     this.status = status;
     this.details = details;
     this.name = "ArkeError";
@@ -88,6 +88,15 @@ function parseApiError(status, body) {
 }
 
 // src/client/ArkeClient.ts
+function isApiKey(token) {
+  return token.startsWith("ak_") || token.startsWith("uk_");
+}
+function getAuthorizationHeader(token) {
+  if (isApiKey(token)) {
+    return `ApiKey ${token}`;
+  }
+  return `Bearer ${token}`;
+}
 var ArkeClient = class {
   constructor(config = {}) {
     this.config = {
@@ -102,7 +111,7 @@ var ArkeClient = class {
       ...this.config.headers
     };
     if (this.config.authToken) {
-      headers["Authorization"] = `Bearer ${this.config.authToken}`;
+      headers["Authorization"] = getAuthorizationHeader(this.config.authToken);
     }
     if (this.config.network === "test") {
       headers["X-Arke-Network"] = "test";
@@ -150,6 +159,603 @@ function createArkeClient(config) {
   return new ArkeClient(config);
 }
 
+// src/operations/upload/cid.ts
+import { CID } from "multiformats/cid";
+import { sha256 } from "multiformats/hashes/sha2";
+import * as raw from "multiformats/codecs/raw";
+async function computeCid(data) {
+  let bytes;
+  if (data instanceof Blob) {
+    const buffer = await data.arrayBuffer();
+    bytes = new Uint8Array(buffer);
+  } else if (data instanceof ArrayBuffer) {
+    bytes = new Uint8Array(data);
+  } else {
+    bytes = data;
+  }
+  const hash = await sha256.digest(bytes);
+  const cid = CID.create(1, raw.code, hash);
+  return cid.toString();
+}
+
+// src/operations/upload/engine.ts
+var PHASE_COUNT = 4;
+var PHASE_INDEX = {
+  "computing-cids": 0,
+  "creating": 1,
+  "backlinking": 2,
+  "uploading": 3,
+  "complete": 4,
+  "error": -1
+};
+async function parallelLimit(items, concurrency, fn) {
+  const results = [];
+  let index = 0;
+  async function worker() {
+    while (index < items.length) {
+      const currentIndex = index++;
+      const item = items[currentIndex];
+      results[currentIndex] = await fn(item, currentIndex);
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
+var TARGET_BYTES_IN_FLIGHT = 200 * 1024 * 1024;
+var BytePool = class {
+  constructor(targetBytes = TARGET_BYTES_IN_FLIGHT) {
+    this.targetBytes = targetBytes;
+    this.bytesInFlight = 0;
+    this.waitQueue = [];
+  }
+  async run(size, fn) {
+    while (this.bytesInFlight > 0 && this.bytesInFlight + size > this.targetBytes) {
+      await new Promise((resolve) => this.waitQueue.push(resolve));
+    }
+    this.bytesInFlight += size;
+    try {
+      return await fn();
+    } finally {
+      this.bytesInFlight -= size;
+      const next = this.waitQueue.shift();
+      if (next) next();
+    }
+  }
+};
+function getParentPath(relativePath) {
+  const lastSlash = relativePath.lastIndexOf("/");
+  if (lastSlash === -1) return null;
+  return relativePath.slice(0, lastSlash);
+}
+function groupFoldersByDepth(folders) {
+  const byDepth = /* @__PURE__ */ new Map();
+  for (const folder of folders) {
+    const depth = folder.relativePath.split("/").length - 1;
+    if (!byDepth.has(depth)) byDepth.set(depth, []);
+    byDepth.get(depth).push(folder);
+  }
+  return byDepth;
+}
+async function uploadTree(client, tree, options) {
+  const { target, onProgress, concurrency = 10, continueOnError = false, note } = options;
+  const errors = [];
+  const createdFolders = [];
+  const createdFiles = [];
+  const foldersByPath = /* @__PURE__ */ new Map();
+  const totalEntities = tree.files.length + tree.folders.length;
+  const totalBytes = tree.files.reduce((sum, f) => sum + f.size, 0);
+  let completedEntities = 0;
+  let bytesUploaded = 0;
+  const reportProgress = (progress) => {
+    if (onProgress) {
+      const phase = progress.phase || "computing-cids";
+      const phaseIndex = PHASE_INDEX[phase] ?? -1;
+      let phasePercent = 0;
+      if (phase === "computing-cids") {
+        const done = progress.completedEntities ?? completedEntities;
+        phasePercent = tree.files.length > 0 ? Math.round(done / tree.files.length * 100) : 100;
+      } else if (phase === "creating") {
+        const done = progress.completedEntities ?? completedEntities;
+        phasePercent = totalEntities > 0 ? Math.round(done / totalEntities * 100) : 100;
+      } else if (phase === "backlinking") {
+        const done = progress.completedParents ?? 0;
+        const total = progress.totalParents ?? 0;
+        phasePercent = total > 0 ? Math.round(done / total * 100) : 100;
+      } else if (phase === "uploading") {
+        const done = progress.bytesUploaded ?? bytesUploaded;
+        phasePercent = totalBytes > 0 ? Math.round(done / totalBytes * 100) : 100;
+      } else if (phase === "complete") {
+        phasePercent = 100;
+      }
+      onProgress({
+        phase,
+        phaseIndex,
+        phaseCount: PHASE_COUNT,
+        phasePercent,
+        totalEntities,
+        completedEntities,
+        totalParents: 0,
+        completedParents: 0,
+        totalBytes,
+        bytesUploaded,
+        ...progress
+      });
+    }
+  };
+  try {
+    let collectionId;
+    let collectionCid;
+    let collectionCreated = false;
+    if (target.createCollection) {
+      const collectionBody = {
+        label: target.createCollection.label,
+        description: target.createCollection.description,
+        roles: target.createCollection.roles,
+        note
+      };
+      const { data, error } = await client.api.POST("/collections", {
+        body: collectionBody
+      });
+      if (error || !data) {
+        throw new Error(`Failed to create collection: ${JSON.stringify(error)}`);
+      }
+      collectionId = data.id;
+      collectionCid = data.cid;
+      collectionCreated = true;
+    } else if (target.collectionId) {
+      collectionId = target.collectionId;
+      const { data, error } = await client.api.GET("/collections/{id}", {
+        params: { path: { id: collectionId } }
+      });
+      if (error || !data) {
+        throw new Error(`Failed to fetch collection: ${JSON.stringify(error)}`);
+      }
+      collectionCid = data.cid;
+    } else {
+      throw new Error("Must provide either collectionId or createCollection in target");
+    }
+    const rootParentId = target.parentId ?? collectionId;
+    reportProgress({ phase: "computing-cids", completedEntities: 0 });
+    const preparedFiles = [];
+    let cidProgress = 0;
+    await parallelLimit(tree.files, Math.max(concurrency, 20), async (file) => {
+      try {
+        const data = await file.getData();
+        const cid = await computeCid(data);
+        preparedFiles.push({ ...file, cid });
+        cidProgress++;
+        reportProgress({
+          phase: "computing-cids",
+          completedEntities: cidProgress,
+          currentItem: file.relativePath
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({ path: file.relativePath, error: `CID computation failed: ${errorMsg}` });
+        } else {
+          throw new Error(`Failed to compute CID for ${file.relativePath}: ${errorMsg}`);
+        }
+      }
+    });
+    reportProgress({ phase: "creating", completedEntities: 0 });
+    const foldersByDepth = groupFoldersByDepth(tree.folders);
+    const sortedDepths = [...foldersByDepth.keys()].sort((a, b) => a - b);
+    for (const depth of sortedDepths) {
+      const foldersAtDepth = foldersByDepth.get(depth);
+      await Promise.all(
+        foldersAtDepth.map(async (folder) => {
+          try {
+            const parentPath = getParentPath(folder.relativePath);
+            const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+            const parentType = parentPath ? "folder" : parentId === collectionId ? "collection" : "folder";
+            const folderBody = {
+              label: folder.name,
+              collection: collectionId,
+              note,
+              relationships: [{ predicate: "in", peer: parentId, peer_type: parentType }]
+            };
+            const { data, error } = await client.api.POST("/folders", {
+              body: folderBody
+            });
+            if (error || !data) {
+              throw new Error(JSON.stringify(error));
+            }
+            foldersByPath.set(folder.relativePath, { id: data.id, cid: data.cid });
+            createdFolders.push({
+              name: folder.name,
+              relativePath: folder.relativePath,
+              id: data.id,
+              entityCid: data.cid
+            });
+            completedEntities++;
+            reportProgress({
+              phase: "creating",
+              completedEntities,
+              currentItem: folder.relativePath
+            });
+          } catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            if (continueOnError) {
+              errors.push({ path: folder.relativePath, error: `Folder creation failed: ${errorMsg}` });
+              completedEntities++;
+            } else {
+              throw new Error(`Failed to create folder ${folder.relativePath}: ${errorMsg}`);
+            }
+          }
+        })
+      );
+    }
+    const FILE_CREATION_CONCURRENCY = 50;
+    await parallelLimit(preparedFiles, FILE_CREATION_CONCURRENCY, async (file) => {
+      try {
+        const parentPath = getParentPath(file.relativePath);
+        const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+        const parentType = parentPath ? "folder" : parentId === collectionId ? "collection" : "folder";
+        const fileBody = {
+          key: file.cid,
+          filename: file.name,
+          content_type: file.mimeType,
+          size: file.size,
+          cid: file.cid,
+          collection: collectionId,
+          relationships: [{ predicate: "in", peer: parentId, peer_type: parentType }]
+        };
+        const { data, error } = await client.api.POST("/files", {
+          body: fileBody
+        });
+        if (error || !data) {
+          throw new Error(`Entity creation failed: ${JSON.stringify(error)}`);
+        }
+        createdFiles.push({
+          ...file,
+          id: data.id,
+          entityCid: data.cid,
+          uploadUrl: data.upload_url,
+          uploadExpiresAt: data.upload_expires_at
+        });
+        completedEntities++;
+        reportProgress({
+          phase: "creating",
+          completedEntities,
+          currentItem: file.relativePath
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({ path: file.relativePath, error: errorMsg });
+          completedEntities++;
+        } else {
+          throw new Error(`Failed to create file ${file.relativePath}: ${errorMsg}`);
+        }
+      }
+    });
+    const childrenByParent = /* @__PURE__ */ new Map();
+    for (const folder of createdFolders) {
+      const parentPath = getParentPath(folder.relativePath);
+      const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+      if (!childrenByParent.has(parentId)) childrenByParent.set(parentId, []);
+      childrenByParent.get(parentId).push({ id: folder.id, type: "folder" });
+    }
+    for (const file of createdFiles) {
+      const parentPath = getParentPath(file.relativePath);
+      const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+      if (!childrenByParent.has(parentId)) childrenByParent.set(parentId, []);
+      childrenByParent.get(parentId).push({ id: file.id, type: "file" });
+    }
+    const totalParents = childrenByParent.size;
+    let completedParents = 0;
+    reportProgress({ phase: "backlinking", totalParents, completedParents: 0 });
+    const parentEntries = [...childrenByParent.entries()];
+    await parallelLimit(parentEntries, concurrency, async ([parentId, children]) => {
+      try {
+        const isCollection = parentId === collectionId;
+        const relationshipsAdd = children.map((child) => ({
+          predicate: "contains",
+          peer: child.id,
+          peer_type: child.type
+        }));
+        if (isCollection) {
+          const { data: collData, error: getError } = await client.api.GET("/collections/{id}", {
+            params: { path: { id: parentId } }
+          });
+          if (getError || !collData) {
+            throw new Error(`Failed to fetch collection: ${JSON.stringify(getError)}`);
+          }
+          const updateBody = {
+            expect_tip: collData.cid,
+            relationships_add: relationshipsAdd,
+            note: note ? `${note} (backlink)` : "Upload backlink"
+          };
+          const { error } = await client.api.PUT("/collections/{id}", {
+            params: { path: { id: parentId } },
+            body: updateBody
+          });
+          if (error) {
+            throw new Error(JSON.stringify(error));
+          }
+        } else {
+          const { data: folderData, error: getError } = await client.api.GET("/folders/{id}", {
+            params: { path: { id: parentId } }
+          });
+          if (getError || !folderData) {
+            throw new Error(`Failed to fetch folder: ${JSON.stringify(getError)}`);
+          }
+          const updateBody = {
+            expect_tip: folderData.cid,
+            relationships_add: relationshipsAdd,
+            note: note ? `${note} (backlink)` : "Upload backlink"
+          };
+          const { error } = await client.api.PUT("/folders/{id}", {
+            params: { path: { id: parentId } },
+            body: updateBody
+          });
+          if (error) {
+            throw new Error(JSON.stringify(error));
+          }
+        }
+        completedParents++;
+        reportProgress({
+          phase: "backlinking",
+          totalParents,
+          completedParents,
+          currentItem: `parent:${parentId}`
+        });
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        if (continueOnError) {
+          errors.push({ path: `parent:${parentId}`, error: `Backlink failed: ${errorMsg}` });
+          completedParents++;
+        } else {
+          throw new Error(`Failed to backlink parent ${parentId}: ${errorMsg}`);
+        }
+      }
+    });
+    reportProgress({ phase: "uploading", bytesUploaded: 0 });
+    const pool = new BytePool();
+    await Promise.all(
+      createdFiles.map(async (file) => {
+        await pool.run(file.size, async () => {
+          try {
+            const fileData = await file.getData();
+            let body;
+            if (fileData instanceof Blob) {
+              body = fileData;
+            } else if (fileData instanceof Uint8Array) {
+              const arrayBuffer = new ArrayBuffer(fileData.byteLength);
+              new Uint8Array(arrayBuffer).set(fileData);
+              body = new Blob([arrayBuffer], { type: file.mimeType });
+            } else {
+              body = new Blob([fileData], { type: file.mimeType });
+            }
+            const uploadResponse = await fetch(file.uploadUrl, {
+              method: "PUT",
+              body,
+              headers: { "Content-Type": file.mimeType }
+            });
+            if (!uploadResponse.ok) {
+              throw new Error(`S3 upload failed with status ${uploadResponse.status}`);
+            }
+            let confirmTip = file.entityCid;
+            let confirmAttempts = 0;
+            const MAX_CONFIRM_ATTEMPTS = 3;
+            while (confirmAttempts < MAX_CONFIRM_ATTEMPTS) {
+              confirmAttempts++;
+              const { error: confirmError } = await client.api.POST("/files/{id}/confirm-upload", {
+                params: { path: { id: file.id } },
+                body: {
+                  expect_tip: confirmTip,
+                  note: note ? `${note} (confirmed)` : "Upload confirmed"
+                }
+              });
+              if (!confirmError) {
+                break;
+              }
+              const errorStr = JSON.stringify(confirmError);
+              if (errorStr.includes("409") || errorStr.includes("CAS") || errorStr.includes("conflict")) {
+                const { data: currentFile, error: fetchError } = await client.api.GET("/files/{id}", {
+                  params: { path: { id: file.id } }
+                });
+                if (fetchError || !currentFile) {
+                  throw new Error(`Failed to fetch file for confirm retry: ${JSON.stringify(fetchError)}`);
+                }
+                confirmTip = currentFile.cid;
+              } else {
+                throw new Error(`Confirm upload failed: ${errorStr}`);
+              }
+            }
+            if (confirmAttempts >= MAX_CONFIRM_ATTEMPTS) {
+              throw new Error(`Confirm upload failed after ${MAX_CONFIRM_ATTEMPTS} CAS retries`);
+            }
+            bytesUploaded += file.size;
+            reportProgress({
+              phase: "uploading",
+              bytesUploaded,
+              currentItem: file.relativePath
+            });
+          } catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            if (continueOnError) {
+              errors.push({ path: file.relativePath, error: `Upload failed: ${errorMsg}` });
+            } else {
+              throw new Error(`Failed to upload ${file.relativePath}: ${errorMsg}`);
+            }
+          }
+        });
+      })
+    );
+    reportProgress({ phase: "complete", totalParents, completedParents, bytesUploaded });
+    const resultFolders = createdFolders.map((f) => ({
+      id: f.id,
+      cid: f.entityCid,
+      type: "folder",
+      relativePath: f.relativePath
+    }));
+    const resultFiles = createdFiles.map((f) => ({
+      id: f.id,
+      cid: f.entityCid,
+      type: "file",
+      relativePath: f.relativePath
+    }));
+    return {
+      success: errors.length === 0,
+      collection: {
+        id: collectionId,
+        cid: collectionCid,
+        created: collectionCreated
+      },
+      folders: resultFolders,
+      files: resultFiles,
+      errors
+    };
+  } catch (err) {
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    reportProgress({
+      phase: "error",
+      error: errorMsg
+    });
+    return {
+      success: false,
+      collection: {
+        id: target.collectionId ?? "",
+        cid: "",
+        created: false
+      },
+      folders: createdFolders.map((f) => ({
+        id: f.id,
+        cid: f.entityCid,
+        type: "folder",
+        relativePath: f.relativePath
+      })),
+      files: createdFiles.map((f) => ({
+        id: f.id,
+        cid: f.entityCid,
+        type: "file",
+        relativePath: f.relativePath
+      })),
+      errors: [...errors, { path: "", error: errorMsg }]
+    };
+  }
+}
+
+// src/operations/upload/scanners.ts
+function getMimeType(filename) {
+  const ext = filename.toLowerCase().split(".").pop() || "";
+  const mimeTypes = {
+    // Images
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    png: "image/png",
+    gif: "image/gif",
+    webp: "image/webp",
+    svg: "image/svg+xml",
+    ico: "image/x-icon",
+    bmp: "image/bmp",
+    tiff: "image/tiff",
+    tif: "image/tiff",
+    // Documents
+    pdf: "application/pdf",
+    doc: "application/msword",
+    docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    xls: "application/vnd.ms-excel",
+    xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ppt: "application/vnd.ms-powerpoint",
+    pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    odt: "application/vnd.oasis.opendocument.text",
+    ods: "application/vnd.oasis.opendocument.spreadsheet",
+    odp: "application/vnd.oasis.opendocument.presentation",
+    // Text
+    txt: "text/plain",
+    md: "text/markdown",
+    csv: "text/csv",
+    html: "text/html",
+    htm: "text/html",
+    css: "text/css",
+    xml: "text/xml",
+    rtf: "application/rtf",
+    // Code
+    js: "text/javascript",
+    mjs: "text/javascript",
+    ts: "text/typescript",
+    jsx: "text/javascript",
+    tsx: "text/typescript",
+    json: "application/json",
+    yaml: "text/yaml",
+    yml: "text/yaml",
+    // Archives
+    zip: "application/zip",
+    tar: "application/x-tar",
+    gz: "application/gzip",
+    rar: "application/vnd.rar",
+    "7z": "application/x-7z-compressed",
+    // Audio
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    ogg: "audio/ogg",
+    m4a: "audio/mp4",
+    flac: "audio/flac",
+    // Video
+    mp4: "video/mp4",
+    webm: "video/webm",
+    avi: "video/x-msvideo",
+    mov: "video/quicktime",
+    mkv: "video/x-matroska",
+    // Fonts
+    woff: "font/woff",
+    woff2: "font/woff2",
+    ttf: "font/ttf",
+    otf: "font/otf",
+    // Other
+    wasm: "application/wasm"
+  };
+  return mimeTypes[ext] || "application/octet-stream";
+}
+async function scanDirectory(directoryPath, options = {}) {
+  const fs = await import("fs/promises");
+  const path = await import("path");
+  const { ignore = ["node_modules", ".git", ".DS_Store"], includeHidden = false } = options;
+  const files = [];
+  const folders = [];
+  const rootName = path.basename(directoryPath);
+  async function scanDir(dirPath, relativePath) {
+    const entries = await fs.readdir(dirPath, { withFileTypes: true });
+    for (const entry of entries) {
+      const name = entry.name;
+      if (!includeHidden && name.startsWith(".")) {
+        continue;
+      }
+      if (ignore.some((pattern) => name === pattern || name.match(pattern))) {
+        continue;
+      }
+      const fullPath = path.join(dirPath, name);
+      const entryRelativePath = relativePath ? `${relativePath}/${name}` : name;
+      if (entry.isDirectory()) {
+        folders.push({
+          name,
+          relativePath: entryRelativePath
+        });
+        await scanDir(fullPath, entryRelativePath);
+      } else if (entry.isFile()) {
+        const stat = await fs.stat(fullPath);
+        files.push({
+          name,
+          relativePath: entryRelativePath,
+          size: stat.size,
+          mimeType: getMimeType(name),
+          getData: async () => {
+            const buffer = await fs.readFile(fullPath);
+            return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+          }
+        });
+      }
+    }
+  }
+  await scanDir(directoryPath, "");
+  folders.sort((a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length);
+  return { files, folders };
+}
+
 // src/operations/folders.ts
 var FolderOperations = class {
   constructor(client) {
@@ -158,15 +764,32 @@ var FolderOperations = class {
   /**
    * Upload a local directory to Arke
    *
-   * TODO: Implement this method
-   * Steps:
-   * 1. Scan directory structure
-   * 2. Create folder hierarchy (depth-first)
-   * 3. Upload files in parallel (with concurrency limit)
-   * 4. Create bidirectional relationships (folder contains file)
+   * @deprecated Use uploadTree and scanDirectory instead
    */
-  async uploadDirectory(_localPath, _options) {
-    throw new Error("FolderOperations.uploadDirectory is not yet implemented");
+  async uploadDirectory(localPath, options) {
+    const tree = await scanDirectory(localPath);
+    const result = await uploadTree(this.client, tree, {
+      target: {
+        collectionId: options.collectionId,
+        parentId: options.parentFolderId
+      },
+      concurrency: options.concurrency,
+      onProgress: options.onProgress ? (p) => {
+        options.onProgress({
+          phase: p.phase === "computing-cids" ? "creating-folders" : p.phase === "creating" ? "uploading-files" : p.phase === "backlinking" ? "linking" : p.phase === "complete" ? "complete" : "scanning",
+          totalFiles: p.totalEntities,
+          completedFiles: p.completedEntities,
+          totalFolders: p.totalParents,
+          completedFolders: p.completedParents,
+          currentFile: p.currentItem
+        });
+      } : void 0
+    });
+    return {
+      rootFolder: result.folders[0] || null,
+      folders: result.folders,
+      files: result.files
+    };
   }
 };
 
@@ -241,6 +864,8 @@ export {
   NotFoundError,
   ValidationError,
   createArkeClient,
+  getAuthorizationHeader,
+  isApiKey,
   parseApiError
 };
 //# sourceMappingURL=index.js.map