@arke-institute/sdk 2.1.0 → 2.3.0

package/dist/operations/index.cjs

@@ -68,6 +68,15 @@ async function verifyCid(data, expectedCid) {
 }
 
 // src/operations/upload/engine.ts
+var PHASE_COUNT = 4;
+var PHASE_INDEX = {
+  "computing-cids": 0,
+  "creating": 1,
+  "backlinking": 2,
+  "uploading": 3,
+  "complete": 4,
+  "error": -1
+};
 async function parallelLimit(items, concurrency, fn) {
   const results = [];
   let index = 0;
@@ -82,24 +91,83 @@ async function parallelLimit(items, concurrency, fn) {
   await Promise.all(workers);
   return results;
 }
+var TARGET_BYTES_IN_FLIGHT = 200 * 1024 * 1024;
+var BytePool = class {
+  constructor(targetBytes = TARGET_BYTES_IN_FLIGHT) {
+    this.targetBytes = targetBytes;
+    this.bytesInFlight = 0;
+    this.waitQueue = [];
+  }
+  async run(size, fn) {
+    while (this.bytesInFlight > 0 && this.bytesInFlight + size > this.targetBytes) {
+      await new Promise((resolve) => this.waitQueue.push(resolve));
+    }
+    this.bytesInFlight += size;
+    try {
+      return await fn();
+    } finally {
+      this.bytesInFlight -= size;
+      const next = this.waitQueue.shift();
+      if (next) next();
+    }
+  }
+};
 function getParentPath(relativePath) {
   const lastSlash = relativePath.lastIndexOf("/");
   if (lastSlash === -1) return null;
   return relativePath.slice(0, lastSlash);
 }
+function groupFoldersByDepth(folders) {
+  const byDepth = /* @__PURE__ */ new Map();
+  for (const folder of folders) {
+    const depth = folder.relativePath.split("/").length - 1;
+    if (!byDepth.has(depth)) byDepth.set(depth, []);
+    byDepth.get(depth).push(folder);
+  }
+  return byDepth;
+}
 async function uploadTree(client, tree, options) {
-  const { target, onProgress, concurrency = 5, continueOnError = false, note } = options;
+  const { target, onProgress, concurrency = 10, continueOnError = false, note } = options;
   const errors = [];
   const createdFolders = [];
   const createdFiles = [];
+  const foldersByPath = /* @__PURE__ */ new Map();
+  const totalEntities = tree.files.length + tree.folders.length;
+  const totalBytes = tree.files.reduce((sum, f) => sum + f.size, 0);
+  let completedEntities = 0;
+  let bytesUploaded = 0;
   const reportProgress = (progress) => {
     if (onProgress) {
+      const phase = progress.phase || "computing-cids";
+      const phaseIndex = PHASE_INDEX[phase] ?? -1;
+      let phasePercent = 0;
+      if (phase === "computing-cids") {
+        const done = progress.completedEntities ?? completedEntities;
+        phasePercent = tree.files.length > 0 ? Math.round(done / tree.files.length * 100) : 100;
+      } else if (phase === "creating") {
+        const done = progress.completedEntities ?? completedEntities;
+        phasePercent = totalEntities > 0 ? Math.round(done / totalEntities * 100) : 100;
+      } else if (phase === "backlinking") {
+        const done = progress.completedParents ?? 0;
+        const total = progress.totalParents ?? 0;
+        phasePercent = total > 0 ? Math.round(done / total * 100) : 100;
+      } else if (phase === "uploading") {
+        const done = progress.bytesUploaded ?? bytesUploaded;
+        phasePercent = totalBytes > 0 ? Math.round(done / totalBytes * 100) : 100;
+      } else if (phase === "complete") {
+        phasePercent = 100;
+      }
       onProgress({
-        phase: "scanning",
-        totalFiles: tree.files.length,
-        completedFiles: 0,
-        totalFolders: tree.folders.length,
-        completedFolders: 0,
+        phase,
+        phaseIndex,
+        phaseCount: PHASE_COUNT,
+        phasePercent,
+        totalEntities,
+        completedEntities,
+        totalParents: 0,
+        completedParents: 0,
+        totalBytes,
+        bytesUploaded,
         ...progress
       });
     }
@@ -109,7 +177,6 @@ async function uploadTree(client, tree, options) {
     let collectionCid;
     let collectionCreated = false;
     if (target.createCollection) {
-      reportProgress({ phase: "scanning", currentFolder: "Creating collection..." });
       const collectionBody = {
         label: target.createCollection.label,
         description: target.createCollection.description,
@@ -138,26 +205,19 @@ async function uploadTree(client, tree, options) {
       throw new Error("Must provide either collectionId or createCollection in target");
     }
     const rootParentId = target.parentId ?? collectionId;
-    reportProgress({
-      phase: "computing-cids",
-      totalFiles: tree.files.length,
-      completedFiles: 0
-    });
+    reportProgress({ phase: "computing-cids", completedEntities: 0 });
     const preparedFiles = [];
     let cidProgress = 0;
-    await parallelLimit(tree.files, concurrency, async (file) => {
+    await parallelLimit(tree.files, Math.max(concurrency, 20), async (file) => {
       try {
         const data = await file.getData();
         const cid = await computeCid(data);
-        preparedFiles.push({
-          ...file,
-          cid
-        });
+        preparedFiles.push({ ...file, cid });
         cidProgress++;
         reportProgress({
           phase: "computing-cids",
-          completedFiles: cidProgress,
-          currentFile: file.relativePath
+          completedEntities: cidProgress,
+          currentItem: file.relativePath
         });
       } catch (err) {
         const errorMsg = err instanceof Error ? err.message : String(err);
@@ -168,74 +228,74 @@ async function uploadTree(client, tree, options) {
         }
       }
     });
-    reportProgress({
-      phase: "creating-folders",
-      totalFolders: tree.folders.length,
-      completedFolders: 0
-    });
-    const sortedFolders = [...tree.folders].sort(
-      (a, b) => a.relativePath.split("/").length - b.relativePath.split("/").length
-    );
-    for (let i = 0; i < sortedFolders.length; i++) {
-      const folder = sortedFolders[i];
-      try {
-        const folderBody = {
-          label: folder.name,
-          collection: collectionId,
-          note
-        };
-        const { data, error } = await client.api.POST("/folders", {
-          body: folderBody
-        });
-        if (error || !data) {
-          throw new Error(JSON.stringify(error));
-        }
-        createdFolders.push({
-          name: folder.name,
-          relativePath: folder.relativePath,
-          id: data.id,
-          entityCid: data.cid
-        });
-        reportProgress({
-          phase: "creating-folders",
-          completedFolders: i + 1,
-          currentFolder: folder.relativePath
-        });
-      } catch (err) {
-        const errorMsg = err instanceof Error ? err.message : String(err);
-        if (continueOnError) {
-          errors.push({ path: folder.relativePath, error: `Folder creation failed: ${errorMsg}` });
-        } else {
-          throw new Error(`Failed to create folder ${folder.relativePath}: ${errorMsg}`);
-        }
-      }
-    }
-    const folderPathToEntity = /* @__PURE__ */ new Map();
-    for (const folder of createdFolders) {
-      folderPathToEntity.set(folder.relativePath, folder);
+    reportProgress({ phase: "creating", completedEntities: 0 });
+    const foldersByDepth = groupFoldersByDepth(tree.folders);
+    const sortedDepths = [...foldersByDepth.keys()].sort((a, b) => a - b);
+    for (const depth of sortedDepths) {
+      const foldersAtDepth = foldersByDepth.get(depth);
+      await Promise.all(
+        foldersAtDepth.map(async (folder) => {
+          try {
+            const parentPath = getParentPath(folder.relativePath);
+            const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+            const parentType = parentPath ? "folder" : parentId === collectionId ? "collection" : "folder";
+            const folderBody = {
+              label: folder.name,
+              collection: collectionId,
+              note,
+              relationships: [{ predicate: "in", peer: parentId, peer_type: parentType }]
+            };
+            const { data, error } = await client.api.POST("/folders", {
+              body: folderBody
+            });
+            if (error || !data) {
+              throw new Error(JSON.stringify(error));
+            }
+            foldersByPath.set(folder.relativePath, { id: data.id, cid: data.cid });
+            createdFolders.push({
+              name: folder.name,
+              relativePath: folder.relativePath,
+              id: data.id,
+              entityCid: data.cid
+            });
+            completedEntities++;
+            reportProgress({
+              phase: "creating",
+              completedEntities,
+              currentItem: folder.relativePath
+            });
+          } catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            if (continueOnError) {
+              errors.push({ path: folder.relativePath, error: `Folder creation failed: ${errorMsg}` });
+              completedEntities++;
+            } else {
+              throw new Error(`Failed to create folder ${folder.relativePath}: ${errorMsg}`);
+            }
+          }
+        })
+      );
     }
-    reportProgress({
-      phase: "creating-files",
-      totalFiles: preparedFiles.length,
-      completedFiles: 0
-    });
-    let fileCreateProgress = 0;
-    await parallelLimit(preparedFiles, concurrency, async (file) => {
+    const FILE_CREATION_CONCURRENCY = 50;
+    await parallelLimit(preparedFiles, FILE_CREATION_CONCURRENCY, async (file) => {
       try {
+        const parentPath = getParentPath(file.relativePath);
+        const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+        const parentType = parentPath ? "folder" : parentId === collectionId ? "collection" : "folder";
         const fileBody = {
           key: file.cid,
-          // Use CID as storage key (best practice)
           filename: file.name,
           content_type: file.mimeType,
           size: file.size,
           cid: file.cid,
-          collection: collectionId
+          collection: collectionId,
+          relationships: [{ predicate: "in", peer: parentId, peer_type: parentType }]
         };
         const { data, error } = await client.api.POST("/files", {
           body: fileBody
         });
         if (error || !data) {
-          throw new Error(JSON.stringify(error));
+          throw new Error(`Entity creation failed: ${JSON.stringify(error)}`);
         }
         createdFiles.push({
           ...file,
@@ -244,166 +304,177 @@ async function uploadTree(client, tree, options) {
           uploadUrl: data.upload_url,
           uploadExpiresAt: data.upload_expires_at
         });
-        fileCreateProgress++;
+        completedEntities++;
         reportProgress({
-          phase: "creating-files",
-          completedFiles: fileCreateProgress,
-          currentFile: file.relativePath
+          phase: "creating",
+          completedEntities,
+          currentItem: file.relativePath
         });
       } catch (err) {
         const errorMsg = err instanceof Error ? err.message : String(err);
         if (continueOnError) {
-          errors.push({ path: file.relativePath, error: `File creation failed: ${errorMsg}` });
+          errors.push({ path: file.relativePath, error: errorMsg });
+          completedEntities++;
         } else {
           throw new Error(`Failed to create file ${file.relativePath}: ${errorMsg}`);
         }
       }
     });
-    const totalBytes = createdFiles.reduce((sum, f) => sum + f.size, 0);
-    let bytesUploaded = 0;
-    reportProgress({
-      phase: "uploading-content",
-      totalFiles: createdFiles.length,
-      completedFiles: 0,
-      totalBytes,
-      bytesUploaded: 0
-    });
-    let uploadProgress = 0;
-    await parallelLimit(createdFiles, concurrency, async (file) => {
-      try {
-        const data = await file.getData();
-        let body;
-        if (data instanceof Blob) {
-          body = data;
-        } else if (data instanceof Uint8Array) {
-          const arrayBuffer = new ArrayBuffer(data.byteLength);
-          new Uint8Array(arrayBuffer).set(data);
-          body = new Blob([arrayBuffer], { type: file.mimeType });
-        } else {
-          body = new Blob([data], { type: file.mimeType });
-        }
-        const response = await fetch(file.uploadUrl, {
-          method: "PUT",
-          body,
-          headers: {
-            "Content-Type": file.mimeType
-          }
-        });
-        if (!response.ok) {
-          throw new Error(`Upload failed with status ${response.status}`);
-        }
-        bytesUploaded += file.size;
-        uploadProgress++;
-        reportProgress({
-          phase: "uploading-content",
-          completedFiles: uploadProgress,
-          currentFile: file.relativePath,
-          bytesUploaded,
-          totalBytes
-        });
-      } catch (err) {
-        const errorMsg = err instanceof Error ? err.message : String(err);
-        if (continueOnError) {
-          errors.push({ path: file.relativePath, error: `Upload failed: ${errorMsg}` });
-        } else {
-          throw new Error(`Failed to upload ${file.relativePath}: ${errorMsg}`);
-        }
-      }
-    });
-    reportProgress({ phase: "linking" });
-    const filePathToEntity = /* @__PURE__ */ new Map();
-    for (const file of createdFiles) {
-      filePathToEntity.set(file.relativePath, file);
-    }
-    const parentGroups = /* @__PURE__ */ new Map();
+    const childrenByParent = /* @__PURE__ */ new Map();
     for (const folder of createdFolders) {
       const parentPath = getParentPath(folder.relativePath);
-      let parentId;
-      if (parentPath === null) {
-        parentId = rootParentId;
-      } else {
-        const parentFolder = folderPathToEntity.get(parentPath);
-        if (!parentFolder) {
-          errors.push({
-            path: folder.relativePath,
-            error: `Parent folder not found: ${parentPath}`
-          });
-          continue;
-        }
-        parentId = parentFolder.id;
-      }
-      if (!parentGroups.has(parentId)) {
-        parentGroups.set(parentId, { folderId: parentId, children: [] });
-      }
-      parentGroups.get(parentId).children.push({ id: folder.id });
+      const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+      if (!childrenByParent.has(parentId)) childrenByParent.set(parentId, []);
+      childrenByParent.get(parentId).push({ id: folder.id, type: "folder" });
     }
     for (const file of createdFiles) {
       const parentPath = getParentPath(file.relativePath);
-      let parentId;
-      if (parentPath === null) {
-        parentId = rootParentId;
-      } else {
-        const parentFolder = folderPathToEntity.get(parentPath);
-        if (!parentFolder) {
-          errors.push({
-            path: file.relativePath,
-            error: `Parent folder not found: ${parentPath}`
-          });
-          continue;
-        }
-        parentId = parentFolder.id;
-      }
-      if (!parentGroups.has(parentId)) {
-        parentGroups.set(parentId, { folderId: parentId, children: [] });
-      }
-      parentGroups.get(parentId).children.push({ id: file.id });
+      const parentId = parentPath ? foldersByPath.get(parentPath).id : rootParentId;
+      if (!childrenByParent.has(parentId)) childrenByParent.set(parentId, []);
+      childrenByParent.get(parentId).push({ id: file.id, type: "file" });
     }
-    for (const [parentId, group] of parentGroups) {
-      if (group.children.length === 0) continue;
+    const totalParents = childrenByParent.size;
+    let completedParents = 0;
+    reportProgress({ phase: "backlinking", totalParents, completedParents: 0 });
+    const parentEntries = [...childrenByParent.entries()];
+    await parallelLimit(parentEntries, concurrency, async ([parentId, children]) => {
       try {
-        let expectTip;
-        if (parentId === collectionId) {
-          const { data, error: error2 } = await client.api.GET("/collections/{id}", {
-            params: { path: { id: collectionId } }
+        const isCollection = parentId === collectionId;
+        const relationshipsAdd = children.map((child) => ({
+          predicate: "contains",
+          peer: child.id,
+          peer_type: child.type
+        }));
+        if (isCollection) {
+          const { data: collData, error: getError } = await client.api.GET("/collections/{id}", {
+            params: { path: { id: parentId } }
+          });
+          if (getError || !collData) {
+            throw new Error(`Failed to fetch collection: ${JSON.stringify(getError)}`);
+          }
+          const updateBody = {
+            expect_tip: collData.cid,
+            relationships_add: relationshipsAdd,
+            note: note ? `${note} (backlink)` : "Upload backlink"
+          };
+          const { error } = await client.api.PUT("/collections/{id}", {
+            params: { path: { id: parentId } },
+            body: updateBody
           });
-          if (error2 || !data) {
-            throw new Error(`Failed to fetch collection CID: ${JSON.stringify(error2)}`);
+          if (error) {
+            throw new Error(JSON.stringify(error));
           }
-          expectTip = data.cid;
         } else {
-          const { data, error: error2 } = await client.api.GET("/folders/{id}", {
+          const { data: folderData, error: getError } = await client.api.GET("/folders/{id}", {
             params: { path: { id: parentId } }
           });
-          if (error2 || !data) {
-            throw new Error(`Failed to fetch folder CID: ${JSON.stringify(error2)}`);
+          if (getError || !folderData) {
+            throw new Error(`Failed to fetch folder: ${JSON.stringify(getError)}`);
+          }
+          const updateBody = {
+            expect_tip: folderData.cid,
+            relationships_add: relationshipsAdd,
+            note: note ? `${note} (backlink)` : "Upload backlink"
+          };
+          const { error } = await client.api.PUT("/folders/{id}", {
+            params: { path: { id: parentId } },
+            body: updateBody
+          });
+          if (error) {
+            throw new Error(JSON.stringify(error));
           }
-          expectTip = data.cid;
         }
-        const bulkBody = {
-          expect_tip: expectTip,
-          children: group.children,
-          note
-        };
-        const { error } = await client.api.POST("/folders/{id}/children/bulk", {
-          params: { path: { id: parentId } },
-          body: bulkBody
+        completedParents++;
+        reportProgress({
+          phase: "backlinking",
+          totalParents,
+          completedParents,
+          currentItem: `parent:${parentId}`
         });
-        if (error) {
-          throw new Error(JSON.stringify(error));
-        }
       } catch (err) {
         const errorMsg = err instanceof Error ? err.message : String(err);
         if (continueOnError) {
-          errors.push({
-            path: `parent:${parentId}`,
-            error: `Bulk linking failed: ${errorMsg}`
-          });
+          errors.push({ path: `parent:${parentId}`, error: `Backlink failed: ${errorMsg}` });
+          completedParents++;
         } else {
-          throw new Error(`Failed to link children to ${parentId}: ${errorMsg}`);
+          throw new Error(`Failed to backlink parent ${parentId}: ${errorMsg}`);
         }
       }
-    }
-    reportProgress({ phase: "complete" });
+    });
+    reportProgress({ phase: "uploading", bytesUploaded: 0 });
+    const pool = new BytePool();
+    await Promise.all(
+      createdFiles.map(async (file) => {
+        await pool.run(file.size, async () => {
+          try {
+            const fileData = await file.getData();
+            let body;
+            if (fileData instanceof Blob) {
+              body = fileData;
+            } else if (fileData instanceof Uint8Array) {
+              const arrayBuffer = new ArrayBuffer(fileData.byteLength);
+              new Uint8Array(arrayBuffer).set(fileData);
+              body = new Blob([arrayBuffer], { type: file.mimeType });
+            } else {
+              body = new Blob([fileData], { type: file.mimeType });
+            }
+            const uploadResponse = await fetch(file.uploadUrl, {
+              method: "PUT",
+              body,
+              headers: { "Content-Type": file.mimeType }
+            });
+            if (!uploadResponse.ok) {
+              throw new Error(`S3 upload failed with status ${uploadResponse.status}`);
+            }
+            let confirmTip = file.entityCid;
+            let confirmAttempts = 0;
+            const MAX_CONFIRM_ATTEMPTS = 3;
+            while (confirmAttempts < MAX_CONFIRM_ATTEMPTS) {
+              confirmAttempts++;
+              const { error: confirmError } = await client.api.POST("/files/{id}/confirm-upload", {
+                params: { path: { id: file.id } },
+                body: {
+                  expect_tip: confirmTip,
+                  note: note ? `${note} (confirmed)` : "Upload confirmed"
+                }
+              });
+              if (!confirmError) {
+                break;
+              }
+              const errorStr = JSON.stringify(confirmError);
+              if (errorStr.includes("409") || errorStr.includes("CAS") || errorStr.includes("conflict")) {
+                const { data: currentFile, error: fetchError } = await client.api.GET("/files/{id}", {
+                  params: { path: { id: file.id } }
+                });
+                if (fetchError || !currentFile) {
+                  throw new Error(`Failed to fetch file for confirm retry: ${JSON.stringify(fetchError)}`);
+                }
+                confirmTip = currentFile.cid;
+              } else {
+                throw new Error(`Confirm upload failed: ${errorStr}`);
+              }
+            }
+            if (confirmAttempts >= MAX_CONFIRM_ATTEMPTS) {
+              throw new Error(`Confirm upload failed after ${MAX_CONFIRM_ATTEMPTS} CAS retries`);
+            }
+            bytesUploaded += file.size;
+            reportProgress({
+              phase: "uploading",
+              bytesUploaded,
+              currentItem: file.relativePath
+            });
+          } catch (err) {
+            const errorMsg = err instanceof Error ? err.message : String(err);
+            if (continueOnError) {
+              errors.push({ path: file.relativePath, error: `Upload failed: ${errorMsg}` });
+            } else {
+              throw new Error(`Failed to upload ${file.relativePath}: ${errorMsg}`);
+            }
+          }
+        });
+      })
+    );
+    reportProgress({ phase: "complete", totalParents, completedParents, bytesUploaded });
     const resultFolders = createdFolders.map((f) => ({
       id: f.id,
       cid: f.entityCid,
@@ -714,12 +785,12 @@ var FolderOperations = class {
       concurrency: options.concurrency,
       onProgress: options.onProgress ? (p) => {
         options.onProgress({
-          phase: p.phase === "computing-cids" || p.phase === "creating-folders" ? "creating-folders" : p.phase === "creating-files" || p.phase === "uploading-content" ? "uploading-files" : p.phase === "linking" ? "linking" : p.phase === "complete" ? "complete" : "scanning",
-          totalFiles: p.totalFiles,
-          completedFiles: p.completedFiles,
-          totalFolders: p.totalFolders,
-          completedFolders: p.completedFolders,
-          currentFile: p.currentFile
+          phase: p.phase === "computing-cids" ? "creating-folders" : p.phase === "creating" ? "uploading-files" : p.phase === "backlinking" ? "linking" : p.phase === "complete" ? "complete" : "scanning",
+          totalFiles: p.totalEntities,
+          completedFiles: p.completedEntities,
+          totalFolders: p.totalParents,
+          completedFolders: p.completedParents,
+          currentFile: p.currentItem
         });
       } : void 0
     });