@rljson/fs-agent 0.0.35 → 0.0.37

package/dist/fs-agent.d.ts

@@ -157,6 +157,15 @@ export declare const AGENT_STATE_FILE = ".fsagent-state.json";
  * that a node joining an idle network is not left waiting.
  */
 export declare const REFUSAL_ANSWER_COOLDOWN_MS = 5000;
+/**
+ * How many tree nodes are fetched at once while walking a tree.
+ *
+ * The walk is latency-bound, so the whole point is to stop waiting for one node
+ * before asking for the next. Bounded because "the whole level at once" on a
+ * 184 000-file catalogue would be tens of thousands of simultaneous requests,
+ * which trades a latency problem for a queueing one.
+ */
+export declare const TREE_FETCH_CONCURRENCY = 64;
 export declare const MASS_DELETE_MIN_FILES = 100;
 /**
  * Above this share of the folder, a prune is treated as suspicious rather than

package/dist/fs-agent.js

@@ -1028,6 +1028,7 @@ const SYNC_ERROR_FILE = ".sync-errors.log";
 const ATOMIC_TMP_PREFIX = ".fsagent-tmp-";
 const AGENT_STATE_FILE = ".fsagent-state.json";
 const REFUSAL_ANSWER_COOLDOWN_MS = 5e3;
+const TREE_FETCH_CONCURRENCY = 64;
 const MASS_DELETE_MIN_FILES = 100;
 const MASS_DELETE_MAX_RATIO = 0.3;
 class RestoreIncompleteError extends Error {
@@ -1690,52 +1691,74 @@ ${err.stack}` : String(err);
    */
   async _fetchTreeRecursively(db, route, treeKey, rootHash) {
     const fetchedNodes = /* @__PURE__ */ new Map();
-    const nodesToFetch = /* @__PURE__ */ new Set([rootHash]);
-    const processed = /* @__PURE__ */ new Set();
-    while (nodesToFetch.size > 0) {
-      const currentHash = Array.from(nodesToFetch)[0];
-      nodesToFetch.delete(currentHash);
-      if (processed.has(currentHash)) {
-        continue;
-      }
-      processed.add(currentHash);
-      let result;
+    const seen = /* @__PURE__ */ new Set([rootHash]);
+    let frontier = [rootHash];
+    while (frontier.length > 0) {
+      const next = [];
+      const collect = (dataArray) => {
+        for (const node of dataArray) {
+          if (!node?._hash) continue;
+          fetchedNodes.set(node._hash, node);
+          if (node.children && Array.isArray(node.children)) {
+            for (const childHash of node.children) {
+              if (typeof childHash !== "string" || seen.has(childHash)) {
+                continue;
+              }
+              seen.add(childHash);
+              next.push(childHash);
+            }
+          }
+        }
+      };
+      let unresolved = frontier;
       try {
-        result = await FsAgent._withTimeout(
-          db.get(route, { _hash: currentHash }),
+        const rowsByHash = await FsAgent._withTimeout(
+          db.core.readRowsByHashes(treeKey, frontier),
           this._timeouts.dbQuery,
-          `db.get(${treeKey}, _hash=${currentHash.slice(0, 8)}…)`
+          `readRowsByHashes(${treeKey}, ${frontier.length})`
         );
-      } catch (error) {
-        if (error instanceof Error && error.message.startsWith("Timeout")) {
-          throw error;
+        if (rowsByHash.size > 0) {
+          collect(Array.from(rowsByHash.values()));
+          unresolved = frontier.filter((h) => !rowsByHash.has(h));
         }
-        const errMsg = error instanceof Error ? error.message : String(error);
-        console.warn(
-          `[FsAgent] _fetchTreeRecursively: db.get failed for hash=${currentHash.slice(0, 8)}…: ${errMsg}`
-        );
-        this._writeSyncError(
-          `fetchTree/db.get(${currentHash.slice(0, 8)}…)`,
-          error
-        );
-        continue;
-      }
-      const treeData = result?.rljson?.[treeKey];
-      if (!treeData || !treeData._data) {
-        continue;
+      } catch {
+        unresolved = frontier;
       }
-      const dataArray = Array.isArray(treeData._data) ? treeData._data : Object.values(treeData._data);
-      for (const node of dataArray) {
-        if (!node._hash) continue;
-        fetchedNodes.set(node._hash, node);
-        if (node.children && Array.isArray(node.children)) {
-          for (const childHash of node.children) {
-            if (typeof childHash === "string" && !processed.has(childHash)) {
-              nodesToFetch.add(childHash);
+      for (let i = 0; i < unresolved.length; i += TREE_FETCH_CONCURRENCY) {
+        const batch = unresolved.slice(i, i + TREE_FETCH_CONCURRENCY);
+        const results = await Promise.all(
+          batch.map(async (hash) => {
+            try {
+              return await FsAgent._withTimeout(
+                db.get(route, { _hash: hash }),
+                this._timeouts.dbQuery,
+                `db.get(${treeKey}, _hash=${hash.slice(0, 8)}…)`
+              );
+            } catch (error) {
+              if (error instanceof Error && error.message.startsWith("Timeout")) {
+                throw error;
+              }
+              const errMsg = error instanceof Error ? error.message : String(error);
+              console.warn(
+                `[FsAgent] _fetchTreeRecursively: db.get failed for hash=${hash.slice(0, 8)}…: ${errMsg}`
+              );
+              this._writeSyncError(
+                `fetchTree/db.get(${hash.slice(0, 8)}…)`,
+                error
+              );
+              return null;
             }
-          }
+          })
+        );
+        for (const result of results) {
+          const treeData = result?.rljson?.[treeKey];
+          if (!treeData || !treeData._data) continue;
+          collect(
+            Array.isArray(treeData._data) ? treeData._data : Object.values(treeData._data)
+          );
         }
       }
+      frontier = next;
     }
     return Array.from(fetchedNodes.values());
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rljson/fs-agent",
-  "version": "0.0.35",
+  "version": "0.0.37",
   "description": "Rljson fs-agent description",
   "homepage": "https://github.com/rljson/fs-agent",
   "bugs": "https://github.com/rljson/fs-agent/issues",