@rljson/fs-agent 0.0.57 → 0.0.59

package/dist/fs-agent.d.ts

@@ -176,6 +176,20 @@ export declare const TREE_FETCH_CONCURRENCY = 64;
  * which is the cheap half of the trade.
  */
 export declare const RESTORED_BLOB_MEMORY_MAX = 50000;
+/**
+ * How many tree children a restore works on at once.
+ *
+ * A restore's cost is dominated by `getBlob`, and a blob this node does not
+ * already hold is a socket round trip. Walking the tree strictly sequentially
+ * makes the whole restore `files × RTT` — invisible on localhost, and about
+ * sixteen files a second on the lab, where a 1 200-file folder then takes
+ * minutes and reads as a stalled node.
+ *
+ * Sixteen rather than sixty-four: blobs carry file CONTENT, so the ceiling is
+ * memory in flight rather than request count, and a folder of large files at
+ * high concurrency is how a client runs out of heap.
+ */
+export declare const RESTORE_FETCH_CONCURRENCY = 16;
 /**
  * What an agent concluded about an inbound ref.
  *
@@ -276,6 +290,14 @@ export declare class FsAgent {
     /** Files written vs left alone by the current {@link restore}. */
     private _restoreWritten;
     private _restoreSkipped;
+    /**
+     * Absolute paths of the files this agent has actually TOLD anyone about.
+     *
+     * A prune may only remove a file that peers could know exists. Anything
+     * written since the last announcement is invisible to every sender, so a
+     * tree that lacks it is not deleting it — it simply predates it.
+     */
+    private readonly _announcedFiles;
     /**
      * Files this restore DELETED.
      *
@@ -724,6 +746,14 @@ export declare class FsAgent {
      * @returns `apply`, or the reason it is not news.
      */
     private _inboundRefVerdict;
+    /**
+     * Records the files an announcement carried.
+     *
+     * Called wherever this agent commits to a state as "what peers know" —
+     * its own pushes, and the states it adopts from theirs.
+     * @param tree - The tree that just went out, or was adopted as ours.
+     */
+    private _rememberAnnounced;
     private _adoptAppliedRef;
     /**
      * Derives a deterministic content key from an FsTree.

package/dist/fs-agent.js

@@ -1079,6 +1079,7 @@ const AGENT_STATE_FILE = ".fsagent-state.json";
 const REFUSAL_ANSWER_COOLDOWN_MS = 5e3;
 const TREE_FETCH_CONCURRENCY = 64;
 const RESTORED_BLOB_MEMORY_MAX = 5e4;
+const RESTORE_FETCH_CONCURRENCY = 16;
 const VERDICT_REASON = {
   "own-echo": "is this agent's own last advertisement echoed back",
   stale: "is not the newest its sender has advertised"
@@ -1152,6 +1153,14 @@ class FsAgent {
   /** Files written vs left alone by the current {@link restore}. */
   _restoreWritten = 0;
   _restoreSkipped = 0;
+  /**
+   * Absolute paths of the files this agent has actually TOLD anyone about.
+   *
+   * A prune may only remove a file that peers could know exists. Anything
+   * written since the last announcement is invisible to every sender, so a
+   * tree that lacks it is not deleting it — it simply predates it.
+   */
+  _announcedFiles = /* @__PURE__ */ new Set();
   /**
    * Files this restore DELETED.
    *
@@ -1629,9 +1638,26 @@ ${err.stack}` : String(err);
       const dirPath = meta.relativePath === "." ? targetPath : join(targetPath, meta.relativePath);
       await mkdir(dirPath, { recursive: true });
       if (treeNode.children && Array.isArray(treeNode.children)) {
-        for (const childHash of treeNode.children) {
-          await this._restoreTree(childHash, trees, targetPath, isOwnRoot);
-        }
+        const children = [...treeNode.children];
+        let next = 0;
+        const worker = async () => {
+          for (; ; ) {
+            const index = next++;
+            if (index >= children.length) return;
+            await this._restoreTree(
+              children[index],
+              trees,
+              targetPath,
+              isOwnRoot
+            );
+          }
+        };
+        await Promise.all(
+          Array.from(
+            { length: Math.min(RESTORE_FETCH_CONCURRENCY, children.length) },
+            worker
+          )
+        );
       }
     }
   }
@@ -1939,6 +1965,9 @@ ${err.stack}` : String(err);
           }
         }
       } else if (!expectedFiles.has(fullPath) && preRestore.has(fullPath)) {
+        if (this._announcedFiles.size > 0 && !this._announcedFiles.has(fullPath)) {
+          continue;
+        }
         await rm(fullPath, { force: true });
         this._restorePruned++;
       }
@@ -1988,12 +2017,14 @@ ${err.stack}` : String(err);
       this._currentRef = initialRef;
       this._persistCurrentRef(initialRef);
       this._lastSentContentKey = this._contentKeyFromTree(initialTree);
+      this._rememberAnnounced(initialTree);
     }
     if (initialRef && !isSilentJoiner) {
       this._lastSentRef = initialRef;
       this._currentRef = initialRef;
       this._persistCurrentRef(initialRef);
       this._lastSentContentKey = this._contentKeyFromTree(initialTree);
+      this._rememberAnnounced(initialTree);
       await this._sendRef(
         connector,
         initialRef,
@@ -2073,6 +2104,7 @@ ${err.stack}` : String(err);
             }
             this._lastSentRef = ref;
             this._lastSentContentKey = contentKey;
+            this._rememberAnnounced(tree);
             if (parentRef && parentRef !== ref) {
               connector.invalidateReceived(parentRef);
             }
@@ -2339,6 +2371,19 @@ ${err.stack}` : String(err);
     if (!isNewestFromSender) return "stale";
     return "apply";
   }
+  /**
+   * Records the files an announcement carried.
+   *
+   * Called wherever this agent commits to a state as "what peers know" —
+   * its own pushes, and the states it adopts from theirs.
+   * @param tree - The tree that just went out, or was adopted as ours.
+   */
+  _rememberAnnounced(tree) {
+    this._announcedFiles.clear();
+    for (const path of this._getFileContentMap(tree).keys()) {
+      this._announcedFiles.add(join(this._rootPath, path));
+    }
+  }
   _adoptAppliedRef(connector, treeRef) {
     if (this._lastAppliedRef && this._lastAppliedRef !== treeRef) {
       connector.invalidateSent?.(this._lastAppliedRef);
@@ -2408,6 +2453,7 @@ ${err.stack}` : String(err);
         const ref = await dbAdapter.storeFsTree(tree, { previous });
         this._lastSentRef = ref;
         this._lastSentContentKey = this._contentKeyFromTree(tree);
+        this._rememberAnnounced(tree);
         this._currentRef = ref;
         return ref;
       }
@@ -2471,6 +2517,7 @@ ${err.stack}` : String(err);
             this._currentRef = treeRef;
             this._persistCurrentRef(treeRef);
             this._lastSentContentKey = this._contentKeyFromTree(currentTree);
+            this._rememberAnnounced(currentTree);
             return;
           }
           if (this._resolveConflicts && this._currentRef && predecessorRefs && predecessorRefs.length > 0) {
@@ -2545,6 +2592,7 @@ ${err.stack}` : String(err);
           if (!hasNewsOfOurOwn) {
             this._lastSentRef = postRestoreRef;
             this._lastSentContentKey = this._contentKeyFromTree(postRestoreTree);
+            this._rememberAnnounced(postRestoreTree);
             if (postRestoreFiles.size < incomingFiles.size) {
               console.log(
                 `[FsAgent] ref=${treeRef.slice(0, 8)}… left this node behind (${postRestoreFiles.size} of ${incomingFiles.size} files) — not announcing a state that is catching up.`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rljson/fs-agent",
-  "version": "0.0.57",
+  "version": "0.0.59",
   "description": "Rljson fs-agent description",
   "homepage": "https://github.com/rljson/fs-agent",
   "bugs": "https://github.com/rljson/fs-agent/issues",