@rljson/fs-agent 0.0.16 → 0.0.18

package/dist/fs-agent.d.ts

@@ -14,6 +14,13 @@ export interface FsAgentOptions {
     maxDepth?: number;
     /** Follow symlinks (default: false) */
     followSymlinks?: boolean;
+    /**
+     * Persist a path→(mtime, size, blobId) scan cache at this file path so a RESTART
+     * does not re-read + re-hash the whole folder (a cold scan of an 80 GB catalog
+     * is ~48 min). Forwarded to the {@link FsScanner}. Requires a PERSISTENT blob
+     * store (e.g. `@rljson/bs-fs`). See {@link FsScanOptions.scanCachePath}.
+     */
+    scanCachePath?: string;
     /** Database instance for automatic syncing */
     db?: Db;
     /** Tree key for database storage */
@@ -126,6 +133,12 @@ export declare class FsAgent {
     private _stopSync?;
     private _stopSyncFromDb?;
     private _lastSentRef?;
+    /**
+     * The incoming ref most recently applied. Retired from the connector's dedup
+     * sets when the next one supersedes it, so a peer returning the tree to that
+     * state can still reach this agent.
+     */
+    private _lastAppliedRef?;
     /** Content fingerprint of the last tree we broadcasted (paths+blobIds) */
     private _lastSentContentKey?;
     private _timeouts;

package/dist/fs-agent.js

@@ -426,15 +426,25 @@ class FsScanner {
   _safetyTimer = null;
   /** Set by stopWatch() so a pending watcher reinstall / rescan bails out. */
   _stopRequested = false;
+  /** Path→content cache backing {@link FsScanOptions.scanCachePath} (unused when unset). */
+  _scanCachePath;
+  /** Last-scan cache (loaded from disk); consulted to skip re-read/re-hash. */
+  _blobCache = /* @__PURE__ */ new Map();
+  /** Cache rebuilt during the CURRENT scan; becomes `_blobCache` at the end (self-pruning). */
+  _nextBlobCache = /* @__PURE__ */ new Map();
+  /** Ensures the persisted cache is read from disk only once. */
+  _cacheLoaded = false;
   constructor(rootPath, options = {}) {
     this._rootPath = rootPath;
     this._options = {
       ignore: options.ignore || ["node_modules", ".git", "dist", "coverage"],
       maxDepth: options.maxDepth,
       followSymlinks: options.followSymlinks ?? false,
-      bs: options.bs
+      bs: options.bs,
+      scanCachePath: options.scanCachePath
     };
     this._bs = options.bs || new BsMem();
+    this._scanCachePath = options.scanCachePath;
   }
   get tree() {
     return this._tree;
@@ -463,6 +473,10 @@ class FsScanner {
         `Cannot access root path "${this._rootPath}": ${error instanceof Error ? error.message : String(error)}`
       );
     }
+    if (this._scanCachePath) {
+      await this._loadPersistedCache();
+      this._nextBlobCache = /* @__PURE__ */ new Map();
+    }
     const trees = /* @__PURE__ */ new Map();
     let rootTree;
     try {
@@ -484,6 +498,10 @@ class FsScanner {
       rootHash: rootHashStr,
       trees
     };
+    if (this._scanCachePath) {
+      this._blobCache = this._nextBlobCache;
+      await this._persistCache();
+    }
     return this._tree;
   }
   async _scanDirectory(absolutePath, relativePath, depth, trees) {
@@ -514,26 +532,41 @@ class FsScanner {
         trees.set(childHashStr, childTree);
         childRefs.push(childHashStr);
       } else if (entry.isFile()) {
-        let fileContent;
-        try {
-          fileContent = await readFile(childPath);
-        } catch (error) {
-          throw new Error(
-            `Failed to read file "${childRelPath}": ${error instanceof Error ? error.message : String(error)}`
-          );
-        }
-        let blobProps;
-        try {
-          blobProps = await this._bs.setBlob(fileContent);
-        } catch (error) {
-          throw new Error(
-            `Failed to store blob for file "${childRelPath}": ${error instanceof Error ? error.message : String(error)}`
-          );
+        const mtimeMs = childStats.mtime.getTime();
+        const cached = this._scanCachePath ? this._blobCache.get(childRelPath) : void 0;
+        let blobId;
+        if (cached && cached.mtime === mtimeMs && cached.size === childStats.size) {
+          blobId = cached.blobId;
+        } else {
+          let fileContent;
+          try {
+            fileContent = await readFile(childPath);
+          } catch (error) {
+            throw new Error(
+              `Failed to read file "${childRelPath}": ${error instanceof Error ? error.message : String(error)}`
+            );
+          }
+          let blobProps;
+          try {
+            blobProps = await this._bs.setBlob(fileContent);
+          } catch (error) {
+            throw new Error(
+              `Failed to store blob for file "${childRelPath}": ${error instanceof Error ? error.message : String(error)}`
+            );
+          }
+          if (!blobProps || !blobProps.blobId) {
+            throw new Error(
+              `Blob storage returned invalid blobId for file "${childRelPath}"`
+            );
+          }
+          blobId = blobProps.blobId;
         }
-        if (!blobProps || !blobProps.blobId) {
-          throw new Error(
-            `Blob storage returned invalid blobId for file "${childRelPath}"`
-          );
+        if (this._scanCachePath) {
+          this._nextBlobCache.set(childRelPath, {
+            mtime: mtimeMs,
+            size: childStats.size,
+            blobId
+          });
         }
         const fileMeta = {
           name: entry.name,
@@ -544,8 +577,8 @@ class FsScanner {
           // the same ref on every client) but NOT for directories (a folder's
           // mtime is per-machine and does not round-trip). The absolute `path`
           // is excluded everywhere — it is folder-specific.
-          mtime: childStats.mtime.getTime(),
-          blobId: blobProps.blobId
+          mtime: mtimeMs,
+          blobId
           // Link to content in Bs
         };
         const fileTree = {
@@ -577,6 +610,41 @@ class FsScanner {
     };
     return dirTree;
   }
+  /**
+   * Load the persisted scan cache from {@link FsScanOptions.scanCachePath} once.
+   * A missing or corrupt cache file is tolerated (the scan just starts cold).
+   */
+  async _loadPersistedCache() {
+    if (this._cacheLoaded) return;
+    this._cacheLoaded = true;
+    if (!this._scanCachePath) return;
+    try {
+      const raw = await readFile(this._scanCachePath, "utf8");
+      const parsed = JSON.parse(raw);
+      if (Array.isArray(parsed.entries)) {
+        this._blobCache = new Map(parsed.entries);
+      }
+    } catch {
+    }
+  }
+  /**
+   * Write the current scan cache to {@link FsScanOptions.scanCachePath} atomically
+   * (temp + rename). Best-effort: a failed cache write never fails the scan.
+   */
+  async _persistCache() {
+    if (!this._scanCachePath) return;
+    const body = JSON.stringify({
+      version: 1,
+      entries: [...this._blobCache.entries()]
+    });
+    const tmp = `${this._scanCachePath}.${Date.now().toString(36)}.tmp`;
+    try {
+      await mkdir(dirname(this._scanCachePath), { recursive: true });
+      await writeFile(tmp, body);
+      await rename(tmp, this._scanCachePath);
+    } catch {
+    }
+  }
   _shouldIgnore(name) {
     if (!this._options.ignore) {
       return false;
@@ -865,6 +933,12 @@ class FsAgent {
   _stopSync;
   _stopSyncFromDb;
   _lastSentRef;
+  /**
+   * The incoming ref most recently applied. Retired from the connector's dedup
+   * sets when the next one supersedes it, so a peer returning the tree to that
+   * state can still reach this agent.
+   */
+  _lastAppliedRef;
   /** Content fingerprint of the last tree we broadcasted (paths+blobIds) */
   _lastSentContentKey;
   _timeouts;
@@ -1051,6 +1125,7 @@ ${err.stack}` : String(err);
    *   via an explicit send, which pre-empts the Connector's db-observer path.
    */
   async _sendRef(connector, ref, predecessorRefs) {
+    connector.invalidateSent?.(ref);
     if (this._resolveConflicts) {
       connector.setPredecessors(predecessorRefs ?? []);
     }
@@ -1802,6 +1877,10 @@ ${err.stack}` : String(err);
             skipNotification: true,
             previous
           });
+          if (this._lastAppliedRef && this._lastAppliedRef !== treeRef) {
+            connector.invalidateSent?.(this._lastAppliedRef);
+          }
+          this._lastAppliedRef = treeRef;
           this._lastSentRef = postRestoreRef;
           this._lastSentContentKey = this._contentKeyFromTree(postRestoreTree);
           this._currentRef = postRestoreRef;

package/dist/fs-scanner.d.ts

@@ -67,6 +67,17 @@ export interface FsScanOptions {
     followSymlinks?: boolean;
     /** Blob storage implementation (defaults to BsMem) */
     bs?: Bs;
+    /**
+     * Persist a path→(mtime, size, blobId) scan cache at this file path. When set,
+     * a file whose mtime AND size are unchanged since the last scan is NOT re-read
+     * or re-hashed — its cached `blobId` is reused. The cache is loaded once on the
+     * first {@link FsScanner.scan} and re-written after every scan, so a RESTART
+     * does not re-read the whole folder (a cold scan of an 80 GB catalog is ~48
+     * min) and the periodic safety-rescan stays cheap. Requires a PERSISTENT blob
+     * store (e.g. `@rljson/bs-fs`) — with an in-RAM store the cached blob is gone
+     * after a restart. Omit to disable (default: full read+hash every scan).
+     */
+    scanCachePath?: string;
 }
 /**
  * Scans and watches file system changes, extracting RLJSON tree structure
@@ -84,12 +95,30 @@ export declare class FsScanner {
     private _safetyTimer;
     /** Set by stopWatch() so a pending watcher reinstall / rescan bails out. */
     private _stopRequested;
+    /** Path→content cache backing {@link FsScanOptions.scanCachePath} (unused when unset). */
+    private _scanCachePath?;
+    /** Last-scan cache (loaded from disk); consulted to skip re-read/re-hash. */
+    private _blobCache;
+    /** Cache rebuilt during the CURRENT scan; becomes `_blobCache` at the end (self-pruning). */
+    private _nextBlobCache;
+    /** Ensures the persisted cache is read from disk only once. */
+    private _cacheLoaded;
     constructor(rootPath: string, options?: FsScanOptions);
     get tree(): FsTree | null;
     get rootPath(): string;
     get bs(): Bs;
     scan(): Promise<FsTree>;
     private _scanDirectory;
+    /**
+     * Load the persisted scan cache from {@link FsScanOptions.scanCachePath} once.
+     * A missing or corrupt cache file is tolerated (the scan just starts cold).
+     */
+    private _loadPersistedCache;
+    /**
+     * Write the current scan cache to {@link FsScanOptions.scanCachePath} atomically
+     * (temp + rename). Best-effort: a failed cache write never fails the scan.
+     */
+    private _persistCache;
     private _shouldIgnore;
     watch(): Promise<void>;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rljson/fs-agent",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "description": "Rljson fs-agent description",
   "homepage": "https://github.com/rljson/fs-agent",
   "bugs": "https://github.com/rljson/fs-agent/issues",
@@ -28,7 +28,7 @@
     "updateGoldens": "cross-env UPDATE_GOLDENS=true pnpm test"
   },
   "devDependencies": {
-    "@rljson/server": "^0.0.10",
+    "@rljson/server": "^0.0.43",
     "@types/node": "^25.3.1",
     "@typescript-eslint/eslint-plugin": "^8.56.1",
     "@typescript-eslint/parser": "^8.56.1",
@@ -51,7 +51,7 @@
   },
   "dependencies": {
     "@rljson/bs": "^0.0.21",
-    "@rljson/db": "^0.0.23",
+    "@rljson/db": "^0.0.30",
     "@rljson/hash": "^0.0.18",
     "@rljson/io": "^0.0.66",
     "@rljson/json": "^0.0.23",