@remnic/core 9.3.517 → 9.3.518

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "9.3.517",
+  "version": "9.3.518",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",

package/src/offline-sync.test.ts

@@ -211,6 +211,8 @@ test("offline sync includes live LCM sqlite artifacts for full-fidelity offline
       "state/lcm.sqlite-shm",
       "state/lcm.sqlite-wal",
     ]);
+    assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/lcm.sqlite-shm"), true);
+    assert.equal(shouldPreferIncomingOfflineRuntimeFile("state/lcm.sqlite-wal"), true);
     const focused = await buildOfflineSyncSnapshotForPaths({
       root,
       sourceId: "remote",
@@ -233,6 +235,14 @@ test("offline sync includes live LCM sqlite artifacts for full-fidelity offline
 
     assert.equal(pull.skipped, 4);
     assert.equal(await readUtf8(root, "state/lcm.sqlite"), "live db");
+
+    await write(root, "state/lcm.sqlite-wal", "local wal churn");
+    const changeset = await buildOfflineSyncChangeset({
+      root,
+      sourceId: "laptop",
+      baseFiles: snapshot.files,
+    });
+    assert.deepEqual(changeset.changes.map((change) => change.path), []);
   } finally {
     await rm(root, { recursive: true, force: true });
   }
@@ -1269,6 +1279,43 @@ test("offline snapshot apply preserves new local runtime files without a base",
   }
 });
 
+test("offline snapshot apply removes current-only sqlite sidecars when absent remotely", async () => {
+  const root = await tempDir("remnic-offline-runtime-sidecar-local-create");
+  try {
+    await write(root, "state/lcm.sqlite-wal", "stale local wal");
+    await write(root, "state/lcm.sqlite-shm", "stale local shm");
+
+    const pull = await applyOfflineSyncSnapshot({
+      root,
+      snapshot: {
+        format: "remnic.offline-sync.snapshot.v1",
+        schemaVersion: 1,
+        createdAt: new Date().toISOString(),
+        sourceId: "remote",
+        includeTranscripts: true,
+        files: [],
+      },
+      baseFiles: [],
+    });
+
+    assert.equal(pull.deleted, 2);
+    assert.equal(pull.pendingLocal, 0);
+    assert.equal(pull.skipped, 0);
+    assert.equal(pull.conflicts.length, 0);
+    assert.deepEqual(pull.nextBaseFiles, []);
+    await assert.rejects(
+      () => readFile(path.join(root, "state/lcm.sqlite-wal")),
+      /ENOENT/,
+    );
+    await assert.rejects(
+      () => readFile(path.join(root, "state/lcm.sqlite-shm")),
+      /ENOENT/,
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
 test("offline snapshot apply restores locally deleted remote-authoritative runtime files", async () => {
   const root = await tempDir("remnic-offline-runtime-local-delete-restore");
   try {
@@ -2222,6 +2269,43 @@ test("offline sync applies and snapshots through secure storage hooks", async ()
   }
 });
 
+test("offline storage digest cache avoids repeat reads and invalidates on ctime changes", async () => {
+  const root = await tempDir("remnic-offline-digest-cache");
+  const relPath = "facts/cache.md";
+  const filePath = path.join(root, relPath);
+  try {
+    await write(root, relPath, "alpha");
+    const storage = new StorageManager(root);
+    const first = await storage.digestOfflineSyncFile(filePath);
+
+    let headerProbes = 0;
+    (storage as unknown as { offlineSyncFileIsEncrypted: () => Promise<boolean> }).offlineSyncFileIsEncrypted =
+      async () => {
+        headerProbes += 1;
+        throw new Error("cache miss");
+      };
+    assert.deepEqual(await storage.digestOfflineSyncFile(filePath), first);
+    assert.equal(headerProbes, 0);
+
+    const before = await stat(filePath);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+    await writeFile(filePath, "bravo");
+    await utimes(filePath, before.atime, before.mtime);
+
+    (storage as unknown as { offlineSyncFileIsEncrypted: () => Promise<boolean> }).offlineSyncFileIsEncrypted =
+      async () => {
+        headerProbes += 1;
+        return false;
+      };
+    const second = await storage.digestOfflineSyncFile(filePath);
+    assert.equal(headerProbes, 1);
+    assert.notEqual(second.sha256, first.sha256);
+    assert.equal(second.bytes, first.bytes);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
 test("offline snapshot fast-base does not probe file headers for trusted metadata", async () => {
   const root = await tempDir("remnic-offline-fast-base-no-header-probe");
   const relPath = "facts/same-size.md";

package/src/offline-sync.ts

@@ -489,16 +489,29 @@ const REMOTE_AUTHORITATIVE_RUNTIME_STATE_FILES = new Set([
   "index_time.json",
   "last_intent.json",
   "last_recall.json",
+  "lcm.sqlite-shm",
+  "lcm.sqlite-wal",
   "memory-lifecycle-ledger.jsonl",
   "recall_impressions.jsonl",
 ]);
 
+const ABSENT_INCOMING_RUNTIME_DELETE_FILES = new Set([
+  "lcm.sqlite-shm",
+  "lcm.sqlite-wal",
+]);
+
 export function shouldPreferIncomingOfflineRuntimeFile(relPosix: string): boolean {
   const parts = relPosix.split("/");
   const basename = parts[parts.length - 1] ?? "";
   return isCanonicalRuntimeStatePath(parts) && REMOTE_AUTHORITATIVE_RUNTIME_STATE_FILES.has(basename);
 }
 
+function shouldDeleteAbsentIncomingOfflineRuntimeFile(relPosix: string): boolean {
+  const parts = relPosix.split("/");
+  const basename = parts[parts.length - 1] ?? "";
+  return isCanonicalRuntimeStatePath(parts) && ABSENT_INCOMING_RUNTIME_DELETE_FILES.has(basename);
+}
+
 function filterBaseFilesForMode(
   files: readonly OfflineSyncFileState[],
   includeTranscripts: boolean,
@@ -1213,7 +1226,10 @@ export async function applyOfflineSyncSnapshot(options: {
       skipped += 1;
       continue;
     }
-    if (shouldPreferIncomingOfflineRuntimeFile(relPath) && base) {
+    if (
+      shouldPreferIncomingOfflineRuntimeFile(relPath) &&
+      (base || shouldDeleteAbsentIncomingOfflineRuntimeFile(relPath))
+    ) {
       await deleteSafeFile(root, relPath, options.deleteFile);
       nextBase.delete(relPath);
       deleted += 1;

package/src/storage.ts

@@ -135,6 +135,15 @@ const ARTIFACT_SEARCH_STOPWORDS = new Set([
   "with",
 ]);
 
+type OfflineSyncDigestCacheEntry = {
+  statBytes: number;
+  mtimeMs: number;
+  ctimeMs: number;
+  encrypted: boolean;
+  sha256: string;
+  bytes: number;
+};
+
 export interface ReextractJobRequest {
   memoryId: string;
   model: string;
@@ -2236,6 +2245,10 @@ export class StorageManager {
   private factHashIndexAuthoritative: boolean | null = null;
   private factHashIndexAuthoritativePromise: Promise<void> | null = null;
   private readonly secureAppendChains = new Map<string, Promise<void>>();
+  private offlineSyncDigestCache: Map<string, OfflineSyncDigestCacheEntry> | null = null;
+  private offlineSyncDigestCacheLoadPromise: Promise<Map<string, OfflineSyncDigestCacheEntry>> | null = null;
+  private offlineSyncDigestCacheWriteChain: Promise<void> = Promise.resolve();
+  private offlineSyncDigestCacheWriteTimer: ReturnType<typeof setTimeout> | null = null;
   /** Optional: set by the orchestrator after construction to enable template-aware citation stripping during legacy hash rebuild. */
   citationTemplate: string = DEFAULT_CITATION_FORMAT;
 
@@ -2516,24 +2529,152 @@ export class StorageManager {
 
   async digestOfflineSyncFile(filePath: string): Promise<{ sha256: string; bytes: number }> {
     const target = this.assertManagedStoragePath(filePath, "storage.digestOfflineSyncFile");
-    if (await this.offlineSyncFileIsEncrypted(target)) {
-      const content = await readMaybeEncryptedFileBuffer(target, this._secureStoreKey, this.baseDir);
+    const st = await stat(target);
+    const relPath = path.relative(this.baseDir, target).split(path.sep).join("/");
+    const cache = await this.loadOfflineSyncDigestCache();
+    const cached = cache.get(relPath);
+    if (
+      cached &&
+      cached.statBytes === st.size &&
+      cached.mtimeMs === st.mtimeMs &&
+      cached.ctimeMs === st.ctimeMs &&
+      !cached.encrypted
+    ) {
       return {
+        sha256: cached.sha256,
+        bytes: cached.bytes,
+      };
+    }
+
+    const encrypted = await this.offlineSyncFileIsEncrypted(target);
+    let digest: { sha256: string; bytes: number };
+    if (encrypted) {
+      const content = await readMaybeEncryptedFileBuffer(target, this._secureStoreKey, this.baseDir);
+      digest = {
         sha256: createHash("sha256").update(content).digest("hex"),
         bytes: content.byteLength,
       };
+    } else {
+      const hash = createHash("sha256");
+      let bytes = 0;
+      for await (const rawChunk of createReadStream(target)) {
+        const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
+        hash.update(chunk);
+        bytes += chunk.length;
+      }
+      digest = {
+        sha256: hash.digest("hex"),
+        bytes,
+      };
     }
-    const hash = createHash("sha256");
-    let bytes = 0;
-    for await (const rawChunk of createReadStream(target)) {
-      const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
-      hash.update(chunk);
-      bytes += chunk.length;
+    if (!encrypted) {
+      this.rememberOfflineSyncDigest(relPath, st, digest);
     }
-    return {
-      sha256: hash.digest("hex"),
-      bytes,
-    };
+    return digest;
+  }
+
+  private async loadOfflineSyncDigestCache(): Promise<Map<string, OfflineSyncDigestCacheEntry>> {
+    if (this.offlineSyncDigestCache) return this.offlineSyncDigestCache;
+    if (!this.offlineSyncDigestCacheLoadPromise) {
+      this.offlineSyncDigestCacheLoadPromise = this.readOfflineSyncDigestCache();
+    }
+    this.offlineSyncDigestCache = await this.offlineSyncDigestCacheLoadPromise;
+    return this.offlineSyncDigestCache;
+  }
+
+  private async readOfflineSyncDigestCache(): Promise<Map<string, OfflineSyncDigestCacheEntry>> {
+    const cache = new Map<string, OfflineSyncDigestCacheEntry>();
+    try {
+      const raw = await readFile(this.offlineSyncDigestCachePath, "utf-8");
+      const parsed = JSON.parse(raw) as unknown;
+      if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return cache;
+      const entries = (parsed as { entries?: unknown }).entries;
+      if (!Array.isArray(entries)) return cache;
+      for (const entry of entries) {
+        if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
+        const record = entry as Record<string, unknown>;
+        const cachePath = typeof record.path === "string" ? record.path : "";
+        const statBytes = typeof record.statBytes === "number" ? record.statBytes : NaN;
+        const mtimeMs = typeof record.mtimeMs === "number" ? record.mtimeMs : NaN;
+        const ctimeMs = typeof record.ctimeMs === "number" ? record.ctimeMs : NaN;
+        const bytes = typeof record.bytes === "number" ? record.bytes : NaN;
+        const sha256 = typeof record.sha256 === "string" ? record.sha256 : "";
+        const encrypted = record.encrypted === true;
+        if (
+          cachePath.length === 0 ||
+          cachePath === ".." ||
+          cachePath.startsWith("../") ||
+          path.isAbsolute(cachePath) ||
+          !Number.isFinite(statBytes) ||
+          !Number.isFinite(mtimeMs) ||
+          !Number.isFinite(ctimeMs) ||
+          !Number.isFinite(bytes) ||
+          !/^[a-f0-9]{64}$/i.test(sha256)
+        ) {
+          continue;
+        }
+        cache.set(cachePath, { statBytes, mtimeMs, ctimeMs, encrypted, sha256, bytes });
+      }
+    } catch (err) {
+      if (!isErrnoCode(err, "ENOENT")) {
+        log.warn(
+          `storage.offlineSyncDigestCache: ignoring unreadable cache: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    }
+    return cache;
+  }
+
+  private rememberOfflineSyncDigest(
+    relPath: string,
+    st: { size: number; mtimeMs: number; ctimeMs: number },
+    digest: { sha256: string; bytes: number },
+  ): void {
+    const cache = this.offlineSyncDigestCache;
+    if (!cache) return;
+    cache.set(relPath, {
+      statBytes: st.size,
+      mtimeMs: st.mtimeMs,
+      ctimeMs: st.ctimeMs,
+      encrypted: false,
+      sha256: digest.sha256,
+      bytes: digest.bytes,
+    });
+    this.scheduleOfflineSyncDigestCacheWrite();
+  }
+
+  private scheduleOfflineSyncDigestCacheWrite(): void {
+    if (this.offlineSyncDigestCacheWriteTimer) {
+      clearTimeout(this.offlineSyncDigestCacheWriteTimer);
+    }
+    this.offlineSyncDigestCacheWriteTimer = setTimeout(() => {
+      this.offlineSyncDigestCacheWriteTimer = null;
+      this.queueOfflineSyncDigestCacheWrite();
+    }, 1_000);
+    this.offlineSyncDigestCacheWriteTimer.unref?.();
+  }
+
+  private queueOfflineSyncDigestCacheWrite(): void {
+    this.offlineSyncDigestCacheWriteChain = this.offlineSyncDigestCacheWriteChain
+      .catch(() => undefined)
+      .then(async () => {
+        const cache = this.offlineSyncDigestCache;
+        if (!cache) return;
+        const entries = [...cache.entries()]
+          .sort(([a], [b]) => a.localeCompare(b))
+          .map(([entryPath, entry]) => ({ path: entryPath, ...entry }));
+        await mkdir(path.dirname(this.offlineSyncDigestCachePath), { recursive: true });
+        await writeFile(
+          this.offlineSyncDigestCachePath,
+          `${JSON.stringify({ version: 1, entries })}\n`,
+          "utf-8",
+        );
+      })
+      .catch((err) => {
+        log.warn(
+          `storage.offlineSyncDigestCache: failed to write cache: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      });
   }
 
   private async offlineSyncFileIsEncrypted(filePath: string): Promise<boolean> {
@@ -2644,6 +2785,9 @@ export class StorageManager {
   private get stateDir(): string {
     return path.join(this.baseDir, "state");
   }
+  private get offlineSyncDigestCachePath(): string {
+    return path.join(this.baseDir, ".offline-sync", "digest-cache.v1.json");
+  }
   private get entitySynthesisQueuePath(): string {
     return path.join(this.stateDir, "entity-synthesis-queue.json");
   }