@remnic/core 9.3.684 → 9.3.686

package/src/orchestrator.ts

@@ -2682,7 +2682,7 @@ export class Orchestrator {
         this.rememberNamespaceStorageDirHint(namespace, storageDir);
         return this.namespaceCatalog.registerResolved(namespace, storageDir);
       },
-    });
+    }, this.namespaceCatalog);
     this.namespaceSearchRouter = new NamespaceSearchRouter(
       config,
       this.storageRouter,
@@ -2691,6 +2691,8 @@ export class Orchestrator {
     // Propagate the inline-attribution template so the storage layer can strip
     // citations from legacy facts during the hash-index rebuild path.
     this.storage.citationTemplate = config.inlineSourceAttributionFormat;
+    // #1522: bind the post-write catalog touch at the storage chokepoint.
+    if (config.defaultNamespace) this.storageRouter.bindCatalogWriteHook(this.storage, config.defaultNamespace);
     // Wire page-level versioning (issue #371)
     this.storage.setVersioningConfig({
       enabled: config.versioningEnabled,
@@ -4479,11 +4481,7 @@ export class Orchestrator {
           // failures where the canonical memory was written but a later archive
           // step throws and the cluster catch continues (codex NY-dK).
           // Best-effort; namespace decoded from the storage dir since this path
-          // has no routed namespace name.
-          this.markCatalogWrite(
-            this.namespaceFromStorageDir(targetStorage.dir),
-            targetStorage.dir,
-          );
+          // #1522: catalog touch handled at the storage chokepoint.
         }
       }
     }
@@ -7992,7 +7990,7 @@ export class Orchestrator {
       recallResultLimit > 0 &&
       !options.abortSignal?.aborted
     ) {
-      for (const ns of recallNamespaces) this.markCatalogRead(ns);
+      for (const ns of recallNamespaces) this.storageRouter.recordRead(ns);
     }
 
     // 0. Shared context (v4.0, optional)
@@ -14200,7 +14198,6 @@ export class Orchestrator {
               );
             }
           }
-          this.markCatalogWrite(target.namespace, targetStorage.dir);
           trackPersistedId(targetStorage, promotedId, { includeReturnedIds: false });
           await this.indexPersistedMemory(targetStorage, promotedId);
           trackBehaviorSignals(
@@ -14391,17 +14388,16 @@ export class Orchestrator {
                   useCallerTimestamp: true,
                 });
                 // Catalog touch (issue #1499 — codex P2 NElSf): this dedup branch
-                // returns WITHOUT reaching the post-write `markCatalogWrite` below,
+                // returns WITHOUT the post-write catalog touch (now at the storage chokepoint #1522),
                 // but `applyTemporalSupersession` mutated the shared namespace
                 // (it rewrote frontmatter to retire stale shared facts). When any
                 // ids were actually superseded, the shared namespace changed, so we
                 // must record the write — otherwise the shared record's
                 // `lastWriteAt` stays stale and `writtenSince` maintenance / QMD
                 // fanout skips the namespace after a supersession-only update.
-                // Best-effort and failure-tolerant (markCatalogWrite swallows
+                // Best-effort and failure-tolerant (the storage chokepoint swallows
                 // errors); only touch when work happened to avoid spurious writes.
                 if (hashDedupSupersession.supersededIds.length > 0) {
-                  this.markCatalogWrite(this.config.sharedNamespace, sharedStorage.dir);
                 }
                 // Active matching fact exists — normal short-circuit is safe.
                 return;
@@ -14505,7 +14501,6 @@ export class Orchestrator {
         // the same promotion pass. The hot-path source-namespace touch uses a
         // different storage dir, so this does not double-count the source.
         // Best-effort and failure-tolerant — it must never crash the promotion.
-        this.markCatalogWrite(this.config.sharedNamespace, sharedStorage.dir);
         trackPersistedId(sharedStorage, promotedId, {
           includeReturnedIds: false,
         });
@@ -15358,7 +15353,6 @@ export class Orchestrator {
             // failure still surfaces the partially durable parent/chunk files to
             // catalog-driven `writtenSince` maintenance. The final touch below
             // still refreshes `lastWriteAt` after later durable writes on success.
-            this.markCatalogWrite(targetNamespaceName, targetStorage.dir);
           }
 
           if (routedRuleId) {
@@ -15502,7 +15496,6 @@ export class Orchestrator {
             // promotion, optional artifact writes, and graph-edge writes — so
             // `lastWriteAt` cannot precede later file changes on successful
             // completion. Use the KNOWN routed name, not a dir-decoded guess.
-            this.markCatalogWrite(targetNamespaceName, targetStorage.dir);
           }
           trackBehaviorSignals(
             targetStorage,
@@ -15732,13 +15725,12 @@ export class Orchestrator {
         // The `finally` preserves the write touch when post-write indexing or
         // promotion fails after the canonical memory is already durable. Use the
         // KNOWN routed name, not a dir-decoded guess (NCQI0).
-        this.markCatalogWrite(targetNamespaceName, targetStorage.dir);
       }
     }
 
     // Tracks whether THIS extraction persisted any durable, non-fact output to the
     // BASE namespace's storage (entity / relationship / profile / question). The
-    // per-fact `markCatalogWrite` only fires inside the fact write loop, so a
+    // per-fact catalog touch (storage chokepoint #1522) only fires inside the fact write loop, so a
     // fact-less extraction that still persists durable data must record exactly one
     // base-namespace catalog touch after all writes complete (NHZEZ, codex P2).
     let durableNonFactWritten = false;
@@ -15748,7 +15740,6 @@ export class Orchestrator {
         baseNamespace && baseNamespace.length > 0
           ? baseNamespace
           : this.namespaceFromStorageDir(storage.dir);
-      this.markCatalogWrite(baseTouchNamespace, storage.dir);
     };
     const recordDurableNonFactWrite = () => {
       durableNonFactWritten = true;
@@ -15863,7 +15854,7 @@ export class Orchestrator {
     }
 
     // Catalog touch for durable NON-FACT outputs (NHZEZ / NIIly, codex P2). The
-    // per-fact `markCatalogWrite` above only fires inside the fact write loop, so
+    // per-fact catalog touch (storage chokepoint #1522) above only fires inside the fact write loop, so
     // an extraction that persists ONLY entities, relationships, profile updates,
     // questions, or an identity reflection (no facts) would record durable data to
     // the BASE namespace's storage without ever touching the catalog — leaving that
@@ -15874,7 +15865,7 @@ export class Orchestrator {
     // KNOWN base namespace name, not a dir-decoded guess (NCQI0). One touch per
     // namespace per extraction — `markWrite` is idempotent, so if the fact path
     // already touched the base namespace this only refreshes `lastWriteAt`.
-    // Best-effort and failure-tolerant (markCatalogWrite swallows errors).
+    // Best-effort and failure-tolerant (storage chokepoint #1522 swallows errors).
     if (durableNonFactWritten) {
       touchBaseNonFactNamespace();
     }
@@ -16604,19 +16595,13 @@ export class Orchestrator {
       }
     }
 
-    // Consolidated catalog write touch (issue #1499 sweep; NIBOi + NIjwl). One
-    // touch covering EVERY durable namespace mutation this pass made — LLM
-    // profile/entity/memory-item actions AND cleanup-only maintenance (entity-file
-    // merges, expired-commitment / ledger-lifecycle / TTL cleanup, fact archival).
-    // Recorded here, after all mutation-producing steps, so a cleanup-only run that
-    // rewrote the store still refreshes `lastWriteAt` (rule #25). The default
-    // namespace is always configured/cataloged; `markWrite` is idempotent so this
-    // only refreshes recency. Best-effort and failure-tolerant.
+    // Consolidated catalog write touch — belt-and-suspenders for cleanup-only
+    // passes that mutate the store via delete-only paths (entity merges, TTL
+    // cleanup) without triggering the storage chokepoint's post-write hook.
+    // Gated on memoryItemMutated (set by every durable mutation including cleanup-only passes)
+    // Best-effort and failure-tolerant.
     if (memoryItemMutated) {
-      this.markCatalogWrite(
-        this.namespaceFromStorageDir(this.storage.dir),
-        this.storage.dir,
-      );
+      this.storageRouter.recordWrite(this.config.defaultNamespace, this.storage.dir);
     }
 
     log.info("consolidation complete");
@@ -17070,7 +17055,6 @@ export class Orchestrator {
     const lifecycleCorpus = await storage.readAllMemories();
     // Record the catalog write when the pass rewrote any frontmatter (codex NR-tS).
     if ((await this.runLifecyclePolicyPass(lifecycleCorpus, storage)) > 0) {
-      this.markCatalogWrite(this.namespaceFromStorageDir(storage.dir), storage.dir);
     }
     return { memoriesAssessed: lifecycleCorpus.length };
   }
@@ -17355,10 +17339,7 @@ export class Orchestrator {
       // summary and then rewrites source-memory archive status, bypassing the
       // extraction write path. Record the touch after both mutations complete so
       // `lastWriteAt` covers the final archived-state write.
-      this.markCatalogWrite(
-        this.namespaceFromStorageDir(this.storage.dir),
-        this.storage.dir,
-      );
+      // #1522: catalog touch handled at the storage chokepoint.
 
       log.info(
         `created summary ${summary.id} from ${batch.length} memories, archived ${archived}`,
@@ -17469,11 +17450,10 @@ export class Orchestrator {
       // a namespace whose sole mutation in the pass is identity consolidation would
       // otherwise keep a stale `lastWriteAt`, making `listNamespaces({ writtenSince })`
       // and catalog-recency consumers miss the write. Best-effort and
-      // failure-tolerant (`markCatalogWrite` swallows errors, never crashing the
+      // failure-tolerant (the storage chokepoint (#1522) swallows errors, never crashing the
       // consolidation; gotcha #13, rule #40). No double-count with the consolidated
       // touch above: that one is gated on `memoryItemMutated` (which identity
       // consolidation does not set), and `markWrite` is idempotent regardless.
-      this.markCatalogWrite(namespace, storage.dir);
       log.info(
         `IDENTITY(${namespace}) consolidated: ${identityContent.length} → ${newContent.length} chars, ${result.learnedPatterns.length} patterns`,
       );
@@ -19917,24 +19897,13 @@ export class Orchestrator {
     return dirName;
   }
 
-  /**
-   * Record a namespace write in the catalog (issue #1499). Best-effort and
-   * failure-tolerant: a catalog write error MUST NOT crash the primary memory
-   * write (CLAUDE.md gotcha #13, rule #40). Fire-and-forget by design.
-   */
-  private markCatalogWrite(namespace: string, storageDir?: string): void {
-    if (!this.namespaceCatalog.enabled) return;
-    this.rememberNamespaceStorageDirHint(namespace, storageDir);
-    void this.namespaceCatalog
-      .markWrite(namespace, { discoveredBy: "write", storageDir })
-      .catch(() => undefined);
-  }
+  // #1522: catalog touch methods removed — touches now happen at the storage chokepoint.
 
   /**
    * Public best-effort catalog write touch (issue #1499). User-facing explicit
    * captures (`memory_store`) and review-queue approvals persist via
    * `persistExplicitCapture()` → `storage.writeMemory()`, which bypasses the
-   * extraction write path that calls `markCatalogWrite`. Without this their
+   * extraction write path that owns the catalog touch. Without this their
    * namespaces never record `lastWriteAt`, so the catalog under-reports write
    * recency (round 5, codex P2). Fire-and-forget and failure-tolerant — a
    * catalog error must never affect the explicit write (gotcha #13, rule #40).
@@ -19944,20 +19913,9 @@ export class Orchestrator {
    * default rather than skipping it (round 6, codex P2 — default `memory_store`
    * and inline-note writes were missing from `writtenSince`/maintenance).
    */
-  recordCatalogWrite(namespace?: string, storageDir?: string): void {
-    const ns = namespace && namespace.trim().length > 0 ? namespace : this.config.defaultNamespace;
-    if (!ns) return;
-    this.markCatalogWrite(ns, storageDir);
-  }
 
-  /** Record a namespace read in the catalog. Best-effort, failure-tolerant. */
-  private markCatalogRead(namespace: string, storageDir?: string): void {
-    if (!this.namespaceCatalog.enabled) return;
-    this.rememberNamespaceStorageDirHint(namespace, storageDir);
-    void this.namespaceCatalog
-      .markRead(namespace, { discoveredBy: "read", storageDir })
-      .catch(() => undefined);
-  }
+
+  // markCatalogRead removed — use storageRouter.recordRead() instead (#1522).
 
   private async readAllMemoriesForNamespaces(
     namespaces: string[],

package/src/storage.ts

@@ -2238,24 +2238,12 @@ export class StorageManager {
 
   // Cache for readAllColdMemories() — keyed by cold root directory path.
   // Prevents an uncached full-tree directory scan on every structured-attribute
-  // write (Finding UOGi, PR #402 round-6).  The cache is only invalidated when
-  // cold-tier content actually changes (via invalidateColdMemoriesCache), NOT
-  // on every hot-tier write.  It also expires after COLD_SCAN_CACHE_TTL_MS as
-  // a safety net.
-  //
-  // Finding UvUy (PR #402 round-11): cache entries now carry a `coldVersion`
-  // sentinel that is bumped (via a file-size counter in state/cold-write.log)
-  // on every write that modifies cold-tier content.  Before serving a cached
-  // result, readAllColdMemories() reads the sentinel from disk and compares.
-  // If they differ the entry is dropped and the cold tree is re-scanned.  This
-  // makes the cache correct across process boundaries (gateway + CLI): a second
-  // process that writes a new cold memory bumps the sentinel on disk, so the
-  // first process's next readAllColdMemories() sees the change within one call
-  // (rather than waiting up to 30s for TTL expiry).
-  //
-  // After Finding UTsP broadened readAllColdMemories to scan the entire cold/
-  // subtree (not just facts/+corrections/), amortizing this I/O across
-  // back-to-back writes in the same burst is even more important.
+  // write (Finding UOGi, PR #402 round-6). Invalidated when cold-tier content
+  // changes (via invalidateColdMemoriesCache) and expires after COLD_SCAN_CACHE_TTL_MS.
+  // Entries carry a `coldVersion` sentinel (Finding UvUy, PR #402 round-11) bumped
+  // on every cold-tier write, making the cache correct across process boundaries
+  // (gateway + CLI). After Finding UTsP broadened the scan to the entire cold/
+  // subtree, amortizing across back-to-back writes is even more important.
   private static readonly COLD_SCAN_CACHE_TTL_MS = 30_000; // 30 seconds
   private static readonly coldMemoriesCache = new Map<string, { memories: MemoryFile[]; loadedAt: number; coldVersion: number }>();
 
@@ -2289,6 +2277,11 @@ export class StorageManager {
   private offlineSyncDigestCacheWriteTimer: ReturnType<typeof setTimeout> | null = null;
   /** Optional: set by the orchestrator after construction to enable template-aware citation stripping during legacy hash rebuild. */
   citationTemplate: string = DEFAULT_CITATION_FORMAT;
+  /** Post-write catalog hook (#1522). Installed by the namespace router; fire-and-forget. */
+  onCatalogWrite?: () => void;
+  private notifyCatalogWrite(): void {
+    try { this.onCatalogWrite?.(); } catch { /* gotcha #13 */ }
+  }
 
   /** Page-versioning configuration.  Set by the orchestrator after construction. */
   private _versioningConfig: VersioningConfig | null = null;
@@ -2576,7 +2569,7 @@ export class StorageManager {
   ): Promise<void> {
     const targetPath = this.wearableTranscriptPath(sourceId, date);
     // writeMaybeEncryptedFile handles mkdir + atomic temp→rename.
-    await writeMaybeEncryptedFile(targetPath, serialized, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(targetPath, serialized);
   }
 
   /** Read a stored day transcript; null when the day has no file. */
@@ -2747,8 +2740,13 @@ export class StorageManager {
   private readStorageSecureFile(filePath: string): Promise<string> {
     return readMaybeEncryptedFile(filePath, this._secureStoreKey, this.baseDir);
   }
-  private writeStorageSecureFile(filePath: string, content: string): Promise<void> {
-    return writeMaybeEncryptedFile(filePath, content, this.resolveWriteKey(), {}, this.baseDir);
+  private writeStorageSecureFile(filePath: string, content: string | Buffer): Promise<void> {
+    return writeMaybeEncryptedFile(filePath, content, this.resolveWriteKey(), {}, this.baseDir)
+      .then(() => this.notifyCatalogWrite());
+  }
+  private writeStorageSecureFileChunks(filePath: string, chunks: AsyncIterable<Buffer>): Promise<void> {
+    return writeMaybeEncryptedFileFromChunks(filePath, chunks, this.resolveWriteKey(), {}, this.baseDir)
+      .then(() => this.notifyCatalogWrite());
   }
 
   private assertManagedStoragePath(filePath: string, method: string): string {
@@ -2929,18 +2927,18 @@ export class StorageManager {
 
   async writeOfflineSyncFile(filePath: string, content: Buffer): Promise<void> {
     const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncFile");
-    await writeMaybeEncryptedFile(target, content, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(target, content);
     await this.invalidateAfterOfflineSyncMutation(target);
   }
 
   async writeOfflineSyncStagingFile(filePath: string, content: Buffer): Promise<void> {
     const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncStagingFile");
-    await writeMaybeEncryptedFile(target, content, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(target, content);
   }
 
   async writeOfflineSyncFileChunks(filePath: string, chunks: AsyncIterable<Buffer>): Promise<void> {
     const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncFileChunks");
-    await writeMaybeEncryptedFileFromChunks(target, chunks, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFileChunks(target, chunks);
     await this.invalidateAfterOfflineSyncMutation(target);
   }
 
@@ -3005,12 +3003,14 @@ export class StorageManager {
         if (isEncryptedFile(await readFile(filePath))) {
           const existing = await this.readStorageSecureFile(filePath);
           await writeMaybeEncryptedFile(filePath, `${existing}${content}`, null, {}, this.baseDir);
+          this.notifyCatalogWrite();
           return;
         }
       } catch (err) {
         if (!isErrnoCode(err, "ENOENT")) throw err;
       }
       await appendFile(filePath, content, "utf-8");
+      this.notifyCatalogWrite();
       return;
     }
 
@@ -3021,6 +3021,7 @@ export class StorageManager {
       if (!isErrnoCode(err, "ENOENT")) throw err;
     }
     await writeMaybeEncryptedFile(filePath, `${existing}${content}`, writeKey, {}, this.baseDir);
+    this.notifyCatalogWrite();
   }
   private get stateDir(): string {
     return path.join(this.baseDir, "state");
@@ -3465,7 +3466,7 @@ export class StorageManager {
     const filePath = await this.resolveCategoryWritePath(category, id, today);
 
     await this.snapshotBeforeWrite(filePath, "write");
-    await writeMaybeEncryptedFile(filePath, fileContent, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(filePath, fileContent);
     this.invalidateAllMemoriesCache();
     await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.writeMemory", {
       memoryId: id,
@@ -3638,7 +3639,7 @@ export class StorageManager {
       return "";
     }
     const filePath = path.join(dir, `${id}.md`);
-    await writeMaybeEncryptedFile(filePath, `${serializeFrontmatter(fm)}\n\n${sanitized.text}\n`, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(filePath, `${serializeFrontmatter(fm)}\n\n${sanitized.text}\n`);
     const actor =
       typeof options.actor === "string" && options.actor.length > 0
         ? options.actor
@@ -3876,7 +3877,7 @@ export class StorageManager {
   async writeProfile(content: string): Promise<void> {
     await this.ensureDirectories();
     await this.snapshotBeforeWrite(this.profilePath, "consolidation");
-    await writeMaybeEncryptedFile(this.profilePath, content, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(this.profilePath, content);
     log.debug("updated profile.md");
   }
 
@@ -4601,10 +4602,9 @@ export class StorageManager {
 
   private async writeMemoryFileAtomic(targetPath: string, memory: MemoryFile): Promise<void> {
     const fileContent = `${serializeFrontmatter(memory.frontmatter)}\n\n${memory.content}\n`;
-    // writeMaybeEncryptedFile handles atomic temp→rename internally and
-    // calls mkdir on the parent directory — no need to duplicate here.
     await writeMaybeEncryptedFile(targetPath, fileContent, this.resolveWriteKey(), {}, this.baseDir);
     this.invalidateAllMemoriesCache();
+    this.notifyCatalogWrite();
   }
 
   async moveMemoryToPath(memory: MemoryFile, targetPath: string): Promise<void> {
@@ -4694,7 +4694,7 @@ export class StorageManager {
       const destPath = path.join(destDir, path.basename(memory.path));
 
       // Write to archive location first (encrypted if applicable), then remove original.
-      await writeMaybeEncryptedFile(destPath, fileContent, this.resolveWriteKey(), {}, this.baseDir);
+      await this.writeStorageSecureFile(destPath, fileContent);
       await unlink(memory.path);
       this.invalidateAllMemoriesCache();
       await this.appendGeneratedMemoryLifecycleEventFailOpen(
@@ -4851,7 +4851,7 @@ export class StorageManager {
       log.warn(`updated memory content sanitized for ${id}; violations=${sanitized.violations.join(", ")}`);
     }
     const fileContent = `${serializeFrontmatter(updated)}\n\n${sanitized.text}\n`;
-    await writeMaybeEncryptedFile(memory.path, fileContent, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(memory.path, fileContent);
     this.invalidateAllMemoriesCache();
     await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.updateMemory", {
       memoryId: id,
@@ -4886,7 +4886,7 @@ export class StorageManager {
     const afterStatus = updated.status ?? "active";
 
     const fileContent = `${serializeFrontmatter(updated)}\n\n${memory.content}\n`;
-    await writeMaybeEncryptedFile(memory.path, fileContent, this.resolveWriteKey(), {}, this.baseDir);
+    await this.writeStorageSecureFile(memory.path, fileContent);
     this.invalidateAllMemoriesCache();
     // If the target file lives in cold/, bump the cold-version sentinel so
     // other processes detect the change on their next readAllColdMemories()
@@ -5826,7 +5826,7 @@ export class StorageManager {
     const content = `---\n${Object.entries(frontmatter).map(([k, v]) => `${k}: ${JSON.stringify(v)}`).join("\n")}\n---\n\n${question}\n\n**Context:** ${context}\n`;
 
     const filePath = path.join(this.questionsDir, `${id}.md`);
-    await writeFile(filePath, content, "utf-8");
+    await this.writeStorageSecureFile(filePath, content);
 
     log.debug(`wrote question ${id} to ${filePath}`);
     this.invalidateQuestionsCache();
@@ -5942,13 +5942,13 @@ export class StorageManager {
     const q = questions.find((q) => q.id === id);
     if (!q) return false;
 
-    let raw = await readFile(q.filePath, "utf-8");
+    let raw = await this.readStorageSecureFile(q.filePath);
     raw = raw.replace(/resolved: false/, "resolved: true");
     raw = raw.replace(
       /---\n\n/,
       `resolvedAt: "${new Date().toISOString()}"\n---\n\n`,
     );
-    await writeFile(q.filePath, raw, "utf-8");
+    await this.writeStorageSecureFile(q.filePath, raw);
     log.debug(`resolved question ${id}`);
     return true;
   }
@@ -7080,7 +7080,7 @@ export class StorageManager {
     const fileContent = `${serializeFrontmatter(updatedFm)}\n\n${oldMemory.content}\n`;
 
     try {
-      await writeMaybeEncryptedFile(oldMemory.path, fileContent, this.resolveWriteKey(), {}, this.baseDir);
+      await this.writeStorageSecureFile(oldMemory.path, fileContent);
       await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.supersedeMemory", {
         memoryId: oldMemoryId,
         eventType: "superseded",