@remnic/core 9.45.4 → 9.46.0

package/src/storage-fact-hash-index-defer.test.ts

@@ -31,7 +31,7 @@ async function withMemoryDir(run: (dir: string) => Promise<void>): Promise<void>
   try {
     await run(dir);
   } finally {
-    await rm(dir, { recursive: true, force: true });
+    await rm(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
   }
 }
 
@@ -2532,7 +2532,7 @@ test("chunked offline sync scans oversized frontmatter before locking its identi
       semanticThreshold: 0.9,
       namespace: "default",
     });
-    const content = `Oversized frontmatter sync ${"x".repeat(900_000)}`;
+    const content = "Oversized frontmatter sync";
     const tombstoneId = await storage.appendTombstone({
       reason: "correction",
       createdBy: "user_correction",
@@ -2548,7 +2548,7 @@ test("chunked offline sync scans oversized frontmatter before locking its identi
     const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
     assert.ok(memory);
 
-    const largeTags = Array.from({ length: 22_000 }, (_, index) => `"tag-${index}"`).join(", ");
+    const largeTags = Array.from({ length: 100_000 }, (_, index) => `"tag-${index}"`).join(", ");
     const updatedFile = [
       "---",
       `id: ${memory.frontmatter.id}`,
@@ -2567,6 +2567,10 @@ test("chunked offline sync scans oversized frontmatter before locking its identi
       content,
       "",
     ].join("\n");
+    assert.ok(
+      updatedFile.indexOf("\n---\n") > 1_048_576,
+      "the streamed body delimiter must appear after the chunk buffer limit",
+    );
     const chunks = (async function* (): AsyncIterable<Buffer> {
       for (let offset = 0; offset < updatedFile.length; offset += 32 * 1024) {
         yield Buffer.from(updatedFile.slice(offset, offset + 32 * 1024), "utf8");
@@ -2651,7 +2655,7 @@ test("chunked offline sync encrypts oversized staging files when secure storage
       semanticThreshold: 0.9,
       namespace: "default",
     });
-    const content = `Oversized encrypted staging content ${"x".repeat(1_048_576)}`;
+    const content = "Oversized encrypted staging content";
     const tombstoneId = await storage.appendTombstone({
       reason: "correction",
       createdBy: "user_correction",
@@ -2666,6 +2670,7 @@ test("chunked offline sync encrypts oversized staging files when secure storage
     assert.equal(result.tombstoneBlocked, true);
     const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
     assert.ok(memory);
+    const largeTags = Array.from({ length: 100_000 }, (_, index) => `"tag-${index}"`).join(", ");
 
     const incomingFile = [
       "---",
@@ -2676,7 +2681,7 @@ test("chunked offline sync encrypts oversized staging files when secure storage
       `source: ${memory.frontmatter.source}`,
       `confidence: ${memory.frontmatter.confidence}`,
       `confidenceTier: ${memory.frontmatter.confidenceTier}`,
-      "tags: []",
+      `tags: [${largeTags}]`,
       "sourceConnector: provider-a",
       "status: pending_review",
       `blockedBy: ${memory.frontmatter.blockedBy}`,
@@ -2685,6 +2690,7 @@ test("chunked offline sync encrypts oversized staging files when secure storage
       content,
       "",
     ].join("\n");
+    assert.ok(incomingFile.length > 1_048_576, "the encrypted staging fixture must exceed the chunk buffer limit");
     const chunks = (async function* (): AsyncIterable<Buffer> {
       for (let offset = 0; offset < incomingFile.length; offset += 32 * 1024) {
         yield Buffer.from(incomingFile.slice(offset, offset + 32 * 1024), "utf8");
@@ -2960,7 +2966,8 @@ test("large blocked offline sync streams through bounded staging and deduplicate
       semanticThreshold: 0.9,
       namespace: "default",
     });
-    const largeContent = "large stream duplicate ".repeat(60_000);
+    const largeContent = "large stream duplicate";
+    const largeTags = Array.from({ length: 100_000 }, (_, index) => `"tag-${index}"`).join(", ");
     const tombstoneId = await storage.appendTombstone({
       reason: "correction",
       createdBy: "user_correction",
@@ -2983,7 +2990,7 @@ test("large blocked offline sync streams through bounded staging and deduplicate
         "source: offline-sync",
         "confidence: 0.8",
         "confidenceTier: implied",
-        "tags: []",
+        `tags: [${largeTags}]`,
         "sourceConnector: provider-a",
         "status: pending_review",
         `blockedBy: ${tombstoneId}`,

package/src/storage.ts

@@ -18,6 +18,7 @@ import { createHash, randomUUID } from "node:crypto";
 import { normalizeContent, computeContentHash } from "./content-hash.js";
 import path from "node:path";
 import { log } from "./logger.js";
+import { createMemorySnapshot } from "./memory-snapshot.js";
 import { assertMemoryFrontmatterId, warnProjectionFallback } from "./storage-guards.js";
 import { MemoryReadStore } from "./storage/memory-read-store.js";
 import { renderProfileWithLastUpdated } from "./storage/profile-header.js";
@@ -1779,6 +1780,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   private knowledgeIndexCache: { result: string; builtAt: number } | null = null;
   private artifactIndexCache: { memories: MemoryFile[]; loadedAtMs: number; writeVersion: number } | null = null;
   private projectionLedgerLagManager = new ProjectionLedgerLagManager();
+  private static readonly loadedMemorySnapshots = new WeakMap<MemoryFile, string>();
   static readonly KNOWLEDGE_INDEX_CACHE_TTL_MS = 600_000; // 10 minutes (entity mutations invalidate)
   /** Read by storage/memory-read-store.ts (decomposition). */
   static readonly ARTIFACT_INDEX_CACHE_TTL_MS = 60_000; // 1 minute
@@ -4504,13 +4506,19 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return lineCount > threshold;
   }
 
+  private rememberMemorySnapshot<T extends MemoryFile | null>(memory: T): T {
+    if (memory !== null && !StorageManager.loadedMemorySnapshots.has(memory)) {
+      StorageManager.loadedMemorySnapshots.set(memory, createMemorySnapshot(memory));
+    }
+    return memory;
+  }
+
+  private rememberMemorySnapshots(memories: MemoryFile[]): MemoryFile[] {
+    for (const memory of memories) this.rememberMemorySnapshot(memory);
+    return memories;
+  }
+
   async readAllMemories(): Promise<MemoryFile[]> {
-    // Version-keyed hot-memories cache (issue #1902): once populated, serve the
-    // full parsed corpus from memory within a version epoch. getCachedMemories
-    // returns null when the sentinel is 0 (fresh/test dirs) or the stored
-    // version differs from the current on-disk sentinel (a peer or local write
-    // bumped it), so a stale entry is never served — the miss falls through to
-    // a fresh disk scan below.
     if (this.hotMemoriesCacheEnabled) {
       const cached = getCachedMemories(
         this.baseDir,
@@ -4518,13 +4526,8 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
         this.hotCacheKeyId(),
         this.hotCacheTtlMs()
       );
-      // Null-check, not truthiness (kilo WARNING): getCachedMemories returns []
-      // for a cached EMPTY corpus, which is falsy — a truthiness guard would
-      // force a full disk scan on every read of an empty/near-empty store.
-      if (cached !== null) return cached;
+      if (cached !== null) return this.rememberMemorySnapshots(cached);
     }
-    // Deduplicate concurrent reads for the same directory so multiple
-    // callers in the same recall share one disk scan.
     // Snapshot the secure-store key identity ONCE (issue #1902, Codex Medium):
     // hotCacheKeyId() can change mid-scan if setSecureStoreKey(null/other) runs
     // during the await. Recomputing it at publish time would store K-decrypted
@@ -4534,13 +4537,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     // key change can't orphan the slot registered under the old identity.
     const keyId = this.hotCacheKeyId();
     const inFlight = getInFlightRead(this.baseDir, keyId);
-    if (inFlight) return inFlight;
+    if (inFlight) return this.rememberMemorySnapshots(await inFlight);
 
     const readPromise = (async (): Promise<MemoryFile[]> => {
-      // Capture the version BEFORE the scan and store under it. If a concurrent
-      // bump (peer or local write) lands during the scan, the stored entry's
-      // version is older than the sentinel, so the next getCachedMemories()
-      // rejects it (version mismatch → rescan) — correct, never stale.
       const version = this.getMemoryCorpusVersion();
       const memories = await this._readAllMemoriesFromDisk();
       // Publish only if neither the corpus version NOR the key identity changed
@@ -4554,11 +4553,8 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     })();
     setInFlightRead(this.baseDir, keyId, readPromise);
     try {
-      return await readPromise;
+      return this.rememberMemorySnapshots(await readPromise);
     } finally {
-      // Only delete if we still own the slot for our snapshot key identity —
-      // invalidateAllMemoriesCache() may have already cleared it and a new read
-      // may have claimed it (the expected-promise guard enforces owner-only).
       deleteInFlightRead(this.baseDir, keyId, readPromise);
     }
   }
@@ -4993,7 +4989,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   }
 
   async readAllColdMemories(): Promise<MemoryFile[]> {
-    return this.memoryReadStore.readAllColdMemories();
+    return this.rememberMemorySnapshots(await this.memoryReadStore.readAllColdMemories());
   }
 
   /**
@@ -5045,11 +5041,11 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     };
 
     await readDir(root);
-    return memories;
+    return this.rememberMemorySnapshots(memories);
   }
 
   async readMemoryByPath(filePath: string): Promise<MemoryFile | null> {
-    return this.memoryReadStore.readMemoryByPath(filePath);
+    return this.rememberMemorySnapshot(await this.memoryReadStore.readMemoryByPath(filePath));
   }
 
   private resolveTierRootDir(tier: "hot" | "cold"): string {
@@ -5077,12 +5073,6 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
       // corrections/ is flat (no date subdir); preserved across tier moves.
       return path.join(root, "corrections", `${memory.frontmatter.id}.md`);
     }
-    // Every other category — decisions/, preferences/, reasoning-traces/, ...
-    // plus the facts/ fallback for fact/entity/unknown — resolves through the
-    // shared categoryDirName() chokepoint so tier moves land in the SAME dir the
-    // writer used, instead of funneling non-{correction,procedure,reasoning_trace}
-    // categories into facts/ (issue #564 PR 3 preserved reasoning-traces/; #1546
-    // generalizes it to every category dir).
     const dir = categoryDirName(memory.frontmatter.category);
     return path.join(root, dir, this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
   }
@@ -5137,14 +5127,24 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     this.invalidateAllMemoriesCache();
     this.notifyCatalogWrite();
   }
-  async moveMemoryToPath(memory: MemoryFile, targetPath: string): Promise<void> {
-    await this.withTombstoneBlockedMemoryPathLock(
+  async moveMemoryToPath(memory: MemoryFile, targetPath: string): Promise<boolean> {
+    const changed = await this.withTombstoneBlockedMemoryPathLock(
       memory.path,
       async (current) => {
+        const destination = await this.readMemoryByPath(targetPath);
         if (current?.frontmatter.id !== memory.frontmatter.id) {
+          if (current === null && destination?.frontmatter.id === memory.frontmatter.id) return false;
+          throw new Error(`memory ${memory.frontmatter.id} changed before its tier move`);
+        }
+        const callerSnapshot = StorageManager.loadedMemorySnapshots.get(memory);
+        const currentSnapshot = createMemorySnapshot(current);
+        if (
+          callerSnapshot === undefined
+            ? current.frontmatter.updated !== memory.frontmatter.updated
+            : currentSnapshot !== callerSnapshot
+        ) {
           throw new Error(`memory ${memory.frontmatter.id} changed before its tier move`);
         }
-        const destination = await this.readMemoryByPath(targetPath);
         const coordinate =
           tombstoneBlocked(current.frontmatter) ||
           isQueuedReviewMemory(current) ||
@@ -5152,26 +5152,26 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
           isQueuedReviewMemory(destination);
         const index = this.getTombstoneBlockedCaptureIndex();
         const marker = coordinate ? await index.prepareWrite() : undefined;
+        const projectedPath = toMemoryPathRel(this.baseDir, targetPath);
         let durable = false;
         try {
           if (destination?.frontmatter.id === current.frontmatter.id) {
             await this.deleteManagedStorageFile(current.path);
             durable = true;
             this.invalidateAllMemoriesCache();
-            updateProjectedMemoryPath(this.baseDir, current.frontmatter.id, toMemoryPathRel(this.baseDir, targetPath));
+            updateProjectedMemoryPath(this.baseDir, current.frontmatter.id, projectedPath);
           } else {
             await this.writeMemoryFileAtomic(targetPath, current);
+            if (memory.frontmatter.entityRef !== current.frontmatter.entityRef) {
+              memory.frontmatter.entityRef = current.frontmatter.entityRef;
+            }
             durable = true;
             const sourcePath = path.resolve(current.path);
             const destPath = path.resolve(targetPath);
             if (sourcePath !== destPath) {
               await this.deleteManagedStorageFile(current.path);
               this.invalidateAllMemoriesCache();
-              updateProjectedMemoryPath(
-                this.baseDir,
-                current.frontmatter.id,
-                toMemoryPathRel(this.baseDir, targetPath)
-              );
+              updateProjectedMemoryPath(this.baseDir, current.frontmatter.id, projectedPath);
             }
           }
           if (marker !== undefined) await this.rebuildTombstoneBlockedCaptureAfterInvalidation(marker);
@@ -5180,9 +5180,14 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
           if (marker !== undefined && durable) index.markUntrusted();
           throw error;
         }
+        return true;
       },
       [targetPath]
     );
+    if (this.isColdOrArchiveTierPath(memory.path) || this.isColdOrArchiveTierPath(targetPath)) {
+      this.invalidateColdMemoriesCache();
+    }
+    return changed;
   }
 
   async migrateMemoryToTier(
@@ -5195,16 +5200,10 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     if (sourcePath === destPath) {
       return { changed: false, targetPath };
     }
-
-    await this.moveMemoryToPath(memory, targetPath);
-    this.invalidateAllMemoriesCache();
-    // If moving to cold, also invalidate the cold-scan cache so the next
-    // readAllColdMemories() call sees the newly-demoted file (Finding UOGi fix).
-    if (targetTier === "cold") {
-      this.invalidateColdMemoriesCache();
-    }
+    const changed = await this.moveMemoryToPath(memory, targetPath);
+    if (!changed) return { changed: false, targetPath };
     this.bumpMemoryStatusVersion();
-    return { changed: true, targetPath };
+    return { changed, targetPath };
   }
 
   private get archiveDir(): string {

package/src/testing/subjects/first-start-tier-migration.test.ts

@@ -0,0 +1,75 @@
+import assert from "node:assert/strict";
+
+import { parseConfig } from "../../config.js";
+import {
+  type FirstStartMigrationResult,
+  runFirstStartMigration,
+} from "../../maintenance/first-start-migration.js";
+import { StorageManager } from "../../storage.js";
+import type { PluginConfig } from "../../types.js";
+import { type LifecycleSubject, type MatrixRow, runLifecycleMatrix } from "../lifecycle-matrix.js";
+import { cleanupDir, mkTempMemoryDir } from "../orchestrator-lite.js";
+
+interface FirstStartTierMigrationState {
+  dir: string;
+  id: string;
+  storage: StorageManager;
+  config: PluginConfig;
+  result?: FirstStartMigrationResult;
+}
+
+const subject: LifecycleSubject<FirstStartTierMigrationState> = {
+  appliesTo(row: MatrixRow): boolean | string {
+    return row.id === "restart-reload-recovery"
+      ? true
+      : "first-start tier migration applies only to startup recovery";
+  },
+
+  async setup(row: MatrixRow): Promise<FirstStartTierMigrationState> {
+    const dir = await mkTempMemoryDir(`first-start-tier-${row.id}`);
+    try {
+      const storage = new StorageManager(dir);
+      await storage.ensureDirectories();
+      const { id } = await storage.writeMemory("fact", "old low-value fact", {
+        confidence: 0.01,
+        source: "test",
+      });
+      const memory = await storage.getMemoryById(id);
+      assert.ok(memory);
+      await storage.writeMemoryFrontmatter(memory, {
+        created: "2020-01-01T00:00:00.000Z",
+        updated: "2020-01-01T00:00:00.000Z",
+      });
+      const config = parseConfig({
+        openaiApiKey: "sk-test",
+        memoryDir: dir,
+        workspaceDir: dir,
+        qmdEnabled: false,
+        lifecyclePolicyEnabled: true,
+        qmdTierMigrationEnabled: true,
+        qmdTierDemotionMinAgeDays: 1,
+        qmdTierDemotionValueThreshold: 0.99,
+      });
+      return { dir, id, storage: new StorageManager(dir), config };
+    } catch (error) {
+      await cleanupDir(dir);
+      throw error;
+    }
+  },
+
+  async exercise(state: FirstStartTierMigrationState): Promise<void> {
+    state.result = await runFirstStartMigration({ storage: state.storage, config: state.config });
+  },
+
+  async invariants(state: FirstStartTierMigrationState): Promise<void> {
+    assert.equal(state.result?.demotedCount, 1);
+    assert.ok((await state.storage.readAllColdMemories()).some((memory) => memory.frontmatter.id === state.id));
+    assert.ok(!(await state.storage.readAllMemories()).some((memory) => memory.frontmatter.id === state.id));
+  },
+
+  async teardown(state: FirstStartTierMigrationState): Promise<void> {
+    await cleanupDir(state.dir);
+  },
+};
+
+runLifecycleMatrix("first-start-tier-migration", subject);