@remnic/core 9.29.1 → 9.30.0

package/src/maintenance/purge.test.ts

@@ -1,5 +1,5 @@
 import assert from "node:assert/strict";
-import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { access, mkdir, mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import test from "node:test";
@@ -9,6 +9,7 @@ import { rebuildMemoryProjection } from "./rebuild-memory-projection.js";
 import { readProjectedMemoryBrowse } from "../memory-projection-store.js";
 import { StorageManager } from "../storage.js";
 import type { MemoryFile } from "../types.js";
+import { buildExplicitCaptureDedupKey } from "../storage/tombstone-blocked-capture-index.js";
 
 test("purgeMemories records audit errors without blocking hard delete", async () => {
   const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-purge-"));
@@ -38,6 +39,10 @@ test("purgeMemories records audit errors without blocking hard delete", async ()
       readAllMemories: async () => [],
       readAllColdMemories: async () => [memory],
       readArchivedMemories: async () => [],
+      deleteMemoryForMaintenance: async (candidate: MemoryFile) => {
+        await unlink(candidate.path);
+        return candidate;
+      },
     };
 
     const result = await purgeMemories({
@@ -108,6 +113,10 @@ test("purgeMemories invalidates projection rows for already-absent files but not
       dir,
       readAllMemories: async () => [firstMemory, secondMemory],
       readAllColdMemories: async () => [],
+      deleteMemoryForMaintenance: async (candidate: MemoryFile) => {
+        await unlink(candidate.path);
+        return candidate;
+      },
       readArchivedMemories: async () => [],
     };
     const future = () => new Date(Date.now() + 86_400_000);
@@ -130,6 +139,153 @@ test("purgeMemories invalidates projection rows for already-absent files but not
   }
 });
 
+test("cleanExpiredCommitments coordinates blocked capture deletion", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-commitment-delete-lock-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    storage.setTombstonesConfig({
+      enabled: true,
+      semanticMatch: false,
+      semanticThreshold: 0.9,
+      namespace: "default",
+    });
+    const content = "A blocked commitment must not disappear under a concurrent capture.";
+    const tombstoneId = await storage.appendTombstone({
+      reason: "correction",
+      createdBy: "user_correction",
+      sourceMemoryId: "commitment-delete-lock",
+      rawContent: content,
+    });
+    assert.ok(tombstoneId);
+    const result = await storage.writeMemory("fact", content, {
+      source: "test",
+      sourceConnector: "provider-a",
+    });
+    assert.equal(result.tombstoneBlocked, true);
+    const blocked = await storage.getMemoryById(result.id);
+    assert.ok(blocked);
+    assert.equal(
+      await storage.writeMemoryFrontmatter(blocked, {
+        category: "commitment",
+        tags: ["fulfilled"],
+      }),
+      true,
+    );
+    const identity = buildExplicitCaptureDedupKey(content, "commitment", "provider-a");
+    const release = Promise.withResolvers<void>();
+    const entered = Promise.withResolvers<void>();
+    const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
+      entered.resolve();
+      await release.promise;
+    }, identity);
+    await entered.promise;
+
+    let completed = false;
+    const pending = storage.cleanExpiredCommitments(-1).then((deleted) => {
+      completed = true;
+      return deleted;
+    });
+    await new Promise((resolve) => setTimeout(resolve, 50));
+    assert.equal(completed, false, "commitment cleanup must wait for the blocked capture identity lock");
+    release.resolve();
+    await held;
+    const deleted = await pending;
+    assert.equal(deleted.length, 1);
+    assert.equal(deleted[0]?.frontmatter.id, result.id);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("purgeMemories coordinates blocked capture deletion", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-purge-delete-lock-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    storage.setTombstonesConfig({
+      enabled: true,
+      semanticMatch: false,
+      semanticThreshold: 0.9,
+      namespace: "default",
+    });
+    const content = "A blocked memory must not disappear under a concurrent capture.";
+    const tombstoneId = await storage.appendTombstone({
+      reason: "correction",
+      createdBy: "user_correction",
+      sourceMemoryId: "purge-delete-lock",
+      rawContent: content,
+    });
+    assert.ok(tombstoneId);
+    const result = await storage.writeMemory("fact", content, {
+      source: "test",
+      sourceConnector: "provider-a",
+    });
+    assert.equal(result.tombstoneBlocked, true);
+    const blocked = await storage.getMemoryById(result.id);
+    assert.ok(blocked);
+    const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
+    const release = Promise.withResolvers<void>();
+    const entered = Promise.withResolvers<void>();
+    const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
+      entered.resolve();
+      await release.promise;
+    }, identity);
+    await entered.promise;
+
+    let completed = false;
+    const pending = purgeMemories({
+      storage,
+      olderThanMs: 1,
+      tier: "all",
+      dryRun: false,
+      now: () => new Date(Date.now() + 86_400_000),
+    }).then((purged) => {
+      completed = true;
+      return purged;
+    });
+    await new Promise((resolve) => setTimeout(resolve, 50));
+    assert.equal(completed, false, "purge must wait for the blocked capture identity lock");
+    release.resolve();
+    await held;
+    const purged = await pending;
+    assert.equal(purged.purgedCount, 1);
+    assert.equal(await fileExists(blocked.path), false);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("maintenance deletion prefers the candidate path when IDs overlap across tiers", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-delete-tier-path-"));
+  try {
+    StorageManager.clearAllStaticCaches();
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const content = "The cold candidate path must win when a hot copy shares its ID.";
+    const result = await storage.writeMemory("fact", content, {
+      source: "test",
+      sourceConnector: "provider-a",
+    });
+    const hot = await storage.getMemoryById(result.id);
+    assert.ok(hot);
+    const coldPath = storage.buildTierMemoryPath(hot, "cold");
+    await mkdir(path.dirname(coldPath), { recursive: true });
+    await writeFile(coldPath, await readFile(hot.path));
+    StorageManager.clearAllStaticCaches();
+
+    const cold = (await storage.readAllColdMemories()).find((memory) => memory.frontmatter.id === result.id);
+    assert.ok(cold);
+    const removed = await storage.deleteMemoryForMaintenance(cold);
+    assert.equal(removed?.path, coldPath);
+    assert.equal(await fileExists(coldPath), false);
+    assert.equal(await fileExists(hot.path), true);
+  } finally {
+    StorageManager.clearAllStaticCaches();
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
 async function fileExists(filePath: string): Promise<boolean> {
   try {
     await access(filePath);

package/src/maintenance/purge.ts

@@ -20,7 +20,7 @@
  */
 
 import path from "node:path";
-import { appendFile, mkdir, unlink } from "node:fs/promises";
+import { appendFile, mkdir } from "node:fs/promises";
 import type { StorageManager } from "../storage.js";
 import type { MemoryFile } from "../types.js";
 import { markProjectedMemoryPathInvalid } from "../memory-projection-store.js";
@@ -168,7 +168,7 @@ export async function purgeMemories(
 
   // Build candidate list
   const candidates: PurgeCandidate[] = [];
-  const candidateMemoriesById = new Map<string, MemoryFile>();
+  const candidateMemoriesByPath = new Map<string, MemoryFile>();
   for (const { memory, resolvedTier } of poolEntries) {
     const ts = resolveTimestamp(memory);
     if (ts.length === 0) continue;
@@ -193,7 +193,7 @@ export async function purgeMemories(
       updatedOrCreated: ts,
       ageMs,
     });
-    candidateMemoriesById.set(memory.frontmatter.id, memory);
+    candidateMemoriesByPath.set(memory.path, memory);
   }
 
   if (dryRun) {
@@ -234,13 +234,19 @@ export async function purgeMemories(
 
   for (const candidate of candidates) {
     try {
-      await unlink(candidate.path);
-      actuallyPurged.push(candidate);
-      addCollectionForCandidate(candidate);
-      markProjectedMemoryPathInvalid(storage.dir, candidate.id);
+      const memory = candidateMemoriesByPath.get(candidate.path);
+      if (!memory) throw new Error(`purge candidate memory missing: ${candidate.id}`);
+      const removed = await storage.deleteMemoryForMaintenance(memory);
+      if (removed) {
+        actuallyPurged.push(candidate);
+        addCollectionForCandidate(candidate);
+      } else {
+        alreadyAbsent.push(candidate);
+        addCollectionForCandidate(candidate);
+        markProjectedMemoryPathInvalid(storage.dir, candidate.id);
+      }
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      // ENOENT is fine — already gone
       if (hasErrorCode(err, "ENOENT")) {
         alreadyAbsent.push(candidate);
         addCollectionForCandidate(candidate);
@@ -302,7 +308,7 @@ export async function purgeMemories(
   }
 
   const purgedFactMemories = resolvedPurges
-    .map((candidate) => candidateMemoriesById.get(candidate.id))
+    .map((candidate) => candidateMemoriesByPath.get(candidate.path))
     .filter((memory): memory is MemoryFile => memory?.frontmatter.category === "fact");
   if (purgedFactMemories.length > 0) {
     const removeFactHashes = (

package/src/secure-store/secure-fs.ts

@@ -43,7 +43,7 @@
  * Naming: `secure-fs.ts` (not `vault-fs.ts`) — see `kdf.ts` naming note.
  */
 
-import { createCipheriv, randomBytes, randomUUID } from "node:crypto";
+import { createCipheriv, createDecipheriv, randomBytes, randomUUID } from "node:crypto";
 import { lstat, mkdir, open as openFile, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
 import path from "node:path";
 
@@ -416,6 +416,83 @@ export async function writeMaybeEncryptedFileFromChunks(
   }
 }
 
+/**
+ * Stream a file through the secure-store reader without buffering the whole
+ * payload. Plaintext files are passed through unchanged; encrypted files are
+ * authenticated and decrypted incrementally.
+ */
+export async function* readMaybeEncryptedFileFromChunks(
+  filePath: string,
+  key: Buffer | null,
+  memoryDir?: string,
+  chunkSize = 64 * 1024
+): AsyncIterable<Buffer> {
+  const handle = await openFile(filePath, "r");
+  const headerSize = MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE;
+  try {
+    const prefix = Buffer.alloc(headerSize);
+    const { bytesRead: prefixBytes } = await handle.read(prefix, 0, prefix.length, 0);
+    const prefixView = prefix.subarray(0, prefixBytes);
+    if (!isEncryptedFile(prefixView)) {
+      if (prefixBytes > 0) yield Buffer.from(prefixView);
+      let position = prefixBytes;
+      const buffer = Buffer.alloc(chunkSize);
+      for (;;) {
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
+        if (bytesRead === 0) break;
+        position += bytesRead;
+        yield Buffer.from(buffer.subarray(0, bytesRead));
+      }
+      return;
+    }
+    if (prefixBytes < headerSize) {
+      throw new Error(`encrypted file header is truncated: ${filePath}`);
+    }
+    if (key === null) {
+      throw new SecureStoreLockedError(
+        `secure-store is locked — cannot read encrypted file at ${filePath}. ` +
+          "Run `remnic secure-store unlock` to decrypt."
+      );
+    }
+    const envelope = prefix.subarray(MAGIC_HEADER_SIZE);
+    const version = envelope.readUInt8(ENVELOPE_LAYOUT.version);
+    if (version !== ENVELOPE_VERSION) {
+      throw new Error(`secure-store: unsupported envelope version ${version}`);
+    }
+    const salt = envelope.subarray(
+      ENVELOPE_LAYOUT.salt,
+      ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
+    );
+    const iv = envelope.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
+    const authTag = envelope.subarray(
+      ENVELOPE_LAYOUT.authTag,
+      ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
+    );
+    const decipher = createDecipheriv("aes-256-gcm", key, iv, { authTagLength: AUTH_TAG_LENGTH });
+    decipher.setAAD(Buffer.concat([buildHeaderAad(salt), filePathAad(filePath, memoryDir)]));
+    decipher.setAuthTag(authTag);
+    let position = headerSize;
+    const buffer = Buffer.alloc(chunkSize);
+    try {
+      for (;;) {
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
+        if (bytesRead === 0) break;
+        position += bytesRead;
+        const plain = decipher.update(buffer.subarray(0, bytesRead));
+        if (plain.length > 0) yield plain;
+      }
+      const final = decipher.final();
+      if (final.length > 0) yield final;
+    } catch (err) {
+      if (err instanceof SecureStoreDecryptError) throw err;
+      const message = err instanceof Error ? err.message : String(err);
+      throw new SecureStoreDecryptError(`secure-store decryption failed: ${message}`);
+    }
+  } finally {
+    await handle.close();
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Migration
 // ---------------------------------------------------------------------------

package/src/storage/content-hash-index.ts

@@ -76,6 +76,14 @@ export class FactHashIndexNotAuthoritativeError extends Error {
   }
 }
 
+type DiskFingerprint = {
+  mtimeMs: number;
+  size: number;
+  dev: number;
+  ino: number;
+  ctimeMs: number;
+  birthtimeMs: number;
+};
 /**
  * Content-hash dedup index for facts.
  * Normalizes content (lowercase, strip punctuation, collapse whitespace),
@@ -120,15 +128,13 @@ export class ContentHashIndex {
   private reconcileRetryBarrier: Promise<void> | null = null;
   private reconcileRetryResolve: (() => void) | null = null;
   /**
-   * On-disk fingerprint (mtime + byte size) of `fact-hashes.txt` captured at the
-   * last point THIS instance's in-memory set matched disk — after `load()`,
-   * `save()`, or a reconcile publish (PR #2016 review). A peer process that
-   * advances the durable index changes this fingerprint;
-   * {@link isDiskFingerprintCurrent} compares a single cheap `stat` against it so
-   * a stale-but-"authoritative" snapshot is detected without an O(file-size)
-   * re-read on the hot path.
+   * On-disk fingerprint captured at the last point THIS instance's in-memory
+   * set matched disk — after `load()`, `save()`, or a reconcile publish. A peer
+   * process that advances the durable index changes its file generation even
+   * when byte size and mtime collide, so the freshness check remains safe
+   * without an O(file-size) read on the hot path.
    */
-  private lastSyncedFingerprint: { mtimeMs: number; size: number } | null = null;
+  private lastSyncedFingerprint: DiskFingerprint | null = null;
 
   constructor(
     stateDir: string,
@@ -166,10 +172,17 @@ export class ContentHashIndex {
   }
 
   /** Cheap `stat` of the durable index file; null when it does not exist yet. */
-  private async statIndexFile(): Promise<{ mtimeMs: number; size: number } | null> {
+  private async statIndexFile(): Promise<DiskFingerprint | null> {
     try {
       const s = await stat(this.filePath);
-      return { mtimeMs: s.mtimeMs, size: s.size };
+      return {
+        mtimeMs: s.mtimeMs,
+        size: s.size,
+        dev: s.dev,
+        ino: s.ino,
+        ctimeMs: s.ctimeMs,
+        birthtimeMs: s.birthtimeMs,
+      };
     } catch (err) {
       if (isErrnoCode(err, "ENOENT")) return null;
       throw err;
@@ -190,14 +203,15 @@ export class ContentHashIndex {
 
   /**
    * True when the durable index file is unchanged since this instance last
-   * synced it (load / save / reconcile). False when a peer advanced it, when the
-   * file appeared or vanished, or when freshness cannot be established (stat
-   * error) — the caller then treats the in-memory snapshot as non-authoritative
-   * and confirms a dedup miss against the durable corpus (PR #2016 review). One
-   * `stat`, never an O(file-size) read, so the hot path stays cheap.
+   * synced it (load / save / reconcile). False when a peer advanced it, when
+   * the file appeared or vanished, or when freshness cannot be established
+   * (stat error) — the caller then treats the in-memory snapshot as
+   * non-authoritative and confirms a dedup miss against the durable corpus.
+   * The stat includes file identity and generation metadata so same-size,
+   * same-mtime atomic replacements are detected without reading the file body.
    */
   async isDiskFingerprintCurrent(): Promise<boolean> {
-    let current: { mtimeMs: number; size: number } | null;
+    let current: DiskFingerprint | null;
     try {
       current = await this.statIndexFile();
     } catch {
@@ -207,7 +221,14 @@ export class ContentHashIndex {
     if (last === null || current === null) {
       return last === null && current === null;
     }
-    return current.mtimeMs === last.mtimeMs && current.size === last.size;
+    return (
+      current.mtimeMs === last.mtimeMs &&
+      current.size === last.size &&
+      current.dev === last.dev &&
+      current.ino === last.ino &&
+      current.ctimeMs === last.ctimeMs &&
+      current.birthtimeMs === last.birthtimeMs
+    );
   }
 
   /** Check if content already exists in the index. */
@@ -475,6 +496,11 @@ export class ContentHashIndex {
     return this.reconcileRetryTimer !== null;
   }
 
+  /** True while this index has in-memory changes not yet published to disk. */
+  get hasPendingChanges(): boolean {
+    return this.dirty;
+  }
+
   /**
    * Resolve once any armed background reconcile-save retry chain has settled —
    * either it published the deferred additions, or it exhausted its bounded