@remnic/core 9.3.761 → 9.3.763

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/core",
-  "version": "9.3.761",
+  "version": "9.3.763",
   "description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
   "type": "module",
   "main": "dist/index.js",
@@ -3000,7 +3000,7 @@
     "core"
   ],
   "peerDependencies": {
-    "@remnic/coding-graph": "^9.3.761"
+    "@remnic/coding-graph": "^9.3.763"
   },
   "peerDependenciesMeta": {
     "@remnic/coding-graph": {

package/src/maintenance/purge.test.ts

@@ -5,6 +5,9 @@ import path from "node:path";
 import test from "node:test";
 
 import { purgeMemories } from "./purge.js";
+import { rebuildMemoryProjection } from "./rebuild-memory-projection.js";
+import { readProjectedMemoryBrowse } from "../memory-projection-store.js";
+import { StorageManager } from "../storage.js";
 import type { MemoryFile } from "../types.js";
 
 test("purgeMemories records audit errors without blocking hard delete", async () => {
@@ -54,6 +57,79 @@ test("purgeMemories records audit errors without blocking hard delete", async ()
   }
 });
 
+test("purgeMemories invalidates projection rows for hard-deleted memories", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-purge-projection-"));
+  try {
+    const storage = new StorageManager(dir);
+    const { id } = await storage.writeMemory("fact", "purge projection target", {
+      source: "test",
+    });
+    const memory = await storage.getMemoryById(id);
+    assert.ok(memory);
+    await rebuildMemoryProjection({ memoryDir: dir, dryRun: false });
+    assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 1);
+
+    const result = await purgeMemories({
+      storage,
+      olderThanMs: 1,
+      tier: "all",
+      dryRun: false,
+      now: () => new Date(Date.now() + 86_400_000),
+    });
+
+    assert.equal(result.purgedCount, 1);
+    assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 0);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("purgeMemories invalidates projection rows for already-absent files but not failed deletes, and dry-run never mutates", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-purge-projection-branches-"));
+  try {
+    // Real projection with two rows; candidates come from a mock storage so the
+    // unlink outcomes are deterministic: one path is already absent (ENOENT),
+    // the other is a non-empty directory (unlink fails, not ENOENT).
+    const realStorage = new StorageManager(dir);
+    const first = await realStorage.writeMemory("fact", "purge branch enoent", { source: "test" });
+    const second = await realStorage.writeMemory("fact", "purge branch failure", { source: "test" });
+    await rebuildMemoryProjection({ memoryDir: dir, dryRun: false });
+    assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 2);
+
+    const firstMemory = await realStorage.getMemoryById(first.id);
+    const secondMemory = await realStorage.getMemoryById(second.id);
+    assert.ok(firstMemory && secondMemory);
+    await rm(firstMemory.path);
+    await rm(secondMemory.path);
+    await mkdir(secondMemory.path);
+    await writeFile(path.join(secondMemory.path, "block.txt"), "x", "utf8");
+
+    const mockStorage = {
+      dir,
+      readAllMemories: async () => [firstMemory, secondMemory],
+      readAllColdMemories: async () => [],
+      readArchivedMemories: async () => [],
+    };
+    const future = () => new Date(Date.now() + 86_400_000);
+
+    // Dry-run: no projection mutation.
+    await purgeMemories({ storage: mockStorage as never, olderThanMs: 1, tier: "all", dryRun: true, now: future });
+    assert.equal(readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 })?.total, 2);
+
+    const result = await purgeMemories({ storage: mockStorage as never, olderThanMs: 1, tier: "all", dryRun: false, now: future });
+
+    assert.equal(result.alreadyAbsentCount, 1);
+    assert.equal(result.purgedCount, 0);
+    assert.equal(result.errors.filter((e) => e.id === second.id).length, 1);
+    const page = readProjectedMemoryBrowse(dir, { limit: 5, offset: 0 });
+    assert.ok(page);
+    // ENOENT row invalidated; failed-delete row keeps its projection entry.
+    assert.equal(page.total, 1);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
 async function fileExists(filePath: string): Promise<boolean> {
   try {
     await access(filePath);

package/src/maintenance/purge.ts

@@ -23,6 +23,7 @@ import path from "node:path";
 import { appendFile, mkdir, unlink } from "node:fs/promises";
 import type { StorageManager } from "../storage.js";
 import type { MemoryFile } from "../types.js";
+import { markProjectedMemoryPathInvalid } from "../memory-projection-store.js";
 import type { SearchBackend } from "../search/port.js";
 
 export type PurgeTierFilter = "cold" | "all";
@@ -236,12 +237,14 @@ export async function purgeMemories(
       await unlink(candidate.path);
       actuallyPurged.push(candidate);
       addCollectionForCandidate(candidate);
+      markProjectedMemoryPathInvalid(storage.dir, candidate.id);
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
       // ENOENT is fine — already gone
       if (hasErrorCode(err, "ENOENT")) {
         alreadyAbsent.push(candidate);
         addCollectionForCandidate(candidate);
+        markProjectedMemoryPathInvalid(storage.dir, candidate.id);
       } else {
         errors.push({ id: candidate.id, path: candidate.path, error: message });
       }

package/src/maintenance/rebuild-memory-lifecycle-ledger.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import { stat } from "node:fs/promises";
 import { StorageManager } from "../storage.js";
+import { log } from "../logger.js";
 import type { MemoryLifecycleEvent } from "../types.js";
 import { toBackupStamp } from "./backup-stamp.js";
 import { writeFileAtomically } from "./atomic-file.js";
@@ -15,12 +16,24 @@ export interface RebuildMemoryLifecycleLedgerOptions {
   now?: Date;
 }
 
+export interface SkippedLifecycleBlankIdMemory {
+  path: string;
+}
+
+export interface SkippedDuplicateLifecycleEvent {
+  eventId: string;
+  keptPath: string;
+  skippedPath: string;
+}
+
 export interface RebuildMemoryLifecycleLedgerResult {
   dryRun: boolean;
   scannedMemories: number;
   rebuiltRows: number;
   outputPath: string;
   backupPath?: string;
+  skippedBlankIdMemories: SkippedLifecycleBlankIdMemory[];
+  skippedDuplicateEvents: SkippedDuplicateLifecycleEvent[];
 }
 
 function isNodeError(err: unknown): err is NodeJS.ErrnoException {
@@ -59,16 +72,52 @@ export async function rebuildMemoryLifecycleLedger(
   const now = options.now ?? new Date();
   const outputPath = path.join(options.memoryDir, "state", "memory-lifecycle-ledger.jsonl");
   const storage = new StorageManager(options.memoryDir);
-  const allMemories = [
-    ...await storage.readAllMemories(),
-    ...await storage.readAllColdMemories(),
-    ...await storage.readArchivedMemories(),
-  ]
-    .sort((a, b) => a.frontmatter.id.localeCompare(b.frontmatter.id));
+  const tiers = [
+    await storage.readAllMemories(),
+    await storage.readAllColdMemories(),
+    await storage.readArchivedMemories(),
+  ];
+  const allMemories = tiers.flat();
+  const skippedBlankIdMemories: SkippedLifecycleBlankIdMemory[] = [];
+  const eventCandidates: Array<{ event: MemoryLifecycleEvent; path: string }> = [];
+  for (const tier of tiers) {
+    for (const memory of [...tier].sort((left, right) => left.path.localeCompare(right.path))) {
+      if (!memory.frontmatter.id.trim()) {
+        skippedBlankIdMemories.push({ path: memory.path });
+        log.warn(`lifecycle ledger rebuild skipped blank memory id: path=${memory.path}`);
+        continue;
+      }
+      for (const event of buildLifecycleEventsForMemory(memory)) {
+        eventCandidates.push({ event, path: memory.path });
+      }
+    }
+  }
 
-  const events: MemoryLifecycleEvent[] = sortMemoryLifecycleEvents(
-    allMemories.flatMap((memory) => buildLifecycleEventsForMemory(memory)),
+  const orderedEvents = sortMemoryLifecycleEvents(eventCandidates.map((candidate) => candidate.event));
+  const candidateByEvent = new Map(
+    eventCandidates.map((candidate) => [candidate.event, candidate]),
   );
+  const keptByEventId = new Map<string, { event: MemoryLifecycleEvent; path: string }>();
+  const events: MemoryLifecycleEvent[] = [];
+  const skippedDuplicateEvents: SkippedDuplicateLifecycleEvent[] = [];
+  for (const event of orderedEvents) {
+    const candidate = candidateByEvent.get(event)!;
+    const kept = keptByEventId.get(event.eventId);
+    if (kept) {
+      skippedDuplicateEvents.push({
+        eventId: event.eventId,
+        keptPath: kept.path,
+        skippedPath: candidate.path,
+      });
+      log.warn(
+        `lifecycle ledger rebuild skipped duplicate event_id=${event.eventId}: `
+        + `kept=${kept.path} skipped=${candidate.path}`,
+      );
+      continue;
+    }
+    keptByEventId.set(event.eventId, candidate);
+    events.push(event);
+  }
 
   let backupPath: string | undefined;
   if (!dryRun) {
@@ -87,5 +136,7 @@ export async function rebuildMemoryLifecycleLedger(
     rebuiltRows: events.length,
     outputPath,
     backupPath,
+    skippedBlankIdMemories,
+    skippedDuplicateEvents,
   };
 }

package/src/maintenance/rebuild-memory-projection.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import { mkdir, rm, stat } from "node:fs/promises";
 import { StorageManager } from "../storage.js";
+import { log } from "../logger.js";
 import {
   listMemoryGovernanceRuns,
   readMemoryGovernanceRunArtifact,
@@ -23,6 +24,7 @@ import {
 import {
   getMemoryProjectionPath,
   initializeMemoryProjectionDb,
+  isProjectedMemoryPathValid,
   type ProjectedEntityMentionRow,
   type MemoryProjectionGovernanceAppliedActionRow,
   type MemoryProjectionGovernanceReviewQueueRow,
@@ -51,6 +53,27 @@ export interface RebuildMemoryProjectionOptions {
   updatedBefore?: string;
 }
 
+export interface SkippedDuplicateMemory {
+  memoryId: string;
+  keptPath: string;
+  skippedPath: string;
+}
+
+export interface SkippedBlankIdMemory {
+  path: string;
+}
+
+export interface SkippedDuplicateTimelineEvent {
+  eventId: string;
+  keptPath: string;
+  skippedPath: string;
+}
+
+export interface SkippedBlankIdTimelineEvent {
+  eventId: string;
+  path: string;
+}
+
 export interface RebuildMemoryProjectionResult {
   dryRun: boolean;
   scannedMemories: number;
@@ -62,6 +85,10 @@ export interface RebuildMemoryProjectionResult {
   outputPath: string;
   backupPath?: string;
   usedLifecycleLedger: boolean;
+  skippedDuplicateMemories: SkippedDuplicateMemory[];
+  skippedBlankIdMemories: SkippedBlankIdMemory[];
+  skippedDuplicateTimelineEvents: SkippedDuplicateTimelineEvent[];
+  skippedBlankIdTimelineEvents: SkippedBlankIdTimelineEvent[];
   scope: {
     updatedAfter: string | null;
     updatedBefore: string | null;
@@ -435,6 +462,91 @@ function serializeGovernanceReviewQueueRow(
   });
 }
 
+function selectProjectionMemories(
+  tiers: MemoryFile[][],
+): {
+  allMemories: MemoryFile[];
+  memories: MemoryFile[];
+  skippedDuplicateMemories: SkippedDuplicateMemory[];
+  skippedBlankIdMemories: SkippedBlankIdMemory[];
+} {
+  const allMemories = tiers.flat();
+  const memories: MemoryFile[] = [];
+  const keptById = new Map<string, MemoryFile>();
+  const skippedDuplicateMemories: SkippedDuplicateMemory[] = [];
+  const skippedBlankIdMemories: SkippedBlankIdMemory[] = [];
+
+  // Tier order is authoritative: hot wins over cold, and cold wins over archive.
+  // Paths break ties within a tier so the winner does not depend on readdir order.
+  for (const tier of tiers) {
+    for (const memory of [...tier].sort((left, right) => left.path.localeCompare(right.path))) {
+      const memoryId = memory.frontmatter.id.trim();
+      if (!memoryId) {
+        skippedBlankIdMemories.push({ path: memory.path });
+        log.warn(`projection rebuild skipped blank memory id: path=${memory.path}`);
+        continue;
+      }
+      const kept = keptById.get(memoryId);
+      if (kept) {
+        skippedDuplicateMemories.push({
+          memoryId,
+          keptPath: kept.path,
+          skippedPath: memory.path,
+        });
+        log.warn(
+          `projection rebuild skipped duplicate memory id=${memoryId}: kept=${kept.path} skipped=${memory.path}`,
+        );
+        continue;
+      }
+      keptById.set(memoryId, memory);
+      memories.push(memory);
+    }
+  }
+
+  return { allMemories, memories, skippedDuplicateMemories, skippedBlankIdMemories };
+}
+
+function deduplicateTimelineEvents(
+  events: MemoryLifecycleEvent[],
+  sourcePathFor: (event: MemoryLifecycleEvent) => string,
+): {
+  events: MemoryLifecycleEvent[];
+  skipped: SkippedDuplicateTimelineEvent[];
+  skippedBlankIds: SkippedBlankIdTimelineEvent[];
+} {
+  const keptById = new Map<string, { event: MemoryLifecycleEvent; path: string }>();
+  const unique: MemoryLifecycleEvent[] = [];
+  const skipped: SkippedDuplicateTimelineEvent[] = [];
+  const skippedBlankIds: SkippedBlankIdTimelineEvent[] = [];
+  for (const event of sortMemoryLifecycleEvents(events)) {
+    const sourcePath = sourcePathFor(event);
+    if (!event.memoryId.trim()) {
+      skippedBlankIds.push({ eventId: event.eventId, path: sourcePath });
+      log.warn(
+        `projection rebuild skipped timeline event with blank memory id: `
+        + `event_id=${event.eventId} path=${sourcePath}`,
+      );
+      continue;
+    }
+    const kept = keptById.get(event.eventId);
+    if (kept) {
+      skipped.push({
+        eventId: event.eventId,
+        keptPath: kept.path,
+        skippedPath: sourcePath,
+      });
+      log.warn(
+        `projection rebuild skipped duplicate timeline event_id=${event.eventId}: `
+        + `kept=${kept.path} skipped=${sourcePath}`,
+      );
+      continue;
+    }
+    keptById.set(event.eventId, { event, path: sourcePath });
+    unique.push(event);
+  }
+  return { events: unique, skipped, skippedBlankIds };
+}
+
 async function loadAuthoritativeProjectionSnapshot(options: {
   memoryDir: string;
   defaultNamespace?: string;
@@ -464,22 +576,43 @@ async function loadAuthoritativeProjectionSnapshot(options: {
     updatedAfter: string | null;
     updatedBefore: string | null;
   };
+  skippedDuplicateMemories: SkippedDuplicateMemory[];
+  skippedBlankIdMemories: SkippedBlankIdMemory[];
+  skippedDuplicateTimelineEvents: SkippedDuplicateTimelineEvent[];
+  skippedBlankIdTimelineEvents: SkippedBlankIdTimelineEvent[];
 }> {
   const storage = new StorageManager(options.memoryDir);
   // Force a fresh disk read — projection verify/rebuild must see the true
   // on-disk state, not a potentially stale in-process cache.
   storage.invalidateAllMemoriesCacheForDir();
-  const allMemories = [
-    ...await storage.readAllMemories(),
-    ...await storage.readAllColdMemories(),
-    ...await storage.readArchivedMemories(),
-  ]
-    .sort((a, b) => a.frontmatter.id.localeCompare(b.frontmatter.id));
+  const selected = selectProjectionMemories([
+    await storage.readAllMemories(),
+    await storage.readAllColdMemories(),
+    await storage.readArchivedMemories(),
+  ]);
+  const allMemories = selected.allMemories;
+  const projectionMemories = selected.memories;
   const lifecycleEvents = await storage.readAllMemoryLifecycleEvents();
-  const { events, usedLifecycleLedger } = loadTimelineEvents(allMemories, lifecycleEvents);
-  const currentRows = allMemories.map((memory) => toCurrentStateRow(options.memoryDir, memory));
+  const timeline = loadTimelineEvents(projectionMemories, lifecycleEvents);
+  const lifecycleLedgerPath = path.join(
+    options.memoryDir,
+    "state",
+    "memory-lifecycle-ledger.jsonl",
+  );
+  const memoryPathById = new Map(
+    projectionMemories.map((memory) => [memory.frontmatter.id, memory.path]),
+  );
+  const deduplicatedTimeline = deduplicateTimelineEvents(
+    timeline.events,
+    (event) => timeline.usedLifecycleLedger
+      ? lifecycleLedgerPath
+      : memoryPathById.get(event.memoryId) ?? lifecycleLedgerPath,
+  );
+  const events = deduplicatedTimeline.events;
+  const usedLifecycleLedger = timeline.usedLifecycleLedger;
+  const currentRows = projectionMemories.map((memory) => toCurrentStateRow(options.memoryDir, memory));
   const scope = normalizeProjectionScope(options);
-  const scopedMemories = filterMemoriesForProjectionScope(allMemories, scope);
+  const scopedMemories = filterMemoriesForProjectionScope(projectionMemories, scope);
   const scopedMemoryIds = new Set(scopedMemories.map((memory) => memory.frontmatter.id));
   const nativeKnowledgeRows = await loadPersistedNativeKnowledgeChunks({
     memoryDir: options.memoryDir,
@@ -516,6 +649,10 @@ async function loadAuthoritativeProjectionSnapshot(options: {
     governance,
     usedLifecycleLedger,
     scope,
+    skippedDuplicateMemories: selected.skippedDuplicateMemories,
+    skippedBlankIdMemories: selected.skippedBlankIdMemories,
+    skippedDuplicateTimelineEvents: deduplicatedTimeline.skipped,
+    skippedBlankIdTimelineEvents: deduplicatedTimeline.skippedBlankIds,
   };
 }
 
@@ -658,6 +795,7 @@ function writeProjectionDb(
   usedLifecycleLedger: boolean,
 ): void {
   const db = openBetterSqlite3(dbPath);
+  const memoryDir = path.dirname(path.dirname(dbPath));
   try {
     initializeMemoryProjectionDb(db);
 
@@ -674,6 +812,7 @@ function writeProjectionDb(
         status,
         lifecycle_state,
         path_rel,
+        path_valid,
         created_at,
         updated_at,
         archived_at,
@@ -687,7 +826,7 @@ function writeProjectionDb(
         last_accessed,
         tags_json,
         preview_text
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
     `);
 
     const insertTimeline = db.prepare(`
@@ -785,6 +924,7 @@ function writeProjectionDb(
           row.status,
           row.lifecycleState ?? null,
           row.pathRel,
+          isProjectedMemoryPathValid(memoryDir, row.pathRel) ? 1 : 0,
           row.created,
           row.updated,
           row.archivedAt ?? null,
@@ -1040,6 +1180,10 @@ export async function rebuildMemoryProjection(
     outputPath,
     backupPath,
     usedLifecycleLedger: snapshot.usedLifecycleLedger,
+    skippedDuplicateMemories: snapshot.skippedDuplicateMemories,
+    skippedBlankIdMemories: snapshot.skippedBlankIdMemories,
+    skippedDuplicateTimelineEvents: snapshot.skippedDuplicateTimelineEvents,
+    skippedBlankIdTimelineEvents: snapshot.skippedBlankIdTimelineEvents,
     scope: snapshot.scope,
   };
 }