@remnic/core 9.3.760 → 9.3.762

package/src/maintenance/rebuild-memory-lifecycle-ledger.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import { stat } from "node:fs/promises";
 import { StorageManager } from "../storage.js";
+import { log } from "../logger.js";
 import type { MemoryLifecycleEvent } from "../types.js";
 import { toBackupStamp } from "./backup-stamp.js";
 import { writeFileAtomically } from "./atomic-file.js";
@@ -15,12 +16,24 @@ export interface RebuildMemoryLifecycleLedgerOptions {
   now?: Date;
 }
 
+export interface SkippedLifecycleBlankIdMemory {
+  path: string;
+}
+
+export interface SkippedDuplicateLifecycleEvent {
+  eventId: string;
+  keptPath: string;
+  skippedPath: string;
+}
+
 export interface RebuildMemoryLifecycleLedgerResult {
   dryRun: boolean;
   scannedMemories: number;
   rebuiltRows: number;
   outputPath: string;
   backupPath?: string;
+  skippedBlankIdMemories: SkippedLifecycleBlankIdMemory[];
+  skippedDuplicateEvents: SkippedDuplicateLifecycleEvent[];
 }
 
 function isNodeError(err: unknown): err is NodeJS.ErrnoException {
@@ -59,16 +72,52 @@ export async function rebuildMemoryLifecycleLedger(
   const now = options.now ?? new Date();
   const outputPath = path.join(options.memoryDir, "state", "memory-lifecycle-ledger.jsonl");
   const storage = new StorageManager(options.memoryDir);
-  const allMemories = [
-    ...await storage.readAllMemories(),
-    ...await storage.readAllColdMemories(),
-    ...await storage.readArchivedMemories(),
-  ]
-    .sort((a, b) => a.frontmatter.id.localeCompare(b.frontmatter.id));
+  const tiers = [
+    await storage.readAllMemories(),
+    await storage.readAllColdMemories(),
+    await storage.readArchivedMemories(),
+  ];
+  const allMemories = tiers.flat();
+  const skippedBlankIdMemories: SkippedLifecycleBlankIdMemory[] = [];
+  const eventCandidates: Array<{ event: MemoryLifecycleEvent; path: string }> = [];
+  for (const tier of tiers) {
+    for (const memory of [...tier].sort((left, right) => left.path.localeCompare(right.path))) {
+      if (!memory.frontmatter.id.trim()) {
+        skippedBlankIdMemories.push({ path: memory.path });
+        log.warn(`lifecycle ledger rebuild skipped blank memory id: path=${memory.path}`);
+        continue;
+      }
+      for (const event of buildLifecycleEventsForMemory(memory)) {
+        eventCandidates.push({ event, path: memory.path });
+      }
+    }
+  }
 
-  const events: MemoryLifecycleEvent[] = sortMemoryLifecycleEvents(
-    allMemories.flatMap((memory) => buildLifecycleEventsForMemory(memory)),
+  const orderedEvents = sortMemoryLifecycleEvents(eventCandidates.map((candidate) => candidate.event));
+  const candidateByEvent = new Map(
+    eventCandidates.map((candidate) => [candidate.event, candidate]),
   );
+  const keptByEventId = new Map<string, { event: MemoryLifecycleEvent; path: string }>();
+  const events: MemoryLifecycleEvent[] = [];
+  const skippedDuplicateEvents: SkippedDuplicateLifecycleEvent[] = [];
+  for (const event of orderedEvents) {
+    const candidate = candidateByEvent.get(event)!;
+    const kept = keptByEventId.get(event.eventId);
+    if (kept) {
+      skippedDuplicateEvents.push({
+        eventId: event.eventId,
+        keptPath: kept.path,
+        skippedPath: candidate.path,
+      });
+      log.warn(
+        `lifecycle ledger rebuild skipped duplicate event_id=${event.eventId}: `
+        + `kept=${kept.path} skipped=${candidate.path}`,
+      );
+      continue;
+    }
+    keptByEventId.set(event.eventId, candidate);
+    events.push(event);
+  }
 
   let backupPath: string | undefined;
   if (!dryRun) {
@@ -87,5 +136,7 @@ export async function rebuildMemoryLifecycleLedger(
     rebuiltRows: events.length,
     outputPath,
     backupPath,
+    skippedBlankIdMemories,
+    skippedDuplicateEvents,
   };
 }

package/src/maintenance/rebuild-memory-projection.ts

@@ -1,6 +1,7 @@
 import path from "node:path";
 import { mkdir, rm, stat } from "node:fs/promises";
 import { StorageManager } from "../storage.js";
+import { log } from "../logger.js";
 import {
   listMemoryGovernanceRuns,
   readMemoryGovernanceRunArtifact,
@@ -51,6 +52,27 @@ export interface RebuildMemoryProjectionOptions {
   updatedBefore?: string;
 }
 
+export interface SkippedDuplicateMemory {
+  memoryId: string;
+  keptPath: string;
+  skippedPath: string;
+}
+
+export interface SkippedBlankIdMemory {
+  path: string;
+}
+
+export interface SkippedDuplicateTimelineEvent {
+  eventId: string;
+  keptPath: string;
+  skippedPath: string;
+}
+
+export interface SkippedBlankIdTimelineEvent {
+  eventId: string;
+  path: string;
+}
+
 export interface RebuildMemoryProjectionResult {
   dryRun: boolean;
   scannedMemories: number;
@@ -62,6 +84,10 @@ export interface RebuildMemoryProjectionResult {
   outputPath: string;
   backupPath?: string;
   usedLifecycleLedger: boolean;
+  skippedDuplicateMemories: SkippedDuplicateMemory[];
+  skippedBlankIdMemories: SkippedBlankIdMemory[];
+  skippedDuplicateTimelineEvents: SkippedDuplicateTimelineEvent[];
+  skippedBlankIdTimelineEvents: SkippedBlankIdTimelineEvent[];
   scope: {
     updatedAfter: string | null;
     updatedBefore: string | null;
@@ -435,6 +461,91 @@ function serializeGovernanceReviewQueueRow(
   });
 }
 
+function selectProjectionMemories(
+  tiers: MemoryFile[][],
+): {
+  allMemories: MemoryFile[];
+  memories: MemoryFile[];
+  skippedDuplicateMemories: SkippedDuplicateMemory[];
+  skippedBlankIdMemories: SkippedBlankIdMemory[];
+} {
+  const allMemories = tiers.flat();
+  const memories: MemoryFile[] = [];
+  const keptById = new Map<string, MemoryFile>();
+  const skippedDuplicateMemories: SkippedDuplicateMemory[] = [];
+  const skippedBlankIdMemories: SkippedBlankIdMemory[] = [];
+
+  // Tier order is authoritative: hot wins over cold, and cold wins over archive.
+  // Paths break ties within a tier so the winner does not depend on readdir order.
+  for (const tier of tiers) {
+    for (const memory of [...tier].sort((left, right) => left.path.localeCompare(right.path))) {
+      const memoryId = memory.frontmatter.id.trim();
+      if (!memoryId) {
+        skippedBlankIdMemories.push({ path: memory.path });
+        log.warn(`projection rebuild skipped blank memory id: path=${memory.path}`);
+        continue;
+      }
+      const kept = keptById.get(memoryId);
+      if (kept) {
+        skippedDuplicateMemories.push({
+          memoryId,
+          keptPath: kept.path,
+          skippedPath: memory.path,
+        });
+        log.warn(
+          `projection rebuild skipped duplicate memory id=${memoryId}: kept=${kept.path} skipped=${memory.path}`,
+        );
+        continue;
+      }
+      keptById.set(memoryId, memory);
+      memories.push(memory);
+    }
+  }
+
+  return { allMemories, memories, skippedDuplicateMemories, skippedBlankIdMemories };
+}
+
+function deduplicateTimelineEvents(
+  events: MemoryLifecycleEvent[],
+  sourcePathFor: (event: MemoryLifecycleEvent) => string,
+): {
+  events: MemoryLifecycleEvent[];
+  skipped: SkippedDuplicateTimelineEvent[];
+  skippedBlankIds: SkippedBlankIdTimelineEvent[];
+} {
+  const keptById = new Map<string, { event: MemoryLifecycleEvent; path: string }>();
+  const unique: MemoryLifecycleEvent[] = [];
+  const skipped: SkippedDuplicateTimelineEvent[] = [];
+  const skippedBlankIds: SkippedBlankIdTimelineEvent[] = [];
+  for (const event of sortMemoryLifecycleEvents(events)) {
+    const sourcePath = sourcePathFor(event);
+    if (!event.memoryId.trim()) {
+      skippedBlankIds.push({ eventId: event.eventId, path: sourcePath });
+      log.warn(
+        `projection rebuild skipped timeline event with blank memory id: `
+        + `event_id=${event.eventId} path=${sourcePath}`,
+      );
+      continue;
+    }
+    const kept = keptById.get(event.eventId);
+    if (kept) {
+      skipped.push({
+        eventId: event.eventId,
+        keptPath: kept.path,
+        skippedPath: sourcePath,
+      });
+      log.warn(
+        `projection rebuild skipped duplicate timeline event_id=${event.eventId}: `
+        + `kept=${kept.path} skipped=${sourcePath}`,
+      );
+      continue;
+    }
+    keptById.set(event.eventId, { event, path: sourcePath });
+    unique.push(event);
+  }
+  return { events: unique, skipped, skippedBlankIds };
+}
+
 async function loadAuthoritativeProjectionSnapshot(options: {
   memoryDir: string;
   defaultNamespace?: string;
@@ -464,22 +575,43 @@ async function loadAuthoritativeProjectionSnapshot(options: {
     updatedAfter: string | null;
     updatedBefore: string | null;
   };
+  skippedDuplicateMemories: SkippedDuplicateMemory[];
+  skippedBlankIdMemories: SkippedBlankIdMemory[];
+  skippedDuplicateTimelineEvents: SkippedDuplicateTimelineEvent[];
+  skippedBlankIdTimelineEvents: SkippedBlankIdTimelineEvent[];
 }> {
   const storage = new StorageManager(options.memoryDir);
   // Force a fresh disk read — projection verify/rebuild must see the true
   // on-disk state, not a potentially stale in-process cache.
   storage.invalidateAllMemoriesCacheForDir();
-  const allMemories = [
-    ...await storage.readAllMemories(),
-    ...await storage.readAllColdMemories(),
-    ...await storage.readArchivedMemories(),
-  ]
-    .sort((a, b) => a.frontmatter.id.localeCompare(b.frontmatter.id));
+  const selected = selectProjectionMemories([
+    await storage.readAllMemories(),
+    await storage.readAllColdMemories(),
+    await storage.readArchivedMemories(),
+  ]);
+  const allMemories = selected.allMemories;
+  const projectionMemories = selected.memories;
   const lifecycleEvents = await storage.readAllMemoryLifecycleEvents();
-  const { events, usedLifecycleLedger } = loadTimelineEvents(allMemories, lifecycleEvents);
-  const currentRows = allMemories.map((memory) => toCurrentStateRow(options.memoryDir, memory));
+  const timeline = loadTimelineEvents(projectionMemories, lifecycleEvents);
+  const lifecycleLedgerPath = path.join(
+    options.memoryDir,
+    "state",
+    "memory-lifecycle-ledger.jsonl",
+  );
+  const memoryPathById = new Map(
+    projectionMemories.map((memory) => [memory.frontmatter.id, memory.path]),
+  );
+  const deduplicatedTimeline = deduplicateTimelineEvents(
+    timeline.events,
+    (event) => timeline.usedLifecycleLedger
+      ? lifecycleLedgerPath
+      : memoryPathById.get(event.memoryId) ?? lifecycleLedgerPath,
+  );
+  const events = deduplicatedTimeline.events;
+  const usedLifecycleLedger = timeline.usedLifecycleLedger;
+  const currentRows = projectionMemories.map((memory) => toCurrentStateRow(options.memoryDir, memory));
   const scope = normalizeProjectionScope(options);
-  const scopedMemories = filterMemoriesForProjectionScope(allMemories, scope);
+  const scopedMemories = filterMemoriesForProjectionScope(projectionMemories, scope);
   const scopedMemoryIds = new Set(scopedMemories.map((memory) => memory.frontmatter.id));
   const nativeKnowledgeRows = await loadPersistedNativeKnowledgeChunks({
     memoryDir: options.memoryDir,
@@ -516,6 +648,10 @@ async function loadAuthoritativeProjectionSnapshot(options: {
     governance,
     usedLifecycleLedger,
     scope,
+    skippedDuplicateMemories: selected.skippedDuplicateMemories,
+    skippedBlankIdMemories: selected.skippedBlankIdMemories,
+    skippedDuplicateTimelineEvents: deduplicatedTimeline.skipped,
+    skippedBlankIdTimelineEvents: deduplicatedTimeline.skippedBlankIds,
   };
 }
 
@@ -1040,6 +1176,10 @@ export async function rebuildMemoryProjection(
     outputPath,
     backupPath,
     usedLifecycleLedger: snapshot.usedLifecycleLedger,
+    skippedDuplicateMemories: snapshot.skippedDuplicateMemories,
+    skippedBlankIdMemories: snapshot.skippedBlankIdMemories,
+    skippedDuplicateTimelineEvents: snapshot.skippedDuplicateTimelineEvents,
+    skippedBlankIdTimelineEvents: snapshot.skippedBlankIdTimelineEvents,
     scope: snapshot.scope,
   };
 }

package/src/namespaces/catalog.test.ts

@@ -1,5 +1,16 @@
 import assert from "node:assert/strict";
-import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, utimes, writeFile } from "node:fs/promises";
+import {
+  appendFile,
+  mkdir,
+  mkdtemp,
+  readFile,
+  realpath,
+  rm,
+  stat,
+  symlink,
+  utimes,
+  writeFile,
+} from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import test from "node:test";
@@ -32,6 +43,21 @@ async function mkMemoryDir(): Promise<string> {
   return await mkdtemp(path.join(os.tmpdir(), "remnic-ns-catalog-"));
 }
 
+class CountingNamespaceCatalog extends NamespaceCatalog {
+  catalogReadCount = 0;
+
+  constructor(config: PluginConfig) {
+    super(config);
+    this.onCatalogReadForTest = () => {
+      this.catalogReadCount++;
+    };
+  }
+
+  setAfterAppendHook(hook: () => Promise<void>): void {
+    this.onAfterCatalogAppendForTest = hook;
+  }
+}
+
 test("markWrite registers a dynamic project namespace in the catalog", async () => {
   const memoryDir = await mkMemoryDir();
   try {
@@ -50,6 +76,85 @@ test("markWrite registers a dynamic project namespace in the catalog", async ()
   }
 });
 
+test("touch reloads the catalog after an external append", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new NamespaceCatalog(makeConfig(memoryDir));
+    await catalog.markWrite("alpha", { discoveredBy: "write" });
+
+    const catalogPath = path.join(memoryDir, "state", "namespaces.jsonl");
+    const alpha = JSON.parse((await readFile(catalogPath, "utf8")).trim());
+    const externalNamespace = "external";
+    await appendFile(
+      catalogPath,
+      `${JSON.stringify({
+        ...alpha,
+        namespace: externalNamespace,
+        identityToken: namespaceIdentityToken(externalNamespace),
+        storageDir: path.join(memoryDir, "namespaces", namespaceIdentityToken(externalNamespace)),
+      })}\n`,
+      "utf8",
+    );
+
+    await catalog.markRead("alpha");
+
+    assert.ok(
+      (await catalog.listNamespaces()).some((record) => record.namespace === externalNamespace),
+      "the touch must fold in a row appended through another file descriptor",
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("repeated touches do not re-read an unchanged catalog", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new CountingNamespaceCatalog(makeConfig(memoryDir));
+
+    await catalog.markWrite("alpha", { discoveredBy: "write" });
+    await catalog.markRead("alpha");
+    await catalog.markMaintenance("alpha", "test");
+
+    assert.equal(catalog.catalogReadCount, 1);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("a foreign append between append and stat invalidates the cache", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const catalog = new CountingNamespaceCatalog(makeConfig(memoryDir));
+    await catalog.markWrite("alpha", { discoveredBy: "write" });
+
+    const catalogPath = path.join(memoryDir, "state", "namespaces.jsonl");
+    const alpha = JSON.parse((await readFile(catalogPath, "utf8")).trim());
+    const externalNamespace = "racing-external";
+    catalog.setAfterAppendHook(async () => {
+      await appendFile(
+        catalogPath,
+        `${JSON.stringify({
+          ...alpha,
+          namespace: externalNamespace,
+          identityToken: namespaceIdentityToken(externalNamespace),
+          storageDir: path.join(memoryDir, "namespaces", namespaceIdentityToken(externalNamespace)),
+        })}\n`,
+        "utf8",
+      );
+    });
+
+    await catalog.markRead("alpha");
+    const readsBeforeReload = catalog.catalogReadCount;
+    const records = await catalog.listNamespaces();
+
+    assert.equal(catalog.catalogReadCount, readsBeforeReload + 1);
+    assert.ok(records.some((record) => record.namespace === externalNamespace));
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 test("configured default and shared namespaces appear in the catalog after register", async () => {
   const memoryDir = await mkMemoryDir();
   try {
@@ -89,6 +194,19 @@ test("rebuildFromDisk finds existing tokenized namespace directories", async ()
   }
 });
 
+test("rebuildFromDisk accepts config without sharedNamespace", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    const config = makeConfig(memoryDir);
+    delete (config as Partial<PluginConfig>).sharedNamespace;
+    const catalog = new NamespaceCatalog(config);
+
+    await assert.doesNotReject(catalog.rebuildFromDisk());
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 test("rebuildFromDisk dry-run does not write the catalog file", async () => {
   const memoryDir = await mkMemoryDir();
   try {
@@ -405,6 +523,20 @@ test("catalog tolerates corrupt / non-object JSONL lines on read", async () => {
   }
 });
 
+test("catalog read fails open when the catalog path is a directory", async () => {
+  const memoryDir = await mkMemoryDir();
+  try {
+    await mkdir(path.join(memoryDir, "state", "namespaces.jsonl"), { recursive: true });
+    const catalog = new NamespaceCatalog(makeConfig(memoryDir));
+
+    await assert.doesNotReject(async () => {
+      assert.deepEqual(await catalog.listNamespaces(), []);
+    });
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 test("catalog quarantines malformed JSONL record fields at the parse boundary", async () => {
   const memoryDir = await mkMemoryDir();
   try {