@remnic/core 10.0.3 → 10.0.4

package/src/maintenance/rebuild-memory-projection.retain-raw.test.ts

@@ -0,0 +1,202 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import {
+  compareMemoryLifecycleEvents,
+  memoryLifecycleEventProjectionIdentity,
+} from "../memory-lifecycle-ledger-utils.js";
+import { getMemoryProjectionPath, openProjectionReadonly } from "../memory-projection-store.js";
+import { readRawMemoryDocument } from "../storage/memory-frontmatter-metadata.js";
+import { StorageManager } from "../storage.js";
+import type { MemoryLifecycleEvent } from "../types.js";
+import { rebuildMemoryProjection } from "./rebuild-memory-projection.js";
+
+const FIXED_NOW = new Date("2026-06-01T00:00:00.000Z");
+
+function fact(id: string, body: string): string {
+  return [
+    "---",
+    `id: ${id}`,
+    "category: fact",
+    "confidence: 0.8",
+    "created: 2026-01-01T00:00:00.000Z",
+    "updated: 2026-01-02T00:00:00.000Z",
+    "status: active",
+    "source: test",
+    "entityRef: project-alpha",
+    "tags:",
+    "  - topic:alpha",
+    "---",
+    "",
+    body,
+    "",
+  ].join("\n");
+}
+
+function ledgerEvent(partial: Pick<MemoryLifecycleEvent, "eventId" | "memoryId" | "eventType" | "timestamp">): MemoryLifecycleEvent {
+  return {
+    actor: "test",
+    ruleVersion: "1",
+    ...partial,
+  };
+}
+
+async function seedCorpus(memoryDir: string, events: MemoryLifecycleEvent[]): Promise<void> {
+  await mkdir(path.join(memoryDir, "facts", "2026-01-01"), { recursive: true });
+  await mkdir(path.join(memoryDir, "cold", "facts", "2026-01-01"), { recursive: true });
+  await mkdir(path.join(memoryDir, "archive", "facts", "2026-01-01"), { recursive: true });
+  await mkdir(path.join(memoryDir, "state"), { recursive: true });
+  await writeFile(path.join(memoryDir, "facts", "2026-01-01", "hot-1.md"), fact("hot-1", "HOTBODY unique hot text"));
+  await writeFile(path.join(memoryDir, "facts", "2026-01-01", "blank.md"), fact("  ", "blank id body"));
+  await writeFile(path.join(memoryDir, "cold", "facts", "2026-01-01", "hot-1.md"), fact("hot-1", "COLDBODY should lose to hot"));
+  await writeFile(path.join(memoryDir, "cold", "facts", "2026-01-01", "cold-only.md"), fact("cold-only", "COLDONLY body"));
+  await writeFile(path.join(memoryDir, "archive", "facts", "2026-01-01", "arch-1.md"), fact("arch-1", "ARCHBODY archived text"));
+  const lines = [
+    ...events.map((event) => JSON.stringify(event)),
+    "{not-json",
+  ];
+  await writeFile(
+    path.join(memoryDir, "state", "memory-lifecycle-ledger.jsonl"),
+    `${lines.join("\n")}\n`,
+    "utf8",
+  );
+}
+
+function expectedKeptEventIds(events: MemoryLifecycleEvent[]): string[] {
+  const kept: string[] = [];
+  const seen = new Set<string>();
+  const sorted = [...events].sort(compareMemoryLifecycleEvents);
+  for (const event of sorted) {
+    if (!event.memoryId.trim()) continue;
+    if (seen.has(event.eventId)) continue;
+    seen.add(event.eventId);
+    kept.push(event.eventId);
+  }
+  return kept.sort();
+}
+
+function projectionDigest(memoryDir: string): { hash: string; counts: Record<string, number>; eventIds: string[]; previews: Record<string, string>; ledgerCount: string | undefined } {
+  const db = openProjectionReadonly(memoryDir);
+  assert.ok(db, "projection opens");
+  try {
+    const hash = createHash("sha256");
+    const counts: Record<string, number> = {};
+    const queries: Array<[string, string]> = [
+      ["memory_current", "SELECT * FROM memory_current ORDER BY memory_id"],
+      ["memory_timeline", "SELECT * FROM memory_timeline ORDER BY event_id"],
+      ["memory_entity_mentions", "SELECT * FROM memory_entity_mentions ORDER BY memory_id, entity_ref, mention_source"],
+    ];
+    for (const [name, sql] of queries) {
+      hash.update(name);
+      hash.update("\n");
+      let count = 0;
+      for (const row of db.prepare(sql).iterate()) {
+        count += 1;
+        hash.update(JSON.stringify(row));
+        hash.update("\n");
+      }
+      counts[name] = count;
+    }
+    const eventIds = (db.prepare("SELECT event_id FROM memory_timeline ORDER BY event_id").all() as Array<{ event_id: string }>)
+      .map((row) => row.event_id);
+    const previews: Record<string, string> = {};
+    for (const row of db.prepare("SELECT memory_id, preview_text FROM memory_current").all() as Array<{ memory_id: string; preview_text: string }>) {
+      previews[row.memory_id] = row.preview_text;
+    }
+    const ledgerCount = (db.prepare("SELECT value FROM meta WHERE key = 'sourceLifecycleLedgerEventCount'").get() as { value?: string } | undefined)?.value;
+    return { hash: hash.digest("hex"), counts, eventIds, previews, ledgerCount };
+  } finally {
+    db.close();
+  }
+}
+
+test("retainRaw false skips the raw copy and does not poison the corpus cache", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-retain-raw-cache-"));
+  try {
+    await mkdir(path.join(memoryDir, "facts", "2026-01-01"), { recursive: true });
+    await mkdir(path.join(memoryDir, "cold", "facts", "2026-01-01"), { recursive: true });
+    await mkdir(path.join(memoryDir, "archive", "facts", "2026-01-01"), { recursive: true });
+    await writeFile(path.join(memoryDir, "facts", "2026-01-01", "hot.md"), fact("hot", "hot body"));
+    await writeFile(path.join(memoryDir, "cold", "facts", "2026-01-01", "cold.md"), fact("cold", "cold body"));
+    await writeFile(path.join(memoryDir, "archive", "facts", "2026-01-01", "arch.md"), fact("arch", "arch body"));
+
+    const storage = new StorageManager(memoryDir);
+    const hot = await storage.readAllMemories();
+    assert.equal(typeof readRawMemoryDocument(hot[0]!), "string");
+    const stripped = await storage.readAllMemories({ retainRaw: false });
+    assert.equal(readRawMemoryDocument(stripped[0]!), undefined);
+    const again = await storage.readAllMemories();
+    assert.equal(typeof readRawMemoryDocument(again[0]!), "string");
+
+    const coldFirst = await storage.readAllColdMemories({ retainRaw: false });
+    assert.equal(readRawMemoryDocument(coldFirst[0]!), undefined);
+    const coldRetained = await storage.readAllColdMemories();
+    assert.equal(typeof readRawMemoryDocument(coldRetained[0]!), "string");
+    const coldAfter = await storage.readAllColdMemories({ retainRaw: false });
+    assert.equal(readRawMemoryDocument(coldAfter[0]!), undefined);
+    const coldStill = await storage.readAllColdMemories();
+    assert.equal(typeof readRawMemoryDocument(coldStill[0]!), "string");
+
+    const archived = await storage.readArchivedMemories({ retainRaw: false });
+    assert.equal(readRawMemoryDocument(archived[0]!), undefined);
+    const archivedKept = await storage.readArchivedMemories();
+    assert.equal(typeof readRawMemoryDocument(archivedKept[0]!), "string");
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("projection rows are byte-identical with retainRaw false", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-retain-raw-bytes-"));
+  const events = [
+    ledgerEvent({
+      eventId: "dup-1",
+      memoryId: "hot-1",
+      eventType: "updated",
+      timestamp: "2026-03-01T00:00:00.000Z",
+    }),
+    ledgerEvent({
+      eventId: "dup-1",
+      memoryId: "hot-1",
+      eventType: "created",
+      timestamp: "2026-01-01T00:00:00.000Z",
+    }),
+    ledgerEvent({
+      eventId: "blank-1",
+      memoryId: "  ",
+      eventType: "created",
+      timestamp: "2026-01-01T00:00:00.000Z",
+    }),
+    ledgerEvent({
+      eventId: "cold-evt",
+      memoryId: "cold-only",
+      eventType: "created",
+      timestamp: "2026-01-15T00:00:00.000Z",
+    }),
+  ];
+  try {
+    await seedCorpus(memoryDir, events);
+    await rebuildMemoryProjection({ memoryDir, dryRun: false, now: FIXED_NOW, retainRaw: true });
+    const withRaw = projectionDigest(memoryDir);
+    await rm(getMemoryProjectionPath(memoryDir), { force: true });
+    await rebuildMemoryProjection({ memoryDir, dryRun: false, now: FIXED_NOW });
+    const withoutRaw = projectionDigest(memoryDir);
+
+    assert.equal(withoutRaw.hash, withRaw.hash);
+    assert.deepEqual(withoutRaw.counts, withRaw.counts);
+    assert.deepEqual(withoutRaw.eventIds, expectedKeptEventIds(events));
+    assert.match(withoutRaw.previews["hot-1"] ?? "", /HOTBODY/);
+    assert.equal((withoutRaw.previews["hot-1"] ?? "").includes("COLDBODY"), false);
+    assert.match(withoutRaw.previews["cold-only"] ?? "", /COLDONLY/);
+    assert.match(withoutRaw.previews["arch-1"] ?? "", /ARCHBODY/);
+    assert.equal(withoutRaw.previews["  "], undefined);
+    const identities = new Set(events.map(memoryLifecycleEventProjectionIdentity));
+    assert.equal(withoutRaw.ledgerCount, String(identities.size));
+    assert.equal(withoutRaw.counts.memory_timeline, expectedKeptEventIds(events).length);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});

package/src/maintenance/rebuild-memory-projection.ts

@@ -15,13 +15,12 @@ import type {
   MemoryStatus,
 } from "../types.js";
 import {
-  buildLifecycleEventsForMemory,
   inferMemoryStatus,
   MEMORY_LIFECYCLE_EVENT_SORT_ORDER,
-  memoryLifecycleEventProjectionIdentity,
   sortMemoryLifecycleEvents,
   toMemoryPathRel,
 } from "../memory-lifecycle-ledger-utils.js";
+import { loadProjectionTimeline } from "./projection-timeline.js";
 import {
   getMemoryProjectionPath,
   initializeMemoryProjectionDb,
@@ -44,6 +43,7 @@ import {
 } from "../memory-projection-format.js";
 import { openBetterSqlite3 } from "../runtime/better-sqlite.js";
 import { commitPreparedFileAtomically } from "./atomic-file.js";
+import { throwIfProjectionRebuildAborted } from "./projection-rebuild-abort.js";
 import { isSupportPassportPrivateMemory } from "../support-passport/card-projection.js";
 
 export interface RebuildMemoryProjectionOptions {
@@ -55,6 +55,14 @@ export interface RebuildMemoryProjectionOptions {
   now?: Date;
   updatedAfter?: string;
   updatedBefore?: string;
+  /**
+   * When true, corpus reads keep the raw file text. Default false: the
+   * projection is built from parsed fields, and skipping the raw copy keeps
+   * the rebuild's peak heap down. Output rows stay the same either way.
+   */
+  retainRaw?: boolean;
+  /** When aborted, the rebuild stops before it publishes the projection file. */
+  signal?: AbortSignal;
 }
 
 export type {
@@ -201,7 +209,8 @@ function toCurrentStateRow(memoryDir: string, memory: MemoryFile): MemoryProject
     accessCount: memory.frontmatter.accessCount,
     lastAccessed: memory.frontmatter.lastAccessed,
     tags: normalizeProjectionTags(memory.frontmatter.tags),
-    preview: normalizeProjectionPreview(memory.content),
+    // Copy the preview off the source string so the full document can be collected.
+    preview: Buffer.from(normalizeProjectionPreview(memory.content), "utf8").toString("utf8"),
     privateRecord: isSupportPassportPrivateMemory(memory),
   };
 }
@@ -295,23 +304,6 @@ async function loadLatestGovernanceProjection(memoryDir: string): Promise<{
   }
 }
 
-function loadTimelineEvents(
-  memories: MemoryFile[],
-  lifecycleEvents: MemoryLifecycleEvent[],
-): { events: MemoryLifecycleEvent[]; usedLifecycleLedger: boolean } {
-  if (lifecycleEvents.length > 0) {
-    return {
-      events: sortMemoryLifecycleEvents(lifecycleEvents),
-      usedLifecycleLedger: true,
-    };
-  }
-
-  return {
-    events: sortMemoryLifecycleEvents(memories.flatMap((memory) => buildLifecycleEventsForMemory(memory))),
-    usedLifecycleLedger: false,
-  };
-}
-
 function normalizeScopedIso(value: string | undefined): string | null {
   if (!value) return null;
   const trimmed = value.trim();
@@ -509,53 +501,14 @@ function selectProjectionMemories(
   return { allMemories, memories, skippedDuplicateMemories, skippedBlankIdMemories };
 }
 
-function deduplicateTimelineEvents(
-  events: MemoryLifecycleEvent[],
-  sourcePathFor: (event: MemoryLifecycleEvent) => string,
-): {
-  events: MemoryLifecycleEvent[];
-  skipped: SkippedDuplicateTimelineEvent[];
-  skippedBlankIds: SkippedBlankIdTimelineEvent[];
-} {
-  const keptById = new Map<string, { event: MemoryLifecycleEvent; path: string }>();
-  const unique: MemoryLifecycleEvent[] = [];
-  const skipped: SkippedDuplicateTimelineEvent[] = [];
-  const skippedBlankIds: SkippedBlankIdTimelineEvent[] = [];
-  for (const event of sortMemoryLifecycleEvents(events)) {
-    const sourcePath = sourcePathFor(event);
-    if (!event.memoryId.trim()) {
-      skippedBlankIds.push({ eventId: event.eventId, path: sourcePath });
-      log.warn(
-        `projection rebuild skipped timeline event with blank memory id: `
-        + `event_id=${event.eventId} path=${sourcePath}`,
-      );
-      continue;
-    }
-    const kept = keptById.get(event.eventId);
-    if (kept) {
-      skipped.push({
-        eventId: event.eventId,
-        keptPath: kept.path,
-        skippedPath: sourcePath,
-      });
-      log.warn(
-        `projection rebuild skipped duplicate timeline event_id=${event.eventId}: `
-        + `kept=${kept.path} skipped=${sourcePath}`,
-      );
-      continue;
-    }
-    keptById.set(event.eventId, { event, path: sourcePath });
-    unique.push(event);
-  }
-  return { events: unique, skipped, skippedBlankIds };
-}
-
 async function loadAuthoritativeProjectionSnapshot(options: {
   memoryDir: string;
   defaultNamespace?: string;
   storage?: StorageManager;
   updatedAfter?: string;
   updatedBefore?: string;
+  retainRaw?: boolean;
+  signal?: AbortSignal;
 }): Promise<{
   allMemories: MemoryFile[];
   currentRows: MemoryProjectionCurrentState[];
@@ -590,35 +543,40 @@ async function loadAuthoritativeProjectionSnapshot(options: {
   // Force a fresh disk read — projection verify/rebuild must see the true
   // on-disk state, not a potentially stale in-process cache.
   storage.invalidateAllMemoriesCacheForDir({ indexedText: false });
-  const selected = selectProjectionMemories([
-    await storage.readAllMemories(),
-    await storage.readAllColdMemories(),
-    await storage.readArchivedMemories(),
-  ]);
+  const corpusRead = {
+    retainRaw: options.retainRaw === true,
+    abortSignal: options.signal,
+  };
+  const hot = await storage.readAllMemories(corpusRead);
+  await throwIfProjectionRebuildAborted(options.signal);
+  const cold = await storage.readAllColdMemories(corpusRead);
+  await throwIfProjectionRebuildAborted(options.signal);
+  const archived = await storage.readArchivedMemories(corpusRead);
+  await throwIfProjectionRebuildAborted(options.signal);
+  const selected = selectProjectionMemories([hot, cold, archived]);
   const allMemories = selected.allMemories;
   const projectionMemories = selected.memories;
-  const lifecycleEvents = await storage.readAllMemoryLifecycleEvents();
-  const timeline = loadTimelineEvents(projectionMemories, lifecycleEvents);
-  const lifecycleLedgerPath = path.join(
-    options.memoryDir,
-    "state",
-    "memory-lifecycle-ledger.jsonl",
-  );
-  const memoryPathById = new Map(
-    projectionMemories.map((memory) => [memory.frontmatter.id, memory.path]),
-  );
-  const deduplicatedTimeline = deduplicateTimelineEvents(
-    timeline.events,
-    (event) => timeline.usedLifecycleLedger
-      ? lifecycleLedgerPath
-      : memoryPathById.get(event.memoryId) ?? lifecycleLedgerPath,
-  );
-  const events = deduplicatedTimeline.events;
-  const usedLifecycleLedger = timeline.usedLifecycleLedger;
-  const sourceLifecycleLedgerEventCount = new Set(
-    lifecycleEvents.map(memoryLifecycleEventProjectionIdentity),
-  ).size;
   const currentRows = projectionMemories.map((memory) => toCurrentStateRow(options.memoryDir, memory));
+  // Preview is already a detached copy. Drop the document bodies before the
+  // ledger scan so peak heap is the larger of the corpus and the timeline,
+  // not both. These objects bypass the shared caches when retainRaw is off,
+  // so clearing them cannot strip a raw copy another caller is holding.
+  if (options.retainRaw !== true) {
+    for (const memory of allMemories) memory.content = "";
+  }
+  const {
+    events,
+    usedLifecycleLedger,
+    sourceLifecycleLedgerEventCount,
+    skipped: skippedDuplicateTimelineEvents,
+    skippedBlankIds: skippedBlankIdTimelineEvents,
+  } = await loadProjectionTimeline({
+    storage,
+    memoryDir: options.memoryDir,
+    memories: projectionMemories,
+    signal: options.signal,
+  });
+  await throwIfProjectionRebuildAborted(options.signal);
   const scope = normalizeProjectionScope(options);
   const scopedMemories = filterMemoriesForProjectionScope(projectionMemories, scope);
   const scopedMemoryIds = new Set(scopedMemories.map((memory) => memory.frontmatter.id));
@@ -660,8 +618,8 @@ async function loadAuthoritativeProjectionSnapshot(options: {
     scope,
     skippedDuplicateMemories: selected.skippedDuplicateMemories,
     skippedBlankIdMemories: selected.skippedBlankIdMemories,
-    skippedDuplicateTimelineEvents: deduplicatedTimeline.skipped,
-    skippedBlankIdTimelineEvents: deduplicatedTimeline.skippedBlankIds,
+    skippedDuplicateTimelineEvents,
+    skippedBlankIdTimelineEvents,
   };
 }
 
@@ -1058,6 +1016,7 @@ export async function rebuildMemoryProjection(
   const now = options.now ?? new Date();
   const outputPath = getMemoryProjectionPath(options.memoryDir);
   const snapshot = await loadAuthoritativeProjectionSnapshot(options);
+  await throwIfProjectionRebuildAborted(options.signal);
 
   let backupPath: string | undefined;
   if (!dryRun) {
@@ -1117,6 +1076,7 @@ export async function rebuildMemoryProjection(
 
     const tempPath = `${outputPath}.tmp`;
     await mkdir(path.dirname(outputPath), { recursive: true });
+    await throwIfProjectionRebuildAborted(options.signal);
     await rm(tempPath, { force: true });
     writeProjectionDb(
       tempPath,
@@ -1130,8 +1090,20 @@ export async function rebuildMemoryProjection(
       snapshot.sourceLifecycleLedgerEventCount,
       !hasScopedProjectionFilter(snapshot.scope),
     );
-    backupPath = await backupExistingProjection(options.memoryDir, outputPath, now);
-    backupPath = await commitPreparedFileAtomically(tempPath, outputPath, backupPath);
+    try {
+      await throwIfProjectionRebuildAborted(options.signal);
+      backupPath = await backupExistingProjection(options.memoryDir, outputPath, now);
+      await throwIfProjectionRebuildAborted(options.signal);
+      backupPath = await commitPreparedFileAtomically(
+        tempPath,
+        outputPath,
+        backupPath,
+        () => throwIfProjectionRebuildAborted(options.signal),
+      );
+    } catch (err) {
+      if (options.signal?.aborted) await rm(tempPath, { force: true });
+      throw err;
+    }
   }
 
   return {

package/src/orchestration/full-corpus-heap-guard.ts

@@ -0,0 +1,38 @@
+/**
+ * Heap guard for scheduled full-corpus work.
+ *
+ * A daemon already near Node's default heap limit must not start another
+ * multi-GB allocation (the scheduled projection rebuild loads hot, cold, and
+ * archived memories plus the lifecycle ledger). Callers postpone the job and
+ * arm their existing failure backoff instead.
+ */
+
+export interface HeapLimitStatistics {
+  used_heap_size: number;
+  heap_size_limit: number;
+}
+
+export interface FullCorpusHeapPressure {
+  /** True when usage is strictly above `limitRatio`. */
+  postpone: boolean;
+  /** `used_heap_size / heap_size_limit`, or 0 when the statistics are unusable. */
+  ratio: number;
+}
+
+/**
+ * Decide whether a scheduled full-corpus job should wait. Non-finite or
+ * non-positive heap limits do not postpone (there is no ratio to trust).
+ * Comparison is strict `>` so a limit of `1` never fires.
+ */
+export function fullCorpusHeapPressure(
+  stats: HeapLimitStatistics,
+  limitRatio: number,
+): FullCorpusHeapPressure {
+  const used = stats.used_heap_size;
+  const cap = stats.heap_size_limit;
+  if (!Number.isFinite(used) || used < 0 || !Number.isFinite(cap) || cap <= 0 || !Number.isFinite(limitRatio)) {
+    return { postpone: false, ratio: 0 };
+  }
+  const ratio = used / cap;
+  return { postpone: ratio > limitRatio, ratio };
+}

package/src/orchestration/maintenance.test.ts

@@ -2968,8 +2968,8 @@ test("maintenance tool request invokes the scheduled projection rebuild wiring (
   });
   try {
     scheduler.requestQmdMaintenanceForTool("test");
-    for (let attempt = 0; attempt < 100 && probeProjectionHealth(memoryDir).state === "absent"; attempt += 1) {
-      await delay(10);
+    for (let attempt = 0; attempt < 60 && probeProjectionHealth(memoryDir).state === "absent"; attempt += 1) {
+      await delay(500);
     }
     assert.equal(
       probeProjectionHealth(memoryDir).state,

package/src/orchestration/maintenance.ts

@@ -55,6 +55,7 @@ import { resolveDaySummaryCronModel } from "../maintenance/day-summary-cron.js";
 import {
   createProjectionRebuildScheduleState,
   maybeRebuildMemoryProjectionScheduled,
+  terminateScheduledProjectionRebuilds,
 } from "./projection-rebuild-schedule.js";
 import {
   drainPendingLifecycleLedgerIfAny,
@@ -1168,6 +1169,7 @@ export class MaintenanceScheduler {
    */
   async dispose(): Promise<void> {
     this.disposed = true;
+    terminateScheduledProjectionRebuilds(this.projectionRebuild);
     const consolidationDrain = this.consolidationRunPromise;
     // The registrar latches teardown first (so a draining tick's onRun cannot
     // re-arm QMD maintenance) and aborts+drains the sync. Capture its drain and