@remnic/core 10.0.3 → 10.0.4

package/src/corpus-read-cancellation.ts

@@ -34,6 +34,13 @@ export interface CorpusReadOptions {
    * work, and on a tree large enough to matter there are thousands of them.
    */
   abortSignal?: AbortSignal;
+  /**
+   * When `false`, parsed memories are returned without the raw file text kept
+   * beside them (`rememberRawFrontmatter`). Default (unset) retains that copy.
+   * A `false` read bypasses the shared corpus caches so it cannot replace a
+   * raw-retaining snapshot other callers still need.
+   */
+  retainRaw?: boolean;
 }
 
 /** Stop a corpus read at a scan boundary when its caller has given up. */

package/src/maintenance/atomic-file.ts

@@ -72,6 +72,7 @@ export async function commitPreparedFileAtomically(
   tempPath: string,
   outputPath: string,
   backupPath?: string,
+  beforeRename?: () => void | Promise<void>,
 ): Promise<string | undefined> {
   const dir = path.dirname(outputPath);
   let resolvedBackupPath: string | undefined;
@@ -79,6 +80,7 @@ export async function commitPreparedFileAtomically(
     if (backupPath) {
       resolvedBackupPath = await copyExistingFileToBackup(outputPath, backupPath);
     }
+    if (beforeRename) await beforeRename();
     await rename(tempPath, outputPath);
     await syncDirectory(dir);
     return resolvedBackupPath;

package/src/maintenance/projection-rebuild-abort.ts

@@ -0,0 +1,40 @@
+/**
+ * Stop a projection-rebuild child when its parent is gone.
+ *
+ * A non-detached fork is reparented, not killed, when the daemon exits. The
+ * IPC channel closes first. The child must observe that and refuse to publish,
+ * or it can rename `memory-projection.sqlite.tmp` over a projection the
+ * replacement daemon has already started serving.
+ */
+
+export function projectionRebuildParentDisconnected(): Error {
+  return new Error("projection rebuild runner parent disconnected");
+}
+
+/** Abort when this process loses its parent IPC channel. */
+export function watchProjectionRebuildParent(): AbortSignal {
+  const abort = new AbortController();
+  const stop = () => {
+    if (abort.signal.aborted) return;
+    abort.abort(projectionRebuildParentDisconnected());
+  };
+  process.on("disconnect", stop);
+  if (process.connected === false) stop();
+  return abort.signal;
+}
+
+/**
+ * Yield so a disconnect that arrived during the previous await is applied,
+ * then refuse to continue. No-op when the caller did not pass a signal.
+ */
+export async function throwIfProjectionRebuildAborted(
+  signal: AbortSignal | undefined,
+): Promise<void> {
+  if (!signal) return;
+  await new Promise<void>((resolve) => {
+    setImmediate(resolve);
+  });
+  if (!signal.aborted) return;
+  const reason = signal.reason;
+  throw reason instanceof Error ? reason : projectionRebuildParentDisconnected();
+}

package/src/maintenance/projection-timeline.ts

@@ -0,0 +1,209 @@
+/**
+ * Projection timeline assembly.
+ *
+ * The lifecycle ledger is streamed one event at a time. Dedup keeps the same
+ * winner as sort-then-first-wins (stable `compareMemoryLifecycleEvents`, file
+ * order on ties) without materializing the unsorted ledger beside the result.
+ */
+import path from "node:path";
+import { throwIfAborted } from "../abort-error.js";
+import { log } from "../logger.js";
+import type { MemoryFile, MemoryLifecycleEvent } from "../types.js";
+import {
+  buildLifecycleEventsForMemory,
+  compareMemoryLifecycleEvents,
+  memoryLifecycleEventProjectionIdentity,
+  sortMemoryLifecycleEvents,
+} from "../memory-lifecycle-ledger-utils.js";
+import type {
+  SkippedBlankIdTimelineEvent,
+  SkippedDuplicateTimelineEvent,
+} from "./projection-support.js";
+
+export interface ProjectionLedgerReader {
+  iterateMemoryLifecycleEvents(): AsyncIterable<MemoryLifecycleEvent>;
+}
+
+export interface ProjectionTimeline {
+  events: MemoryLifecycleEvent[];
+  usedLifecycleLedger: boolean;
+  sourceLifecycleLedgerEventCount: number;
+  skipped: SkippedDuplicateTimelineEvent[];
+  skippedBlankIds: SkippedBlankIdTimelineEvent[];
+}
+
+function loadTimelineEvents(
+  memories: MemoryFile[],
+  lifecycleEvents: MemoryLifecycleEvent[],
+): { events: MemoryLifecycleEvent[]; usedLifecycleLedger: boolean } {
+  if (lifecycleEvents.length > 0) {
+    return {
+      events: sortMemoryLifecycleEvents(lifecycleEvents),
+      usedLifecycleLedger: true,
+    };
+  }
+
+  return {
+    events: sortMemoryLifecycleEvents(memories.flatMap((memory) => buildLifecycleEventsForMemory(memory))),
+    usedLifecycleLedger: false,
+  };
+}
+
+function deduplicateTimelineEvents(
+  events: MemoryLifecycleEvent[],
+  sourcePathFor: (event: MemoryLifecycleEvent) => string,
+): {
+  events: MemoryLifecycleEvent[];
+  skipped: SkippedDuplicateTimelineEvent[];
+  skippedBlankIds: SkippedBlankIdTimelineEvent[];
+} {
+  const keptById = new Map<string, { event: MemoryLifecycleEvent; path: string }>();
+  const unique: MemoryLifecycleEvent[] = [];
+  const skipped: SkippedDuplicateTimelineEvent[] = [];
+  const skippedBlankIds: SkippedBlankIdTimelineEvent[] = [];
+  for (const event of sortMemoryLifecycleEvents(events)) {
+    const sourcePath = sourcePathFor(event);
+    if (!event.memoryId.trim()) {
+      skippedBlankIds.push({ eventId: event.eventId, path: sourcePath });
+      log.warn(
+        `projection rebuild skipped timeline event with blank memory id: `
+        + `event_id=${event.eventId} path=${sourcePath}`,
+      );
+      continue;
+    }
+    const kept = keptById.get(event.eventId);
+    if (kept) {
+      skipped.push({
+        eventId: event.eventId,
+        keptPath: kept.path,
+        skippedPath: sourcePath,
+      });
+      log.warn(
+        `projection rebuild skipped duplicate timeline event_id=${event.eventId}: `
+        + `kept=${kept.path} skipped=${sourcePath}`,
+      );
+      continue;
+    }
+    keptById.set(event.eventId, { event, path: sourcePath });
+    unique.push(event);
+  }
+  return { events: unique, skipped, skippedBlankIds };
+}
+
+function internPool(): <T extends string>(value: T) => T {
+  const pool = new Map<string, string>();
+  return <T extends string>(value: T): T => {
+    const existing = pool.get(value);
+    if (existing !== undefined) return existing as T;
+    pool.set(value, value);
+    return value;
+  };
+}
+
+/** Share repeated ledger scalars so a long ledger does not keep one string per row. */
+function internLedgerEvent(event: MemoryLifecycleEvent, intern: <T extends string>(value: T) => T): void {
+  event.memoryId = intern(event.memoryId);
+  event.eventType = intern(event.eventType);
+  event.timestamp = intern(event.timestamp);
+  event.actor = intern(event.actor);
+  event.ruleVersion = intern(event.ruleVersion);
+  if (event.reasonCode) event.reasonCode = intern(event.reasonCode);
+  if (event.correlationId) event.correlationId = intern(event.correlationId);
+}
+
+/**
+ * Stream ledger events into the deduped timeline. Returns null when the ledger
+ * contributed no parsable row, so the caller can fall back to frontmatter.
+ */
+async function consumeLifecycleLedger(
+  events: AsyncIterable<MemoryLifecycleEvent>,
+  ledgerPath: string,
+  signal?: AbortSignal,
+): Promise<ProjectionTimeline | null> {
+  const intern = internPool();
+  const best = new Map<string, MemoryLifecycleEvent>();
+  const losers: MemoryLifecycleEvent[] = [];
+  const blanks: MemoryLifecycleEvent[] = [];
+  const identities = new Set<string>();
+  let sawEvent = false;
+  for await (const event of events) {
+    throwIfAborted(signal, "projection rebuild aborted");
+    sawEvent = true;
+    internLedgerEvent(event, intern);
+    identities.add(memoryLifecycleEventProjectionIdentity(event));
+    if (!event.memoryId.trim()) {
+      blanks.push(event);
+      continue;
+    }
+    const prev = best.get(event.eventId);
+    if (!prev) {
+      best.set(event.eventId, event);
+      continue;
+    }
+    if (compareMemoryLifecycleEvents(event, prev) < 0) {
+      losers.push(prev);
+      best.set(event.eventId, event);
+    } else {
+      losers.push(event);
+    }
+  }
+  if (!sawEvent) return null;
+  const sourceLifecycleLedgerEventCount = identities.size;
+  identities.clear();
+  const skippedBlankIds = sortMemoryLifecycleEvents(blanks).map((event) => {
+    log.warn(
+      `projection rebuild skipped timeline event with blank memory id: `
+      + `event_id=${event.eventId} path=${ledgerPath}`,
+    );
+    return { eventId: event.eventId, path: ledgerPath };
+  });
+  const skipped = sortMemoryLifecycleEvents(losers).map((event) => {
+    log.warn(
+      `projection rebuild skipped duplicate timeline event_id=${event.eventId}: `
+      + `kept=${ledgerPath} skipped=${ledgerPath}`,
+    );
+    return {
+      eventId: event.eventId,
+      keptPath: ledgerPath,
+      skippedPath: ledgerPath,
+    };
+  });
+  return {
+    events: sortMemoryLifecycleEvents([...best.values()]),
+    usedLifecycleLedger: true,
+    sourceLifecycleLedgerEventCount,
+    skipped,
+    skippedBlankIds,
+  };
+}
+
+export async function loadProjectionTimeline(options: {
+  storage: ProjectionLedgerReader;
+  memoryDir: string;
+  memories: MemoryFile[];
+  signal?: AbortSignal;
+}): Promise<ProjectionTimeline> {
+  const ledgerPath = path.join(options.memoryDir, "state", "memory-lifecycle-ledger.jsonl");
+  const streamed = await consumeLifecycleLedger(
+    options.storage.iterateMemoryLifecycleEvents(),
+    ledgerPath,
+    options.signal,
+  );
+  if (streamed) return streamed;
+
+  const timeline = loadTimelineEvents(options.memories, []);
+  const memoryPathById = new Map(
+    options.memories.map((memory) => [memory.frontmatter.id, memory.path]),
+  );
+  const deduplicated = deduplicateTimelineEvents(
+    timeline.events,
+    (event) => memoryPathById.get(event.memoryId) ?? ledgerPath,
+  );
+  return {
+    events: deduplicated.events,
+    usedLifecycleLedger: false,
+    sourceLifecycleLedgerEventCount: 0,
+    skipped: deduplicated.skipped,
+    skippedBlankIds: deduplicated.skippedBlankIds,
+  };
+}

package/src/maintenance/rebuild-memory-projection-runner.ts

@@ -0,0 +1,94 @@
+/**
+ * Child-process entry for the scheduled memory-projection rebuild.
+ *
+ * The daemon forks this file so the corpus and lifecycle ledger are allocated
+ * in a fresh heap. Invoke only with `--remnic-projection-rebuild-runner`.
+ * Arguments are the memory directory, default namespace, and an ISO timestamp.
+ * Do not put secrets, passphrases, or secure-store unlock material on argv
+ * or in the environment — an encrypted store rebuilds in the parent instead.
+ */
+import { initLogger, log } from "../logger.js";
+import { watchProjectionRebuildParent } from "./projection-rebuild-abort.js";
+import { rebuildMemoryProjection } from "./rebuild-memory-projection.js";
+
+export const PROJECTION_REBUILD_RUNNER_FLAG = "--remnic-projection-rebuild-runner";
+
+export interface ProjectionRebuildRunnerSuccess {
+  ok: true;
+  currentRows: number;
+  timelineRows: number;
+  entityMentionRows: number;
+}
+
+interface ProjectionRebuildRunnerFailure {
+  ok: false;
+  error: string;
+}
+
+function runnerArgs(): { memoryDir: string; defaultNamespace: string; now: Date } {
+  const index = process.argv.indexOf(PROJECTION_REBUILD_RUNNER_FLAG);
+  const memoryDir = index >= 0 ? process.argv[index + 1] : undefined;
+  const defaultNamespace = index >= 0 ? process.argv[index + 2] : undefined;
+  const nowIso = index >= 0 ? process.argv[index + 3] : undefined;
+  if (!memoryDir || !defaultNamespace || !nowIso) {
+    throw new Error(
+      "projection rebuild runner requires memoryDir, defaultNamespace, and now",
+    );
+  }
+  const now = new Date(nowIso);
+  if (Number.isNaN(now.getTime())) {
+    throw new Error(`projection rebuild runner now is not a timestamp: ${nowIso}`);
+  }
+  return { memoryDir, defaultNamespace, now };
+}
+
+function send(message: ProjectionRebuildRunnerSuccess | ProjectionRebuildRunnerFailure): Promise<void> {
+  if (typeof process.send !== "function") {
+    throw new Error("projection rebuild runner requires an IPC channel");
+  }
+  const sendMessage = process.send.bind(process);
+  return new Promise((resolve, reject) => {
+    sendMessage(message, (err) => (err ? reject(err) : resolve()));
+  });
+}
+
+async function main(): Promise<void> {
+  initLogger();
+  const { memoryDir, defaultNamespace, now } = runnerArgs();
+  const signal = watchProjectionRebuildParent();
+  const result = await rebuildMemoryProjection({
+    memoryDir,
+    defaultNamespace,
+    dryRun: false,
+    now,
+    signal,
+  });
+  if (signal.aborted || process.connected === false) {
+    throw new Error("projection rebuild runner parent disconnected");
+  }
+  await send({
+    ok: true,
+    currentRows: result.currentRows,
+    timelineRows: result.timelineRows,
+    entityMentionRows: result.entityMentionRows,
+  });
+}
+
+const invokedAsRunner = process.argv.includes(PROJECTION_REBUILD_RUNNER_FLAG);
+if (invokedAsRunner) {
+  main().then(
+    () => process.exit(0),
+    (err: unknown) => {
+      const error = err instanceof Error ? err.message : String(err);
+      if (process.connected === false) {
+        process.exit(1);
+        return;
+      }
+      log.warn(`projection rebuild runner failed: ${error}`);
+      send({ ok: false, error }).then(
+        () => process.exit(1),
+        () => process.exit(1),
+      );
+    },
+  );
+}

package/src/maintenance/rebuild-memory-projection.memory.test.ts

@@ -0,0 +1,159 @@
+/**
+ * Heap regression for the scheduled projection rebuild.
+ *
+ * About 20k synthetic facts (~2 KB each) and a 100k-line lifecycle ledger are
+ * rebuilt once under `--max-old-space-size=192` and once with the default
+ * heap. Row counts and the SHA-256 of the projection rows must match. The
+ * child is a fresh process so the fixture writer does not share its heap
+ * with the rebuild.
+ */
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { once } from "node:events";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import v8 from "node:v8";
+
+const FACT_COUNT = 20_000;
+const LEDGER_LINES = 100_000;
+const BODY_BYTES = 2048;
+const FIXED_NOW = "2026-06-01T00:00:00.000Z";
+const CHILD_FLAG = "--projection-memory-child";
+
+function factId(index: number): string {
+  return `fact-${String(index).padStart(5, "0")}`;
+}
+
+async function writeCorpus(memoryDir: string): Promise<void> {
+  const factDir = path.join(memoryDir, "facts", "2026-01-01");
+  await mkdir(factDir, { recursive: true });
+  await mkdir(path.join(memoryDir, "state"), { recursive: true });
+  let next = 0;
+  const workers = Array.from({ length: 32 }, async () => {
+    for (;;) {
+      const index = next;
+      next += 1;
+      if (index >= FACT_COUNT) return;
+      const id = factId(index);
+      const body = Buffer.alloc(BODY_BYTES, 97);
+      body.write(`fact ${id} `);
+      for (let offset = 16; offset < body.length; offset += 1) {
+        body[offset] = 97 + ((index + offset) % 26);
+      }
+      const markdown = `---\nid: ${id}\ncategory: fact\nconfidence: 0.5\ncreated: 2026-01-01T00:00:00.000Z\nupdated: 2026-01-01T00:00:00.000Z\nstatus: active\nsource: synthetic\n---\n\n${body.toString("utf8")}\n`;
+      await writeFile(path.join(factDir, `${id}.md`), markdown);
+    }
+  });
+  await Promise.all(workers);
+
+  const ledger = createWriteStream(path.join(memoryDir, "state", "memory-lifecycle-ledger.jsonl"));
+  for (let index = 0; index < LEDGER_LINES; index += 1) {
+    const line = `{"eventId":"evt-${String(index).padStart(6, "0")}","memoryId":"${factId(index % FACT_COUNT)}","eventType":"created","timestamp":"${new Date(Date.UTC(2026, 0, 1) + index * 1000).toISOString()}","actor":"synthetic","ruleVersion":"1"}\n`;
+    if (!ledger.write(line)) await once(ledger, "drain");
+  }
+  ledger.end();
+  await once(ledger, "finish");
+}
+
+async function runChild(memoryDir: string, maxOldSpace?: number): Promise<void> {
+  const args: string[] = [];
+  if (maxOldSpace !== undefined) args.push(`--max-old-space-size=${maxOldSpace}`);
+  args.push("--import", "tsx", fileURLToPath(import.meta.url), CHILD_FLAG, memoryDir);
+  const nodeOptions = (process.env.NODE_OPTIONS ?? "")
+    .split(" ")
+    .filter((part) => part.length > 0 && !part.startsWith("--max-old-space-size"))
+    .join(" ");
+  await new Promise<void>((resolve, reject) => {
+    const child = spawn(process.execPath, args, {
+      env: { ...process.env, NODE_OPTIONS: nodeOptions },
+      stdio: "inherit",
+    });
+    child.once("error", reject);
+    child.once("exit", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`projection memory child exited ${code ?? "null"}`));
+    });
+  });
+}
+
+async function projectionDigest(memoryDir: string): Promise<{ hash: string; counts: Record<string, number> }> {
+  const { openProjectionReadonly } = await import("../memory-projection-store.js");
+  const db = openProjectionReadonly(memoryDir);
+  if (!db) throw new Error("projection was not written");
+  try {
+    const hash = createHash("sha256");
+    const counts: Record<string, number> = {};
+    const queries: Array<[string, string]> = [
+      ["memory_current", "SELECT * FROM memory_current ORDER BY memory_id"],
+      ["memory_timeline", "SELECT * FROM memory_timeline ORDER BY event_id"],
+      ["memory_entity_mentions", "SELECT * FROM memory_entity_mentions ORDER BY memory_id, entity_ref, mention_source"],
+    ];
+    for (const [name, sql] of queries) {
+      hash.update(name);
+      hash.update("\n");
+      let count = 0;
+      for (const row of db.prepare(sql).iterate()) {
+        count += 1;
+        hash.update(JSON.stringify(row));
+        hash.update("\n");
+      }
+      counts[name] = count;
+    }
+    return { hash: hash.digest("hex"), counts };
+  } finally {
+    db.close();
+  }
+}
+
+async function rebuildInThisProcess(memoryDir: string): Promise<void> {
+  const { rebuildMemoryProjection } = await import("./rebuild-memory-projection.js");
+  const result = await rebuildMemoryProjection({
+    memoryDir,
+    dryRun: false,
+    now: new Date(FIXED_NOW),
+  });
+  const stats = v8.getHeapStatistics();
+  console.log(
+    `projection-memory-child rows current=${result.currentRows} timeline=${result.timelineRows} `
+    + `heap used=${stats.used_heap_size} limit=${stats.heap_size_limit}`,
+  );
+}
+
+const childDir = process.argv.indexOf(CHILD_FLAG);
+if (childDir >= 0) {
+  const memoryDir = process.argv[childDir + 1];
+  if (!memoryDir) {
+    console.error("projection memory child requires a memory directory");
+    process.exit(2);
+  }
+  try {
+    await rebuildInThisProcess(memoryDir);
+  } catch (err) {
+    console.error(err);
+    process.exit(1);
+  }
+  process.exit(0);
+} else {
+  const { default: test } = await import("node:test");
+
+  test("slow: 20k-fact projection rebuild matches under a 192MB heap", { timeout: 600_000 }, async () => {
+    const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-proj-heap-corpus-"));
+    try {
+      await writeCorpus(memoryDir);
+      await runChild(memoryDir, 192);
+      const constrained = await projectionDigest(memoryDir);
+      await runChild(memoryDir);
+      const unconstrained = await projectionDigest(memoryDir);
+      assert.equal(constrained.counts.memory_current, FACT_COUNT);
+      assert.equal(constrained.counts.memory_timeline, LEDGER_LINES);
+      assert.deepEqual(unconstrained.counts, constrained.counts);
+      assert.equal(unconstrained.hash, constrained.hash);
+    } finally {
+      await rm(memoryDir, { recursive: true, force: true });
+    }
+  });
+}