@remnic/core 9.69.51 → 9.69.53

package/src/orchestration/recall-recognition-index.test.ts

@@ -0,0 +1,249 @@
+/**
+ * recall-recognition-index.test.ts — issue #2975 write-path maintenance slice.
+ *
+ * Pins: when recallRecognitionTier is on, a namespace write upserts the
+ * memory id into state/index_recognition.json so loadRecognitionIndex
+ * sees a fresh entry. Gate-off is zero index I/O. Descriptions are the
+ * first non-empty body line (no recognition-trigger generation here).
+ */
+import assert from "node:assert/strict";
+import { mock } from "node:test";
+import os from "node:os";
+import path from "node:path";
+import { promises as fsp } from "node:fs";
+import { mkdtemp, rm } from "node:fs/promises";
+import test from "node:test";
+
+import { parseConfig } from "../config.js";
+import { loadRecognitionIndex, recognitionIndexPath } from "../recall-recognition-tier.js";
+import { PersistenceIndexCoordinator } from "./persistence-index.js";
+import { maintainRecognitionIndexAfterWrite } from "./recall-recognition-index.js";
+import type { MemoryFile, PluginConfig } from "../types.js";
+
+async function tmpNamespace(): Promise<string> {
+  return mkdtemp(path.join(os.tmpdir(), "remnic-recognition-index-"));
+}
+
+async function exists(filePath: string): Promise<boolean> {
+  try {
+    await fsp.stat(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+test("acceptance: gate-off write is zero index I/O", async () => {
+  const dir = await tmpNamespace();
+  const spies = [
+    mock.method(fsp, "readFile"),
+    mock.method(fsp, "writeFile"),
+    mock.method(fsp, "mkdir"),
+    mock.method(fsp, "open"),
+    mock.method(fsp, "rename"),
+    mock.method(fsp, "rm"),
+    mock.method(fsp, "unlink"),
+    mock.method(fsp, "stat"),
+    mock.method(fsp, "access"),
+    mock.method(fsp, "readdir"),
+  ];
+  try {
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: false,
+      changes: [{ action: "upsert", id: "fact-001", content: "SSD inference chips in the lab rack.\nMore body." }],
+    });
+    for (const spy of spies) {
+      assert.equal(spy.mock.calls.length, 0, `${String(spy.mock.calls.length)} unexpected fs calls on the off path`);
+    }
+    assert.equal(await exists(recognitionIndexPath(dir)), false);
+    assert.equal(await loadRecognitionIndex(dir), null);
+  } finally {
+    for (const spy of spies) spy.mock.restore();
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("acceptance: gate-on write upserts id so loadRecognitionIndex sees a fresh entry", async () => {
+  const dir = await tmpNamespace();
+  try {
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: true,
+      changes: [
+        {
+          action: "upsert",
+          id: "fact-001",
+          content: "SSD inference chips in the lab rack.\nRunning the model off a local tier.",
+        },
+      ],
+    });
+    const index = await loadRecognitionIndex(dir);
+    assert.ok(index, "index must exist after an enabled write");
+    assert.deepEqual(index.entries, [
+      { id: "fact-001", description: "SSD inference chips in the lab rack." },
+    ]);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("incremental write updates an existing id in place and appends a new id", async () => {
+  const dir = await tmpNamespace();
+  try {
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: true,
+      changes: [{ action: "upsert", id: "m-001", content: "first line original" }],
+    });
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: true,
+      changes: [
+        { action: "upsert", id: "m-001", content: "first line rewritten\nbody" },
+        { action: "upsert", id: "m-002", content: "second memory trigger" },
+      ],
+    });
+    const index = await loadRecognitionIndex(dir);
+    assert.deepEqual(index?.entries, [
+      { id: "m-001", description: "first line rewritten" },
+      { id: "m-002", description: "second memory trigger" },
+    ]);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("remove drops the id; blank ids are ignored without writing", async () => {
+  const dir = await tmpNamespace();
+  try {
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: true,
+      changes: [
+        { action: "upsert", id: "keep", content: "kept" },
+        { action: "upsert", id: "drop", content: "gone" },
+      ],
+    });
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: true,
+      changes: [{ action: "remove", id: "drop" }],
+    });
+    assert.deepEqual((await loadRecognitionIndex(dir))?.entries, [
+      { id: "keep", description: "kept" },
+    ]);
+
+    const before = await fsp.readFile(recognitionIndexPath(dir), "utf8");
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: true,
+      changes: [{ action: "upsert", id: "   ", content: "blank id is dropped" }],
+    });
+    assert.equal(await fsp.readFile(recognitionIndexPath(dir), "utf8"), before);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+function memory(id: string, content: string): MemoryFile {
+  return {
+    path: `/tmp/${id}.md`,
+    content,
+    frontmatter: { id, category: "fact", created: "2026-08-25T00:00:00.000Z", updated: "2026-08-25T00:00:00.000Z", source: "test", confidence: 1, confidenceTier: "explicit", tags: [] },
+  };
+}
+
+function stubCoordinator(config: PluginConfig) {
+  return new PersistenceIndexCoordinator({
+    config,
+    contentHashIndex: null,
+    contentHashIndexForStorage: async () => null,
+    contentHashIndexesByStorageDir: new Map(),
+    embeddingFallback: { isAvailable: async () => false } as never,
+    graphIndexFor: () => {
+      throw new Error("graph unused");
+    },
+    readAllMemoriesForNamespaces: async () => [],
+    semanticDedupScopeFor: () => ({}),
+  });
+}
+
+test("persist-path: enabled writes update the namespace index; disabled stays quiet", async () => {
+  const dir = await tmpNamespace();
+  const memories = new Map<string, MemoryFile>([
+    ["fact-ssd", memory("fact-ssd", "Running the model off an SSD tier.\nDetails.")],
+  ]);
+  const storage = {
+    dir,
+    getMemoryById: async (id: string) => memories.get(id) ?? null,
+    getMemoryByIdIncludingArchived: async (id: string) => memories.get(id) ?? null,
+    readAllMemories: async () => [...memories.values()],
+    readAllColdMemories: async () => [],
+  };
+  try {
+    const off = stubCoordinator(parseConfig({}));
+    await off.updateTemporalTagIndexes(storage as never, ["fact-ssd"]);
+    assert.equal(await exists(recognitionIndexPath(dir)), false, "off-path persist must not mint an index");
+
+    const on = stubCoordinator(parseConfig({ recallRecognitionTier: true }));
+    await on.updateTemporalTagIndexes(storage as never, ["fact-ssd"]);
+    const index = await loadRecognitionIndex(dir);
+    assert.deepEqual(index?.entries, [
+      { id: "fact-ssd", description: "Running the model off an SSD tier." },
+    ]);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("persist-path: first post-enable persist bootstraps the active corpus, not just the write", async () => {
+  const dir = await tmpNamespace();
+  const memories = new Map<string, MemoryFile>([
+    ["fact-old", memory("fact-old", "Existing SSD rack layout.\nOlder note.")],
+    ["fact-ssd", memory("fact-ssd", "Running the model off an SSD tier.\nDetails.")],
+  ]);
+  const storage = {
+    dir,
+    getMemoryById: async (id: string) => memories.get(id) ?? null,
+    getMemoryByIdIncludingArchived: async (id: string) => memories.get(id) ?? null,
+    readAllMemories: async () => [...memories.values()],
+    readAllColdMemories: async () => [],
+  };
+  try {
+    const on = stubCoordinator(parseConfig({ recallRecognitionTier: true }));
+    await on.updateTemporalTagIndexes(storage as never, ["fact-ssd"]);
+    const index = await loadRecognitionIndex(dir);
+    assert.deepEqual(
+      index?.entries.map((entry) => entry.id).sort(),
+      ["fact-old", "fact-ssd"],
+    );
+    assert.deepEqual(
+      index?.entries.find((entry) => entry.id === "fact-old"),
+      { id: "fact-old", description: "Existing SSD rack layout." },
+    );
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("stale recognition-index lock is broken so a later write can publish", async () => {
+  const dir = await tmpNamespace();
+  const lockDir = `${recognitionIndexPath(dir)}.lock.d`;
+  try {
+    await fsp.mkdir(lockDir, { recursive: true });
+    const stale = new Date(Date.now() - 120_000);
+    await fsp.utimes(lockDir, stale, stale);
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: dir,
+      enabled: true,
+      changes: [{ action: "upsert", id: "fact-001", content: "SSD inference chips in the lab rack." }],
+    });
+    const index = await loadRecognitionIndex(dir);
+    assert.deepEqual(index?.entries, [
+      { id: "fact-001", description: "SSD inference chips in the lab rack." },
+    ]);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});

package/src/orchestration/recall-recognition-index.ts

@@ -0,0 +1,169 @@
+/**
+ * Recognition-index write-path maintenance (issue #2975).
+ *
+ * Sibling of recall-recognition-search.ts at the memory write-path seam:
+ * when recallRecognitionTier is enabled, a namespace write upserts (or
+ * removes) that memory's id in `<memoryDir>/state/index_recognition.json`
+ * so the recall slice's loadRecognitionIndex sees a fresh entry. Off-path
+ * (`enabled !== true`) returns immediately — zero index I/O.
+ *
+ * Descriptions: layer 1 has no generator. This slice keeps ids current and
+ * fills description from the first non-empty body line (the compact
+ * `id: description` form the tier already renders). Discriminability
+ * rewrite is a later tidy pass.
+ *
+ * Serialization matches temporal-index: an in-process op chain per index
+ * path plus a directory lock around load → mutate → save. saveRecognitionIndex
+ * is single-writer and does not lock on its own.
+ */
+import { promises as fsp } from "node:fs";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+
+import {
+  buildRecognitionIndex,
+  loadRecognitionIndex,
+  recognitionIndexPath,
+  saveRecognitionIndex,
+  type RecognitionIndexEntry,
+} from "../recall-recognition-tier.js";
+
+export type RecognitionIndexChange =
+  | { readonly action: "upsert"; readonly id: string; readonly content: string }
+  | { readonly action: "remove"; readonly id: string };
+
+const writeChains = new Map<string, Promise<void>>();
+const LOCK_POLL_MS = 10;
+const LOCK_ATTEMPTS = 50;
+const LOCK_STALE_MS = 60_000;
+
+/**
+ * First non-empty line of a memory body. Layer 1 has no description
+ * generator; this is the compact line the tier already renders.
+ */
+export function recognitionDescriptionFromContent(content: string): string {
+  const trimmed = content.trim();
+  if (trimmed.length === 0) return "";
+  const newline = trimmed.indexOf("\n");
+  return newline === -1 ? trimmed : trimmed.slice(0, newline).trimEnd();
+}
+
+function withIndexChain<T>(indexPath: string, op: () => Promise<T>): Promise<T> {
+  const previous = writeChains.get(indexPath) ?? Promise.resolve();
+  const run = previous.then(op, op);
+  const tail = run.then(() => undefined, () => undefined);
+  writeChains.set(indexPath, tail);
+  void tail.then(() => {
+    if (writeChains.get(indexPath) === tail) writeChains.delete(indexPath);
+  });
+  return run;
+}
+
+async function removeAbandonedRecognitionLock(
+  lockDir: string,
+): Promise<"removed" | "wait" | "blocked"> {
+  try {
+    const info = await fsp.lstat(lockDir);
+    if (info.isSymbolicLink()) return "blocked";
+    if (!info.isDirectory()) {
+      await fsp.rm(lockDir, { force: true });
+      return "removed";
+    }
+    if (Date.now() - info.mtimeMs < LOCK_STALE_MS) return "wait";
+    await fsp.rm(lockDir, { recursive: true, force: true });
+    return "removed";
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException)?.code === "ENOENT") return "removed";
+    return "blocked";
+  }
+}
+
+async function withRecognitionIndexLock<T>(memoryDir: string, op: () => Promise<T>): Promise<T> {
+  const lockDir = `${recognitionIndexPath(memoryDir)}.lock.d`;
+  for (let attempt = 0; attempt < LOCK_ATTEMPTS; attempt++) {
+    try {
+      await fsp.mkdir(lockDir);
+      try {
+        return await op();
+      } finally {
+        await fsp.rm(lockDir, { recursive: true, force: true }).catch(() => undefined);
+      }
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException)?.code;
+      if (code === "ENOENT") {
+        await fsp.mkdir(path.dirname(lockDir), { recursive: true }).catch(() => undefined);
+        continue;
+      }
+      if (code !== "EEXIST") throw error;
+      const cleanup = await removeAbandonedRecognitionLock(lockDir);
+      if (cleanup === "blocked") {
+        throw new Error(`recognition index lock blocked at ${lockDir}`);
+      }
+      if (cleanup === "removed") continue;
+      await delay(LOCK_POLL_MS);
+    }
+  }
+  throw new Error(`recognition index lock timed out at ${lockDir}`);
+}
+
+function applyChanges(
+  entries: readonly RecognitionIndexEntry[],
+  changes: readonly RecognitionIndexChange[],
+): RecognitionIndexEntry[] {
+  const next = entries.map((entry) => ({ id: entry.id, description: entry.description }));
+  const byId = new Map(next.map((entry, index) => [entry.id, index]));
+  for (const change of changes) {
+    const id = change.id.trim();
+    if (id.length === 0) continue;
+    if (change.action === "remove") {
+      const index = byId.get(id);
+      if (index === undefined) continue;
+      next.splice(index, 1);
+      byId.delete(id);
+      for (const [key, value] of byId) {
+        if (value > index) byId.set(key, value - 1);
+      }
+      continue;
+    }
+    const description = recognitionDescriptionFromContent(change.content);
+    const existing = byId.get(id);
+    if (existing !== undefined) {
+      next[existing] = { id, description };
+    } else {
+      byId.set(id, next.length);
+      next.push({ id, description });
+    }
+  }
+  return next;
+}
+
+function meaningfulChanges(changes: readonly RecognitionIndexChange[]): RecognitionIndexChange[] {
+  return changes.filter((change) => change.id.trim().length > 0);
+}
+
+/**
+ * Incrementally maintain a namespace recognition index after memory writes.
+ * `enabled: false` is a proven no-op: no reads, no writes, no locks.
+ */
+export async function maintainRecognitionIndexAfterWrite(args: {
+  memoryDir: string;
+  enabled: boolean;
+  changes: readonly RecognitionIndexChange[];
+}): Promise<void> {
+  if (args.enabled !== true) return;
+  const changes = meaningfulChanges(args.changes);
+  if (changes.length === 0) return;
+
+  const indexPath = recognitionIndexPath(args.memoryDir);
+  await withIndexChain(indexPath, () =>
+    withRecognitionIndexLock(args.memoryDir, async () => {
+      const loaded = await loadRecognitionIndex(args.memoryDir);
+      // `changes` is the full set the caller wants published. Persist-path
+      // bootstraps from the active corpus when the file is absent so a
+      // first post-enable write cannot mint a 1-id index that recall would
+      // treat as authoritative.
+      const entries = applyChanges(loaded?.entries ?? [], changes);
+      await saveRecognitionIndex(args.memoryDir, buildRecognitionIndex(entries));
+    }),
+  );
+}

package/src/orchestrator.ts

@@ -1711,6 +1711,7 @@ export class Orchestrator {
       removeContentHashForMemory: (targetStorage, memory, context) =>
         this.removeContentHashForMemory(targetStorage, memory, context),
       saveContentHashIndexes: () => this.saveContentHashIndexes(),
+      getHandleHistory: () => this.handleHistory,
     });
     this.threading = new ThreadingManager(
       path.join(config.memoryDir, "threads"),

package/src/procedural/procedure-miner.ts

@@ -4,14 +4,11 @@
 
 import type { PluginConfig } from "../types.js";
 import type { StorageManager } from "../storage.js";
-import type { CausalTrajectoryRecord } from "../causal-trajectory.js";
 import { createHash } from "node:crypto";
 import { mkdir, rm, stat } from "node:fs/promises";
 import path from "node:path";
-import {
-  readCausalTrajectoryRecords,
-  filterTrajectoriesByLookbackDays,
-} from "../causal-trajectory.js";
+import { readCausalTrajectoryRecords, type CausalTrajectoryRecord } from "../causal-trajectory.js";
+import { collectProcedureMiningRecords } from "../experience/session-experience-mining.js";
 import { buildProcedurePersistBody, normalizeProcedureSteps, type ProcedureStep } from "./procedure-types.js";
 import { clusterByKey } from "./reinforcement-core.js";
 import { log } from "../logger.js";
@@ -207,7 +204,12 @@ export async function runProcedureMining(options: {
     memoryDir: options.memoryDir,
     causalTrajectoryStoreDir: trajectoryDir,
   });
-  const recent = filterTrajectoriesByLookbackDays(trajectories, cfg.lookbackDays);
+  const recent = await collectProcedureMiningRecords({
+    trajectories,
+    lookbackDays: cfg.lookbackDays,
+    storage: options.storage,
+    experienceEnabled: options.config.sessionExperience?.enabled === true,
+  });
 
   const clusters = clusterByKey(recent, clusterKey);
 

package/src/testing/subjects/recall-recognition-index.test.ts

@@ -0,0 +1,97 @@
+/**
+ * Recognition-index write-path subject for the scenario-matrix harness
+ * (issue #2975).
+ *
+ * Every canonical row drives the real maintainRecognitionIndexAfterWrite
+ * seam twice: gate off (zero index I/O, no index file) and gate on
+ * (the write's id is visible to loadRecognitionIndex). Restart rows
+ * reload from disk; replay rows prove an id upsert is idempotent.
+ */
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import { promises as fsp } from "node:fs";
+import { mkdtemp, rm } from "node:fs/promises";
+
+import { loadRecognitionIndex, recognitionIndexPath } from "../../recall-recognition-tier.js";
+import { maintainRecognitionIndexAfterWrite } from "../../orchestration/recall-recognition-index.js";
+import { type LifecycleSubject, type MatrixRow, runLifecycleMatrix } from "../lifecycle-matrix.js";
+
+interface IndexState {
+  rowId: string;
+  dir: string;
+  restart: boolean;
+  dedupeOrReplay: boolean;
+}
+
+const subject: LifecycleSubject<IndexState> = {
+  async setup(row: MatrixRow): Promise<IndexState> {
+    const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-recognition-index-subject-"));
+    return {
+      rowId: String(row.id),
+      dir,
+      restart: row.dimensions.restart,
+      dedupeOrReplay: row.dimensions.dedupeOrReplay,
+    };
+  },
+
+  async exercise(state): Promise<void> {
+    const id = `fact-${state.rowId}`;
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: state.dir,
+      enabled: false,
+      changes: [{ action: "upsert", id, content: `off-path body for ${state.rowId}` }],
+    });
+    assert.equal(await loadRecognitionIndex(state.dir), null);
+    try {
+      await fsp.access(recognitionIndexPath(state.dir));
+      assert.fail("off-path must not create the recognition index file");
+    } catch (err) {
+      assert.equal((err as NodeJS.ErrnoException).code, "ENOENT");
+    }
+
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: state.dir,
+      enabled: true,
+      changes: [{ action: "upsert", id, content: `SSD inference chips — ${state.rowId}\nbody` }],
+    });
+    if (state.dedupeOrReplay) {
+      await maintainRecognitionIndexAfterWrite({
+        memoryDir: state.dir,
+        enabled: true,
+        changes: [{ action: "upsert", id, content: `SSD inference chips — ${state.rowId} replayed\nbody` }],
+      });
+    }
+  },
+
+  async invariants(state): Promise<void> {
+    const id = `fact-${state.rowId}`;
+    const index = await loadRecognitionIndex(state.dir);
+    assert.ok(index, "enabled write must persist an index loadable after the row");
+    assert.equal(index.entries.length, 1);
+    assert.equal(index.entries[0]?.id, id);
+    const expected = state.dedupeOrReplay
+      ? `SSD inference chips — ${state.rowId} replayed`
+      : `SSD inference chips — ${state.rowId}`;
+    assert.equal(index.entries[0]?.description, expected);
+
+    if (state.restart) {
+      const reloaded = await loadRecognitionIndex(state.dir);
+      assert.deepEqual(reloaded, index, "restart must see the same on-disk index");
+    }
+
+    const before = { ...index };
+    await maintainRecognitionIndexAfterWrite({
+      memoryDir: state.dir,
+      enabled: false,
+      changes: [{ action: "remove", id }],
+    });
+    assert.deepEqual(await loadRecognitionIndex(state.dir), before, "off-path after on-path must not mutate the index");
+  },
+
+  async teardown(state): Promise<void> {
+    await rm(state.dir, { recursive: true, force: true });
+  },
+};
+
+runLifecycleMatrix("recall-recognition-index", subject);

package/src/types.ts

@@ -23,6 +23,7 @@ import type {
 import type { DeepRecallSettings } from "./deep-recall-config.js";
 import type { RecallNavigationSettings } from "./recall-navigation-config.js";
 import type { RecognitionTierSettings } from "./recall-recognition-tier.js";
+import type { SeedGraduationSettings } from "./lifecycle/seed-graduation-config-types.js";
 import type { ProceduralMaintenanceConfig } from "./procedural/maintenance-config.js";
 import type { SkillProjectionConfig } from "./procedural/skill-projection.js";
 import type { SessionExperienceConfig } from "./experience/session-experience-config.js";
@@ -698,6 +699,7 @@ export interface PluginConfig
     DeepRecallSettings,
     RecallNavigationSettings,
     RecognitionTierSettings,
+    SeedGraduationSettings,
     LocalLlmConfig {
   openaiApiKey: string | undefined;
   openaiBaseUrl: string | undefined;
@@ -3877,12 +3879,8 @@ export interface HourlySummary {
   generatedAt: string;
 }
 
-// ============================================================================
-// Dreams Pipeline Telemetry (issue #678 PR 3/4)
-// ============================================================================
-// Re-export from the authoritative source to avoid duplicate definitions.
-// dreams-ledger.ts is the single source of truth; types.ts re-exports so
-// callers that import from types.js continue to work unchanged.
+// Dreams Pipeline Telemetry (issue #678 PR 3/4). Re-export from
+// dreams-ledger.ts so callers importing types.js stay unchanged.
 export type {
   DreamsPhase,
   DreamsPhaseStatus,