@remnic/core 9.69.47 → 9.69.49

package/src/directory-sidecars.ts

@@ -0,0 +1,383 @@
+/**
+ * @remnic/core — per-directory abstract/overview sidecars (issue #2977).
+ *
+ * One derived `overview.md` per memory directory (category roots and every
+ * directory beneath them, including `namespaces/<ns>/` subtrees), rendered
+ * bottom-up from existing memory content:
+ *
+ *   - the ABSTRACT is a single line (<=256 chars) composed from child titles;
+ *   - the OVERVIEW body lists one preview line per child memory and embeds
+ *     each child DIRECTORY's abstract, so a parent summarizes its subtree
+ *     without embedding full child overviews.
+ *
+ * Determinism: no LLM, no embeddings. Content is assembled with the same
+ * preview machinery the OKF index files use (`normalizeProjectionPreview`).
+ *
+ * Invalidation: each sidecar stores a content-free fingerprint of its subtree
+ * (child name|size|mtime for files, recursive fingerprints for child dirs).
+ * Any add/edit/delete anywhere below a directory changes its fingerprint, so
+ * `loadDirectorySidecar` can detect staleness with `readdir`+`stat` alone and
+ * refresh without reading child memory files first. Eager regeneration runs
+ * through {@link runDirectorySidecarMaintenance}; lazy refresh on read covers
+ * direct-write paths that bypass scheduled maintenance (Gotcha #43).
+ *
+ * Isolation: sidecars are derived artifacts, excluded from generic recall via
+ * `isGenericRecallExcludedPath` (basename match, the same tradeoff OKF makes
+ * for `index.md`/`log.md`) while staying full-text searchable. Files at the
+ * sidecar path WITHOUT the marker are user content: never overwritten, never
+ * removed.
+ */
+
+import { createHash } from "node:crypto";
+import { lstatSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs";
+import path from "node:path";
+
+import { writeFileAtomically } from "./maintenance/atomic-file.js";
+import { normalizeProjectionPreview } from "./memory-projection-format.js";
+import { OKF_RESERVED_BASENAMES } from "./okf/type-mapping.js";
+import { RECALL_FALLBACK_DIRS } from "./utils/category-dir.js";
+
+export const DIRECTORY_SIDECAR_BASENAME = "overview.md";
+export const DIRECTORY_SIDECAR_MARKER = "<!-- remnic-directory-sidecar -->";
+
+const ABSTRACT_MAX_CHARS = 256;
+const OVERVIEW_MAX_CHARS = 4000;
+const DEFAULT_QUERY_LIMIT = 5;
+
+/** Basenames never treated as child memories (ours plus OKF's reserved ones). */
+const RESERVED_CHILD_BASENAMES: Readonly<Record<string, true>> = Object.freeze({
+  ...OKF_RESERVED_BASENAMES,
+  [DIRECTORY_SIDECAR_BASENAME]: true,
+});
+
+export interface DirectorySidecar {
+  /** Absolute directory the sidecar describes. */
+  readonly dir: string;
+  /** Single-line L0 abstract (<=256 chars). */
+  readonly abstract: string;
+  /** Abstract plus the per-child preview body (<=4000 chars). */
+  readonly overview: string;
+  /** Number of memory `.md` children in this directory. */
+  readonly childCount: number;
+  /** False when the stored fingerprint no longer matches the subtree. */
+  readonly fresh: boolean;
+}
+
+export interface DirectorySidecarMatch extends DirectorySidecar {
+  readonly score: number;
+}
+
+export interface DirectorySidecarReport {
+  written: string[];
+  removed: string[];
+}
+
+interface ParsedSidecar {
+  abstract: string;
+  overview: string;
+  childCount: number;
+  fingerprint: string;
+}
+
+function listEntries(dir: string) {
+  try {
+    return readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+}
+
+/** Root check that refuses symlinks (fs-boundary rule, PRs #1938/#1999). */
+function isRealDirectory(entryPath: string): boolean {
+  try {
+    const stats = lstatSync(entryPath);
+    return !stats.isSymbolicLink() && stats.isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+function memoryChildren(dir: string): string[] {
+  // Dirent.isFile() is false for symlinks, so a linked memory never enters the corpus.
+  return listEntries(dir)
+    .filter((entry) => entry.isFile() && !RESERVED_CHILD_BASENAMES[entry.name] && entry.name.endsWith(".md"))
+    .map((entry) => entry.name)
+    .sort((a, b) => a.localeCompare(b));
+}
+
+function directoryChildren(dir: string): string[] {
+  // Dirent.isDirectory() is false for symlinks (fs-boundary rule, PRs #1938/#1999).
+  return listEntries(dir)
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => entry.name)
+    .sort((a, b) => a.localeCompare(b));
+}
+
+/**
+ * Content-free subtree fingerprint: sha256 over sorted child descriptors.
+ * Files contribute `name|size|mtimeMs`; directories contribute their own
+ * recursive fingerprint, so any change below a directory invalidates every
+ * ancestor. Reads no file contents.
+ */
+export function directoryFingerprint(dir: string): string {
+  const hash = createHash("sha256");
+  for (const name of memoryChildren(dir)) {
+    try {
+      const stats = statSync(path.join(dir, name));
+      hash.update(`f|${name}|${stats.size}|${Math.trunc(stats.mtimeMs)}\n`);
+    } catch {
+      hash.update(`f|${name}|missing\n`);
+    }
+  }
+  for (const name of directoryChildren(dir)) {
+    hash.update(`d|${name}|${directoryFingerprint(path.join(dir, name))}\n`);
+  }
+  return hash.digest("hex");
+}
+
+function readFileOrNull(target: string): string | null {
+  try {
+    return readFileSync(target, "utf8");
+  } catch {
+    return null;
+  }
+}
+
+function parseSidecar(content: string): ParsedSidecar | null {
+  if (!content.startsWith(DIRECTORY_SIDECAR_MARKER)) return null;
+  const fingerprint = /^fingerprint: ([0-9a-f]{64})$/m.exec(content)?.[1];
+  if (!fingerprint) return null;
+  const abstract = /^## Abstract\n([^\n]+)$/m.exec(content)?.[1] ?? "";
+  const overviewStart = content.indexOf("## Abstract");
+  const overview =
+    overviewStart === -1 ? "" : content.slice(overviewStart, overviewStart + OVERVIEW_MAX_CHARS).trim();
+  // The `* name — preview` child bullets are the rendered child count.
+  const childCount = (content.match(/^\* .+$/gm) ?? []).length;
+  return { abstract, overview, childCount, fingerprint };
+}
+
+function sidecarPath(dir: string): string {
+  return path.join(dir, DIRECTORY_SIDECAR_BASENAME);
+}
+
+function renderSidecar(dir: string, fingerprint: string, childAbstracts: Map<string, string>): string | null {
+  const children = memoryChildren(dir);
+  const childDirs = directoryChildren(dir).filter((name) => childAbstracts.has(path.join(dir, name)));
+  if (children.length === 0 && childDirs.length === 0) return null;
+
+  const previews = children.map((name) => {
+    try {
+      return normalizeProjectionPreview(readFileSync(path.join(dir, name), "utf8"), 120);
+    } catch {
+      return "";
+    }
+  });
+  const titles = children.map(
+    (name, index) => previews[index].split(" ").slice(0, 8).join(" ") || name.replace(/\.md$/, ""),
+  );
+  const abstract = normalizeProjectionPreview(
+    `${path.basename(dir)}/ — ${children.length} memories: ${titles.slice(0, 5).join(" · ")}`,
+    ABSTRACT_MAX_CHARS,
+  );
+
+  const lines = [
+    DIRECTORY_SIDECAR_MARKER,
+    `fingerprint: ${fingerprint}`,
+    `# ${path.basename(dir)}/ overview`,
+    "",
+    "## Abstract",
+    abstract,
+    "",
+    "## Children",
+  ];
+  for (const [index, name] of children.entries()) {
+    lines.push(`* ${name} — ${previews[index]}`);
+  }
+  for (const name of childDirs) {
+    lines.push(`### ${name}/`);
+    lines.push(childAbstracts.get(path.join(dir, name)) ?? "");
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+
+/** Scope roots whose category subtrees carry sidecars: the memory root plus every namespace root. */
+function scopeRoots(memoryDir: string): string[] {
+  const roots = [memoryDir];
+  const namespacesRoot = path.join(memoryDir, "namespaces");
+  for (const entry of listEntries(namespacesRoot)) {
+    if (entry.isDirectory() && isRealDirectory(path.join(namespacesRoot, entry.name))) {
+      roots.push(path.join(namespacesRoot, entry.name));
+    }
+  }
+  return roots;
+}
+
+/**
+ * Every directory that should carry a sidecar, deepest first (post-order) so
+ * parents render after their children and can embed child abstracts.
+ */
+function collectSidecarDirs(scopeRoot: string): string[] {
+  const dirs: string[] = [];
+  const walk = (dir: string, isCategoryRoot: boolean): boolean => {
+    let carries = memoryChildren(dir).length > 0;
+    for (const name of directoryChildren(dir)) {
+      carries = walk(path.join(dir, name), false) || carries;
+    }
+    // A directory that stopped carrying memories must still be visited so its
+    // orphaned sidecar can be pruned or rewritten (cache-invalidation completeness).
+    const hasSidecar = readFileOrNull(sidecarPath(dir))?.startsWith(DIRECTORY_SIDECAR_MARKER) === true;
+    if (isCategoryRoot || carries || hasSidecar) dirs.push(dir);
+    return carries;
+  };
+  for (const category of RECALL_FALLBACK_DIRS) {
+    const categoryDir = path.join(scopeRoot, category);
+    if (!isRealDirectory(categoryDir)) continue;
+    walk(categoryDir, true);
+  }
+  return dirs;
+}
+
+/**
+ * Regenerate directory sidecars incrementally. Unchanged sidecars are left
+ * untouched; emptied directories lose theirs; user files at the sidecar path
+ * (no marker) are never written or removed. With `enabled: false` every
+ * generated sidecar is removed — the OKF index-maintenance contract.
+ */
+export async function runDirectorySidecarMaintenance(
+  memoryDir: string,
+  enabled: boolean,
+): Promise<DirectorySidecarReport> {
+  const written: string[] = [];
+  const removed: string[] = [];
+  for (const scope of scopeRoots(memoryDir)) {
+    const childAbstracts = new Map<string, string>();
+    for (const dir of collectSidecarDirs(scope)) {
+      const target = sidecarPath(dir);
+      const raw = readFileOrNull(target);
+      const existing = raw !== null && raw.startsWith(DIRECTORY_SIDECAR_MARKER) ? raw : null;
+      if (!enabled) {
+        if (existing !== null) {
+          unlinkSync(target);
+          removed.push(target);
+        }
+        continue;
+      }
+      const next = renderSidecar(dir, directoryFingerprint(dir), childAbstracts);
+      if (next === null) {
+        if (existing !== null) {
+          unlinkSync(target);
+          removed.push(target);
+        }
+        continue;
+      }
+      const parsedNext = parseSidecar(next);
+      if (parsedNext) childAbstracts.set(dir, parsedNext.abstract);
+      if (raw === next) continue;
+      if (existing === null && raw !== null) continue; // user-authored file: hands off
+      await writeFileAtomically(target, next);
+      written.push(target);
+    }
+  }
+  return { written, removed };
+}
+
+/**
+ * Read a directory's sidecar for retrieval. By default a stale sidecar is
+ * refreshed in place (child directories first), which is what makes direct
+ * writes that bypass maintenance still converge. With `{ refresh: false }`
+ * the stored sidecar is returned as-is with `fresh: false` so callers can
+ * observe staleness without writes.
+ */
+export async function loadDirectorySidecar(
+  dir: string,
+  options: { refresh?: boolean } = {},
+): Promise<DirectorySidecar | null> {
+  const refresh = options.refresh !== false;
+  const target = sidecarPath(dir);
+  const stored = readFileOrNull(target);
+  const generated = stored !== null && stored.startsWith(DIRECTORY_SIDECAR_MARKER);
+  const parsed = generated ? parseSidecar(stored) : null;
+  const fingerprint = directoryFingerprint(dir);
+  if (parsed !== null && (parsed.fingerprint === fingerprint || !refresh)) {
+    return {
+      dir,
+      abstract: parsed.abstract,
+      overview: parsed.overview,
+      childCount: parsed.childCount,
+      fresh: parsed.fingerprint === fingerprint,
+    };
+  }
+  if (!refresh) return null;
+
+  // Refresh bottom-up so this directory's body embeds current child abstracts.
+  const childAbstracts = new Map<string, string>();
+  for (const name of directoryChildren(dir)) {
+    const child = await loadDirectorySidecar(path.join(dir, name));
+    if (child) childAbstracts.set(path.join(dir, name), child.abstract);
+  }
+  const next = renderSidecar(dir, directoryFingerprint(dir), childAbstracts);
+  if (next === null) {
+    if (generated) unlinkSync(target);
+    return null;
+  }
+  if (readFileOrNull(target) !== next) {
+    if (stored !== null && !generated) {
+      // User-authored file at the sidecar path: serve nothing rather than clobber it.
+      return null;
+    }
+    await writeFileAtomically(target, next);
+  }
+  const refreshed = parseSidecar(next);
+  return refreshed
+    ? {
+        dir,
+        abstract: refreshed.abstract,
+        overview: refreshed.overview,
+        childCount: refreshed.childCount,
+        fresh: true,
+      }
+    : null;
+}
+
+/**
+ * Overview-query primitive for hierarchical retrieval (layer 1 of #2977):
+ * score every sidecar's abstract (+3 per query term) and overview (+1 per
+ * term) lexically, then return the most specific matches — when both a
+ * directory and its descendant match, only the descendant is kept, matching
+ * the drill-down semantics the full retrieval mode will build on.
+ */
+export async function findDirectorySidecarsForQuery(
+  memoryDir: string,
+  query: string,
+  options: { namespace?: string; limit?: number } = {},
+): Promise<DirectorySidecarMatch[]> {
+  const scope = options.namespace ? path.join(memoryDir, "namespaces", options.namespace) : memoryDir;
+  if (!isRealDirectory(scope)) return [];
+  const terms = query
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((term) => term.length >= 2);
+
+  const matches: DirectorySidecarMatch[] = [];
+  for (const root of scopeRoots(scope)) {
+    for (const dir of collectSidecarDirs(root)) {
+      const sidecar = await loadDirectorySidecar(dir);
+      if (!sidecar) continue;
+      const abstractLower = sidecar.abstract.toLowerCase();
+      const overviewLower = sidecar.overview.toLowerCase();
+      let score = 0;
+      for (const term of terms) {
+        if (abstractLower.includes(term)) score += 3;
+        else if (overviewLower.includes(term)) score += 1;
+      }
+      if (score > 0) matches.push({ ...sidecar, score });
+    }
+  }
+  matches.sort((a, b) => b.score - a.score || a.dir.localeCompare(b.dir));
+
+  const limit = options.limit ?? DEFAULT_QUERY_LIMIT;
+  return matches
+    .filter((candidate) => !matches.some((other) => other.dir.startsWith(`${candidate.dir}${path.sep}`)))
+    .slice(0, limit);
+}

package/src/experience/session-experience-config.ts

@@ -0,0 +1,51 @@
+/**
+ * Session-end experience extraction config (issue #2979) — the
+ * `sessionExperience` block.
+ *
+ * Parsed like `recallNavigation`: shape-validated, fail-closed on unknown
+ * keys, no silent clamping. Defaults OFF (a machine-generated memory write
+ * from the completed session must be an explicit operator opt-in); the
+ * `conservative` preset pins `enabled: false`.
+ */
+
+import { coerceBool } from "../connectors/coerce.js";
+
+export interface SessionExperienceConfig {
+  /**
+   * Master switch for session-end Situation/Approach/Reflection episode
+   * extraction. Default false.
+   */
+  enabled: boolean;
+}
+
+export const SESSION_EXPERIENCE_CONFIG_DEFAULTS: SessionExperienceConfig = {
+  enabled: false,
+};
+
+function parseFlag(src: Record<string, unknown>, key: string, fallback: boolean): boolean {
+  const value = src[key];
+  if (value === undefined || value === null) return fallback;
+  const coerced = coerceBool(value);
+  if (coerced === undefined) {
+    throw new Error(
+      `sessionExperience.${key} must be a boolean (or "true"/"false"/"1"/"0"); got ${JSON.stringify(value)}`,
+    );
+  }
+  return coerced;
+}
+
+export function parseSessionExperienceConfig(raw: unknown): SessionExperienceConfig {
+  if (raw === undefined || raw === null) return { ...SESSION_EXPERIENCE_CONFIG_DEFAULTS };
+  if (typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error(`sessionExperience must be an object; got ${JSON.stringify(raw)}`);
+  }
+  const src = raw as Record<string, unknown>;
+  const { enabled: _enabled, ...unknown } = src;
+  const unknownKey = Object.keys(unknown)[0];
+  if (unknownKey !== undefined) {
+    throw new Error(`sessionExperience contains unknown key ${JSON.stringify(unknownKey)}`);
+  }
+  return {
+    enabled: parseFlag(src, "enabled", SESSION_EXPERIENCE_CONFIG_DEFAULTS.enabled),
+  };
+}

package/src/experience/session-experience-lifecycle.test.ts

@@ -0,0 +1,139 @@
+/**
+ * Session-end experience extraction through the real flush lifecycle
+ * (issue #2979): `flushSession(reason: "session_end")` drains the buffer
+ * and then derives at most one agent-experience episode; `before_reset`
+ * must not write one; gate off must write nothing anywhere.
+ */
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+import { Orchestrator } from "../orchestrator.js";
+import {
+  cleanupDir,
+  makeHarmonicLifecycleConfig,
+  markdownFilesUnder,
+  mkTempMemoryDir,
+  singleFactResult,
+  stubExtraction,
+} from "../testing/orchestrator-lite.js";
+
+async function sessionEndMemories(memoryDir: string): Promise<string[]> {
+  const matches: string[] = [];
+  for (const file of await markdownFilesUnder(`${memoryDir}/procedures`)) {
+    if ((await readFile(file, "utf8")).includes("Reflection: ")) matches.push(file);
+  }
+  return matches;
+}
+
+async function withOrchestrator(
+  label: string,
+  overrides: Record<string, unknown>,
+  run: (orchestrator: Orchestrator, memoryDir: string) => Promise<void>,
+): Promise<void> {
+  const memoryDir = await mkTempMemoryDir(label);
+  let orchestrator: Orchestrator | undefined;
+  try {
+    orchestrator = new Orchestrator(makeHarmonicLifecycleConfig(memoryDir, overrides));
+    stubExtraction(orchestrator, (turns) => singleFactResult("The harmonic probe fact persists."));
+    await run(orchestrator, memoryDir);
+  } finally {
+    await orchestrator?.destroy().catch(() => undefined);
+    await cleanupDir(memoryDir);
+  }
+}
+
+const SESSION = "session-end-experience";
+
+const SUCCESS_TURNS: Array<[role: "user" | "assistant", content: string]> = [
+  ["user", "Fix the flaky payments integration test that times out after thirty seconds."],
+  ["assistant", "I will reproduce the timeout and inspect the payments test setup."],
+  ["assistant", "Added the missing await to the setup call. Verified: the payments test passes now."],
+];
+
+test("session_end flush writes exactly one pending_review agent-experience episode", async () => {
+  await withOrchestrator(
+    "sx-e2e-on",
+    { sessionExperience: { enabled: true } },
+    async (orchestrator, memoryDir) => {
+      for (const [role, content] of SUCCESS_TURNS) {
+        await orchestrator.processTurn(role, content, SESSION);
+      }
+      await orchestrator.flushSession(SESSION, { reason: "session_end" });
+      const experiences = await sessionEndMemories(memoryDir);
+      assert.equal(experiences.length, 1, "one session = at most one experience episode");
+      const body = await readFile(experiences[0], "utf8");
+      assert.match(body, /Situation: Fix the flaky payments integration test/);
+      assert.match(body, /Approach: /);
+      assert.match(body, /Reflection: /);
+      assert.match(body, /status: pending_review/);
+      assert.match(body, /subject: agent/);
+      assert.match(body, /category: procedure/);
+      assert.match(body, /experience_session_hash/);
+      // A second session_end drain sees an empty buffer and must not duplicate.
+      await orchestrator.flushSession(SESSION, { reason: "session_end" });
+      assert.equal((await sessionEndMemories(memoryDir)).length, 1, "replay must not duplicate");
+    },
+  );
+});
+
+test("a failed session_end records the failure reflection", async () => {
+  await withOrchestrator(
+    "sx-e2e-fail",
+    { sessionExperience: { enabled: true } },
+    async (orchestrator, memoryDir) => {
+      await orchestrator.processTurn(
+        "user",
+        "Migrate the billing export job to the new queue library before the release cut.",
+        SESSION,
+      );
+      await orchestrator.processTurn(
+        "assistant",
+        "The migration failed: the new queue library cannot serialize the billing payload format.",
+        SESSION,
+      );
+      await orchestrator.flushSession(SESSION, { reason: "session_end" });
+      const experiences = await sessionEndMemories(memoryDir);
+      assert.equal(experiences.length, 1);
+      const body = await readFile(experiences[0], "utf8");
+      assert.match(body, /experience_outcome: failure|experience_outcome=failure/);
+      assert.match(body, /fail/i);
+    },
+  );
+});
+
+test("gate off (default): a session_end flush writes no experience anywhere", async () => {
+  await withOrchestrator("sx-e2e-off", {}, async (orchestrator, memoryDir) => {
+    for (const [role, content] of SUCCESS_TURNS) {
+      await orchestrator.processTurn(role, content, SESSION);
+    }
+    await orchestrator.flushSession(SESSION, { reason: "session_end" });
+    assert.equal((await sessionEndMemories(memoryDir)).length, 0, "gate off must write nothing");
+    for (const file of await markdownFilesUnder(memoryDir)) {
+      const body = await readFile(file, "utf8");
+      assert.doesNotMatch(
+        body,
+        /Reflection: /,
+        `no experience body may appear anywhere while gated off (saw ${file})`,
+      );
+    }
+  });
+});
+
+test("before_reset drains the buffer but writes no experience episode", async () => {
+  await withOrchestrator(
+    "sx-e2e-reset",
+    { sessionExperience: { enabled: true } },
+    async (orchestrator, memoryDir) => {
+      for (const [role, content] of SUCCESS_TURNS) {
+        await orchestrator.processTurn(role, content, SESSION);
+      }
+      await orchestrator.flushSession(SESSION, { reason: "before_reset" });
+      assert.equal(
+        (await sessionEndMemories(memoryDir)).length,
+        0,
+        "experiences are session_end-only in v1",
+      );
+    },
+  );
+});