@remnic/core 1.1.11 → 1.1.13

package/src/semantic-consolidation.test.ts

@@ -0,0 +1,182 @@
+/**
+ * Unit tests for the ConsolidationOperator vocabulary and derived_from
+ * validator introduced in issue #561 PR 1.
+ */
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  CONSOLIDATION_OPERATORS,
+  isConsolidationOperator,
+  isSemanticConsolidationLlmOperator,
+  isValidDerivedFromEntry,
+  type ConsolidationOperator,
+  type SemanticConsolidationLlmOperator,
+} from "./semantic-consolidation.js";
+// The standalone module is the source of truth; semantic-consolidation.ts
+// re-exports it.  This test import proves both surfaces work.
+import {
+  CONSOLIDATION_OPERATORS as CONSOLIDATION_OPERATORS_DIRECT,
+  isConsolidationOperator as isConsolidationOperatorDirect,
+  isSemanticConsolidationLlmOperator as isSemanticConsolidationLlmOperatorDirect,
+  isValidDerivedFromEntry as isValidDerivedFromEntryDirect,
+} from "./consolidation-operator.js";
+
+test("semantic-consolidation.ts re-exports match consolidation-operator.ts", () => {
+  assert.deepEqual([...CONSOLIDATION_OPERATORS], [...CONSOLIDATION_OPERATORS_DIRECT]);
+  assert.equal(isConsolidationOperator, isConsolidationOperatorDirect);
+  assert.equal(
+    isSemanticConsolidationLlmOperator,
+    isSemanticConsolidationLlmOperatorDirect,
+  );
+  assert.equal(isValidDerivedFromEntry, isValidDerivedFromEntryDirect);
+});
+
+test("CONSOLIDATION_OPERATORS enumerates split/merge/update/pattern-reinforcement", () => {
+  // `pattern-reinforcement` joined the operator vocabulary in issue
+  // #687 PR 2/4 so the maintenance job in
+  // `maintenance/pattern-reinforcement.ts` can stamp `derived_via`
+  // through the same write-time validator the rest of consolidation
+  // uses.
+  assert.deepEqual(
+    [...CONSOLIDATION_OPERATORS],
+    ["split", "merge", "update", "pattern-reinforcement"],
+  );
+});
+
+test("isConsolidationOperator accepts every defined operator", () => {
+  for (const op of CONSOLIDATION_OPERATORS) {
+    assert.equal(isConsolidationOperator(op), true, `${op} should be accepted`);
+  }
+});
+
+test("isConsolidationOperator rejects unknown and non-string values", () => {
+  assert.equal(isConsolidationOperator("MERGE"), false); // case-sensitive
+  assert.equal(isConsolidationOperator("annihilate"), false);
+  assert.equal(isConsolidationOperator(""), false);
+  assert.equal(isConsolidationOperator(undefined), false);
+  assert.equal(isConsolidationOperator(null), false);
+  assert.equal(isConsolidationOperator(42), false);
+  assert.equal(isConsolidationOperator({ op: "merge" }), false);
+});
+
+test("isValidDerivedFromEntry accepts well-formed path:version strings", () => {
+  assert.equal(isValidDerivedFromEntry("facts/a.md:0"), true);
+  assert.equal(isValidDerivedFromEntry("facts/a.md:2"), true);
+  assert.equal(isValidDerivedFromEntry("facts/2026-01-15/pref-001.md:17"), true);
+  assert.equal(isValidDerivedFromEntry("entities/person-alice.md:1"), true);
+  // Paths containing colons are still parseable because only the final
+  // `:<digits>` is consumed as the version.
+  assert.equal(isValidDerivedFromEntry("facts/weird:name.md:3"), true);
+});
+
+test("isValidDerivedFromEntry rejects malformed entries", () => {
+  assert.equal(isValidDerivedFromEntry(""), false, "empty string");
+  // `facts/a.md` has no `:`, fails the snapshot regex, and fails the
+  // memory-id regex (contains `/` and `.`).
+  assert.equal(isValidDerivedFromEntry("facts/a.md"), false, "no version, path-shaped");
+  assert.equal(isValidDerivedFromEntry("facts/a.md:"), false, "missing digits");
+  assert.equal(isValidDerivedFromEntry("facts/a.md:abc"), false, "non-numeric version");
+  assert.equal(isValidDerivedFromEntry("facts/a.md:-1"), false, "negative version");
+  assert.equal(isValidDerivedFromEntry("facts/a.md:1.5"), false, "fractional version");
+  assert.equal(isValidDerivedFromEntry(":3"), false, "empty path");
+  assert.equal(isValidDerivedFromEntry("   :3"), false, "whitespace-only path");
+  // Memory-id form must reject leading non-alphanumeric characters
+  // (so it cannot accidentally swallow a malformed snapshot like
+  // `_-bad`).
+  assert.equal(isValidDerivedFromEntry("-not-an-id"), false, "leading hyphen");
+  assert.equal(isValidDerivedFromEntry("has spaces"), false, "spaces in id");
+  assert.equal(isValidDerivedFromEntry("has/slash"), false, "slash without version");
+});
+
+test("isValidDerivedFromEntry accepts memory-id-shaped entries (issue #687 PR 2/4)", () => {
+  // Pattern reinforcement records source memory IDs directly rather
+  // than page-versioning snapshots; the validator widened to accept
+  // either shape.
+  assert.equal(isValidDerivedFromEntry("m-abc123-de"), true);
+  assert.equal(isValidDerivedFromEntry("m-1"), true);
+  assert.equal(isValidDerivedFromEntry("alpha"), true);
+  assert.equal(isValidDerivedFromEntry("Mem_Underscored-Allowed"), true);
+  // Common bare-id form used by extraction.
+  assert.equal(isValidDerivedFromEntry("fact-abc-123"), true);
+});
+
+test("isValidDerivedFromEntry accepts namespace-prefixed memory IDs containing colons (PR #730 Codex P1)", () => {
+  // Namespace-prefixed IDs like `global:fact-abc-123` are a real
+  // shape this codebase already handles in graph retrieval
+  // (`stripDerivedFromVersion`).  The validator must NOT reject them
+  // just because they contain `:`.  Disambiguation against the
+  // `<path>:<version>` form is by checking whether the trailing
+  // segment is integer AND the left side looks like a path.
+  assert.equal(
+    isValidDerivedFromEntry("global:fact-abc-123"),
+    true,
+    "namespace-prefixed memory id must be accepted",
+  );
+  assert.equal(
+    isValidDerivedFromEntry("entity:person-alice"),
+    true,
+    "entity-prefixed memory id must be accepted",
+  );
+  assert.equal(
+    isValidDerivedFromEntry("tenant:proj:fact-xyz"),
+    true,
+    "multi-colon memory id must be accepted",
+  );
+});
+
+test("isValidDerivedFromEntry: snapshot-vs-id disambiguation handles ambiguous tail-numeric ids", () => {
+  // `path/file.md:42` is a valid snapshot reference (left side has
+  // `/` and `.`).  `global:42` is a valid memory id whose tail
+  // happens to be numeric — it must NOT be misclassified as a
+  // snapshot because there is no path-shaped left side.  This
+  // matches the precedence used by `stripDerivedFromVersion` in
+  // `graph-retrieval.ts`.
+  assert.equal(isValidDerivedFromEntry("path/file.md:42"), true, "valid snapshot reference");
+  assert.equal(isValidDerivedFromEntry("global:42"), true, "valid memory id with numeric tail");
+  // A memory id with no colon and no path characters is also valid.
+  assert.equal(isValidDerivedFromEntry("fact-abc-123"), true, "bare memory id");
+});
+
+test("isValidDerivedFromEntry rejects non-string values", () => {
+  assert.equal(isValidDerivedFromEntry(undefined), false);
+  assert.equal(isValidDerivedFromEntry(null), false);
+  assert.equal(isValidDerivedFromEntry(42), false);
+  assert.equal(isValidDerivedFromEntry(["facts/a.md:2"]), false);
+  assert.equal(isValidDerivedFromEntry({ path: "facts/a.md", version: 2 }), false);
+});
+
+test("ConsolidationOperator type includes split/merge/update/pattern-reinforcement", () => {
+  // Compile-time-only: ensure every literal is assignable to the type.
+  const split: ConsolidationOperator = "split";
+  const merge: ConsolidationOperator = "merge";
+  const update: ConsolidationOperator = "update";
+  const reinforcement: ConsolidationOperator = "pattern-reinforcement";
+  assert.equal([split, merge, update, reinforcement].length, 4);
+});
+
+test("isSemanticConsolidationLlmOperator rejects pattern-reinforcement (Cursor Bugbot, PR #730)", () => {
+  // The maintenance-only operator must NEVER be acceptable as LLM
+  // output — a hallucinated `{"operator":"pattern-reinforcement"}`
+  // would otherwise stamp misleading provenance on a
+  // semantic-consolidation memory.
+  assert.equal(isSemanticConsolidationLlmOperator("pattern-reinforcement"), false);
+  // Legacy operators stay valid.
+  assert.equal(isSemanticConsolidationLlmOperator("split"), true);
+  assert.equal(isSemanticConsolidationLlmOperator("merge"), true);
+  assert.equal(isSemanticConsolidationLlmOperator("update"), true);
+  assert.equal(isSemanticConsolidationLlmOperator("annihilate"), false);
+  assert.equal(isSemanticConsolidationLlmOperator(""), false);
+  assert.equal(isSemanticConsolidationLlmOperator(undefined), false);
+});
+
+test("SemanticConsolidationLlmOperator type literally excludes pattern-reinforcement", () => {
+  // Compile-time-only: the legacy operators are assignable, but the
+  // pattern-reinforcement literal is not.  The latter is enforced
+  // implicitly — if this stops compiling we'll catch it in CI.
+  const split: SemanticConsolidationLlmOperator = "split";
+  const merge: SemanticConsolidationLlmOperator = "merge";
+  const update: SemanticConsolidationLlmOperator = "update";
+  assert.equal([split, merge, update].length, 3);
+});

package/src/semantic-consolidation.ts

@@ -0,0 +1,432 @@
+/**
+ * semantic-consolidation.ts — Semantic Consolidation Engine
+ *
+ * Finds clusters of semantically similar memories using token overlap,
+ * synthesizes canonical versions via LLM, and archives the originals.
+ * Reduces memory store bloat while preserving all unique information.
+ */
+
+import type { MemoryFile, PluginConfig } from "./types.js";
+import { normalizeRecallTokens, countRecallTokenOverlap } from "./recall-tokenization.js";
+import { runPostConsolidationMaterialize } from "./connectors/codex-materialize-runner.js";
+import type { MaterializeResult, RolloutSummaryInput } from "./connectors/codex-materialize.js";
+import { discoverMemoryExtensions, renderExtensionsBlock, resolveExtensionsRoot } from "./memory-extension-host/index.js";
+import { log } from "./logger.js";
+
+// Re-export resolveExtensionsRoot for backward compatibility — existing
+// consumers that import from semantic-consolidation.ts continue to work.
+export { resolveExtensionsRoot } from "./memory-extension-host/index.js";
+
+// Operator vocabulary (issue #561).  The types and validators live in a
+// standalone `consolidation-operator.ts` module so `storage.ts` can import
+// them without creating a `storage → semantic-consolidation →
+// codex-materialize-runner → storage` cycle.  Re-exported here so existing
+// consumers that reach for `./semantic-consolidation.js` keep working.
+export {
+  CONSOLIDATION_OPERATORS,
+  isConsolidationOperator,
+  isSemanticConsolidationLlmOperator,
+  isValidDerivedFromEntry,
+  type ConsolidationOperator,
+  type SemanticConsolidationLlmOperator,
+} from "./consolidation-operator.js";
+
+import {
+  CONSOLIDATION_OPERATORS as _CONSOLIDATION_OPERATORS,
+  isConsolidationOperator as _isConsolidationOperator,
+  isSemanticConsolidationLlmOperator as _isSemanticConsolidationLlmOperator,
+  type ConsolidationOperator as _ConsolidationOperator,
+  type SemanticConsolidationLlmOperator as _SemanticConsolidationLlmOperator,
+} from "./consolidation-operator.js";
+
+export interface ConsolidationCluster {
+  category: string;
+  memories: MemoryFile[];
+  overlapScore: number;
+  canonicalContent?: string;
+}
+
+export interface SemanticConsolidationResult {
+  clustersFound: number;
+  memoriesConsolidated: number;
+  memoriesArchived: number;
+  errors: number;
+  clusters: ConsolidationCluster[];
+}
+
+/**
+ * Find clusters of semantically similar memories using token overlap.
+ */
+export function findSimilarClusters(
+  memories: MemoryFile[],
+  config: {
+    threshold: number;
+    minClusterSize: number;
+    excludeCategories: string[];
+    maxPerRun: number;
+  },
+): ConsolidationCluster[] {
+  const excluded = new Set(config.excludeCategories);
+
+  // Group by category first
+  const byCategory = new Map<string, MemoryFile[]>();
+  for (const m of memories) {
+    const cat = m.frontmatter.category;
+    if (excluded.has(cat)) continue;
+    if (m.frontmatter.status && m.frontmatter.status !== "active") continue;
+    const list = byCategory.get(cat) ?? [];
+    list.push(m);
+    byCategory.set(cat, list);
+  }
+
+  const clusters: ConsolidationCluster[] = [];
+  let totalCandidates = 0;
+
+  for (const [category, mems] of byCategory) {
+    if (totalCandidates >= config.maxPerRun) break;
+
+    // Token-normalize all memories in this category
+    const tokenized = mems.map((m) => ({
+      memory: m,
+      tokens: new Set(normalizeRecallTokens(m.content, [])),
+    }));
+
+    // Track which memories are already clustered
+    const clustered = new Set<string>();
+
+    for (let i = 0; i < tokenized.length && totalCandidates < config.maxPerRun; i++) {
+      if (clustered.has(tokenized[i].memory.frontmatter.id)) continue;
+
+      const cluster: MemoryFile[] = [tokenized[i].memory];
+      let totalOverlap = 0;
+      let comparisons = 0;
+
+      for (let j = i + 1; j < tokenized.length; j++) {
+        if (clustered.has(tokenized[j].memory.frontmatter.id)) continue;
+
+        const aTokens = tokenized[i].tokens;
+        const bTokens = tokenized[j].tokens;
+        if (aTokens.size === 0 || bTokens.size === 0) continue;
+
+        // Bidirectional overlap: what fraction of tokens are shared
+        const overlap = countRecallTokenOverlap(aTokens, [...bTokens].join(" "));
+        const maxTokens = Math.max(aTokens.size, bTokens.size);
+        const score = maxTokens > 0 ? overlap / maxTokens : 0;
+
+        if (score >= config.threshold) {
+          cluster.push(tokenized[j].memory);
+          totalOverlap += score;
+          comparisons++;
+          // Enforce maxPerRun within a single cluster
+          if (totalCandidates + cluster.length >= config.maxPerRun) break;
+        }
+      }
+
+      if (cluster.length >= config.minClusterSize) {
+        for (const m of cluster) clustered.add(m.frontmatter.id);
+        clusters.push({
+          category,
+          memories: cluster,
+          overlapScore: comparisons > 0 ? totalOverlap / comparisons : 0,
+        });
+        totalCandidates += cluster.length;
+      }
+    }
+  }
+
+  return clusters;
+}
+
+/**
+ * Build the LLM prompt for synthesizing a canonical memory from a cluster.
+ */
+export function buildConsolidationPrompt(cluster: ConsolidationCluster): string {
+  const memoryTexts = cluster.memories
+    .map(
+      (m, i) =>
+        `Memory ${i + 1} (${m.frontmatter.id}, created ${m.frontmatter.created}):\n${m.content}`,
+    )
+    .join("\n\n");
+
+  return `You are a memory consolidation system. The following ${cluster.memories.length} memories in the "${cluster.category}" category contain overlapping information.
+
+Synthesize them into ONE canonical memory that:
+1. Preserves ALL unique information from every source memory
+2. Removes redundancy and repetition
+3. Uses clear, concise language
+4. Maintains the same category and tone
+5. Does NOT add information that isn't in the sources
+
+${memoryTexts}
+
+Write ONLY the consolidated memory content (no metadata, no explanation, no preamble):`;
+}
+
+/**
+ * Parse the LLM response to extract the canonical content.
+ */
+export function parseConsolidationResponse(response: string): string {
+  return response.trim();
+}
+
+// ─── Operator-aware prompt / parse (issue #561 PR 3) ─────────────────────────
+
+/**
+ * Structured result from an operator-aware consolidation LLM call.
+ *
+ * - `operator` — the consolidation operator the LLM chose for this cluster.
+ *   Falls back to the heuristic default when the LLM omits or returns an
+ *   unknown value (the parser never surfaces an invalid operator; see
+ *   `parseOperatorAwareConsolidationResponse`).
+ * - `output` — the canonical content (same format the legacy prompt
+ *   returns).  Callers persist this as the body of the new memory.
+ */
+export interface OperatorAwareConsolidationResult {
+  // Restricted to the LLM-allowed subset (Cursor Bugbot, PR #730):
+  // `pattern-reinforcement` is reserved for the maintenance job and
+  // must never reach this struct from a consolidation LLM response.
+  operator: _SemanticConsolidationLlmOperator;
+  output: string;
+}
+
+/**
+ * Heuristic default operator for a cluster.  Used as the fallback when the
+ * LLM does not return a parseable operator, and as the "floor" decision in
+ * `parseOperatorAwareConsolidationResponse`.
+ *
+ * Current heuristic (kept deliberately conservative — PR 3 only):
+ *
+ *   - Two or more memories being collapsed into one canonical blob is a
+ *     MERGE by definition.  This is the path the current clustering
+ *     pipeline exercises.
+ *   - A cluster of size 1 that still reaches consolidation (future path,
+ *     e.g. supersession of a single older memory by a newer value) is an
+ *     UPDATE.
+ *   - SPLIT is never selected by the heuristic because the current write
+ *     path emits exactly one canonical memory per cluster.  The prompt
+ *     reserves SPLIT for future cluster shapes where the LLM decides one
+ *     logical source actually encodes several distinct facts — at which
+ *     point the orchestrator would need to write multiple outputs.
+ */
+export function chooseConsolidationOperator(
+  cluster: ConsolidationCluster,
+): _SemanticConsolidationLlmOperator {
+  if (cluster.memories.length <= 1) return "update";
+  return "merge";
+}
+
+/**
+ * Build the operator-aware LLM prompt.  The LLM is asked to return a
+ * JSON object `{ "operator": <split|merge|update>, "output": <content> }`.
+ *
+ * The prompt is additive: it still asks for a single canonical blob under
+ * `output`, so the upstream write path does not change.  Future expansions
+ * (SPLIT emitting multiple outputs) are explicitly documented as
+ * out-of-scope for the parser — `parseOperatorAwareConsolidationResponse`
+ * collapses any SPLIT response into a single canonical output for now.
+ */
+export function buildOperatorAwareConsolidationPrompt(
+  cluster: ConsolidationCluster,
+): string {
+  const memoryTexts = cluster.memories
+    .map(
+      (m, i) =>
+        `Memory ${i + 1} (${m.frontmatter.id}, created ${m.frontmatter.created}):\n${m.content}`,
+    )
+    .join("\n\n");
+
+  return `You are a memory consolidation system.  The following ${cluster.memories.length} memories in the "${cluster.category}" category contain overlapping information.
+
+Pick exactly ONE consolidation operator for this cluster and return a JSON object.
+
+Operator vocabulary:
+  - "merge"  — multiple distinct source memories overlap and should be collapsed into one canonical memory (most common).
+  - "update" — one source memory carries a stale value that a newer source supersedes within the same logical fact.
+  - "split"  — a single logical source really encodes multiple distinct facts that should be separated (rare; if you pick split, still emit ONE canonical body — the write path will chunk it later).
+
+Output JSON ONLY, no prose before or after.  The "operator" key MUST be set to exactly one of the three strings "merge", "update", or "split" — never a pipe-separated placeholder like "merge|update|split".  Example shape:
+  {
+    "operator": "merge",
+    "output": "<the canonical memory text>"
+  }
+
+The "output" value must:
+1. Preserve ALL unique information from every source memory
+2. Remove redundancy and repetition
+3. Use clear, concise language
+4. Match the "${cluster.category}" category and tone
+5. NOT add information that isn't in the sources
+
+${memoryTexts}
+
+Return ONLY the JSON object:`;
+}
+
+/**
+ * Parse an operator-aware consolidation response.
+ *
+ * Contract:
+ *   - Accepts strict JSON `{ "operator": "...", "output": "..." }`.
+ *   - Tolerates a JSON payload wrapped in a fenced code block (```json ...```).
+ *   - Falls back to the heuristic operator when the JSON is malformed,
+ *     the `operator` field is missing / unknown, or the raw response is
+ *     a plain blob with no JSON at all.  This keeps PR 3 backwards
+ *     compatible with older models that ignore the JSON instruction.
+ *   - Never throws.  A missing / empty `output` field falls back to the
+ *     trimmed raw response so the caller still writes something rather
+ *     than dropping the cluster.
+ */
+export function parseOperatorAwareConsolidationResponse(
+  response: string,
+  cluster: ConsolidationCluster,
+): OperatorAwareConsolidationResult {
+  const fallback: OperatorAwareConsolidationResult = {
+    operator: chooseConsolidationOperator(cluster),
+    output: response.trim(),
+  };
+
+  const trimmed = response.trim();
+  if (trimmed.length === 0) return fallback;
+
+  // Strip a fenced code block if present.
+  const fenced = /^```(?:json)?\s*([\s\S]*?)```\s*$/u.exec(trimmed);
+  const payload = fenced ? fenced[1].trim() : trimmed;
+
+  // Find a balanced brace-delimited JSON object that has an `operator`
+  // key (PR #632 round-4 review, codex P1).  A first/last-brace slice
+  // breaks when the model prepends an earlier brace block (e.g.
+  // `Example: {"note":"..."} ... {"operator":"merge",...}`).  Walk the
+  // payload tracking nesting + string/escape state and skip past
+  // objects that don't look like our target shape.
+  const parsed = findLastJsonObjectWithOperator(payload);
+  if (parsed === undefined) return fallback;
+  if (typeof parsed !== "object" || parsed === null) return fallback;
+
+  const obj = parsed as Record<string, unknown>;
+  const rawOperator = typeof obj.operator === "string" ? obj.operator.trim().toLowerCase() : "";
+  const rawOutput = typeof obj.output === "string" ? obj.output : "";
+
+  // Narrow gate (Cursor Bugbot review on PR #730 head `aa1c2a8`):
+  // accept ONLY the legacy split/merge/update LLM vocabulary here.
+  // `pattern-reinforcement` joined the broader `ConsolidationOperator`
+  // type in #687 PR 2/4 but is reserved for the maintenance job — if
+  // an LLM hallucinates that operator we must NOT promote it onto
+  // `derived_via`.
+  const operator = _isSemanticConsolidationLlmOperator(rawOperator)
+    ? rawOperator
+    : chooseConsolidationOperator(cluster);
+  const output = rawOutput.trim().length > 0 ? rawOutput.trim() : response.trim();
+
+  return { operator, output };
+}
+
+/**
+ * Walk `text`, find all balanced top-level `{ ... }` blocks whose
+ * JSON.parse result is an object with an `operator` key, and return
+ * the LAST one (function name reflects this — PR #632 review,
+ * cursor Low).  Returns `undefined` when nothing matches.  Tracks
+ * string state + escape sequences so braces inside string values
+ * don't throw off the depth counter.
+ *
+ * Used by `parseOperatorAwareConsolidationResponse` to tolerate models
+ * that prepend an explanatory JSON example block before the real
+ * payload (PR #632 round-4 + round-5 review, codex P1).  We take the
+ * LAST candidate so that an instructional example with an `operator`
+ * key ahead of the real answer doesn't steal precedence — models
+ * typically write the example first and the real answer last.
+ */
+function findLastJsonObjectWithOperator(text: string): unknown {
+  let searchFrom = 0;
+  let last: unknown = undefined;
+  while (searchFrom < text.length) {
+    const start = text.indexOf("{", searchFrom);
+    if (start < 0) return last;
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+    let closed = false;
+    let endIdx = -1;
+    for (let i = start; i < text.length; i++) {
+      const ch = text[i];
+      if (inString) {
+        if (escape) {
+          escape = false;
+        } else if (ch === "\\") {
+          escape = true;
+        } else if (ch === '"') {
+          inString = false;
+        }
+        continue;
+      }
+      if (ch === '"') {
+        inString = true;
+      } else if (ch === "{") {
+        depth += 1;
+      } else if (ch === "}") {
+        depth -= 1;
+        if (depth === 0) {
+          closed = true;
+          endIdx = i;
+          break;
+        }
+      }
+    }
+    if (!closed) return last;
+    const slice = text.slice(start, endIdx + 1);
+    try {
+      const parsed = JSON.parse(slice);
+      if (
+        typeof parsed === "object" &&
+        parsed !== null &&
+        "operator" in (parsed as Record<string, unknown>)
+      ) {
+        last = parsed;
+      }
+    } catch {
+      // Fall through to the next candidate.
+    }
+    searchFrom = endIdx + 1;
+  }
+  return last;
+}
+
+// Silence unused-import warnings when tsup tree-shakes: these are used
+// above in chooseConsolidationOperator + parse helpers.
+void _CONSOLIDATION_OPERATORS;
+
+/**
+ * Discover extensions and build the block to append to a consolidation prompt.
+ * Returns "" when extensions are disabled or none are found.
+ */
+export async function buildExtensionsBlockForConsolidation(
+  config: PluginConfig,
+): Promise<string> {
+  if (!config.memoryExtensionsEnabled) return "";
+  const root = resolveExtensionsRoot(config);
+  const extensions = await discoverMemoryExtensions(root, log);
+  if (extensions.length === 0) return "";
+  return renderExtensionsBlock(extensions);
+}
+
+/**
+ * Optional post-consolidation hook — materializes the namespace into Codex's
+ * native memory layout when the consolidation run finishes. Kept here (rather
+ * than in orchestrator.ts) so #378 doesn't conflict with Wave 1 edits.
+ *
+ * Safe to call regardless of config state: honors `codexMaterializeMemories`
+ * and `codexMaterializeOnConsolidation` and silently becomes a no-op when
+ * either is disabled.
+ */
+export async function materializeAfterSemanticConsolidation(options: {
+  config: PluginConfig;
+  namespace?: string;
+  memories?: MemoryFile[];
+  memoryDir?: string;
+  codexHome?: string;
+  rolloutSummaries?: RolloutSummaryInput[];
+  now?: Date;
+}): Promise<MaterializeResult | null> {
+  // Delegates to the shared post-consolidation helper so semantic and causal
+  // flows stay in lock-step — any guard/logging change happens in one place.
+  return runPostConsolidationMaterialize("[semantic-consolidation]", options);
+}