@remnic/core 9.6.30 → 9.6.32

package/src/lcm/engine.ts

@@ -3,7 +3,11 @@ import { openLcmDatabase, ensureLcmStateDir } from "./schema.js";
 import { LcmArchive, type LcmStructuredRecallMatch } from "./archive.js";
 import { LcmDag } from "./dag.js";
 import { LcmSummarizer, type SummarizeFn } from "./summarizer.js";
-import { assembleCompressedHistory, type LcmRecallConfig } from "./recall.js";
+import {
+  assembleCompressedHistoryWithTrace,
+  type LcmRecallConfig,
+  type LcmRecallWithTraceResult,
+} from "./recall.js";
 import { LcmWorkQueue, type LcmObserveMessage } from "./queue.js";
 import type { PluginConfig } from "../types.js";
 import { log } from "../logger.js";
@@ -336,17 +340,25 @@ export class LcmEngine {
     sessionId: string,
     budgetChars: number,
   ): Promise<string> {
-    if (!this.config.enabled) return "";
+    return (await this.assembleRecallWithTrace(sessionId, budgetChars)).text;
+  }
+
+  /** Build compressed recall plus content-free selected-summary receipts. */
+  async assembleRecallWithTrace(
+    sessionId: string,
+    budgetChars: number,
+  ): Promise<LcmRecallWithTraceResult> {
+    if (!this.config.enabled) return { text: "", selectedSummaries: [] };
     const normalizedSessionId = normalizeLcmSessionId(sessionId);
-    if (!normalizedSessionId) return "";
+    if (!normalizedSessionId) return { text: "", selectedSummaries: [] };
     await this.ensureInitialized();
 
     const effectiveBudget = Math.ceil(
       budgetChars * this.config.recallBudgetShare,
     );
-    if (effectiveBudget <= 0) return "";
+    if (effectiveBudget <= 0) return { text: "", selectedSummaries: [] };
 
-    return assembleCompressedHistory(this.dag!, this.archive!, normalizedSessionId, {
+    return assembleCompressedHistoryWithTrace(this.dag!, this.archive!, normalizedSessionId, {
       freshTailTurns: this.config.freshTailTurns,
       budgetChars: effectiveBudget,
     });

package/src/lcm/index.ts

@@ -14,6 +14,11 @@ export {
 export { LcmArchive, estimateTokens } from "./archive.js";
 export { LcmDag, type SummaryNode } from "./dag.js";
 export { LcmSummarizer, type SummarizeFn } from "./summarizer.js";
-export { assembleCompressedHistory } from "./recall.js";
+export {
+  assembleCompressedHistory,
+  assembleCompressedHistoryWithTrace,
+  type LcmRecallWithTraceResult,
+  type LcmSummarySelectionReceipt,
+} from "./recall.js";
 export { registerLcmTools } from "./tools.js";
 export { openLcmDatabase, ensureLcmStateDir } from "./schema.js";

package/src/lcm/recall.ts

@@ -7,6 +7,21 @@ export interface LcmRecallConfig {
   budgetChars: number;
 }
 
+export interface LcmSummarySelectionReceipt {
+  id: string;
+  depth: number;
+  msgStart: number;
+  msgEnd: number;
+  /** Half-open offsets within the returned compressed-history string. */
+  entryStart: number;
+  entryEnd: number;
+}
+
+export interface LcmRecallWithTraceResult {
+  text: string;
+  selectedSummaries: LcmSummarySelectionReceipt[];
+}
+
 /**
  * Assemble a compressed session history section from the summary DAG.
  *
@@ -21,14 +36,24 @@ export function assembleCompressedHistory(
   sessionId: string,
   config: LcmRecallConfig,
 ): string {
+  return assembleCompressedHistoryWithTrace(dag, archive, sessionId, config).text;
+}
+
+export function assembleCompressedHistoryWithTrace(
+  dag: LcmDag,
+  archive: LcmArchive,
+  sessionId: string,
+  config: LcmRecallConfig,
+): LcmRecallWithTraceResult {
   const maxTurn = archive.getMaxTurnIndex(sessionId);
-  if (maxTurn < 0) return "";
+  if (maxTurn < 0) return { text: "", selectedSummaries: [] };
 
   const allNodes = dag.getAllNodes(sessionId);
-  if (allNodes.length === 0) return "";
+  if (allNodes.length === 0) return { text: "", selectedSummaries: [] };
 
   const freshTailStart = Math.max(0, maxTurn - config.freshTailTurns + 1);
   const sections: string[] = [];
+  const selectedNodes: SummaryNode[] = [];
   let usedChars = 0;
 
   // Collect nodes covering the "old" portion (before fresh tail)
@@ -39,6 +64,7 @@ export function assembleCompressedHistory(
       const entry = `**${label}** (depth ${node.depth}):\n${node.summary_text}`;
       if (usedChars + entry.length > config.budgetChars) break;
       sections.push(entry);
+      selectedNodes.push(node);
       usedChars += entry.length;
     }
   }
@@ -58,12 +84,29 @@ export function assembleCompressedHistory(
     const entry = `**${label}**:\n${node.summary_text}`;
     if (usedChars + entry.length > config.budgetChars) break;
     sections.push(entry);
+    selectedNodes.push(node);
     usedChars += entry.length;
   }
 
-  if (sections.length === 0) return "";
-
-  return `## Session History (Compressed)\n\n${sections.join("\n\n")}`;
+  if (sections.length === 0) return { text: "", selectedSummaries: [] };
+
+  const prefix = "## Session History (Compressed)\n\n";
+  const text = `${prefix}${sections.join("\n\n")}`;
+  let entryStart = prefix.length;
+  const selectedSummaries = selectedNodes.map((node, index) => {
+    const entryEnd = entryStart + sections[index]!.length;
+    const receipt: LcmSummarySelectionReceipt = {
+      id: node.id,
+      depth: node.depth,
+      msgStart: node.msg_start,
+      msgEnd: node.msg_end,
+      entryStart,
+      entryEnd,
+    };
+    entryStart = entryEnd + 2;
+    return receipt;
+  });
+  return { text, selectedSummaries };
 }
 
 /**

package/src/lcm-engine.test.ts

@@ -676,6 +676,52 @@ test("expandContext preserves first, middle, and last row identity when truncate
   }
 });
 
+test("assembleRecallWithTrace preserves text and reports selected summary metadata", async () => {
+  const memoryDir = await mkdtemp(
+    path.join(os.tmpdir(), "engram-lcm-summary-receipts-"),
+  );
+
+  try {
+    const engine = new LcmEngine(
+      createPluginConfig(memoryDir),
+      async () => "private summary body",
+    );
+    await engine.observeMessages(
+      "private-session",
+      Array.from({ length: 8 }, (_, index) => ({
+        role: index % 2 === 0 ? "user" : "assistant",
+        content: `private message ${index}`,
+      })),
+    );
+    await engine.waitForSessionObserveIdle("private-session");
+
+    const baseline = await engine.assembleRecall("private-session", 10_000);
+    const traced = await engine.assembleRecallWithTrace(
+      "private-session",
+      10_000,
+    );
+
+    assert.equal(traced.text, baseline);
+    assert.ok(traced.selectedSummaries.length > 0);
+    assert.ok(
+      traced.selectedSummaries.every((receipt) =>
+        receipt.entryStart < receipt.entryEnd &&
+        receipt.msgStart <= receipt.msgEnd
+      ),
+    );
+    assert.equal(
+      JSON.stringify(traced.selectedSummaries).includes("private summary body"),
+      false,
+    );
+    assert.equal(
+      JSON.stringify(traced.selectedSummaries).includes("private-session"),
+      false,
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 test("close prevents deferred observe work from reinitializing the engine", async () => {
   const memoryDir = await mkdtemp(
     path.join(os.tmpdir(), "engram-lcm-engine-close-"),

package/src/lcm-recall.test.ts

@@ -6,7 +6,10 @@ import test from "node:test";
 
 import { LcmArchive } from "./lcm/archive.js";
 import { LcmDag } from "./lcm/dag.js";
-import { assembleCompressedHistory } from "./lcm/recall.js";
+import {
+  assembleCompressedHistory,
+  assembleCompressedHistoryWithTrace,
+} from "./lcm/recall.js";
 import { ensureLcmStateDir, openLcmDatabase } from "./lcm/schema.js";
 
 test("assembleCompressedHistory excludes old summaries that straddle the fresh tail", async () => {
@@ -58,12 +61,56 @@ test("assembleCompressedHistory excludes old summaries that straddle the fresh t
       freshTailTurns: 16,
       budgetChars: 10_000,
     });
+    const traced = assembleCompressedHistoryWithTrace(
+      dag,
+      archive,
+      sessionId,
+      { freshTailTurns: 16, budgetChars: 10_000 },
+    );
 
+    assert.equal(traced.text, section);
     assert.doesNotMatch(section, /parent summary 0-31/);
     assert.match(section, /leaf summary 0-7/);
     assert.match(section, /leaf summary 8-15/);
     assert.match(section, /leaf summary 16-23/);
     assert.match(section, /leaf summary 24-31/);
+    assert.deepEqual(
+      traced.selectedSummaries.map(({ id, depth, msgStart, msgEnd }) => ({
+        id,
+        depth,
+        msgStart,
+        msgEnd,
+      })),
+      [
+        { id: "leaf-0-7", depth: 0, msgStart: 0, msgEnd: 7 },
+        { id: "leaf-8-15", depth: 0, msgStart: 8, msgEnd: 15 },
+        { id: "leaf-16-23", depth: 0, msgStart: 16, msgEnd: 23 },
+        { id: "leaf-24-31", depth: 0, msgStart: 24, msgEnd: 31 },
+      ],
+    );
+    for (const receipt of traced.selectedSummaries) {
+      assert.ok(receipt.entryStart < receipt.entryEnd);
+      const expectedSummary = receipt.id.replace("leaf-", "leaf summary ");
+      assert.equal(
+        traced.text.slice(receipt.entryStart, receipt.entryEnd).includes(expectedSummary),
+        true,
+      );
+      assert.equal("summary" in receipt, false);
+      assert.equal("sessionId" in receipt, false);
+    }
+
+    const first = traced.selectedSummaries[0]!;
+    const firstEntryBudget = first.entryEnd - first.entryStart;
+    const limited = assembleCompressedHistoryWithTrace(
+      dag,
+      archive,
+      sessionId,
+      { freshTailTurns: 16, budgetChars: firstEntryBudget },
+    );
+    assert.deepEqual(
+      limited.selectedSummaries.map((receipt) => receipt.id),
+      ["leaf-0-7"],
+    );
   } finally {
     db.close();
     await rm(memoryDir, { recursive: true, force: true });

package/src/recall-trace-receipts.test.ts

@@ -0,0 +1,380 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  buildExplicitCueRecallSection,
+  buildTrajectoryAnalysisRecallSection,
+  type ExplicitCueRecallEngine,
+  type TrajectoryAnalysisLineReceipt,
+} from "./explicit-cue-recall.js";
+import type { EvidencePackSelectionReceipt } from "./evidence-pack.js";
+
+interface ReceiptRow {
+  id?: number;
+  session_id: string;
+  turn_index: number;
+  role: string;
+  content: string;
+}
+
+class ReceiptEngine implements ExplicitCueRecallEngine {
+  constructor(private readonly rows: ReceiptRow[]) {}
+
+  async expandContext(
+    sessionId: string,
+    fromTurn: number,
+    toTurn: number,
+  ): Promise<ReceiptRow[]> {
+    return this.rows.filter((row) =>
+      row.session_id === sessionId &&
+      row.turn_index >= fromTurn &&
+      row.turn_index <= toTurn
+    );
+  }
+
+  async searchContextFull(): Promise<Array<ReceiptRow & { score: number }>> {
+    return [];
+  }
+
+  async getStats(): Promise<{ totalMessages: number; maxTurnIndex?: number }> {
+    return {
+      totalMessages: this.rows.length,
+      maxTurnIndex: this.rows.length > 0
+        ? Math.max(...this.rows.map((row) => row.turn_index))
+        : undefined,
+    };
+  }
+}
+
+test("explicit-cue receipts identify only blocks that survive selection", async () => {
+  const engine = new ReceiptEngine([
+    {
+      id: 101,
+      session_id: "s",
+      turn_index: 7,
+      role: "user",
+      content: "first selected row",
+    },
+    {
+      id: 102,
+      session_id: "s",
+      turn_index: 7,
+      role: "assistant",
+      content: "second row that the deliberately narrow section budget rejects",
+    },
+  ]);
+  const receipts: EvidencePackSelectionReceipt[] = [];
+  const options = {
+    engine,
+    sessionId: "s",
+    query: "Review turn 7",
+    maxChars:
+      "## Explicit Cue Evidence".length +
+      2 +
+      "[s, turn 7, user]: first selected row".length,
+    maxItemChars: 200,
+  };
+
+  const baseline = await buildExplicitCueRecallSection(options);
+  const traced = await buildExplicitCueRecallSection({
+    ...options,
+    onEvidenceSelected: (receipt) => receipts.push(receipt),
+  });
+
+  assert.equal(traced, baseline);
+  assert.deepEqual(receipts.map((receipt) => receipt.item.archiveRowId), [101]);
+  assert.equal(
+    traced.slice(receipts[0]!.blockStart, receipts[0]!.blockEnd).includes(
+      "first selected row",
+    ),
+    true,
+  );
+  assert.equal(JSON.stringify(receipts).includes("first selected row"), false);
+  assert.equal(JSON.stringify(receipts).includes("second row"), false);
+});
+
+test("trajectory receipts retain row lineage only for emitted clipped lines", async () => {
+  const engine = new ReceiptEngine([
+    {
+      id: 201,
+      session_id: "ama",
+      turn_index: 1,
+      role: "user",
+      content: "[Action 1]: move right",
+    },
+    {
+      id: 202,
+      session_id: "ama",
+      turn_index: 1,
+      role: "assistant",
+      content: "[Observation 1]: reached the wall",
+    },
+    {
+      id: 203,
+      session_id: "ama",
+      turn_index: 2,
+      role: "user",
+      content: "[Action 2]: move left",
+    },
+    {
+      id: 204,
+      session_id: "ama",
+      turn_index: 2,
+      role: "assistant",
+      content: "[Observation 2]: reached the door",
+    },
+  ]);
+  const receipts: TrajectoryAnalysisLineReceipt[] = [];
+  const options = {
+    engine,
+    sessionId: "ama",
+    query:
+      "What will be the resulting state at action 1? Provide the full observation.",
+    maxChars: 4_000,
+  };
+
+  const baseline = await buildTrajectoryAnalysisRecallSection(options);
+  const traced = await buildTrajectoryAnalysisRecallSection({
+    ...options,
+    onLineSelected: (receipt) => receipts.push(receipt),
+  });
+
+  assert.equal(traced, baseline);
+  const actionOne = receipts.find((receipt) =>
+    traced.slice(receipt.lineStart, receipt.lineEnd).startsWith("[Action 1]")
+  );
+  const observationOne = receipts.find((receipt) =>
+    traced.slice(receipt.lineStart, receipt.lineEnd).startsWith("[Observation 1]")
+  );
+  assert.ok(actionOne);
+  assert.ok(observationOne);
+  assert.equal(actionOne.lineageStatus, "exact");
+  assert.equal(observationOne.lineageStatus, "exact");
+  assert.deepEqual(actionOne.actionArchiveRowIds, [201]);
+  assert.deepEqual(actionOne.observationArchiveRowIds, []);
+  assert.deepEqual(observationOne.actionArchiveRowIds, []);
+  assert.deepEqual(observationOne.observationArchiveRowIds, [202]);
+  assert.equal(actionOne.actionArchiveRowIds.includes(203), false);
+  assert.equal(observationOne.observationArchiveRowIds.includes(204), false);
+
+  const clippedReceipts: TrajectoryAnalysisLineReceipt[] = [];
+  const clipped = await buildTrajectoryAnalysisRecallSection({
+    ...options,
+    maxChars: actionOne.lineStart + 8,
+    onLineSelected: (receipt) => clippedReceipts.push(receipt),
+  });
+  const partialAction = clippedReceipts.at(-1)!;
+  assert.equal(partialAction.lineStart, actionOne.lineStart);
+  assert.equal(partialAction.lineEnd, clipped.length);
+  assert.equal(partialAction.lineageStatus, "exact");
+  assert.deepEqual(partialAction.actionArchiveRowIds, [201]);
+  assert.equal(partialAction.actionArchiveRowIds.includes(203), false);
+  assert.equal(JSON.stringify(receipts).includes("move right"), false);
+  assert.equal(JSON.stringify(receipts).includes("reached the wall"), false);
+});
+
+test("spatial derived-line receipts mark lineage unavailable without guessing siblings", async () => {
+  const engine = new ReceiptEngine([
+    {
+      id: 301,
+      session_id: "ama",
+      turn_index: 20,
+      role: "user",
+      content: "[Action 20]: right",
+    },
+    {
+      id: 302,
+      session_id: "ama",
+      turn_index: 20,
+      role: "assistant",
+      content: [
+        "[Observation 20]: Active rules:",
+        "baba is you",
+        "",
+        "Objects on the map:",
+        "rule `win` 3 step to the left",
+      ].join("\n"),
+    },
+    {
+      id: 303,
+      session_id: "ama",
+      turn_index: 21,
+      role: "user",
+      content: "[Action 21]: left",
+    },
+    {
+      id: 304,
+      session_id: "ama",
+      turn_index: 21,
+      role: "assistant",
+      content: [
+        "[Observation 21]: Active rules:",
+        "baba is you",
+        "",
+        "Objects on the map:",
+        "rule `win` 2 step to the left",
+      ].join("\n"),
+    },
+  ]);
+  const receipts: TrajectoryAnalysisLineReceipt[] = [];
+  const options = {
+    engine,
+    sessionId: "ama",
+    query:
+      "Between steps 20 and 21, the rule 'win' block's relative position changed from 3 step to the left to 2 step to the left. What was the actual movement?",
+    maxChars: 4_000,
+  };
+
+  const baseline = await buildTrajectoryAnalysisRecallSection(options);
+  const traced = await buildTrajectoryAnalysisRecallSection({
+    ...options,
+    onLineSelected: (receipt) => receipts.push(receipt),
+  });
+
+  assert.equal(traced, baseline);
+  const heading = receipts.find((receipt) =>
+    traced.slice(receipt.lineStart, receipt.lineEnd) === "Relative-position movement cues:"
+  );
+  const movement = receipts.find((receipt) =>
+    traced.slice(receipt.lineStart, receipt.lineEnd).startsWith("Observation 20->21:")
+  );
+  assert.ok(heading);
+  assert.ok(movement);
+  assert.equal(heading.lineageStatus, "exact");
+  assert.deepEqual(heading.actionArchiveRowIds, []);
+  assert.deepEqual(heading.observationArchiveRowIds, []);
+  assert.equal(movement.lineageStatus, "unavailable");
+  assert.deepEqual(movement.actionArchiveRowIds, []);
+  assert.deepEqual(movement.observationArchiveRowIds, []);
+  assert.equal(JSON.stringify(movement).includes("rule win"), false);
+});
+
+test("trajectory receipts mark id-less and partially id-less direct sources unavailable", async () => {
+  const engine = new ReceiptEngine([
+    {
+      session_id: "ama",
+      turn_index: 1,
+      role: "user",
+      content: "[Action 1]: move right",
+    },
+    {
+      id: 402,
+      session_id: "ama",
+      turn_index: 1,
+      role: "assistant",
+      content: "[Observation 1]: reached the wall",
+    },
+    {
+      session_id: "ama",
+      turn_index: 2,
+      role: "user",
+      content: "[Action 2]: move left",
+    },
+    {
+      session_id: "ama",
+      turn_index: 2,
+      role: "assistant",
+      content: "[Observation 2]: reached the door",
+    },
+  ]);
+  const receipts: TrajectoryAnalysisLineReceipt[] = [];
+  const text = await buildTrajectoryAnalysisRecallSection({
+    engine,
+    sessionId: "ama",
+    query: "Compare steps 1 and 2 and provide the full observations.",
+    maxChars: 4_000,
+    onLineSelected: (receipt) => receipts.push(receipt),
+  });
+  const findLine = (prefix: string) => receipts.find((receipt) =>
+    text.slice(receipt.lineStart, receipt.lineEnd).startsWith(prefix)
+  );
+
+  const idLessAction = findLine("[Action 1]");
+  const identifiedObservation = findLine("[Observation 1]");
+  const idLessObservation = findLine("[Observation 2]");
+  assert.ok(idLessAction);
+  assert.ok(identifiedObservation);
+  assert.ok(idLessObservation);
+  assert.equal(idLessAction.lineageStatus, "unavailable");
+  assert.deepEqual(idLessAction.actionArchiveRowIds, []);
+  assert.equal(identifiedObservation.lineageStatus, "exact");
+  assert.deepEqual(identifiedObservation.observationArchiveRowIds, [402]);
+  assert.equal(idLessObservation.lineageStatus, "unavailable");
+  assert.deepEqual(idLessObservation.observationArchiveRowIds, []);
+});
+
+test("inferred-location receipts identify only the action that establishes the final location", async () => {
+  const engine = new ReceiptEngine([
+    {
+      id: 501,
+      session_id: "ama",
+      turn_index: 1,
+      role: "user",
+      content: "[Action 1]: open cabinet 1",
+    },
+    {
+      id: 502,
+      session_id: "ama",
+      turn_index: 2,
+      role: "user",
+      content: "[Action 2]: move cabinet 1 to kitchen",
+    },
+  ]);
+  const receipts: TrajectoryAnalysisLineReceipt[] = [];
+  const text = await buildTrajectoryAnalysisRecallSection({
+    engine,
+    sessionId: "ama",
+    query:
+      "What is the state of cabinet 1 at step 2 and what was its prior whole changes history?",
+    maxChars: 4_000,
+    onLineSelected: (receipt) => receipts.push(receipt),
+  });
+  const location = receipts.find((receipt) =>
+    text.slice(receipt.lineStart, receipt.lineEnd).startsWith(
+      "Inferred cabinet 1 location at step 2: kitchen",
+    )
+  );
+
+  assert.ok(location);
+  assert.equal(location.lineageStatus, "exact");
+  assert.deepEqual(location.actionArchiveRowIds, [502]);
+  assert.equal(location.actionArchiveRowIds.includes(501), false);
+  assert.deepEqual(location.observationArchiveRowIds, []);
+});
+
+test("stateful inventory-change lines do not publish partial current-row lineage", async () => {
+  const engine = new ReceiptEngine([
+    {
+      id: 601,
+      session_id: "ama",
+      turn_index: 1,
+      role: "user",
+      content: "[Action 1]: take apple 1 from shelf 1",
+    },
+    {
+      id: 602,
+      session_id: "ama",
+      turn_index: 2,
+      role: "user",
+      content: "[Action 2]: place apple 1 into bowl 1",
+    },
+  ]);
+  const receipts: TrajectoryAnalysisLineReceipt[] = [];
+  const text = await buildTrajectoryAnalysisRecallSection({
+    engine,
+    sessionId: "ama",
+    query: "What changes occurred to the inventory throughout the trajectory?",
+    maxChars: 4_000,
+    onLineSelected: (receipt) => receipts.push(receipt),
+  });
+  const placed = receipts.find((receipt) =>
+    text.slice(receipt.lineStart, receipt.lineEnd).startsWith(
+      "[Action 2]: place apple 1 into bowl 1 => inventory removed apple 1",
+    )
+  );
+
+  assert.ok(placed);
+  assert.equal(placed.lineageStatus, "unavailable");
+  assert.deepEqual(placed.actionArchiveRowIds, []);
+  assert.deepEqual(placed.observationArchiveRowIds, []);
+});