@remnic/core 9.6.28 → 9.6.30

package/src/focused-list-recall.ts

@@ -8,6 +8,7 @@ import {
   gatherAcrossReadSessions,
   resolveLcmReadSessionIds,
 } from "./lcm-fallback-read.js";
+import { isSameLcmRow, lcmEvidenceIdentity } from "./lcm/evidence-identity.js";
 
 export interface FocusedListRecallOptions {
   engine: ExplicitCueRecallEngine | null | undefined;
@@ -134,27 +135,35 @@ async function collectFocusedListItems(
       result.turn_index + searchWindowAfter,
       searchWindowTokens,
     );
+    const searchIdentity = lcmEvidenceIdentity(result, result.session_id);
     const searchHit: EvidencePackItem = {
-      id: `${result.session_id}:${result.turn_index}`,
+      ...searchIdentity,
       sessionId: result.session_id,
       turnIndex: result.turn_index,
       role: result.role,
       content: result.content,
       ...(typeof result.score === "number" ? { score: result.score } : {}),
     };
-    const candidates: EvidencePackItem[] = expanded.map((message) => ({
-      id: `${result.session_id}:${message.turn_index}`,
-      sessionId: result.session_id,
-      turnIndex: message.turn_index,
-      role: message.role,
-      content: message.content,
-      ...(message.turn_index === result.turn_index &&
-      typeof result.score === "number"
-        ? { score: result.score }
-        : {}),
-    }));
-    const hitIndex = candidates.findIndex((candidate) =>
-      candidate.turnIndex === result.turn_index
+    const candidates: EvidencePackItem[] = expanded.map((message) => {
+      const matchesSearchHit = isSameLcmRow(
+        message,
+        result.session_id,
+        result,
+        result.session_id,
+      );
+      return {
+        ...lcmEvidenceIdentity(message, result.session_id),
+        sessionId: message.session_id ?? result.session_id,
+        turnIndex: message.turn_index,
+        role: message.role,
+        content: message.content,
+        ...(matchesSearchHit && typeof result.score === "number"
+          ? { score: result.score }
+          : {}),
+      };
+    });
+    const hitIndex = expanded.findIndex((message) =>
+      isSameLcmRow(message, result.session_id, result, result.session_id)
     );
     if (hitIndex >= 0) {
       candidates[hitIndex] = searchHit;
@@ -221,8 +230,8 @@ async function collectFocusedListScanItems(
   );
   for (const message of messages) {
     const candidate = {
-      id: `${options.sessionId}:${message.turn_index}`,
-      sessionId: options.sessionId,
+      ...lcmEvidenceIdentity(message, options.sessionId),
+      sessionId: message.session_id ?? options.sessionId,
       turnIndex: message.turn_index,
       role: message.role,
       content: message.content,

package/src/lcm/evidence-identity.ts

@@ -0,0 +1,59 @@
+export interface LcmRowIdentity {
+  id?: number;
+  session_id?: string;
+  turn_index: number;
+}
+
+export interface LcmEvidenceIdentity {
+  id: string;
+  archiveRowId?: number;
+}
+
+export function lcmArchiveRowId(row: Pick<LcmRowIdentity, "id">): number | undefined {
+  const value = row.id;
+  return typeof value === "number" && Number.isSafeInteger(value) && value > 0
+    ? value
+    : undefined;
+}
+
+function resolveSessionId(row: LcmRowIdentity, fallbackSessionId: string): string {
+  return row.session_id ?? fallbackSessionId;
+}
+
+/**
+ * Build the non-rendered evidence identity for an LCM archive row.
+ *
+ * Older adapters and test doubles may omit the SQLite row id. Those callers
+ * retain the historical session+turn identity until they adopt row lineage.
+ */
+export function lcmEvidenceIdentity(
+  row: LcmRowIdentity,
+  fallbackSessionId: string,
+): LcmEvidenceIdentity {
+  const sessionId = resolveSessionId(row, fallbackSessionId);
+  const archiveRowId = lcmArchiveRowId(row);
+  if (archiveRowId === undefined) {
+    return { id: `${sessionId}:${row.turn_index}` };
+  }
+  return {
+    id: JSON.stringify(["lcm-row", sessionId, archiveRowId]),
+    archiveRowId,
+  };
+}
+
+/** Match a search hit to its expanded archive row without conflating siblings. */
+export function isSameLcmRow(
+  left: LcmRowIdentity,
+  leftFallbackSessionId: string,
+  right: LcmRowIdentity,
+  rightFallbackSessionId: string,
+): boolean {
+  const leftRowId = lcmArchiveRowId(left);
+  const rightRowId = lcmArchiveRowId(right);
+  const leftSessionId = resolveSessionId(left, leftFallbackSessionId);
+  const rightSessionId = resolveSessionId(right, rightFallbackSessionId);
+  if (leftRowId !== undefined && rightRowId !== undefined) {
+    return leftRowId === rightRowId && leftSessionId === rightSessionId;
+  }
+  return leftSessionId === rightSessionId && left.turn_index === right.turn_index;
+}

package/src/lcm/index.ts

@@ -4,6 +4,13 @@ export {
   type LcmEngineConfig,
   type LcmExpandedMessage,
 } from "./engine.js";
+export {
+  isSameLcmRow,
+  lcmArchiveRowId,
+  lcmEvidenceIdentity,
+  type LcmEvidenceIdentity,
+  type LcmRowIdentity,
+} from "./evidence-identity.js";
 export { LcmArchive, estimateTokens } from "./archive.js";
 export { LcmDag, type SummaryNode } from "./dag.js";
 export { LcmSummarizer, type SummarizeFn } from "./summarizer.js";

package/src/lcm-retrieval-lineage.test.ts

@@ -0,0 +1,236 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { AccessLcmSurface, type AccessLcmSurfaceDeps } from "./access-lcm-surface.js";
+import {
+  AccessRecallSurface,
+  type AccessRecallSurfaceDeps,
+} from "./access-recall-surface.js";
+import { buildEventOrderRecallSection } from "./event-order-recall.js";
+import { buildExplicitCueRecallSection, type ExplicitCueRecallEngine } from "./explicit-cue-recall.js";
+import { buildFocusedListRecallSection } from "./focused-list-recall.js";
+import {
+  isSameLcmRow,
+  lcmEvidenceIdentity,
+} from "./lcm/evidence-identity.js";
+import { buildResponseGuidanceRecallSection } from "./response-guidance-recall.js";
+import { buildTargetedFactRecallSection } from "./targeted-fact-recall.js";
+
+interface TestRow {
+  id?: number;
+  session_id: string;
+  turn_index: number;
+  role: string;
+  content: string;
+}
+
+class LineageEngine implements ExplicitCueRecallEngine {
+  constructor(
+    private readonly rows: TestRow[],
+    private readonly searchRowId?: number,
+    private readonly searchContent?: string,
+  ) {}
+
+  async expandContext(
+    sessionId: string,
+    fromTurn: number,
+    toTurn: number,
+    _maxTokens: number,
+  ): Promise<TestRow[]> {
+    return this.rows.filter((row) =>
+      row.session_id === sessionId &&
+      row.turn_index >= fromTurn &&
+      row.turn_index <= toTurn
+    );
+  }
+
+  async searchContextFull(): Promise<
+    Array<TestRow & { score: number }>
+  > {
+    const row = this.rows.find((candidate) => candidate.id === this.searchRowId);
+    return row
+      ? [{
+          ...row,
+          content: this.searchContent ?? row.content,
+          score: 99,
+        }]
+      : [];
+  }
+
+  async getStats(sessionId?: string): Promise<{
+    totalMessages: number;
+    maxTurnIndex?: number;
+  }> {
+    const rows = sessionId
+      ? this.rows.filter((row) => row.session_id === sessionId)
+      : this.rows;
+    return {
+      totalMessages: rows.length,
+      maxTurnIndex: rows.length > 0
+        ? Math.max(...rows.map((row) => row.turn_index))
+        : undefined,
+    };
+  }
+}
+
+test("LCM evidence identity distinguishes sibling rows and preserves the legacy fallback", () => {
+  const first = { id: 41, session_id: "session", turn_index: 7 };
+  const second = { id: 42, session_id: "session", turn_index: 7 };
+  assert.notEqual(
+    lcmEvidenceIdentity(first, "session").id,
+    lcmEvidenceIdentity(second, "session").id,
+  );
+  assert.equal(isSameLcmRow(first, "session", second, "session"), false);
+  assert.equal(
+    lcmEvidenceIdentity({ turn_index: 7 }, "session").id,
+    "session:7",
+  );
+  assert.equal(
+    isSameLcmRow(
+      { turn_index: 7 },
+      "session",
+      { turn_index: 7 },
+      "session",
+    ),
+    true,
+  );
+});
+
+test("explicit-cue recall keeps distinct archive rows that share a turn", async () => {
+  const engine = new LineageEngine([
+    { id: 101, session_id: "s", turn_index: 7, role: "user", content: "First row at the referenced turn." },
+    { id: 102, session_id: "s", turn_index: 7, role: "assistant", content: "Second row at the referenced turn." },
+  ]);
+  const recalled = await buildExplicitCueRecallSection({
+    engine,
+    sessionId: "s",
+    query: "Review turn 7",
+    maxChars: 4_000,
+  });
+  assert.match(recalled, /First row at the referenced turn/);
+  assert.match(recalled, /Second row at the referenced turn/);
+});
+
+test("targeted recall replaces and scores only the matching same-turn row", async () => {
+  const engine = new LineageEngine([
+    { id: 101, session_id: "s", turn_index: 7, role: "user", content: "I earn approximately $100,000 CAD annually as a senior engineer." },
+    { id: 102, session_id: "s", turn_index: 7, role: "user", content: "My recent raise to $110,000 CAD as a senior engineer at Saint Pierre Manufacturing Ltd is now effective." },
+  ], 102, "The matching row records my recent raise to $110,000 CAD as a senior engineer at Saint Pierre Manufacturing Ltd.");
+  const recalled = await buildTargetedFactRecallSection({
+    engine,
+    sessionId: "s",
+    query: "What is my annual salary as a senior engineer at Saint Pierre Manufacturing Ltd?",
+    maxChars: 6_000,
+  });
+  assert.match(recalled, /earn approximately \$100,000 CAD annually/);
+  assert.match(recalled, /matching row records my recent raise to \$110,000 CAD/);
+});
+
+test("focused-list recall keeps distinct same-turn calculations while matching the search row", async () => {
+  const engine = new LineageEngine([
+    { id: 101, session_id: "s", turn_index: 7, role: "user", content: "Can you help me calculate P(both heads) = 1/2 x 1/2 = 1/4 so I can make sure I get it right?" },
+    { id: 102, session_id: "s", turn_index: 7, role: "user", content: "I am trying to verify if P(rolling a number greater than 4) = 2/6 = 1/3 is correct." },
+  ], 102, "I confirmed P(rolling a number greater than 4) = 2/6 = 1/3.");
+  const recalled = await buildFocusedListRecallSection({
+    engine,
+    sessionId: "s",
+    query: "In my questions about tossing coins and rolling dice, how many different probability calculations did I try to confirm?",
+    maxChars: 6_000,
+  });
+  assert.match(recalled, /P\(both heads\)/);
+  assert.match(recalled, /P\(rolling a number greater than 4\)/);
+});
+
+test("response guidance keeps distinct same-turn rows while preserving its existing filters", async () => {
+  const engine = new LineageEngine([
+    { id: 101, session_id: "s", turn_index: 7, role: "user", content: "I prefer a structured daily routine with a written priority list each morning." },
+    { id: 102, session_id: "s", turn_index: 7, role: "user", content: "My structured daily routine schedules focused work blocks and buffer time between meetings." },
+  ], 102, "My structured daily routine schedules focused work blocks and buffer time between meetings.");
+  const recalled = await buildResponseGuidanceRecallSection({
+    engine,
+    sessionId: "s",
+    query: "How should I organize my day to stay on track?",
+    maxChars: 6_000,
+  });
+  assert.match(recalled, /written priority list each morning/);
+  assert.match(recalled, /focused work blocks/);
+});
+
+test("event-order recall keeps distinct same-turn chronological rows", async () => {
+  const engine = new LineageEngine([
+    { id: 101, session_id: "s", turn_index: 7, role: "user", content: "Patrick suggested a March workshop on workflow optimization." },
+    { id: 102, session_id: "s", turn_index: 7, role: "user", content: "Patrick and I planned an interview tips meeting for tomorrow." },
+  ]);
+  const recalled = await buildEventOrderRecallSection({
+    engine,
+    sessionId: "s",
+    query: "Walk me through my interactions with Patrick in order.",
+    maxChars: 6_000,
+  });
+  assert.match(recalled, /workshop suggestion/);
+  assert.match(recalled, /interview tips meeting/);
+});
+
+function accessSearchDeps(rows: TestRow[]): AccessLcmSurfaceDeps {
+  const orchestrator = {
+    config: {
+      defaultNamespace: "default",
+      namespaces: false,
+    },
+    lcmEngine: {
+      enabled: true,
+      searchContextFull: async () => rows,
+    },
+  };
+  return {
+    orchestrator,
+    lcmSessionIdsForNamespaces: () => ["primary", "fallback"],
+    resolveImplicitLcmReadFallbackNamespace: () => "default",
+    resolveLcmReadNamespace: () => "default",
+    resolveLcmReadSessionIds: () => ["primary", "fallback"],
+    resolveLcmReadSessionKey: () => "primary",
+    resolveReadableNamespace: () => "default",
+    resolveRequestPrincipal: () => "default",
+    resolveScopeProfileLcmReadNamespaces: () => null,
+  } as unknown as AccessLcmSurfaceDeps;
+}
+
+test("access LCM search dedupes repeated rows but retains same-turn siblings", async () => {
+  const rows: TestRow[] = [
+    { id: 101, session_id: "s", turn_index: 7, role: "user", content: "first sibling" },
+    { id: 102, session_id: "s", turn_index: 7, role: "assistant", content: "second sibling" },
+  ];
+  const response = await new AccessLcmSurface(accessSearchDeps(rows)).lcmSearch({
+    query: "sibling",
+    sessionKey: "s",
+  });
+  assert.deepEqual(response.results.map((row) => row.content), [
+    "first sibling",
+    "second sibling",
+  ]);
+});
+
+test("raw excerpts dedupe repeated rows but retain same-turn siblings", async () => {
+  const rows: TestRow[] = [
+    { id: 101, session_id: "s", turn_index: 7, role: "user", content: "first sibling" },
+    { id: 102, session_id: "s", turn_index: 7, role: "assistant", content: "second sibling" },
+  ];
+  const deps = {
+    orchestrator: {
+      config: { defaultNamespace: "default" },
+      lcmEngine: {
+        enabled: true,
+        searchContextFull: async () => rows,
+      },
+    },
+  } as unknown as AccessRecallSurfaceDeps;
+  const excerpts = await new AccessRecallSurface(deps).fetchRawExcerpts("raw", {
+    query: "sibling",
+    sessionKey: "s",
+    lcmSessionIds: ["primary", "fallback"],
+  });
+  assert.deepEqual(excerpts?.map((row) => row.content), [
+    "first sibling",
+    "second sibling",
+  ]);
+});

package/src/orchestration/xray-capture-queue.test.ts

@@ -0,0 +1,42 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { XrayCaptureQueue } from "./xray-capture-queue.js";
+
+test("initial snapshot read failure releases the next queued capture", async () => {
+  const queue = new XrayCaptureQueue();
+  let failNextRead = true;
+  let snapshot: string | null = "prior";
+  const operations: string[] = [];
+  const state = {
+    read(): string | null {
+      if (failNextRead) {
+        failNextRead = false;
+        throw new Error("snapshot clone failed");
+      }
+      return snapshot;
+    },
+    clear(): void {
+      snapshot = null;
+    },
+    restore(value: string | null): void {
+      snapshot = value;
+    },
+  };
+
+  const failed = queue.run(async () => {
+    operations.push("failed");
+    return "unexpected";
+  }, state);
+  const next = queue.run(async () => {
+    operations.push("next");
+    snapshot = "fresh";
+    return "next-result";
+  }, state);
+
+  await assert.rejects(failed, /snapshot clone failed/);
+  const result = await next;
+  assert.deepEqual(operations, ["next"]);
+  assert.equal(result.result, "next-result");
+  assert.equal(result.snapshot, "fresh");
+});

package/src/orchestration/xray-capture-queue.ts

@@ -0,0 +1,78 @@
+import { abortError, throwIfAborted } from "../abort-error.js";
+
+export interface AtomicCaptureState<TSnapshot> {
+  read(): TSnapshot | null;
+  clear(): void;
+  restore(snapshot: TSnapshot | null): void;
+}
+
+export interface AtomicCaptureResult<TResult, TSnapshot> {
+  result: TResult;
+  snapshot: TSnapshot | null;
+  recallStartedAt: number;
+}
+
+/** Per-owner abortable FIFO for operations that publish through one mutable slot. */
+export class XrayCaptureQueue {
+  private tail: Promise<void> = Promise.resolve();
+
+  async run<TResult, TSnapshot>(
+    operation: () => Promise<TResult>,
+    state: AtomicCaptureState<TSnapshot>,
+    signal?: AbortSignal,
+  ): Promise<AtomicCaptureResult<TResult, TSnapshot>> {
+    throwIfAborted(signal, "x-ray capture aborted before queueing");
+    const previous = this.tail;
+    let release: () => void = () => {};
+    this.tail = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+
+    try {
+      await this.waitForTurn(previous, signal);
+    } catch (error) {
+      // Keep this abandoned node as a barrier until its predecessor settles,
+      // so later callers cannot overtake the still-active operation.
+      void previous.then(release, release);
+      throw error;
+    }
+
+    let previousSnapshot: TSnapshot | null = null;
+    let didReadPreviousSnapshot = false;
+    try {
+      previousSnapshot = state.read();
+      didReadPreviousSnapshot = true;
+      const recallStartedAt = Date.now();
+      state.clear();
+      const result = await operation();
+      const snapshot = state.read();
+      if (!snapshot) state.restore(previousSnapshot);
+      return { result, snapshot, recallStartedAt };
+    } catch (error) {
+      if (didReadPreviousSnapshot) state.restore(previousSnapshot);
+      throw error;
+    } finally {
+      release();
+    }
+  }
+
+  private async waitForTurn(previous: Promise<void>, signal?: AbortSignal): Promise<void> {
+    throwIfAborted(signal, "x-ray capture aborted while queued");
+    if (!signal) {
+      await previous;
+      return;
+    }
+
+    let onAbort: (() => void) | undefined;
+    const aborted = new Promise<never>((_resolve, reject) => {
+      onAbort = () => reject(abortError("x-ray capture aborted while queued"));
+      signal.addEventListener("abort", onAbort, { once: true });
+    });
+    try {
+      await Promise.race([previous, aborted]);
+      throwIfAborted(signal, "x-ray capture aborted while queued");
+    } finally {
+      if (onAbort) signal.removeEventListener("abort", onAbort);
+    }
+  }
+}