@remnic/core 9.6.27 → 9.6.29

package/src/access-recall-surface.ts

@@ -14,6 +14,7 @@
 import { createHash } from "node:crypto";
 import * as nodePath from "node:path";
 import { AccessAuditAdapter, type AccessAuditResult } from "./access-audit.js";
+import { throwIfAborted } from "./abort-error.js";
 import { resolveNamespaceCapabilities } from "./capabilities.js";
 import { resolveCodingNamespaceOverlay } from "./coding/coding-namespace.js";
 import { type BudgetDecision, CrossNamespaceBudget } from "./cross-namespace-budget.js";
@@ -173,7 +174,6 @@ export interface AccessRecallSurfaceDeps {
     primaryNamespace: string,
     recallNamespaces?: readonly string[],
   ): Promise<{ storage: StorageManager; dir: string } | null>;
-  xrayQueue: Promise<void>;
 }
 
 export class AccessRecallSurface {
@@ -1117,6 +1117,8 @@ export class AccessRecallSurface {
      * regular X-ray API/CLI/MCP surfaces keep their existing payload shape.
      */
     includeRecall?: boolean;
+    /** Cancel the capture before it starts and propagate cancellation to recall. */
+    abortSignal?: AbortSignal;
   }): Promise<{
     snapshotFound: boolean;
     snapshot?: RecallXraySnapshot;
@@ -1195,21 +1197,9 @@ export class AccessRecallSurface {
     const mode = this.deps.normalizeRecallMode(request.mode);
     const disclosure = request.disclosure ?? DEFAULT_RECALL_DISCLOSURE;
 
-    // Serialize x-ray invocations behind a per-service mutex so the
-    // per-process `getLastXraySnapshot()` slot cannot be clobbered by
-    // a concurrent capturing call before this caller reads it back.
-    // Budget and principal are now threaded through
-    // `RecallInvocationOptions`, so global config mutation is gone
-    // (CLAUDE.md rule 47: no shared mutable state across async
-    // boundaries).  The mutex stays only for the snapshot-slot
-    // ordering guarantee.
-    const previousQueue = this.deps.xrayQueue;
-    let release: () => void = () => {};
-    this.deps.xrayQueue = new Promise<void>((resolve) => {
-      release = resolve;
-    });
-    await previousQueue;
-    const recallStartedAt = Date.now();
+    // Reset when the orchestrator-owned snapshot lock is actually acquired so
+    // queue wait is not misreported as recall latency.
+    let recallStartedAt = Date.now();
 
     const recallSessionKey = request.sessionKey?.trim() || undefined;
     let xrayResponse: {
@@ -1217,37 +1207,46 @@ export class AccessRecallSurface {
       snapshot?: RecallXraySnapshot;
     } = { snapshotFound: false };
 
-    try {
-      // Clear any prior snapshot so a capture failure surfaces as
-      // `{snapshotFound: false}` rather than returning stale data
-      // from an earlier call on the same orchestrator.
-      this.deps.orchestrator.clearLastXraySnapshot();
-      await this.deps.orchestrator.recall(query, recallSessionKey, {
-        xrayCapture: true,
-        ...(requestedNamespace ? { namespace: requestedNamespace } : {}),
-        ...(budgetOverride !== undefined
-          ? { budgetCharsOverride: budgetOverride }
-          : {}),
-        ...(mode !== undefined ? { mode } : {}),
-        // When the caller supplies an authenticated principal, forward
-        // it via the dedicated override channel so orchestrator-side
-        // ACL decisions use the SAME principal the access-surface
-        // pre-check above authorized.  Threading an
-        // `authenticatedPrincipal` through `sessionKey` would be wrong:
-        // `resolvePrincipal(sessionKey)` only maps configured raw
-        // session keys and otherwise collapses to `"default"`, which
-        // in namespace-enabled deployments produces false denials /
-        // wrong-scope serving despite the pre-check passing
-        // (CLAUDE.md rule 42).
-        ...(authenticatedPrincipal
-          ? { principalOverride: authenticatedPrincipal }
-          : {}),
-        ...(request.currentContextScopes !== undefined
-          ? { currentContextScopes: request.currentContextScopes }
-          : {}),
-      });
-
-      const rawSnapshot = this.deps.orchestrator.getLastXraySnapshot();
+    {
+      // Capture through the orchestrator-owned critical section so every
+      // consumer of its mutable snapshot slot shares the same queue. The
+      // returned clone remains owned by this call after the lock is released.
+      const {
+        snapshot: rawSnapshot,
+        recallStartedAt: capturedRecallStartedAt,
+      } = await this.deps.orchestrator.recallWithXrayCapture(
+        query,
+        recallSessionKey,
+        {
+          ...(requestedNamespace ? { namespace: requestedNamespace } : {}),
+          ...(budgetOverride !== undefined
+            ? { budgetCharsOverride: budgetOverride }
+            : {}),
+          ...(mode !== undefined ? { mode } : {}),
+          // When the caller supplies an authenticated principal, forward
+          // it via the dedicated override channel so orchestrator-side
+          // ACL decisions use the SAME principal the access-surface
+          // pre-check above authorized.  Threading an
+          // `authenticatedPrincipal` through `sessionKey` would be wrong:
+          // `resolvePrincipal(sessionKey)` only maps configured raw
+          // session keys and otherwise collapses to `"default"`, which
+          // in namespace-enabled deployments produces false denials /
+          // wrong-scope serving despite the pre-check passing
+          // (CLAUDE.md rule 42).
+          ...(authenticatedPrincipal
+            ? { principalOverride: authenticatedPrincipal }
+            : {}),
+          ...(request.currentContextScopes !== undefined
+            ? { currentContextScopes: request.currentContextScopes }
+            : {}),
+          ...(request.abortSignal ? { abortSignal: request.abortSignal } : {}),
+        },
+      );
+      recallStartedAt = capturedRecallStartedAt;
+      // Cancellation covers queueing + recall/capture. Post-capture shaping is
+      // intentionally not wired for mid-I/O abort in this change, but do not
+      // begin that work after the caller has already canceled.
+      throwIfAborted(request.abortSignal, "recall X-ray aborted before postprocessing");
       // Re-check namespace after capture: the recall may have served
       // from a different namespace than the caller requested.  Drop
       // the snapshot rather than leak cross-tenant data (CLAUDE.md
@@ -1483,8 +1482,6 @@ export class AccessRecallSurface {
           };
         }
       }
-    } finally {
-      release();
     }
 
     if (

package/src/access-service.ts

@@ -2917,6 +2917,8 @@ export class EngramAccessService {
      * regular X-ray API/CLI/MCP surfaces keep their existing payload shape.
      */
     includeRecall?: boolean;
+    /** Cancel the capture before it starts and propagate cancellation to recall. */
+    abortSignal?: AbortSignal;
   }): Promise<{
     snapshotFound: boolean;
     snapshot?: RecallXraySnapshot;
@@ -2926,12 +2928,6 @@ export class EngramAccessService {
       request,
     );
   }
-  // Sequence lock for `recallXray` — see comment inside the method.
-  // Lives on the instance so every x-ray call on the same service
-  // shares it, and so separate services in the same process (e.g.
-  // per-tenant) do not block each other.
-  private xrayQueue: Promise<void> = Promise.resolve();
-
   async memoryStore(
     request: EngramAccessMemoryStoreRequest,
     hooks?: { enforceWriteQuota?: () => void | Promise<void> },
@@ -5906,4 +5902,3 @@ export class EngramAccessService {
     );
   }
 }
-

package/src/cli.ts

@@ -5756,16 +5756,10 @@ export function registerCli(
             args[0],
             (args[1] ?? {}) as Record<string, unknown>,
           );
-          // Route the xray capture through `EngramAccessService` so
-          // the CLI shares the same `xrayQueue` mutex that the HTTP
-          // and MCP surfaces use — otherwise the
-          // `clearLastXraySnapshot() → recall() → getLastXraySnapshot()`
-          // sequence races with concurrent callers (e.g., a gateway
-          // agent hitting the same orchestrator) and could swap in
-          // their snapshot mid-flight, or our capture could overwrite
-          // theirs (cursor Medium + codex P1 review on #597).  The
-          // service enforces CLAUDE.md rules 40 (serialized state) and
-          // 47 (no shared mutable state across async boundaries).
+          // Delegate through `EngramAccessService` to the orchestrator's atomic
+          // capture API, so CLI, HTTP, and MCP callers sharing an orchestrator
+          // also share one X-ray ordering domain and receive the snapshot owned
+          // by their invocation.
           const xrayService = new EngramAccessService(orchestrator);
           const response = await xrayService.recallXray({
             query: parsed.query,

package/src/lcm/engine.ts

@@ -23,6 +23,15 @@ export interface LcmEngineConfig {
   messagePartsRecallMaxResults: number;
 }
 
+/** An expanded archive row with stable lineage identity; content may be budget-truncated. */
+export interface LcmExpandedMessage {
+  id: number;
+  session_id: string;
+  turn_index: number;
+  role: string;
+  content: string;
+}
+
 function positiveInteger(value: unknown, fallback: number, min = 1): number {
   if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
   return Math.max(min, Math.floor(value));
@@ -526,7 +535,7 @@ export class LcmEngine {
     fromTurn: number,
     toTurn: number,
     maxTokens: number,
-  ): Promise<Array<{ turn_index: number; role: string; content: string }>> {
+  ): Promise<LcmExpandedMessage[]> {
     if (!this.config.enabled) return [];
     const normalizedSessionId = normalizeLcmSessionId(sessionId);
     if (!normalizedSessionId) return [];
@@ -542,6 +551,8 @@ export class LcmEngine {
 
     if (totalChars <= maxChars) {
       return messages.map((m) => ({
+        id: m.id,
+        session_id: m.session_id,
         turn_index: m.turn_index,
         role: m.role,
         content: m.content,
@@ -549,8 +560,7 @@ export class LcmEngine {
     }
 
     // Keep first and last messages, truncate from middle
-    const result: Array<{ turn_index: number; role: string; content: string }> =
-      [];
+    const result: LcmExpandedMessage[] = [];
     let budget = maxChars;
 
     // Reserve space for the last message
@@ -567,6 +577,8 @@ export class LcmEngine {
       const m = messages[i];
       const truncated = m.content.slice(0, budget);
       result.push({
+        id: m.id,
+        session_id: m.session_id,
         turn_index: m.turn_index,
         role: m.role,
         content: truncated,
@@ -576,6 +588,8 @@ export class LcmEngine {
 
     // Always append the last message
     result.push({
+      id: lastMsg.id,
+      session_id: lastMsg.session_id,
       turn_index: lastMsg.turn_index,
       role: lastMsg.role,
       content: lastMsg.content.slice(0, lastMsgChars + Math.max(0, budget)),

package/src/lcm/index.ts

@@ -1,4 +1,9 @@
-export { LcmEngine, extractLcmConfig, type LcmEngineConfig } from "./engine.js";
+export {
+  LcmEngine,
+  extractLcmConfig,
+  type LcmEngineConfig,
+  type LcmExpandedMessage,
+} from "./engine.js";
 export { LcmArchive, estimateTokens } from "./archive.js";
 export { LcmDag, type SummaryNode } from "./dag.js";
 export { LcmSummarizer, type SummarizeFn } from "./summarizer.js";

package/src/lcm-engine.test.ts

@@ -4,7 +4,8 @@ import os from "node:os";
 import path from "node:path";
 import test from "node:test";
 
-import { extractLcmConfig, LcmEngine } from "./lcm/engine.js";
+import { LcmArchive } from "./lcm/archive.js";
+import { LcmEngine, extractLcmConfig } from "./lcm/engine.js";
 import { openLcmDatabase } from "./lcm/schema.js";
 import type { PluginConfig } from "./types.js";
 
@@ -546,6 +547,135 @@ test("LCM normalizes session IDs across observe, stats, and clear", async () =>
   }
 });
 
+test("expandContext preserves distinct archive-row identity for duplicate turns", async () => {
+  const memoryDir = await mkdtemp(
+    path.join(os.tmpdir(), "engram-lcm-expand-identity-"),
+  );
+
+  try {
+    const engine = new LcmEngine(
+      createPluginConfig(memoryDir),
+      async () => "summary",
+    );
+    await engine.ensureInitialized();
+
+    const db = openLcmDatabase(memoryDir);
+    let expected: ReturnType<LcmArchive["getMessages"]>;
+    try {
+      const archive = new LcmArchive(db);
+      archive.appendMessages("session-expanded", [
+        { turnIndex: 7, role: "user", content: "duplicate turn user" },
+        {
+          turnIndex: 7,
+          role: "assistant",
+          content: "duplicate turn assistant",
+        },
+      ]);
+      expected = archive.getMessages("session-expanded", 7, 7);
+    } finally {
+      db.close();
+    }
+
+    assert.equal(expected.length, 2);
+    const [firstDuplicate, secondDuplicate] = expected;
+    assert.ok(firstDuplicate);
+    assert.ok(secondDuplicate);
+    assert.notEqual(firstDuplicate.id, secondDuplicate.id);
+
+    const expanded = await engine.expandContext(
+      "  session-expanded  ",
+      7,
+      7,
+      100,
+    );
+
+    assert.deepEqual(
+      expanded,
+      expected.map(({ id, session_id, turn_index, role, content }) => ({
+        id,
+        session_id,
+        turn_index,
+        role,
+        content,
+      })),
+    );
+    assert.deepEqual(
+      expanded.map(({ session_id }) => session_id),
+      ["session-expanded", "session-expanded"],
+    );
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("expandContext preserves first, middle, and last row identity when truncated", async () => {
+  const memoryDir = await mkdtemp(
+    path.join(os.tmpdir(), "engram-lcm-expand-truncated-identity-"),
+  );
+
+  try {
+    const engine = new LcmEngine(
+      createPluginConfig(memoryDir),
+      async () => "summary",
+    );
+    await engine.ensureInitialized();
+
+    const db = openLcmDatabase(memoryDir);
+    let archived: ReturnType<LcmArchive["getMessages"]>;
+    try {
+      const archive = new LcmArchive(db);
+      archive.appendMessages("session-truncated", [
+        { turnIndex: 1, role: "user", content: "AAAAAAAA" },
+        { turnIndex: 2, role: "assistant", content: "BBBBBBBB" },
+        { turnIndex: 3, role: "user", content: "CCCCCCCC" },
+        { turnIndex: 4, role: "assistant", content: "DDDDDDDD" },
+      ]);
+      archived = archive.getMessages("session-truncated", 1, 4);
+    } finally {
+      db.close();
+    }
+
+    assert.equal(archived.length, 4);
+    const [first, middle, , last] = archived;
+    assert.ok(first);
+    assert.ok(middle);
+    assert.ok(last);
+
+    const expanded = await engine.expandContext(
+      "  session-truncated  ",
+      1,
+      4,
+      5,
+    );
+
+    assert.deepEqual(expanded, [
+      {
+        id: first.id,
+        session_id: "session-truncated",
+        turn_index: 1,
+        role: "user",
+        content: "AAAAAAAA",
+      },
+      {
+        id: middle.id,
+        session_id: "session-truncated",
+        turn_index: 2,
+        role: "assistant",
+        content: "BBBBBB",
+      },
+      {
+        id: last.id,
+        session_id: "session-truncated",
+        turn_index: 4,
+        role: "assistant",
+        content: "DDDDDD",
+      },
+    ]);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
 test("close prevents deferred observe work from reinitializing the engine", async () => {
   const memoryDir = await mkdtemp(
     path.join(os.tmpdir(), "engram-lcm-engine-close-"),

package/src/orchestration/xray-capture-queue.test.ts

@@ -0,0 +1,42 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { XrayCaptureQueue } from "./xray-capture-queue.js";
+
+test("initial snapshot read failure releases the next queued capture", async () => {
+  const queue = new XrayCaptureQueue();
+  let failNextRead = true;
+  let snapshot: string | null = "prior";
+  const operations: string[] = [];
+  const state = {
+    read(): string | null {
+      if (failNextRead) {
+        failNextRead = false;
+        throw new Error("snapshot clone failed");
+      }
+      return snapshot;
+    },
+    clear(): void {
+      snapshot = null;
+    },
+    restore(value: string | null): void {
+      snapshot = value;
+    },
+  };
+
+  const failed = queue.run(async () => {
+    operations.push("failed");
+    return "unexpected";
+  }, state);
+  const next = queue.run(async () => {
+    operations.push("next");
+    snapshot = "fresh";
+    return "next-result";
+  }, state);
+
+  await assert.rejects(failed, /snapshot clone failed/);
+  const result = await next;
+  assert.deepEqual(operations, ["next"]);
+  assert.equal(result.result, "next-result");
+  assert.equal(result.snapshot, "fresh");
+});

package/src/orchestration/xray-capture-queue.ts

@@ -0,0 +1,78 @@
+import { abortError, throwIfAborted } from "../abort-error.js";
+
+export interface AtomicCaptureState<TSnapshot> {
+  read(): TSnapshot | null;
+  clear(): void;
+  restore(snapshot: TSnapshot | null): void;
+}
+
+export interface AtomicCaptureResult<TResult, TSnapshot> {
+  result: TResult;
+  snapshot: TSnapshot | null;
+  recallStartedAt: number;
+}
+
+/** Per-owner abortable FIFO for operations that publish through one mutable slot. */
+export class XrayCaptureQueue {
+  private tail: Promise<void> = Promise.resolve();
+
+  async run<TResult, TSnapshot>(
+    operation: () => Promise<TResult>,
+    state: AtomicCaptureState<TSnapshot>,
+    signal?: AbortSignal,
+  ): Promise<AtomicCaptureResult<TResult, TSnapshot>> {
+    throwIfAborted(signal, "x-ray capture aborted before queueing");
+    const previous = this.tail;
+    let release: () => void = () => {};
+    this.tail = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+
+    try {
+      await this.waitForTurn(previous, signal);
+    } catch (error) {
+      // Keep this abandoned node as a barrier until its predecessor settles,
+      // so later callers cannot overtake the still-active operation.
+      void previous.then(release, release);
+      throw error;
+    }
+
+    let previousSnapshot: TSnapshot | null = null;
+    let didReadPreviousSnapshot = false;
+    try {
+      previousSnapshot = state.read();
+      didReadPreviousSnapshot = true;
+      const recallStartedAt = Date.now();
+      state.clear();
+      const result = await operation();
+      const snapshot = state.read();
+      if (!snapshot) state.restore(previousSnapshot);
+      return { result, snapshot, recallStartedAt };
+    } catch (error) {
+      if (didReadPreviousSnapshot) state.restore(previousSnapshot);
+      throw error;
+    } finally {
+      release();
+    }
+  }
+
+  private async waitForTurn(previous: Promise<void>, signal?: AbortSignal): Promise<void> {
+    throwIfAborted(signal, "x-ray capture aborted while queued");
+    if (!signal) {
+      await previous;
+      return;
+    }
+
+    let onAbort: (() => void) | undefined;
+    const aborted = new Promise<never>((_resolve, reject) => {
+      onAbort = () => reject(abortError("x-ray capture aborted while queued"));
+      signal.addEventListener("abort", onAbort, { once: true });
+    });
+    try {
+      await Promise.race([previous, aborted]);
+      throwIfAborted(signal, "x-ray capture aborted while queued");
+    } finally {
+      if (onAbort) signal.removeEventListener("abort", onAbort);
+    }
+  }
+}