@remnic/core 9.6.28 → 9.6.29

package/src/access-service.ts

@@ -2917,6 +2917,8 @@ export class EngramAccessService {
      * regular X-ray API/CLI/MCP surfaces keep their existing payload shape.
      */
     includeRecall?: boolean;
+    /** Cancel the capture before it starts and propagate cancellation to recall. */
+    abortSignal?: AbortSignal;
   }): Promise<{
     snapshotFound: boolean;
     snapshot?: RecallXraySnapshot;
@@ -2926,12 +2928,6 @@ export class EngramAccessService {
       request,
     );
   }
-  // Sequence lock for `recallXray` — see comment inside the method.
-  // Lives on the instance so every x-ray call on the same service
-  // shares it, and so separate services in the same process (e.g.
-  // per-tenant) do not block each other.
-  private xrayQueue: Promise<void> = Promise.resolve();
-
   async memoryStore(
     request: EngramAccessMemoryStoreRequest,
     hooks?: { enforceWriteQuota?: () => void | Promise<void> },
@@ -5906,4 +5902,3 @@ export class EngramAccessService {
     );
   }
 }
-

package/src/cli.ts

@@ -5756,16 +5756,10 @@ export function registerCli(
             args[0],
             (args[1] ?? {}) as Record<string, unknown>,
           );
-          // Route the xray capture through `EngramAccessService` so
-          // the CLI shares the same `xrayQueue` mutex that the HTTP
-          // and MCP surfaces use — otherwise the
-          // `clearLastXraySnapshot() → recall() → getLastXraySnapshot()`
-          // sequence races with concurrent callers (e.g., a gateway
-          // agent hitting the same orchestrator) and could swap in
-          // their snapshot mid-flight, or our capture could overwrite
-          // theirs (cursor Medium + codex P1 review on #597).  The
-          // service enforces CLAUDE.md rules 40 (serialized state) and
-          // 47 (no shared mutable state across async boundaries).
+          // Delegate through `EngramAccessService` to the orchestrator's atomic
+          // capture API, so CLI, HTTP, and MCP callers sharing an orchestrator
+          // also share one X-ray ordering domain and receive the snapshot owned
+          // by their invocation.
           const xrayService = new EngramAccessService(orchestrator);
           const response = await xrayService.recallXray({
             query: parsed.query,

package/src/orchestration/xray-capture-queue.test.ts

@@ -0,0 +1,42 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { XrayCaptureQueue } from "./xray-capture-queue.js";
+
+test("initial snapshot read failure releases the next queued capture", async () => {
+  const queue = new XrayCaptureQueue();
+  let failNextRead = true;
+  let snapshot: string | null = "prior";
+  const operations: string[] = [];
+  const state = {
+    read(): string | null {
+      if (failNextRead) {
+        failNextRead = false;
+        throw new Error("snapshot clone failed");
+      }
+      return snapshot;
+    },
+    clear(): void {
+      snapshot = null;
+    },
+    restore(value: string | null): void {
+      snapshot = value;
+    },
+  };
+
+  const failed = queue.run(async () => {
+    operations.push("failed");
+    return "unexpected";
+  }, state);
+  const next = queue.run(async () => {
+    operations.push("next");
+    snapshot = "fresh";
+    return "next-result";
+  }, state);
+
+  await assert.rejects(failed, /snapshot clone failed/);
+  const result = await next;
+  assert.deepEqual(operations, ["next"]);
+  assert.equal(result.result, "next-result");
+  assert.equal(result.snapshot, "fresh");
+});

package/src/orchestration/xray-capture-queue.ts

@@ -0,0 +1,78 @@
+import { abortError, throwIfAborted } from "../abort-error.js";
+
+export interface AtomicCaptureState<TSnapshot> {
+  read(): TSnapshot | null;
+  clear(): void;
+  restore(snapshot: TSnapshot | null): void;
+}
+
+export interface AtomicCaptureResult<TResult, TSnapshot> {
+  result: TResult;
+  snapshot: TSnapshot | null;
+  recallStartedAt: number;
+}
+
+/** Per-owner abortable FIFO for operations that publish through one mutable slot. */
+export class XrayCaptureQueue {
+  private tail: Promise<void> = Promise.resolve();
+
+  async run<TResult, TSnapshot>(
+    operation: () => Promise<TResult>,
+    state: AtomicCaptureState<TSnapshot>,
+    signal?: AbortSignal,
+  ): Promise<AtomicCaptureResult<TResult, TSnapshot>> {
+    throwIfAborted(signal, "x-ray capture aborted before queueing");
+    const previous = this.tail;
+    let release: () => void = () => {};
+    this.tail = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+
+    try {
+      await this.waitForTurn(previous, signal);
+    } catch (error) {
+      // Keep this abandoned node as a barrier until its predecessor settles,
+      // so later callers cannot overtake the still-active operation.
+      void previous.then(release, release);
+      throw error;
+    }
+
+    let previousSnapshot: TSnapshot | null = null;
+    let didReadPreviousSnapshot = false;
+    try {
+      previousSnapshot = state.read();
+      didReadPreviousSnapshot = true;
+      const recallStartedAt = Date.now();
+      state.clear();
+      const result = await operation();
+      const snapshot = state.read();
+      if (!snapshot) state.restore(previousSnapshot);
+      return { result, snapshot, recallStartedAt };
+    } catch (error) {
+      if (didReadPreviousSnapshot) state.restore(previousSnapshot);
+      throw error;
+    } finally {
+      release();
+    }
+  }
+
+  private async waitForTurn(previous: Promise<void>, signal?: AbortSignal): Promise<void> {
+    throwIfAborted(signal, "x-ray capture aborted while queued");
+    if (!signal) {
+      await previous;
+      return;
+    }
+
+    let onAbort: (() => void) | undefined;
+    const aborted = new Promise<never>((_resolve, reject) => {
+      onAbort = () => reject(abortError("x-ray capture aborted while queued"));
+      signal.addEventListener("abort", onAbort, { once: true });
+    });
+    try {
+      await Promise.race([previous, aborted]);
+      throwIfAborted(signal, "x-ray capture aborted while queued");
+    } finally {
+      if (onAbort) signal.removeEventListener("abort", onAbort);
+    }
+  }
+}

package/src/orchestrator-xray-capture.test.ts

@@ -0,0 +1,319 @@
+import assert from "node:assert/strict";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { parseConfig } from "./config.js";
+import { Orchestrator } from "./orchestrator.js";
+import { buildXraySnapshot, type RecallXraySnapshot } from "./recall-xray.js";
+
+let orchestratorSequence = 0;
+
+function makeOrchestrator(): Orchestrator {
+  orchestratorSequence += 1;
+  const memoryDir = path.join(
+    tmpdir(),
+    `remnic-xray-capture-unit-${process.pid}-${orchestratorSequence}`,
+  );
+  return new Orchestrator(parseConfig({
+    memoryDir,
+    workspaceDir: memoryDir,
+    qmdEnabled: false,
+    embeddingFallbackEnabled: false,
+  }));
+}
+
+function snapshot(query: string, snapshotId: string): RecallXraySnapshot {
+  return buildXraySnapshot({
+    query,
+    snapshotIdGenerator: () => snapshotId,
+    now: () => 1,
+  });
+}
+
+function setSnapshot(orchestrator: Orchestrator, value: RecallXraySnapshot): void {
+  (orchestrator as unknown as { lastXraySnapshot: RecallXraySnapshot | null })
+    .lastXraySnapshot = value;
+}
+
+function stubRecall(
+  orchestrator: Orchestrator,
+  invoke: (
+    prompt: string,
+    sessionKey: string | undefined,
+    options: Record<string, unknown>,
+  ) => Promise<string>,
+): void {
+  (orchestrator as unknown as { invokeRecall: typeof invoke }).invokeRecall = invoke;
+}
+
+function deferred<T>(): {
+  promise: Promise<T>;
+  resolve: (value: T) => void;
+  reject: (reason?: unknown) => void;
+} {
+  let resolve!: (value: T) => void;
+  let reject!: (reason?: unknown) => void;
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  return { promise, resolve, reject };
+}
+
+test("recallWithXrayCapture clears stale state and returns an owned nested clone", async () => {
+  const orchestrator = makeOrchestrator();
+  setSnapshot(orchestrator, snapshot("stale", "stale-id"));
+  const captured = buildXraySnapshot({
+    query: "normalized query",
+    snapshotIdGenerator: () => "fresh-id",
+    now: () => 1,
+    results: [{
+      memoryId: "memory-1",
+      path: "memories/memory-1.md",
+      servedBy: "hybrid",
+      scoreDecomposition: { final: 1 },
+      admittedBy: ["retrieval"],
+      graphPath: ["a", "b"],
+    }],
+    filters: [{ name: "retrieval", considered: 1, admitted: 1 }],
+  });
+  stubRecall(orchestrator, async (_prompt, _sessionKey, options) => {
+    assert.equal(options.xrayCapture, true);
+    assert.equal(orchestrator.getLastXraySnapshot(), null);
+    setSnapshot(orchestrator, captured);
+    return "recall-result";
+  });
+
+  const result = await orchestrator.recallWithXrayCapture("raw query", "session");
+  assert.equal(result.result, "recall-result");
+  assert.deepEqual(result.snapshot, captured);
+  assert.notEqual(result.snapshot, captured);
+
+  result.snapshot!.results[0]!.admittedBy.push("tampered");
+  result.snapshot!.results[0]!.graphPath!.push("tampered");
+  result.snapshot!.filters[0]!.reason = "tampered";
+  const stored = orchestrator.getLastXraySnapshot();
+  assert.deepEqual(stored?.results[0]?.admittedBy, ["retrieval"]);
+  assert.deepEqual(stored?.results[0]?.graphPath, ["a", "b"]);
+  assert.equal(stored?.filters[0]?.reason, undefined);
+});
+
+test("recallWithXrayCapture serializes every consumer of one orchestrator slot", async () => {
+  const orchestrator = makeOrchestrator();
+  const firstGate = deferred<void>();
+  const firstStarted = deferred<void>();
+  const order: string[] = [];
+  stubRecall(orchestrator, async (prompt) => {
+    order.push(`${prompt}:start`);
+    if (prompt === "first") {
+      firstStarted.resolve();
+      await firstGate.promise;
+    }
+    setSnapshot(orchestrator, snapshot(prompt, `${prompt}-id`));
+    order.push(`${prompt}:end`);
+    return prompt;
+  });
+
+  const first = orchestrator.recallWithXrayCapture("first");
+  const second = orchestrator.recallWithXrayCapture("second");
+  await firstStarted.promise;
+  assert.deepEqual(order, ["first:start"]);
+  firstGate.resolve();
+
+  const [firstResult, secondResult] = await Promise.all([first, second]);
+  assert.deepEqual(order, ["first:start", "first:end", "second:start", "second:end"]);
+  assert.equal(firstResult.snapshot?.snapshotId, "first-id");
+  assert.equal(secondResult.snapshot?.snapshotId, "second-id");
+});
+
+test("recallWithXrayCapture queues are independent across orchestrators", async () => {
+  const firstOrchestrator = makeOrchestrator();
+  const secondOrchestrator = makeOrchestrator();
+  const firstGate = deferred<void>();
+  let secondStarted = false;
+  stubRecall(firstOrchestrator, async () => {
+    await firstGate.promise;
+    return "first";
+  });
+  stubRecall(secondOrchestrator, async () => {
+    secondStarted = true;
+    return "second";
+  });
+
+  const first = firstOrchestrator.recallWithXrayCapture("first");
+  await secondOrchestrator.recallWithXrayCapture("second");
+  assert.equal(secondStarted, true);
+  firstGate.resolve();
+  await first;
+});
+
+test("recallWithXrayCapture rejects a pre-aborted signal without clearing or invoking", async () => {
+  const orchestrator = makeOrchestrator();
+  const stale = snapshot("stale", "stale-id");
+  setSnapshot(orchestrator, stale);
+  const abortController = new AbortController();
+  abortController.abort();
+  let invoked = false;
+  stubRecall(orchestrator, async () => {
+    invoked = true;
+    return "unexpected";
+  });
+
+  await assert.rejects(
+    orchestrator.recallWithXrayCapture("aborted", undefined, {
+      abortSignal: abortController.signal,
+    }),
+    { name: "AbortError" },
+  );
+  assert.equal(invoked, false);
+  assert.deepEqual(orchestrator.getLastXraySnapshot(), stale);
+});
+
+test("queued abort rejects promptly while its barrier prevents a third capture overtaking", async () => {
+  const orchestrator = makeOrchestrator();
+  const firstGate = deferred<void>();
+  const abortController = new AbortController();
+  const order: string[] = [];
+  stubRecall(orchestrator, async (prompt) => {
+    order.push(`${prompt}:start`);
+    if (prompt === "first") await firstGate.promise;
+    setSnapshot(orchestrator, snapshot(prompt, `${prompt}-id`));
+    order.push(`${prompt}:end`);
+    return prompt;
+  });
+
+  const first = orchestrator.recallWithXrayCapture("first");
+  await Promise.resolve();
+  const queued = orchestrator.recallWithXrayCapture("aborted", undefined, {
+    abortSignal: abortController.signal,
+  });
+  const third = orchestrator.recallWithXrayCapture("third");
+  abortController.abort();
+
+  await assert.rejects(queued, { name: "AbortError" });
+  assert.deepEqual(order, ["first:start"]);
+  firstGate.resolve();
+
+  const [firstResult, thirdResult] = await Promise.all([first, third]);
+  assert.deepEqual(order, ["first:start", "first:end", "third:start", "third:end"]);
+  assert.equal(firstResult.snapshot?.snapshotId, "first-id");
+  assert.equal(thirdResult.snapshot?.snapshotId, "third-id");
+});
+
+test("started abort retains ownership until recall settles", async () => {
+  const orchestrator = makeOrchestrator();
+  const activeGate = deferred<void>();
+  const activeStarted = deferred<void>();
+  const abortController = new AbortController();
+  let nextStarted = false;
+  stubRecall(orchestrator, async (prompt) => {
+    if (prompt === "active") {
+      activeStarted.resolve();
+      await activeGate.promise;
+      setSnapshot(orchestrator, snapshot("active", "active-id"));
+    } else {
+      nextStarted = true;
+      setSnapshot(orchestrator, snapshot("next", "next-id"));
+    }
+    return prompt;
+  });
+
+  const active = orchestrator.recallWithXrayCapture("active", undefined, {
+    abortSignal: abortController.signal,
+  });
+  await activeStarted.promise;
+  abortController.abort();
+  const next = orchestrator.recallWithXrayCapture("next");
+  await Promise.resolve();
+  assert.equal(nextStarted, false);
+
+  activeGate.resolve();
+  const activeResult = await active;
+  const nextResult = await next;
+  assert.equal(activeResult.snapshot?.snapshotId, "active-id");
+  assert.equal(nextResult.snapshot?.snapshotId, "next-id");
+});
+
+test("failed and empty captures restore prior shared snapshot but return no stale snapshot", async () => {
+  const orchestrator = makeOrchestrator();
+  const prior = snapshot("prior", "prior-id");
+  setSnapshot(orchestrator, prior);
+  stubRecall(orchestrator, async (prompt) => {
+    if (prompt === "failure") throw new Error("recall failed");
+    return "empty";
+  });
+
+  await assert.rejects(
+    orchestrator.recallWithXrayCapture("failure"),
+    /recall failed/,
+  );
+  assert.equal(orchestrator.getLastXraySnapshot()?.snapshotId, "prior-id");
+
+  const empty = await orchestrator.recallWithXrayCapture("empty");
+  assert.equal(empty.snapshot, null);
+  assert.equal(orchestrator.getLastXraySnapshot()?.snapshotId, "prior-id");
+});
+
+test("legacy capturing recall shares the queue and preserves its string return", async () => {
+  const orchestrator = makeOrchestrator();
+  const firstGate = deferred<void>();
+  const order: string[] = [];
+  stubRecall(orchestrator, async (prompt) => {
+    order.push(`${prompt}:start`);
+    if (prompt === "atomic") await firstGate.promise;
+    setSnapshot(orchestrator, snapshot(prompt, `${prompt}-id`));
+    order.push(`${prompt}:end`);
+    return `${prompt}-result`;
+  });
+
+  const atomic = orchestrator.recallWithXrayCapture("atomic");
+  await Promise.resolve();
+  const legacy = orchestrator.recall("legacy", undefined, { xrayCapture: true });
+  await Promise.resolve();
+  assert.deepEqual(order, ["atomic:start"]);
+  firstGate.resolve();
+
+  const [atomicResult, legacyResult] = await Promise.all([atomic, legacy]);
+  assert.equal(atomicResult.result, "atomic-result");
+  assert.equal(legacyResult, "legacy-result");
+  assert.deepEqual(order, ["atomic:start", "atomic:end", "legacy:start", "legacy:end"]);
+});
+
+test("ordinary recall remains independent and does not overwrite a captured snapshot", async () => {
+  const orchestrator = makeOrchestrator();
+  const captureGate = deferred<void>();
+  const captureStarted = deferred<void>();
+  const order: string[] = [];
+  stubRecall(orchestrator, async (prompt, _sessionKey, options) => {
+    order.push(prompt);
+    if (prompt === "capture") {
+      captureStarted.resolve();
+      await captureGate.promise;
+      setSnapshot(orchestrator, snapshot("capture", "capture-id"));
+    }
+    assert.equal(options.xrayCapture === true, prompt === "capture");
+    return `${prompt}-result`;
+  });
+
+  const capture = orchestrator.recallWithXrayCapture("capture");
+  await captureStarted.promise;
+  const ordinary = await orchestrator.recall("ordinary");
+  assert.equal(ordinary, "ordinary-result");
+  assert.deepEqual(order, ["capture", "ordinary"]);
+  captureGate.resolve();
+  await capture;
+  await orchestrator.recall("ordinary-after");
+  assert.equal(orchestrator.getLastXraySnapshot()?.snapshotId, "capture-id");
+});
+
+test("recallWithXrayCapture accepts normalized snapshot queries without equality checks", async () => {
+  const orchestrator = makeOrchestrator();
+  stubRecall(orchestrator, async () => {
+    setSnapshot(orchestrator, snapshot("daily briefing", "normalized-id"));
+    return "context";
+  });
+
+  const result = await orchestrator.recallWithXrayCapture(":cron: daily briefing");
+  assert.equal(result.snapshot?.query, "daily briefing");
+});

package/src/orchestrator.ts

@@ -121,6 +121,7 @@ import { selfDeps } from "./orchestration/self-deps.js";
 import { RecallEntryCoordinator } from "./orchestration/recall-entry.js";
 import { SessionContextCoordinator } from "./orchestration/session-context.js";
 import { drainRecallWrites, trackRecallWrite } from "./orchestration/recall-background-writes.js";
+import { XrayCaptureQueue } from "./orchestration/xray-capture-queue.js";
 import {
   abortRecallError,
   buildCompressionGuidelinesMarkdown,
@@ -258,10 +259,6 @@ import {
   type EvalShadowRecallRecord,
 } from "./evals.js";
 import { SessionObserverState } from "./session-observer-state.js";
-import {
-  abortError as sharedAbortError,
-  throwIfAborted as sharedThrowIfAborted,
-} from "./abort-error.js";
 import { CODEX_THREAD_KEY_PREFIX } from "./thread-key.js";
 import { isDisagreementPrompt } from "./signal.js";
 import { lintWorkspaceFiles, rotateMarkdownFileToArchive } from "./hygiene.js";
@@ -633,6 +630,8 @@ export class Orchestrator {
    * capturing caller can still read their snapshot back.
    */
   private lastXraySnapshot: RecallXraySnapshot | null = null;
+  /** Per-instance ordering domain for every writer of `lastXraySnapshot`. */
+  private readonly xrayCaptureQueue = new XrayCaptureQueue();
   readonly embeddingFallback: EmbeddingFallback;
   private readonly conversationIndexDir: string;
   private readonly extraction: ExtractionEngine;
@@ -2182,7 +2181,7 @@ export class Orchestrator {
     );
   }
 
-  async recall(
+  private invokeRecall(
     prompt: string,
     sessionKey?: string,
     options: RecallInvocationOptions = {},
@@ -2196,23 +2195,92 @@ export class Orchestrator {
     );
   }
 
+  async recall(
+    prompt: string,
+    sessionKey?: string,
+    options: RecallInvocationOptions = {},
+  ): Promise<string> {
+    if (options.xrayCapture === true) {
+      // Preserve the legacy string-returning surface and its soft-abort
+      // behavior while still placing every X-ray writer in the same ordering
+      // domain. In particular, a pre-aborted legacy recall continues to
+      // resolve with an empty string rather than rejecting.
+      const { result } = await this.runRecallWithXrayCapture(
+        prompt,
+        sessionKey,
+        options,
+      );
+      return result;
+    }
+    return this.invokeRecall(prompt, sessionKey, options);
+  }
+
   /**
-   * Return the most recent X-ray snapshot captured during a
-   * `recall()` call that passed `xrayCapture: true` (issue #570 PR 1).
+   * Return the most recent X-ray snapshot captured during a recall.
    * Returns `null` when no such capture has occurred on this
    * orchestrator instance.  Returned snapshot is a deep copy so
    * caller mutation cannot tear the stored value.
+   *
+   * @deprecated Reading this after a separate `recall()` is not atomic. Use
+   * `recallWithXrayCapture()` when the snapshot must belong to that call.
    */
   getLastXraySnapshot(): RecallXraySnapshot | null {
     if (!this.lastXraySnapshot) return null;
     return structuredClone(this.lastXraySnapshot);
   }
 
-  /** Clear the captured X-ray snapshot.  Exposed for tests / explicit reset. */
+  /**
+   * Clear the captured X-ray snapshot. Exposed for tests / explicit reset.
+   *
+   * @deprecated A separate clear → recall → get sequence is not atomic. Use
+   * `recallWithXrayCapture()` for capture operations.
+   */
   clearLastXraySnapshot(): void {
     this.lastXraySnapshot = null;
   }
 
+  /** Atomically run recall and return the cloned X-ray snapshot it published. */
+  async recallWithXrayCapture(
+    prompt: string,
+    sessionKey?: string,
+    options: Omit<RecallInvocationOptions, "xrayCapture"> = {},
+  ): Promise<{
+    result: string;
+    snapshot: RecallXraySnapshot | null;
+    recallStartedAt: number;
+  }> {
+    return this.runRecallWithXrayCapture(
+      prompt,
+      sessionKey,
+      options,
+      options.abortSignal,
+    );
+  }
+
+  private async runRecallWithXrayCapture(
+    prompt: string,
+    sessionKey: string | undefined,
+    options: Omit<RecallInvocationOptions, "xrayCapture"> | RecallInvocationOptions,
+    queueAbortSignal?: AbortSignal,
+  ): Promise<{
+    result: string;
+    snapshot: RecallXraySnapshot | null;
+    recallStartedAt: number;
+  }> {
+    return this.xrayCaptureQueue.run(
+      () => this.invokeRecall(prompt, sessionKey, {
+        ...options,
+        xrayCapture: true,
+      }),
+      {
+        read: () => this.getLastXraySnapshot(),
+        clear: () => this.clearLastXraySnapshot(),
+        restore: (snapshot) => { this.lastXraySnapshot = snapshot; },
+      },
+      queueAbortSignal,
+    );
+  }
+
   async waitForDirectAnswerObservationIdle(
     timeoutMs: number = 60_000,
   ): Promise<boolean> {