@remnic/core 9.7.15 → 9.8.0

package/src/recall-state.ts

@@ -1,8 +1,11 @@
-import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { appendFile, mkdir, readFile, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 import { log } from "./logger.js";
 import { writeFileAtomically } from "./maintenance/atomic-file.js";
+import { isErrnoCode } from "./utils/errno.js";
+import { withHeldFileLock, type HeldFileLockOptions } from "./utils/serialize-mutations.js";
+import { ensureContainedSpillDir, listContainedSpillFiles } from "./utils/path-containment.js";
 import type { SearchDegradation } from "./search/port.js";
 import type {
   IdentityInjectionMode,
@@ -21,6 +24,10 @@ export interface LastRecallBudgetSummary {
   truncated?: boolean;
   includedSections?: string[];
   omittedSections?: string[];
+  includedMemoryIds?: string[];
+  includedMemoryPaths?: string[];
+  includedMemoryNamespaces?: Array<string | undefined>;
+  omittedMemoryIds?: string[];
 }
 
 export interface LastRecallSnapshot {
@@ -40,6 +47,7 @@ export interface LastRecallSnapshot {
   budgetsApplied?: LastRecallBudgetSummary;
   latencyMs?: number;
   resultPaths?: string[];
+  resultNamespaces?: Array<string | undefined>;
   policyVersion?: string;
   identityInjectionMode?: IdentityInjectionMode | "none";
   identityInjectedChars?: number;
@@ -227,16 +235,67 @@ const DEFAULT_TIER_MIGRATION_STATUS: TierMigrationStatusSnapshot = {
   },
 };
 
+/**
+ * Suffix marking an impression spill CLAIMED for commit but not yet deleted.
+ * The crash-safe drain renames `<uuid>.jsonl` -> `<uuid>.jsonl.claimed` before
+ * committing its rows, mirroring the lifecycle ledger's claim/commit protocol
+ * (#2033).
+ */
+const CLAIMED_IMPRESSION_SPILL_SUFFIX = ".claimed";
+
+/**
+ * Outcome of {@link LastRecallStore.drainPendingImpressions}. Distinguishes a
+ * completed drain (or nothing pending) from a DEFERRED drain that left durable
+ * spills in the offline-sync-EXCLUDED queue, so the caller can tell the
+ * data-safe cases apart from the one where an offline-sync snapshot would
+ * silently omit recorded impressions (#2033).
+ */
+export interface DrainPendingImpressionsResult {
+  /** Pending spill rows were folded into the synced active file this call. */
+  folded: boolean;
+  /**
+   * Durable spill rows remain in the offline-sync-EXCLUDED pending queue after
+   * this drain: the rotation lock could not be acquired, a spill could not be
+   * claimed, or the drain-only fold left rows that would only fit by rotating
+   * into a sync-excluded archive. A snapshot taken now would silently omit them
+   * (#2033): the caller MUST treat the active file as INCOMPLETE and retry or
+   * abort the snapshot rather than report success. `false` means the active file
+   * is complete — every pending row was folded or nothing was pending.
+   */
+  pendingDeferred: boolean;
+}
+
 export class LastRecallStore {
   private readonly statePath: string;
   private readonly impressionsPath: string;
   private readonly writeStateFile: StateFileWriter;
+  private readonly impressionsRotateBytes: number;
+  private readonly impressionsRotateKeep: number;
+  // Lock-acquisition timing for the cross-process impression rotation lock.
+  // Only tests set this (small maxWaitMs) to force the acquisition-timeout
+  // (`acquired=false`) branch deterministically; production uses the defaults.
+  private readonly impressionsLockOptions?: Pick<HeldFileLockOptions, "maxWaitMs" | "pollMs">;
   private state: LastRecallState = {};
   private stateWriteChain: Promise<void> = Promise.resolve();
-
-  constructor(memoryDir: string, options: { writeStateFile?: StateFileWriter } = {}) {
+  // Serializes the rotate-then-append critical section so concurrent record()
+  // calls never interleave rotation renames with each other's appends (#1910).
+  private impressionsWriteChain: Promise<void> = Promise.resolve();
+
+  constructor(
+    memoryDir: string,
+    options: {
+      writeStateFile?: StateFileWriter;
+      impressionsRotateBytes?: number;
+      impressionsRotateKeep?: number;
+      impressionsLockOptions?: Pick<HeldFileLockOptions, "maxWaitMs" | "pollMs">;
+    } = {},
+  ) {
     this.statePath = path.join(memoryDir, "state", "last_recall.json");
     this.impressionsPath = path.join(memoryDir, "state", "recall_impressions.jsonl");
+    // 0 disables rotation (never coerced to a default); keep floors at 1.
+    this.impressionsRotateBytes = Math.max(0, Math.floor(options.impressionsRotateBytes ?? 0));
+    this.impressionsRotateKeep = Math.max(1, Math.floor(options.impressionsRotateKeep ?? 5));
+    this.impressionsLockOptions = options.impressionsLockOptions;
     this.writeStateFile =
       options.writeStateFile ??
       (async (filePath, content) => {
@@ -292,6 +351,7 @@ export class LastRecallStore {
     budgetsApplied?: LastRecallBudgetSummary;
     latencyMs?: number;
     resultPaths?: string[];
+    resultNamespaces?: Array<string | undefined>;
     policyVersion?: string;
     appendImpression?: boolean;
     identityInjection?: {
@@ -318,8 +378,8 @@ export class LastRecallStore {
 
     // Build the snapshot from opts, then deep-copy it via
     // cloneLastRecallSnapshot so caller arrays/objects passed in
-    // `opts` cannot retain a live reference to the persisted state
-    // and tear it after record() returns.
+    // `opts` cannot retain a live reference to the persisted state and
+    // tear it after `record()` returns.
     const liveSnapshot: LastRecallSnapshot = {
       sessionKey: opts.sessionKey,
       recordedAt: now,
@@ -338,6 +398,7 @@ export class LastRecallStore {
       budgetsApplied: opts.budgetsApplied,
       latencyMs: opts.latencyMs,
       resultPaths: opts.resultPaths,
+      resultNamespaces: opts.resultNamespaces,
       policyVersion: opts.policyVersion,
       identityInjectionMode: opts.identityInjection?.mode,
       identityInjectedChars: opts.identityInjection?.injectedChars,
@@ -373,15 +434,353 @@ export class LastRecallStore {
     }
 
     if (opts.appendImpression !== false) {
+      const line = JSON.stringify(snapshot) + "\n";
+      // Chain onto the previous impression write so rotation + append run as one
+      // serialized critical section — concurrent record() calls can never
+      // interleave a rename with another call's append (#1910). `.catch` before
+      // the link keeps a prior failure from poisoning the chain.
+      const chained = this.impressionsWriteChain
+        .catch(() => {})
+        .then(() => this.appendImpressionSerialized(line));
+      this.impressionsWriteChain = chained;
       try {
-        await mkdir(path.dirname(this.impressionsPath), { recursive: true });
-        await appendFile(this.impressionsPath, JSON.stringify(snapshot) + "\n", "utf-8");
+        await chained;
       } catch (err) {
         log.debug(`recall impressions append failed: ${err}`);
       }
     }
   }
 
+  /**
+   * Drain pending spills, rotate (best-effort, sized to the projected payload),
+   * then append one impression line under ONE shared cross-process lock (#1910,
+   * #2033). Both the archive-shift renames AND the append run inside the same
+   * held lock so a peer process holding the lock can never rename the active
+   * inode to `.1` between our open and write — which would land the impression in
+   * a rotated archive that offline-sync excludes, silently dropping it from
+   * active-state/sync consumers.
+   *
+   * On lock-acquisition timeout the impression is NEVER written to the active
+   * file unlocked (#2033): a peer holding the lock is likely mid-rotation, so an
+   * unlocked append can land in the inode it is about to rename to `.1` — the
+   * exact silent-drop this lock prevents. Instead the row is spilled to a
+   * durable node-local per-event file under `<impressions>.pending.d/` and
+   * folded back into the active file by the next append that DOES hold the lock,
+   * preserving durability without racing rotation. Rotation failure under the
+   * held lock is logged and the append still runs. Callers invoke this only
+   * through `impressionsWriteChain` so the sequence is serialized in-process.
+   */
+  private get impressionsPendingDir(): string {
+    return `${this.impressionsPath}.pending.d`;
+  }
+
+  private async spillImpression(line: string): Promise<void> {
+    await ensureContainedSpillDir(this.impressionsPendingDir);
+    // Write to a temp name the drain's `.jsonl` lister ignores, then atomically
+    // rename to the final `<uuid>.jsonl` (#2033). writeFile creates the path
+    // before its bytes are durable, so writing straight to the final name lets a
+    // concurrent lock holder's drain read/rename/delete a partial spill and
+    // append malformed JSON — permanently losing that impression. The temp lives
+    // in the SAME contained dir, so the rename is atomic and stays symlink-safe.
+    const id = randomUUID();
+    const tempPath = path.join(this.impressionsPendingDir, `${id}.jsonl.tmp`);
+    const finalPath = path.join(this.impressionsPendingDir, `${id}.jsonl`);
+    await writeFile(tempPath, line, "utf-8");
+    await rename(tempPath, finalPath);
+  }
+
+  /**
+   * Recover a claim a crashed drain left mid-flight (#2033): the crash-safe
+   * drain renames `<uuid>.jsonl` -> `<uuid>.jsonl.claimed` BEFORE committing, so
+   * a `.claimed` file with no matching commit is the ONLY durable copy of those
+   * rows. Rename each orphan back to `<uuid>.jsonl` so it re-enters the normal
+   * collect+claim+commit flow. MUST run under the held rotation lock, before
+   * collecting. listContainedSpillFiles skips symlinked/escaping entries.
+   */
+  private async recoverOrphanedImpressionClaims(): Promise<void> {
+    const orphans = await listContainedSpillFiles(
+      this.impressionsPendingDir,
+      `.jsonl${CLAIMED_IMPRESSION_SPILL_SUFFIX}`,
+    );
+    for (const claimedPath of orphans) {
+      const original = claimedPath.slice(0, -CLAIMED_IMPRESSION_SPILL_SUFFIX.length);
+      await rename(claimedPath, original).catch(() => undefined);
+    }
+  }
+
+  /**
+   * Recover a crash-orphaned spill temp after its write completed but before
+   * the atomic `.tmp` → `.jsonl` rename. Valid JSONL temps are promoted back
+   * into the normal queue; incomplete temps remain pending and keep offline sync
+   * deferred rather than being discarded.
+   */
+  private async recoverOrphanedImpressionTemps(): Promise<void> {
+    const temps = await listContainedSpillFiles(this.impressionsPendingDir, ".jsonl.tmp");
+    for (const tempPath of temps) {
+      let content: string;
+      try {
+        content = await readFile(tempPath, "utf-8");
+        for (const line of content.split("\n")) {
+          if (line.trim().length > 0) JSON.parse(line);
+        }
+      } catch {
+        continue;
+      }
+      const finalPath = tempPath.slice(0, -".tmp".length);
+      await rename(tempPath, finalPath).catch(() => undefined);
+    }
+  }
+
+  private async appendImpressionSerialized(line: string): Promise<void> {
+    await mkdir(path.dirname(this.impressionsPath), { recursive: true });
+    // Every active-file write goes under the shared cross-process rotation lock,
+    // even when THIS writer's local rotation is disabled (`impressionsRotateBytes
+    // <= 0`). A peer process with rotation enabled shares this lock and may
+    // rename the active inode to `.1` at any moment, so an unlocked append could
+    // land in an offline-sync-excluded archive and be silently dropped (#2033).
+    // Only the local rotate DECISION is disabled — the bounded fold skips
+    // rotation when this writer's limit is 0 — the lock itself is not.
+    const lockPath = `${this.impressionsPath}.lock`;
+    await withHeldFileLock(
+      lockPath,
+      { staleMs: 30_000, ...this.impressionsLockOptions },
+      async (acquired) => {
+        // withHeldFileLock falls back to task(false) when it cannot acquire the
+        // lock within the budget. Do NOT append to the active file unlocked: a
+        // peer holding the lock may be mid-rotation, so an unlocked append could
+        // land in the inode it renames to `.1` (an offline-sync-excluded
+        // archive), silently dropping the impression. Spill it to the durable
+        // pending queue instead — the next lock holder folds it back in (#2033).
+        if (!acquired) {
+          log.debug("recall impressions rotation lock not acquired; spilling impression to the durable pending queue");
+          await this.spillImpression(line);
+          return;
+        }
+        await this.foldPendingImpressionsAndAppend(line);
+      },
+    );
+  }
+
+  /**
+   * Fold any durable pending recall-impression spills into the synced active
+   * `recall_impressions.jsonl` WITHOUT recording a new impression (#2033).
+   *
+   * The pending spill dir is offline-sync-excluded, so an impression that a
+   * lock-timed-out `record()` spilled there is absent from an offline-sync
+   * snapshot and lost if this node is discarded before the next `record()`
+   * folds it back. Offline sync calls this before building a snapshot so a
+   * recorded impression always reaches the remote. Fast no-op — no lock and no
+   * dir/lock-file creation — when nothing is pending.
+   *
+   * Returns a {@link DrainPendingImpressionsResult}: `folded` is true when rows
+   * were folded into the active file; `pendingDeferred` is true when durable
+   * spills still remain in the offline-sync-EXCLUDED queue after this call —
+   * because the rotation lock could not be acquired, a spill could not be
+   * claimed, or the drain-only fold deferred rows that would only fit by
+   * rotating into a sync-excluded archive (#2033). A snapshot taken while
+   * `pendingDeferred` is true would silently drop those rows, so the caller MUST
+   * NOT report a clean snapshot — it either retries or aborts.
+   */
+  async drainPendingImpressions(): Promise<DrainPendingImpressionsResult> {
+    if (!(await this.pendingSpillsRemain())) {
+      return { folded: false, pendingDeferred: false };
+    }
+    const lockPath = `${this.impressionsPath}.lock`;
+    let folded = false;
+    let acquiredLock = false;
+    let spillsRemain = false;
+    await withHeldFileLock(
+      lockPath,
+      { staleMs: 30_000, ...this.impressionsLockOptions },
+      async (acquired) => {
+        // A peer holds the lock (mid-rotation or mid-append). We MUST NOT touch
+        // the active file unlocked (#2033), so the spills stay in the
+        // offline-sync-EXCLUDED pending queue. Report the drain as DEFERRED
+        // (pendingDeferred=true) so the caller does not build a snapshot that
+        // silently omits them; the next lock holder — or a caller retry — folds
+        // them in.
+        if (!acquired) return;
+        acquiredLock = true;
+        folded = await this.foldPendingImpressionsAndAppend(null);
+        // Even WITH the lock, a spill the fold could not claim or a `.claimed`
+        // orphan a failed commit left behind keeps durable rows in the excluded
+        // queue. Treat that as an incomplete drain so the caller retries/aborts
+        // rather than snapshotting an active file that omits them (#2033).
+        spillsRemain = await this.pendingSpillsRemain();
+      },
+    );
+    return { folded, pendingDeferred: !acquiredLock || spillsRemain };
+  }
+
+  /**
+   * Claim/commit fold of durable pending impression spills into the active file
+   * under the held rotation lock (#2033). `line` is a new impression to append
+   * after the drained rows, or null for a drain-only pre-sync fold.
+   *
+   * Each spill is folded as an INDEPENDENT claim -> append -> finalize unit
+   * rather than materializing the whole queue as one joined payload and
+   * splitting it in a single synchronous pass (#2033). A multi-hundred-MB drain
+   * done as one join/split blocks the event loop long enough for
+   * `withHeldFileLock`'s ownership-refresh timer to miss its window, so a peer
+   * stale-breaks the 30s lock and rotates/appends concurrently — defeating the
+   * rotation lock this drain relies on. The per-file `await`s keep the payload
+   * bounded to one spill at a time and let the refresh timer fire between files.
+   *
+   * The DRAIN-ONLY path (`line === null`, run before an offline-sync snapshot)
+   * NEVER rotates a drained row into an archive. `recall_impressions.jsonl.*`
+   * is offline-sync-EXCLUDED, so rotating a just-drained row into `.1`/`.2` and
+   * then deleting its spill would leave that row only in a path sync omits —
+   * silently lost when the node is discarded after a "successful" sync (#2033).
+   * It folds only the rows that fit the synced active file under the cap and
+   * leaves the rest durable in the pending queue; {@link drainPendingImpressions}
+   * then reports the drain DEFERRED so the caller retries or aborts. The
+   * record() path (`line !== null`) keeps rotating normally — its archives stay
+   * on a live node that is not being discarded.
+   *
+   * Recovers any crash-orphaned claim first. Each impression carries a unique
+   * writeNonce, so a crash-recovered re-commit is a collapsible duplicate, never
+   * a lost row. listContainedSpillFiles rejects symlinked/escaping entries; a
+   * spill that cannot be claimed is skipped this pass. An append failure rolls
+   * that spill's claim back so the row retries on the next lock holder's drain.
+   * Returns true when the active file was written. MUST run under the held
+   * rotation lock.
+   */
+  private async foldPendingImpressionsAndAppend(line: string | null): Promise<boolean> {
+    await this.recoverOrphanedImpressionTemps();
+    await this.recoverOrphanedImpressionClaims();
+    const spillFiles = await listContainedSpillFiles(this.impressionsPendingDir);
+    const drainOnly = line === null;
+    const cap = this.impressionsRotateBytes;
+    let activeSize = 0;
+    if (cap > 0) {
+      try {
+        activeSize = (await stat(this.impressionsPath)).size;
+      } catch (err) {
+        if (!isErrnoCode(err, "ENOENT")) throw err;
+      }
+    }
+    let wrote = false;
+    for (const filePath of spillFiles) {
+      const content = await readFile(filePath, "utf-8");
+      const unit = content.length === 0 || content.endsWith("\n") ? content : `${content}\n`;
+      const unitBytes = Buffer.byteLength(unit, "utf-8");
+      // Drain-only fold must never push a drained row into a sync-excluded
+      // archive (#2033). Once the active file cannot take this spill under the
+      // cap without rotating, stop: the remaining spills stay durable in the
+      // pending queue and the drain reports itself deferred. A lone spill larger
+      // than the cap on an empty active file is still folded whole (it cannot be
+      // split without corrupting JSONL) so it reaches sync rather than stranding.
+      if (drainOnly && cap > 0 && activeSize > 0 && activeSize + unitBytes >= cap) break;
+      const claimedPath = `${filePath}${CLAIMED_IMPRESSION_SPILL_SUFFIX}`;
+      try {
+        await rename(filePath, claimedPath);
+      } catch {
+        continue; // could not claim → do not commit; a later drain retries it.
+      }
+      try {
+        activeSize = await this.appendImpressionUnit(unit, activeSize, drainOnly);
+      } catch (err) {
+        // Commit failed: roll this spill's claim back so it retries on the next
+        // lock holder's drain instead of being lost.
+        await rename(claimedPath, filePath).catch(() => undefined);
+        throw err;
+      }
+      // Commit is durable: delete the claimed file. A delete that fails leaves a
+      // `.claimed` orphan the next drain recovers and re-commits — a
+      // writeNonce-collapsible duplicate, never a lost row (#2033).
+      await unlink(claimedPath).catch(() => undefined);
+      wrote = true;
+    }
+    if (line !== null) {
+      activeSize = await this.appendImpressionUnit(line, activeSize, false);
+      wrote = true;
+    }
+    return wrote;
+  }
+
+  /**
+   * Append one whole JSONL unit (a claimed spill's rows or a fresh impression
+   * line) to the active impressions file, returning the resulting active-file
+   * byte size. On the rotating (record) path, when the unit would push the
+   * current active generation to/over `impressionsRotateBytes`, the full active
+   * file is rotated aside FIRST so no generation exceeds the cap (#2033); a lone
+   * unit larger than the cap on an empty active file is written whole because
+   * splitting a JSONL row would corrupt it. `drainOnly` and disabled rotation
+   * (`impressionsRotateBytes <= 0`) both skip rotation — drain-only defers its
+   * overflow in {@link foldPendingImpressionsAndAppend} rather than rotating into
+   * a sync-excluded archive. A rotation error is logged and the append still
+   * runs so a row is never lost; an append failure propagates so the caller
+   * rolls its claim back. MUST run under the held rotation lock.
+   */
+  private async appendImpressionUnit(
+    unit: string,
+    activeSize: number,
+    drainOnly: boolean,
+  ): Promise<number> {
+    const cap = this.impressionsRotateBytes;
+    const unitBytes = Buffer.byteLength(unit, "utf-8");
+    if (cap > 0 && !drainOnly && activeSize > 0 && activeSize + unitBytes >= cap) {
+      try {
+        await this.shiftImpressionArchives();
+        activeSize = 0;
+      } catch (err) {
+        log.debug(`recall impressions rotation failed (append preserved): ${err}`);
+      }
+    }
+    await appendFile(this.impressionsPath, unit, "utf-8");
+    return activeSize + unitBytes;
+  }
+
+  /**
+   * True when any live `<uuid>.jsonl` spill, crash-orphaned `.claimed` file,
+   * or crash-orphaned `.jsonl.tmp` spill remains in the offline-sync-EXCLUDED
+   * pending queue (#2033). After a drain acquires the rotation lock, a spill
+   * the fold could not claim (rename race / read-only dir) or a `.claimed`
+   * orphan a failed commit left behind means durable rows are STILL excluded
+   * from a snapshot — the drain must report itself INCOMPLETE so the caller
+   * retries or aborts rather than snapshotting an active file that omits them.
+   */
+  private async pendingSpillsRemain(): Promise<boolean> {
+    if ((await listContainedSpillFiles(this.impressionsPendingDir, ".jsonl.tmp")).length > 0) {
+      return true;
+    }
+    const liveSpills = await listContainedSpillFiles(this.impressionsPendingDir);
+    if (liveSpills.length > 0) return true;
+    const claimed = await listContainedSpillFiles(
+      this.impressionsPendingDir,
+      `.jsonl${CLAIMED_IMPRESSION_SPILL_SUFFIX}`,
+    );
+    return claimed.length > 0;
+  }
+
+  /**
+   * Shift `.1..N` archives down one slot and move the active file to `.1`,
+   * dropping anything beyond `keep`. Callers invoke this only while holding the
+   * cross-process rotation lock (issue #1910).
+   */
+  private async shiftImpressionArchives(): Promise<void> {
+    const keep = this.impressionsRotateKeep;
+    // Drop the archive that would fall off the end, then shift down: .(keep-1)
+    // -> .keep, ..., .1 -> .2, active -> .1.
+    try {
+      await rm(`${this.impressionsPath}.${keep}`, { force: true });
+    } catch {
+      // best-effort cleanup of the oldest archive
+    }
+    for (let i = keep - 1; i >= 1; i -= 1) {
+      try {
+        await rename(`${this.impressionsPath}.${i}`, `${this.impressionsPath}.${i + 1}`);
+      } catch (err) {
+        if (!isErrnoCode(err, "ENOENT")) throw err;
+      }
+    }
+    try {
+      await rename(this.impressionsPath, `${this.impressionsPath}.1`);
+    } catch (err) {
+      if (!isErrnoCode(err, "ENOENT")) throw err;
+    }
+  }
+
   /**
    * Attach a RecallTierExplain block to the existing snapshot for a
    * session without rewriting the entire snapshot.  Used by the

package/src/recall-timings-access.test.ts

@@ -201,6 +201,47 @@ test("recall timing records expose only the fixed telemetry schema", () => {
   });
 });
 
+test("trustStage is allowlisted and its ms is parsed (issue #1905)", () => {
+  const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-trust"));
+  recordRecallTiming(config, {
+    timestamp: new Date(0).toISOString(),
+    namespace: "default",
+    total: "10ms",
+    trustStage: "3ms",
+    recallPlan: "full",
+    queryPolicy: "general/full",
+  });
+  recordRecallTiming(config, {
+    timestamp: new Date(1).toISOString(),
+    namespace: "default",
+    total: "10ms",
+    trustStage: "2ms-cache",
+    recallPlan: "full",
+    queryPolicy: "general/full",
+  });
+  const records = getRecallTimings(config);
+  // newest-first ordering
+  assert.equal(records[0]?.timingsMs.trustStage, 2, "\"2ms-cache\" parses to numeric 2");
+  assert.equal(records[1]?.timingsMs.trustStage, 3, "\"3ms\" parses to numeric 3");
+});
+
+test("queueWaitMs is allowlisted and parsed while a non-allowlisted sibling is dropped (issue #1906)", () => {
+  const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-queuewait"));
+  recordRecallTiming(config, {
+    timestamp: new Date(0).toISOString(),
+    namespace: "default",
+    total: "42ms",
+    queueWaitMs: "12ms",
+    queueWaitDebug: "99ms",
+    recallPlan: "full",
+    queryPolicy: "general/full",
+  });
+  const record = getRecallTimings(config)[0];
+  assert.ok(record);
+  assert.equal(record.timingsMs.queueWaitMs, 12);
+  assert.equal("queueWaitDebug" in record.timingsMs, false);
+});
+
 test("phases that did not run are omitted while measured zeros survive", () => {
   const config = makeConfig(path.join(os.tmpdir(), "remnic-recall-timing-phases"));
   recordRecallTiming(config, {

package/src/recall-timings.ts

@@ -44,6 +44,8 @@ const TIMING_FIELD_ALLOWLIST = [
   "compounding",
   "graphShadow",
   "qmdPost",
+  "trustStage",
+  "queueWaitMs",
 ] as const;
 // The ring buffer is process-local: a daemon restart or haproxy failover to
 // the other backend starts an empty history. processStartedAt lets consumers
@@ -85,6 +87,20 @@ function sanitizeRecallTiming(
   };
 }
 
+/**
+ * Build the recall `timings` map, seeding the additive queue-wait phase (issue
+ * #1906) when the caller measured a real per-principal slot / single-flight
+ * wait. `queueWaitMs` 0/undefined/non-finite omits the phase so recall-timings
+ * stays byte-identical to the uncontended pre-#1906 path.
+ */
+export function foldQueueWaitTiming(queueWaitMs?: number): Record<string, string> {
+  const timings: Record<string, string> = {};
+  if (typeof queueWaitMs === "number" && Number.isFinite(queueWaitMs) && queueWaitMs >= 0) {
+    timings.queueWaitMs = `${queueWaitMs}ms`;
+  }
+  return timings;
+}
+
 export function recordRecallTiming(
   config: PluginConfig,
   input: Record<string, unknown>,

package/src/relay/mission-access.test.ts

@@ -367,25 +367,25 @@ test("Relay write quota releases the exact reservation when timestamps collide",
     adminConsoleEnabled: false,
   });
   const internals = server as unknown as {
-    writeRequestSlots: Array<{ readonly recordedAt: number }>;
+    writeLimiter: { slotsFor(principal?: string): ReadonlyArray<{ readonly recordedAt: number }> };
     reserveWriteRateLimitSlot: () => () => void;
   };
   const originalNow = Date.now;
   try {
     Date.now = () => 1_754_000_000_000;
     const releaseCommitted = internals.reserveWriteRateLimitSlot();
-    const committedSlot = internals.writeRequestSlots[0];
+    const committedSlot = internals.writeLimiter.slotsFor()[0];
     const releaseFailed = internals.reserveWriteRateLimitSlot();
-    const failedSlot = internals.writeRequestSlots[1];
+    const failedSlot = internals.writeLimiter.slotsFor()[1];
     assert.ok(committedSlot);
     assert.ok(failedSlot);
     assert.notEqual(committedSlot, failedSlot);
 
     releaseFailed();
-    assert.deepEqual(internals.writeRequestSlots, [committedSlot]);
+    assert.deepEqual(internals.writeLimiter.slotsFor(), [committedSlot]);
 
     releaseCommitted();
-    assert.deepEqual(internals.writeRequestSlots, []);
+    assert.deepEqual(internals.writeLimiter.slotsFor(), []);
   } finally {
     Date.now = originalNow;
   }

package/src/rerank.ts

@@ -173,3 +173,25 @@ export async function rerankLocalOrNoop(opts: {
 
   return rankedIds;
 }
+
+/**
+ * Reorder `results` to match `rankedKeys` (a ranker's output order), keying by
+ * `keyOf`. Items not present in `rankedKeys` are appended in their original
+ * order. `keyOf` must return a stable identity (e.g. namespace-qualified) so
+ * same-relative-path items across namespaces are not conflated.
+ */
+export function reorderByRankedKeys<T>(
+  results: readonly T[],
+  rankedKeys: readonly string[],
+  keyOf: (item: T) => string,
+): T[] {
+  const byKey = new Map(results.map((r) => [keyOf(r), r]));
+  const reordered: T[] = [];
+  for (const key of rankedKeys) {
+    const it = byKey.get(key);
+    if (it) reordered.push(it);
+  }
+  const rankedSet = new Set(rankedKeys);
+  for (const r of results) if (!rankedSet.has(keyOf(r))) reordered.push(r);
+  return reordered;
+}