@remnic/core 10.0.29 → 10.0.31

package/src/access-observe-write-surface.ts

@@ -30,6 +30,7 @@ import {
   NamespaceNotWritableError,
 } from "./access-service.js";
 import { applyBriefingLocationContext, reapplyCategoryCoercion, splitCanonicalWriteRequest } from "./access-observe-write-category.js";
+import { abortError } from "./abort-error.js";
 import { extractionForceFlush } from "./access-extraction-force-flush.js";
 import { ObserveTranscriptPersister, observeTranscriptSessionKey } from "./access-observe-transcript.js";
 import { FileCalendarSource, buildBriefing, parseBriefingFocus, parseBriefingWindow } from "./briefing.js";
@@ -841,19 +842,12 @@ export class AccessObserveWriteSurface {
         // (the #1495 drift). Passing writeNamespaceOverride makes the extraction
         // target deterministic and identical to LCM/objective-state (rule 39).
         //
-        // Pin WHENEVER namespaces are enabled, not only when writeNamespace differs
-        // from the default store (#1505 round 3, codex "Pin default-store extraction
-        // writes too"). For an unqualified/no-overlay observe by a principal that
-        // HAS a self namespace, writeNamespace is `config.defaultNamespace` but an
-        // unpinned `runExtraction` would fall back to
-        // `defaultNamespaceForPrincipal(principal)` = the SELF namespace — diverging
-        // from where LCM/objective-state/response wrote (`default`). Pinning the
-        // resolved writeNamespace forces all side effects onto the one scope-plan
-        // namespace. When namespaces are DISABLED the router collapses every
-        // namespace to one store, so leaving the override undefined preserves the
-        // existing single-store routing byte-for-byte.
-        const writeNamespaceOverride =
-          resolveNamespaceCapabilities(this.deps.orchestrator.config).namespaces === true ? writeNamespace : undefined;
+        // #3251: ALWAYS pin the already-resolved writeNamespace — including
+        // when namespaces are OFF (where the resolved value IS the collapsed
+        // default store). Accepted captures need an original actual storage
+        // target so a later namespaces-ON restart cannot retarget them to a
+        // newly-active principal/project namespace.
+        const writeNamespaceOverride = writeNamespace;
         // Pin provenance PRINCIPAL to the scope plan's resolved principal (#1505
         // thread 1). The scope plan already applied auth precedence
         // (authenticatedPrincipal/principalOverride > resolvePrincipal(original
@@ -863,38 +857,39 @@ export class AccessObserveWriteSurface {
         // existing behavior.
         const principalOverride =
           typeof scope.principal === "string" && scope.principal.length > 0 ? scope.principal : undefined;
-        // Fire-and-forget: queue extraction in the background so the HTTP
-        // response returns immediately; extraction LLM calls can take minutes.
-        //
-        // Backpressure: the orchestrator's queue limits extraction concurrency
-        // (one per session); this just decouples the HTTP response from the drain.
-        if (!observePreparation.isCancelled()) {
-          try {
-            const observeAbortController = new AbortController();
-            const extractionPromise = this.deps.orchestrator.ingestReplayBatch(turns, {
-              archiveLcm: false,
-              writeNamespaceOverride,
-              principalOverride,
-              scopeProfileWritePlan: scope.scopeProfilePlan,
-              abortSignal: observeAbortController.signal,
-              ...(typeof request.authenticatedPrincipal === "string" && request.authenticatedPrincipal.trim().length > 0
-                ? { sessionOwnerPrincipal: request.authenticatedPrincipal.trim() }
-                : {}),
-            });
-            extractionPromise.catch((err) => {
+        // Acceptance waits for durable capture; cancellation only stops extraction.
+        const observeAbortController = new AbortController();
+        const preparationCancelled = observePreparation.isCancelled();
+        if (preparationCancelled) {
+          // Preserve capture even when its preparation was cancelled.
+          observeAbortController.abort(abortError("observe preparation cancelled"));
+        }
+        const trackedExtractionKey = this.pendingObserveExtractions.key(request.sessionKey, scope.principal, writeNamespace);
+        const ingestOptions: NonNullable<Parameters<Orchestrator["ingestReplayBatch"]>[1]> = {
+          archiveLcm: false,
+          extractionMode: "buffered",
+          writeNamespaceOverride,
+          principalOverride,
+          scopeProfileWritePlan: scope.scopeProfilePlan,
+          // Re-authorize current policy against the original project binding.
+          codingContext: this.deps.orchestrator.getCodingContextForSession(request.sessionKey) ?? undefined,
+          abortSignal: observeAbortController.signal,
+        };
+        if (request.authenticatedPrincipal !== undefined && request.authenticatedPrincipal.trim().length > 0) {
+          ingestOptions.sessionOwnerPrincipal = request.authenticatedPrincipal.trim();
+        }
+        // Track every job immediately, but don't retain cancelled preparation barriers.
+        if (!preparationCancelled) {
+          ingestOptions.onExtractionQueued = (extraction: Promise<void>) => {
+            extraction.catch((err) => {
               log.error(`access-observe background extraction failed: ${err}`);
             });
-            this.pendingObserveExtractions.track(
-              this.pendingObserveExtractions.key(request.sessionKey, scope.principal, writeNamespace),
-              extractionPromise,
-              observeAbortController
-            );
+            this.pendingObserveExtractions.track(trackedExtractionKey, extraction, observeAbortController);
             extractionQueued = true;
-          } catch (err) {
-            // Synchronous enqueue failure (e.g. orchestrator disposed)
-            log.error(`access-observe extraction enqueue failed: ${err}`);
-          }
+          };
         }
+        await this.deps.orchestrator.ingestReplayBatch(turns, ingestOptions);
+        // A native trigger may leave the durable capture pending rather than queued.
       }
 
       log.info(

package/src/access-service-observe-lcm-parity.test.ts

@@ -512,7 +512,11 @@ test("#2128: force-flush waits for a pending observe extraction", async () => {
   const pendingExtraction = new Promise<void>((resolve) => {
     resolveExtraction = resolve;
   });
-  probe.orch.ingestReplayBatch = async () => pendingExtraction;
+  // #3252 contract: observe awaits only durable capture; the actual extraction
+  // promise is handed to onExtractionQueued and tracked for force-flush.
+  probe.orch.ingestReplayBatch = async (_turns, options = {}) => {
+    options.onExtractionQueued?.(pendingExtraction);
+  };
   const service = new EngramAccessService(probe.orch);
   const sessionKey = "pending-observe-extraction";
 
@@ -547,10 +551,11 @@ test("#2128: pending extraction barriers stay isolated by authenticated principa
     ],
   } as Partial<PluginConfig>);
   const pending = new Map<string, () => void>();
-  probe.orch.ingestReplayBatch = async (_turns, options = {}) =>
-    new Promise<void>((resolve) => {
-      pending.set(options.principalOverride ?? "", resolve);
-    });
+  probe.orch.ingestReplayBatch = async (_turns, options = {}) => {
+    const { promise, resolve } = Promise.withResolvers<void>();
+    pending.set(options.principalOverride ?? "", resolve);
+    options.onExtractionQueued?.(promise);
+  };
   const service = new EngramAccessService(probe.orch);
   const sessionKey = "shared-session";
 
@@ -595,10 +600,11 @@ test("#2128: pending extraction barriers stay isolated by authenticated principa
 test("#2128: chained observes remain behind one force-flush barrier", async () => {
   const probe = makeParityProbe({ namespacesEnabled: false } as Partial<PluginConfig>);
   const resolvers: Array<() => void> = [];
-  probe.orch.ingestReplayBatch = async () =>
-    new Promise<void>((resolve) => {
-      resolvers.push(resolve);
-    });
+  probe.orch.ingestReplayBatch = async (_turns, options = {}) => {
+    const { promise, resolve } = Promise.withResolvers<void>();
+    resolvers.push(resolve);
+    options.onExtractionQueued?.(promise);
+  };
   const service = new EngramAccessService(probe.orch);
   const sessionKey = "chained-observes";
 
@@ -628,12 +634,12 @@ test("#2128: a failed observe extraction is consumed after force-flush reports i
   const probe = makeParityProbe({ namespacesEnabled: false } as Partial<PluginConfig>);
   let rejectExtraction!: (error: Error) => void;
   let extractionAttempt = 0;
-  probe.orch.ingestReplayBatch = async () => {
+  probe.orch.ingestReplayBatch = async (_turns, options = {}) => {
     extractionAttempt += 1;
     if (extractionAttempt > 1) return;
-    return new Promise<void>((_resolve, reject) => {
-      rejectExtraction = reject;
-    });
+    const { promise, reject } = Promise.withResolvers<void>();
+    rejectExtraction = reject;
+    options.onExtractionQueued?.(promise);
   };
   const service = new EngramAccessService(probe.orch);
   const sessionKey = "failed-observe-extraction";
@@ -655,13 +661,13 @@ test("#2128: force-flush deadline cancels the pending observe extraction", async
   let extractionSignal: AbortSignal | undefined;
   probe.orch.ingestReplayBatch = async (_turns, options = {}) => {
     extractionSignal = options.abortSignal;
-    return new Promise<void>((_resolve, reject) => {
-      options.abortSignal?.addEventListener(
-        "abort",
-        () => reject(new Error("extraction cancelled")),
-        { once: true },
-      );
-    });
+    const { promise, reject } = Promise.withResolvers<void>();
+    options.abortSignal?.addEventListener(
+      "abort",
+      () => reject(new Error("extraction cancelled")),
+      { once: true },
+    );
+    options.onExtractionQueued?.(promise);
   };
   const service = new EngramAccessService(probe.orch);
   const sessionKey = "deadline-cancelled-extraction";
@@ -689,13 +695,13 @@ test("#2128: force-flush abort signal cancels the pending observe extraction", a
   let extractionSignal: AbortSignal | undefined;
   probe.orch.ingestReplayBatch = async (_turns, options = {}) => {
     extractionSignal = options.abortSignal;
-    return new Promise<void>((_resolve, reject) => {
-      options.abortSignal?.addEventListener(
-        "abort",
-        () => reject(new Error("extraction cancelled")),
-        { once: true },
-      );
-    });
+    const { promise, reject } = Promise.withResolvers<void>();
+    options.abortSignal?.addEventListener(
+      "abort",
+      () => reject(new Error("extraction cancelled")),
+      { once: true },
+    );
+    options.onExtractionQueued?.(promise);
   };
   const service = new EngramAccessService(probe.orch);
   const sessionKey = "abort-cancelled-extraction";
@@ -727,14 +733,14 @@ test("#2128: force-flush abort only cancels the resolved namespace barrier", asy
   probe.orch.ingestReplayBatch = async (_turns, options = {}) => {
     const namespace = options.writeNamespaceOverride ?? "";
     extractionSignals.set(namespace, options.abortSignal);
-    return new Promise<void>((resolve, reject) => {
-      extractionResolvers.set(namespace, resolve);
-      options.abortSignal?.addEventListener(
-        "abort",
-        () => reject(options.abortSignal?.reason ?? new Error("extraction cancelled")),
-        { once: true },
-      );
-    });
+    const { promise, resolve, reject } = Promise.withResolvers<void>();
+    extractionResolvers.set(namespace, resolve);
+    options.abortSignal?.addEventListener(
+      "abort",
+      () => reject(options.abortSignal?.reason ?? new Error("extraction cancelled")),
+      { once: true },
+    );
+    options.onExtractionQueued?.(promise);
   };
   const service = new EngramAccessService(probe.orch);
   const sessionKey = "opaque-shared-session";
@@ -960,7 +966,14 @@ test("#2128: post-flush retained cleanup stops on abort or deadline", async (t)
   assert.deepEqual(cleanupCalls[0], {
     sessionKey: "pi-geek:cleanup-abort",
     ownerPrincipal: "pi-geek",
-    options: { abortSignal: abortController.signal, deadlineMs: undefined },
+    options: {
+      abortSignal: abortController.signal,
+      deadlineMs: undefined,
+      // #3252: cleanup targets the pinned accepted-capture storage target.
+      writeNamespaceOverride: "default",
+      codingContext: undefined,
+      scopeProfileWritePlan: undefined,
+    },
   });
 
   const cleanupStartedPromise = new Promise<void>((resolve) => {
@@ -981,7 +994,14 @@ test("#2128: post-flush retained cleanup stops on abort or deadline", async (t)
   assert.deepEqual(cleanupCalls[1], {
     sessionKey: "pi-geek:cleanup-deadline",
     ownerPrincipal: "pi-geek",
-    options: { abortSignal: undefined, deadlineMs: 5 },
+    options: {
+      abortSignal: undefined,
+      deadlineMs: 5,
+      // #3252: cleanup targets the pinned accepted-capture storage target.
+      writeNamespaceOverride: "default",
+      codingContext: undefined,
+      scopeProfileWritePlan: undefined,
+    },
   });
   t.mock.timers.tick(5);
   await assert.rejects(
@@ -1086,8 +1106,10 @@ test("#2206: abort during scope resolution cancels pre-resolution observe prepar
   await assert.rejects(flush, /extraction force-flush aborted/);
   releaseObserveScope();
   const response = await observe;
+  // Cancelled preparation aborts only the extraction; the durable capture is
+  // preserved (#3252), so ingest is still called once with nothing queued.
   assert.equal(response.extractionQueued, false);
-  assert.equal(probe.extractionCalls.length, 0);
+  assert.equal(probe.extractionCalls.length, 1);
 });
 
 test("#2206: scope-resolution deadline cancels preparations by raw projectTag and cwd hints", async (t) => {
@@ -1164,8 +1186,10 @@ test("#2206: scope-resolution deadline cancels preparations by raw projectTag an
       releaseObserveScope();
     }
     const response = await observe;
+    // Cancelled preparation aborts only the extraction; the durable capture
+    // is preserved (#3252), so ingest is still called once with nothing queued.
     assert.equal(response.extractionQueued, false);
-    assert.equal(probe.extractionCalls.length, 0);
+    assert.equal(probe.extractionCalls.length, 1);
   }
 });
 

package/src/access-service-observe-scope.test.ts

@@ -649,8 +649,9 @@ test("#1495 namespacesEnabled:false ⇒ single-store behavior preserved", async
   assert.equal(res.namespace, "default");
   // No namespace prefix on the LCM key when the effective ns is the default.
   assert.equal(probe.lcmCalls[0].sessionKey, "pi-geek:abc123");
-  // No override needed when there is only one store.
-  assert.equal(probe.extractionCalls[0].writeNamespaceOverride, undefined);
+  // #3251: ALWAYS pin the resolved writeNamespace even when namespaces are
+  // disabled — the flat capture stores its original actual storage target.
+  assert.equal(probe.extractionCalls[0].writeNamespaceOverride, "default");
   assert.deepEqual(
     new Set(probe.extractionCalls[0].sessionKeys),
     new Set(["pi-geek:abc123"]),

package/src/buffer-turn-helpers.ts

@@ -64,6 +64,43 @@ export function copyBufferTurn(turn: BufferTurn): BufferTurn {
   if (typeof turn.ambientCapture === "boolean") {
     copy.ambientCapture = turn.ambientCapture;
   }
+  if (turn.sourceValidAt !== undefined) {
+    copy.sourceValidAt = turn.sourceValidAt;
+  }
+  if (turn.parts !== undefined) {
+    copy.parts = turn.parts;
+  }
+  if (turn.rawContent !== undefined) {
+    copy.rawContent = turn.rawContent;
+  }
+  if (turn.sourceFormat !== undefined) {
+    copy.sourceFormat = turn.sourceFormat;
+  }
+  // #3251 scope pins: strings copy by value; the plan object is an immutable
+  // config-derived identity passed by reference.
+  if (turn.writeNamespaceOverride !== undefined) {
+    copy.writeNamespaceOverride = turn.writeNamespaceOverride;
+  }
+  if (turn.principalOverride !== undefined) {
+    copy.principalOverride = turn.principalOverride;
+  }
+  if (turn.scopeProfileWritePlan !== undefined) {
+    copy.scopeProfileWritePlan = turn.scopeProfileWritePlan;
+  }
+  if (turn.acceptedCapture === true) {
+    copy.acceptedCapture = true;
+  }
+  const ctx = turn.codingContext;
+  if (ctx) {
+    // All four fields are identity inputs (branchScope uses branch) — copy
+    // explicitly as-is; never a spread of external input.
+    copy.codingContext = {
+      projectId: ctx.projectId,
+      branch: ctx.branch,
+      rootPath: ctx.rootPath,
+      defaultBranch: ctx.defaultBranch,
+    };
+  }
   return copy;
 }
 
@@ -81,8 +118,90 @@ export function bufferTurnsEqual(left: BufferTurn | undefined, right: BufferTurn
     left.persistProcessedFingerprint === right.persistProcessedFingerprint &&
     left.sourceConnector === right.sourceConnector &&
     left.originRole === right.originRole &&
-    left.ambientCapture === right.ambientCapture
+    left.ambientCapture === right.ambientCapture &&
+    left.sourceValidAt === right.sourceValidAt &&
+    left.acceptedCapture === right.acceptedCapture &&
+    left.codingContext?.projectId === right.codingContext?.projectId &&
+    left.codingContext?.rootPath === right.codingContext?.rootPath &&
+    left.codingContext?.branch === right.codingContext?.branch &&
+    left.codingContext?.defaultBranch === right.codingContext?.defaultBranch &&
+    // #3251 scope pins compare by identity triple, not deep plan equality: a
+    // restart-rehydrated plan must still match the pin it was accepted under.
+    left.writeNamespaceOverride === right.writeNamespaceOverride &&
+    left.principalOverride === right.principalOverride &&
+    scopeProfilePlanIdentity(left.scopeProfileWritePlan) ===
+      scopeProfilePlanIdentity(right.scopeProfileWritePlan)
+  );
+}
+
+/** Replace only the processed snapshot; callers without one retain legacy whole-slot semantics. */
+export function selectRetainedDeferredTurns(
+  existing: readonly BufferTurn[],
+  turns: BufferTurn[],
+  max: number,
+  originalTurns?: readonly BufferTurn[],
+): BufferTurn[] {
+  const others = originalTurns === undefined ? [] : existing.filter(
+    (turn) => !originalTurns.some((original) => bufferTurnsEqual(turn, original)),
   );
+  if (!Array.isArray(turns) || max <= 0) return others;
+  return [...others, ...turns.slice(-max).map(copyBufferTurn)];
+}
+
+export function hasPinnedCaptureScope(turn: BufferTurn): boolean {
+  return turn.acceptedCapture === true ||
+    turn.writeNamespaceOverride !== undefined ||
+    turn.scopeProfileWritePlan !== undefined;
+}
+
+/** Stable identity string for a pinned scope-profile plan (#3251). */
+export function scopeProfilePlanIdentity(
+  plan: BufferTurn["scopeProfileWritePlan"],
+): string {
+  if (!plan) return "";
+  return `${plan.profileId}\u0000${plan.writeLayer}\u0000${plan.writeNamespace}`;
+}
+
+export interface ExtractionScopeGroup {
+  writeNamespaceOverride?: string;
+  principalOverride?: string;
+  scopeProfileWritePlan?: BufferTurn["scopeProfileWritePlan"];
+  codingContext?: BufferTurn["codingContext"];
+  turns: BufferTurn[];
+}
+
+/**
+ * Partition extraction turns by their durable scope pins (#3251). Turns
+ * captured under different namespaces / principals / scope profiles /
+ * project bindings must never be extracted together — each group runs its
+ * own extraction under its own scope. Native (unpinned) turns form their own
+ * group and keep today's single-store routing.
+ */
+export function groupTurnsByExtractionScope(turns: BufferTurn[]): ExtractionScopeGroup[] {
+  const groups = new Map<string, ExtractionScopeGroup>();
+  for (const turn of turns) {
+    const key = [
+      turn.writeNamespaceOverride ?? "",
+      turn.principalOverride ?? "",
+      scopeProfilePlanIdentity(turn.scopeProfileWritePlan),
+      turn.codingContext?.projectId ?? "",
+      turn.codingContext?.rootPath ?? "",
+      turn.codingContext?.branch ?? "",
+      turn.codingContext?.defaultBranch ?? "",
+    ].join("\u0000");
+    const group = groups.get(key);
+    if (group) {
+      group.turns.push(turn);
+    } else {
+      const created: ExtractionScopeGroup = { turns: [turn] };
+      if (turn.writeNamespaceOverride !== undefined) created.writeNamespaceOverride = turn.writeNamespaceOverride;
+      if (turn.principalOverride !== undefined) created.principalOverride = turn.principalOverride;
+      if (turn.scopeProfileWritePlan !== undefined) created.scopeProfileWritePlan = turn.scopeProfileWritePlan;
+      if (turn.codingContext !== undefined) created.codingContext = turn.codingContext;
+      groups.set(key, created);
+    }
+  }
+  return [...groups.values()];
 }
 
 export function bufferTurnArraysEqual(

package/src/buffer-turn.ts

@@ -0,0 +1,50 @@
+/**
+ * BufferTurn — the durable extraction-buffer turn shape (#3251). Extracted to
+ * a sibling module per the size-ratchet rule (AGENTS rule 7); `types.ts`
+ * re-exports it so existing `import type { BufferTurn } from "./types.js"`
+ * callers are unchanged.
+ */
+import type { SourceConnectorProvenance } from "./types.js";
+import type { AmbientCaptureProvenance, BufferTurnOwner } from "./buffer-turn-helpers.js";
+
+export interface BufferTurn extends SourceConnectorProvenance, BufferTurnOwner, AmbientCaptureProvenance {
+  role: "user" | "assistant";
+  content: string;
+  timestamp: string;
+  sourceValidAt?: string;
+  sessionKey?: string;
+  logicalSessionKey?: string;
+  providerThreadId?: string | null;
+  turnFingerprint?: string;
+  persistProcessedFingerprint?: boolean;
+  extractionContextOnly?: boolean;
+  parts?: import("./message-parts/index.js").LcmMessagePartInput[];
+  rawContent?: unknown;
+  sourceFormat?: import("./message-parts/index.js").MessagePartSourceFormat;
+  importProvenance?: import("./bulk-import/types.js").ImportTurnProvenance;
+  /**
+   * Scope pins stamped by the access `observe` capture path (#3251) so a
+   * durably pending turn can be drained under EXACTLY the scope it was
+   * accepted under — never merged with the same raw session under a
+   * different namespace / owner / scope profile. Absent on native
+   * `processTurn` turns.
+   */
+  writeNamespaceOverride?: string;
+  principalOverride?: string;
+  scopeProfileWritePlan?: import("./namespaces/scope-profiles.js").ResolvedScopeProfilePlan | null;
+  /**
+   * Intrinsic marker for an ACCEPTED durable capture (#3251): set by the
+   * buffered capture path on every accepted turn, independent of namespace
+   * presence, so single-store accepted captures are equally protected from
+   * cap-pressure eviction.
+   */
+  acceptedCapture?: boolean;
+  /**
+   * ORIGINAL native coding context at capture time (#3251) — the identity
+   * input (projectId/branch/rootPath/defaultBranch) the scope resolver
+   * consumed, stored as-is so a drain can re-resolve CURRENT policy for the
+   * SAME project binding. Never a frozen authorization: current config is
+   * re-applied at the execution boundary.
+   */
+  codingContext?: import("./types.js").CodingContext;
+}