@bitkyc08/opencodex 2.80.0 → 2.82.0

package/src/responses/spill-store.ts

@@ -72,6 +72,7 @@ export interface ResponseSpillPayload {
    */
   providerOutputStart?: number;
   providers?: OcxProviderContinuationState;
+  unforcedStoreFalse?: boolean;
 }
 
 export interface ResponseSpillRef {
@@ -496,6 +497,7 @@ function serializedSpill(
     items: state.items,
     ...(state.providerOutputStart !== undefined ? { providerOutputStart: state.providerOutputStart } : {}),
     ...(state.providers ? { providers: state.providers } : {}),
+    ...(state.unforcedStoreFalse ? { unforcedStoreFalse: state.unforcedStoreFalse } : {}),
   };
   const serialized = JSON.stringify(payload);
   if (serialized === undefined) throw new Error("Response spill serialization failed");
@@ -549,12 +551,14 @@ function validPayload(value: unknown, responseId: string): value is ResponseSpil
   if (!value || typeof value !== "object" || Array.isArray(value)) return false;
   const payload = value as Record<string, unknown>;
   const keys = Object.keys(payload);
-  if (keys.some(key => !["version", "responseId", "createdAt", "clientThreadId", "items", "providerOutputStart", "providers"].includes(key))) return false;
+  if (keys.some(key => !["version", "responseId", "createdAt", "clientThreadId", "items", "providerOutputStart", "providers", "unforcedStoreFalse"].includes(key))) return false;
   if (payload.version !== 1 || payload.responseId !== responseId) return false;
   if (typeof payload.createdAt !== "number" || !Number.isFinite(payload.createdAt)) return false;
   if (payload.clientThreadId !== undefined
     && (typeof payload.clientThreadId !== "string" || payload.clientThreadId.trim().length === 0)) return false;
+  if (payload.unforcedStoreFalse !== undefined && typeof payload.unforcedStoreFalse !== "boolean") return false;
   if (!Array.isArray(payload.items)) return false;
+  if (payload.unforcedStoreFalse === true && payload.providerOutputStart === undefined) return false;
   // A malformed boundary must degrade to "never skip", never to a bad index: reject the
   // payload outright so materialization treats it as corrupt rather than trusting it.
   if (payload.providerOutputStart !== undefined) {
@@ -761,10 +765,23 @@ export function readResponseSpill(responseId: string, ref: ResponseSpillRef): Re
 export function deleteResponseSpill(ref: ResponseSpillRef): void {
   if (!validSpillRef(ref)) return;
   const dir = responseSpillDirectory();
+  const path = join(dir, ref.fileName);
   try {
-    unlink(join(dir, ref.fileName));
+    unlink(path);
     fsyncDirectoryBestEffort(dir);
-  } catch { /* best effort */ }
+  } catch (error) {
+    // Still best effort, but no longer invisible: a file a failed unlink left behind occupies
+    // the volume, so the accounting owner keeps pricing it until the path is gone (#6747).
+    if (!isErrno(error, "ENOENT")) spillUnlinkFailureObserver?.(path, ref.payloadBytes);
+  }
+}
+
+type SpillUnlinkFailureObserver = (path: string, bytes: number) => void;
+let spillUnlinkFailureObserver: SpillUnlinkFailureObserver | null = null;
+
+/** The owner of spill accounting registers here so a failed unlink cannot under-report disk use. */
+export function setResponseSpillUnlinkFailureObserver(next: SpillUnlinkFailureObserver | null): void {
+  spillUnlinkFailureObserver = next;
 }
 
 type SpillDirNameKind = "spill" | "temp";

package/src/responses/state/metrics.ts

@@ -0,0 +1,95 @@
+import type { StoredResponseState } from "../state";
+import type { ResponseSpillWriteStatus, ResponseSpillWriteFailureCode, ResponseSpillWriteFailureOrigin } from "./spill-failure";
+
+export interface ResponseStateMetrics {
+  count: number;
+  residentCount: number;
+  spillStubCount: number;
+  tombstoneCount: number;
+  totalBytes: number;
+  spillPayloadBytes: number;
+  largestBytes: number;
+  oldestAgeMs: number;
+  spillWrites: number;
+  spillWriteFailures: number;
+  spillWriteStatus: ResponseSpillWriteStatus;
+  spillWriteConsecutiveFailures: number;
+  spillLastWriteFailureCode: ResponseSpillWriteFailureCode | null;
+  spillLastWriteFailureOrigin: ResponseSpillWriteFailureOrigin | null;
+  spillAclRetryReturnedTimeouts: number;
+  spillAclTimeoutMemoRefusals: number;
+  /** Publications refused because the durable spill cap could not make room (cumulative). */
+  spillCapacityRefusals: number;
+  /** Spills evicted by admission to make room for a newer publication (cumulative). */
+  spillHeadroomEvictions: number;
+  spillLastWriteFailureAt: number | null;
+  spillLastWriteSuccessAt: number | null;
+  spillReadFailures: number;
+  replayScopeMismatchDrops: number;
+}
+
+export function computeResponseStateMetrics(params: {
+  states: Iterable<StoredResponseState>;
+  stateCount: number;
+  now: number;
+  totalBytes: number;
+  spillWrites: number;
+  spillWriteFailures: number;
+  spillWriteConsecutiveFailures: number;
+  spillLastFailureCode: ResponseSpillWriteFailureCode | null;
+  spillLastFailureOrigin: ResponseSpillWriteFailureOrigin | null;
+  spillLastSuccessAt: number | null;
+  spillAclRetryReturnedTimeouts: number;
+  spillAclTimeoutMemoRefusals: number;
+  spillCapacityRefusals: number;
+  spillHeadroomEvictions: number;
+  spillLastFailureAt: number | null;
+  spillReadFailures: number;
+  replayScopeMismatchDrops: number;
+}): ResponseStateMetrics {
+  let largestBytes = 0;
+  let oldestCreatedAt = params.now;
+  let residentCount = 0;
+  let spillStubCount = 0;
+  let tombstoneCount = 0;
+  let spillPayloadBytes = 0;
+  for (const state of params.states) {
+    const bytes = state.sizeBytes;
+    if (bytes > largestBytes) largestBytes = bytes;
+    if (state.createdAt < oldestCreatedAt) oldestCreatedAt = state.createdAt;
+    if (state.kind === "resident") {
+      residentCount += 1;
+    } else if (state.kind === "spill") {
+      spillStubCount += 1;
+      spillPayloadBytes += state.spill.payloadBytes;
+    } else tombstoneCount += 1;
+  }
+  return {
+    count: params.stateCount,
+    residentCount,
+    spillStubCount,
+    tombstoneCount,
+    totalBytes: params.totalBytes,
+    spillPayloadBytes,
+    largestBytes,
+    oldestAgeMs: params.stateCount > 0 ? params.now - oldestCreatedAt : 0,
+    spillWrites: params.spillWrites,
+    spillWriteFailures: params.spillWriteFailures,
+    spillWriteStatus: params.spillWriteConsecutiveFailures > 0
+      ? "degraded"
+      : params.spillLastSuccessAt !== null
+        ? "healthy"
+        : "initial",
+    spillWriteConsecutiveFailures: params.spillWriteConsecutiveFailures,
+    spillLastWriteFailureCode: params.spillLastFailureCode,
+    spillLastWriteFailureOrigin: params.spillLastFailureOrigin,
+    spillAclRetryReturnedTimeouts: params.spillAclRetryReturnedTimeouts,
+    spillAclTimeoutMemoRefusals: params.spillAclTimeoutMemoRefusals,
+    spillCapacityRefusals: params.spillCapacityRefusals,
+    spillHeadroomEvictions: params.spillHeadroomEvictions,
+    spillLastWriteFailureAt: params.spillLastFailureAt,
+    spillLastWriteSuccessAt: params.spillLastSuccessAt,
+    spillReadFailures: params.spillReadFailures,
+    replayScopeMismatchDrops: params.replayScopeMismatchDrops,
+  };
+}

package/src/responses/state/snapshot-codec.ts

@@ -24,6 +24,7 @@ interface LegacySnapshotState {
   providers?: OcxProviderContinuationState;
   conversationId?: unknown;
   cursorCheckpointUsable?: unknown;
+  unforcedStoreFalse?: unknown;
 }
 
 function isSpillRef(value: unknown): value is ResponseSpillRef {
@@ -51,6 +52,9 @@ export function loadSnapshotEntry(id: string, value: unknown, store: SnapshotLoa
       ? raw as number
       : undefined;
   };
+  // Restricted retention requires proven provider output; legacy unrestricted rows stay readable.
+  if (rec.unforcedStoreFalse === true
+    && anchorFor(rec.kind === "spill" ? Number.MAX_SAFE_INTEGER : Array.isArray(rec.items) ? rec.items.length : 0) === undefined) return;
   if (rec.kind === "spill") {
     if (!isSpillRef(rec.spill)) return;
     const base: Omit<SpilledResponseState, "sizeBytes"> = {
@@ -61,6 +65,7 @@ export function loadSnapshotEntry(id: string, value: unknown, store: SnapshotLoa
       // here; the spill payload validator re-checks it against the real array.
       ...(anchorFor(Number.MAX_SAFE_INTEGER) !== undefined ? { providerOutputStart: anchorFor(Number.MAX_SAFE_INTEGER) } : {}),
       ...(rec.providers ? { providers: rec.providers } : {}),
+      ...(rec.unforcedStoreFalse === true ? { unforcedStoreFalse: true } : {}),
       spill: rec.spill,
     };
     store.replaceMapEntry(id, { ...base, sizeBytes: store.stubSize(id, base) });
@@ -88,6 +93,7 @@ export function loadSnapshotEntry(id: string, value: unknown, store: SnapshotLoa
     items: rec.items,
     ...(anchorFor(rec.items.length) !== undefined ? { providerOutputStart: anchorFor(rec.items.length) } : {}),
     ...(providers ? { providers } : {}),
+    ...(rec.unforcedStoreFalse === true ? { unforcedStoreFalse: true } : {}),
   });
   if (!resident) {
     store.replaceMapEntry(id, store.tombstone(id, rec.createdAt));

package/src/responses/state/spill-failure.ts

@@ -1,8 +1,14 @@
 export const spillCounters = {
   writes: 0, writeFailures: 0, readFailures: 0,
   aclRetryReturnedTimeouts: 0, aclTimeoutMemoRefusals: 0,
+  capacityRefusals: 0, headroomEvictions: 0,
 };
 
+/** Test-only: zero every spill counter, including ones added later. */
+export function resetSpillCountersForTests(): void {
+  for (const key of Object.keys(spillCounters) as Array<keyof typeof spillCounters>) spillCounters[key] = 0;
+}
+
 export type ResponseSpillWriteFailureCode =
   | "EACLRETRYEXHAUSTED"
   | "ETIMEDOUT"
@@ -102,6 +108,12 @@ export function noteSpillWriteFailure(
   if (origin === "retry_returned_timeout") spillCounters.aclRetryReturnedTimeouts += 1;
   else if (origin === "timeout_memo_refusal") spillCounters.aclTimeoutMemoRefusals += 1;
 }
+
+/** A publication refused because the durable spill cap could not make room for it (#6747). */
+export function noteSpillCapacityRefusal(): void {
+  spillCounters.capacityRefusals += 1;
+  noteSpillWriteFailure(null, "ECAPACITY");
+}
 /**
  * Admission-boundary observability (test-visible). directSpills: oversized
  * candidates routed straight to durable spill without a resident stay or

package/src/responses/state/spill-queue.ts

@@ -11,12 +11,14 @@ import {
   responseSpillNow,
   type ResponseSpillPublicationControl,
   type ResponseSpillRef,
+  setResponseSpillUnlinkFailureObserver,
   writeResponseSpillDurably,
   writeResponseSpillDurablyAsync,
 } from "../spill-store";
 import { enforceAppOwnedMemoryBudget } from "../../lib/app-owned-memory";
 import {
   admissionCounters,
+  noteSpillCapacityRefusal,
   noteSpillWriteFailure,
   noteSpillWriteSuccess,
   spillAclMemoRefusalOrigin,
@@ -42,7 +44,8 @@ export interface SpillQueueStore {
   pruneResponses(): void;
   accountedResponseSpillBytes(): number;
   spillByteCap(): number;
-  enforceSpilledResponseBudget(): number;
+  /** No argument: plain cap pruning. A number (0 included): admission reclaim of that headroom. */
+  enforceSpilledResponseBudget(requiredHeadroomBytes?: number): number;
   terminalizationMaxPasses(): number;
 }
 
@@ -122,6 +125,8 @@ function chargeUnreclaimableSpillPath(path: string | null | undefined, bytes: nu
   if (!path || bytes <= 0) return;
   unreclaimableSpillPaths.set(path, bytes);
 }
+// A failed unlink of an installed or deferred spill leaves a real file too; price it the same way.
+setResponseSpillUnlinkFailureObserver(chargeUnreclaimableSpillPath);
 
 /** Drop charges for paths that have since disappeared; returns the surviving total. */
 function reconcileUnreclaimableSpillPaths(): number {
@@ -191,6 +196,7 @@ function spillPayloadForResident(candidate: ResidentResponseState): Parameters<t
     items: candidate.items,
     ...(candidate.providerOutputStart !== undefined ? { providerOutputStart: candidate.providerOutputStart } : {}),
     ...(candidate.providers ? { providers: candidate.providers } : {}),
+    ...(candidate.unforcedStoreFalse ? { unforcedStoreFalse: candidate.unforcedStoreFalse } : {}),
   };
 }
 
@@ -285,9 +291,10 @@ export function queuePendingResponseSpill(
   // against a total that is short by a whole envelope.
   const inheritedBytes = inheritedSpill?.payloadBytes ?? 0;
   if (requireStore().accountedResponseSpillBytes() + footprint + inheritedBytes > requireStore().spillByteCap()) {
-    requireStore().enforceSpilledResponseBudget();
+    // An envelope over the replay ceiling ends in EFBIG anyway; never evict a continuation for it.
+    if (footprint / 2 <= responseSpillPayloadCap()) requireStore().enforceSpilledResponseBudget(footprint + inheritedBytes);
     if (requireStore().accountedResponseSpillBytes() + footprint + inheritedBytes > requireStore().spillByteCap()) {
-      noteSpillWriteFailure(null, "ECAPACITY");
+      noteSpillCapacityRefusal();
       requireStore().replaceWithSpillFailure(id, candidate);
       requireStore().deferSupersededSpill(inheritedSpill);
       return;
@@ -420,10 +427,11 @@ function installShutdownFallbackSpill(
     // over budget. `replaceWithSpillFailure` is the same fail-closed ending the budget
     // exhaustion path uses, so replay reports `spill_failed` and the client resends.
     if (requireStore().accountedResponseSpillBytes() + supersededBytes > requireStore().spillByteCap()) {
-      requireStore().enforceSpilledResponseBudget();
+      // The footprint is already reserved above, so only the superseded generation is extra room.
+      if (footprint / 2 <= responseSpillPayloadCap()) requireStore().enforceSpilledResponseBudget(supersededBytes);
       if (requireStore().accountedResponseSpillBytes() + supersededBytes > requireStore().spillByteCap()) {
         if (requireStore().currentEntry(job.id) === candidate) {
-          noteSpillWriteFailure(null, "ECAPACITY");
+          noteSpillCapacityRefusal();
           requireStore().replaceWithSpillFailure(job.id, candidate);
           requireStore().deferSupersededSpill(job.supersededSpill);
         }

package/src/responses/state/unforced-store-false.ts

@@ -0,0 +1,55 @@
+/**
+ * Closed provenance and replay gating for unforced store:false client tool call responses.
+ *
+ * An unforced store:false response is cached exclusively to allow the client to send a matching
+ * function/custom tool output continuation on the next turn. Normal replay, text continuation, or
+ * mismatched output without a matching pending call fails replay and requires the client to
+ * supply the full conversation.
+ */
+
+export function hasPendingClientToolCall(output: readonly unknown[]): boolean {
+  return output.some(item =>
+    !!item && typeof item === "object" && clientToolOutputType((item as { type?: unknown }).type) !== undefined,
+  );
+}
+
+export function allowsUnforcedStoreFalseReplay(
+  stored: readonly unknown[],
+  providerOutputStart: number | undefined,
+  clientInput: readonly unknown[],
+  carried: number,
+): boolean {
+  if (typeof providerOutputStart !== "number" || !Number.isSafeInteger(providerOutputStart)
+    || providerOutputStart < 0 || providerOutputStart > stored.length) return false;
+  const anchor = providerOutputStart;
+  const pendingCallIds = new Map<string, string>();
+  for (const item of stored.slice(anchor)) {
+    if (item && typeof item === "object") {
+      const outputType = clientToolOutputType((item as { type?: unknown }).type);
+      const callId = (item as { call_id?: unknown }).call_id;
+      if (outputType && typeof callId === "string" && callId) pendingCallIds.set(callId, outputType);
+    }
+  }
+  for (let i = 0; i < carried; i++) {
+    const item = clientInput[i];
+    if (item && typeof item === "object") {
+      const callId = (item as { call_id?: unknown }).call_id;
+      if (typeof callId === "string" && pendingCallIds.get(callId) === (item as { type?: unknown }).type) pendingCallIds.delete(callId);
+    }
+  }
+  return clientInput.slice(carried).some(item => {
+    if (item && typeof item === "object") {
+      const callId = (item as { call_id?: unknown }).call_id;
+      return typeof callId === "string" && pendingCallIds.has(callId)
+        && pendingCallIds.get(callId) === (item as { type?: unknown }).type;
+    }
+    return false;
+  });
+}
+
+/** Closed call/output pairs supported by the client-owned continuation path. */
+function clientToolOutputType(type: unknown): string | undefined {
+  if (type === "function_call") return "function_call_output";
+  if (type === "custom_tool_call") return "custom_tool_call_output";
+  return undefined;
+}