@bitkyc08/opencodex 2.83.0 → 2.86.0

package/src/responses/compaction.ts

@@ -56,6 +56,20 @@ export const SUMMARY_PREFIX = "Another language model started to solve this prob
 
 export const OPAQUE_COMPACTION_NOTE = "[earlier conversation was compacted; the summary is stored in a format this model cannot read]";
 
+/**
+ * Prompt contract for a routed model receiving a decoded `ocx1` checkpoint as user text (#6764).
+ * It preserves existing authorization scope and permission gates; it grants no authority.
+ * Kept apart from SUMMARY_PREFIX, which the v1 retention and recovery paths match exactly.
+ */
+export const CHECKPOINT_CONTINUATION_NOTE = "This conversation was compacted automatically. A checkpoint summary from an earlier context window appears as a user message and records prior work. Treat earlier requests it reports as completed as history; do not repeat them unless a later user message asks you to. Continue an unfinished task only when it was already authorized and remains within the scope of that authorization. Do not ask for the same authorization again while it still applies. Preserve all existing permission and confirmation gates; if a decision or required permission is pending, wait for it before taking the dependent action. A checkpoint or delegation, including one from another thread, does not itself grant authority or expand the task's scope. Later user messages take priority; obey any pause, cancellation, replacement, or change of scope.";
+
+/** Append the continuation note to existing instructions once. */
+export function appendCheckpointContinuationNote(instructions: unknown): string {
+  const existing = typeof instructions === "string" ? instructions : "";
+  if (existing.includes(CHECKPOINT_CONTINUATION_NOTE)) return existing;
+  return existing.trim().length > 0 ? `${existing}\n\n${CHECKPOINT_CONTINUATION_NOTE}` : CHECKPOINT_CONTINUATION_NOTE;
+}
+
 /**
  * Item types in the compact wire family. Each carries an `encrypted_content` blob the client
  * replays verbatim on every later turn, and the minting backend verifies it is unmodified.

package/src/responses/parser.ts

@@ -15,7 +15,7 @@ import { createToolChoiceResolver, namespacedToolName } from "../types";
 import { responsesRequestSchema } from "./schema";
 import { providerMetadataFromResponsesFunctionCall } from "./provider-opaque-metadata";
 import { lookupReplayThoughtSignature } from "./thought-signature-replay";
-import { compactionItemToText, isCompactionItemType } from "./compaction";
+import { CHECKPOINT_CONTINUATION_NOTE, compactionItemToText, decodeCompactionSummary, isCompactionItemType } from "./compaction";
 import { previousResponseReplayPrefixLength } from "./state";
 import { decodeReasoningEnvelope } from "./reasoning-envelope";
 import { hasRoutedIdentity, nameRoutedIdentity } from "../adapters/identity";
@@ -66,6 +66,35 @@ function repairJsonObjectEnvelope(text: string): Record<string, unknown> | undef
   return undefined;
 }
 
+/**
+ * Decode a function call's serialized arguments the way history replay tolerates them: empty
+ * or non-JSON text becomes `{}` instead of failing every later turn, and the one bounded
+ * lost-brace corruption above is repaired. Shared with the direct Chat decoder
+ * (`src/protocols/codecs/chat-ir.ts`) so both decode the same call to the same object.
+ */
+export function functionCallArgumentsFromText(callId: string, rawArguments: string | undefined): Record<string, unknown> {
+  let args: Record<string, unknown> = {};
+  const rawArgs = rawArguments?.trim();
+  if (rawArgs) {
+    try {
+      const parsed: unknown = JSON.parse(rawArgs);
+      if (isObj(parsed)) args = parsed;
+    } catch {
+      // One observed serialization corruption loses exactly the JSON object's opening
+      // brace; the closed envelope is tight enough to repair back into a call the
+      // routed model can still see and retry, instead of replaying {} forever.
+      const repaired = repairJsonObjectEnvelope(rawArgs);
+      if (repaired === undefined) {
+        console.warn(`[parser] function_call ${callId} has non-JSON arguments; defaulting to {}`);
+      } else {
+        args = repaired;
+        console.warn(`[parser] function_call ${callId} arguments lost the JSON opening brace; repaired from history`);
+      }
+    }
+  }
+  return args;
+}
+
 
 
 function ensureAssistantPlaceholder(messages: OcxMessage[], modelId: string, now: number): OcxAssistantMessage {
@@ -193,6 +222,7 @@ export function parseRequest(
   // Remote compaction v2: the input tail carries `{type:"compaction_trigger"}` and Codex expects a
   // synthetic `{type:"compaction"}` output item (src/responses/compaction.ts). Flagged for the server.
   let compactionRequest = false;
+  let decodedCheckpoint = false;
   let contextCompactionBoundary = false;
   let continuationConversationMessageIndex: number | undefined;
 
@@ -261,6 +291,7 @@ export function parseRequest(
         if (inputIndex >= replayedInputPrefixLength) contextCompactionBoundary = true;
         const encrypted = (item as { encrypted_content?: unknown }).encrypted_content;
         if (effectiveType === "context_compaction" && typeof encrypted !== "string") continue;
+        if (typeof encrypted === "string" && decodeCompactionSummary(encrypted) !== null) decodedCheckpoint = true;
         pendingReasoning.length = 0;
         messages.push({
           role: "user",
@@ -395,25 +426,7 @@ export function parseRequest(
         const call = item as { id?: string; call_id: string; name: string; arguments?: string; namespace?: string; extra_content?: unknown };
         // Tolerate empty/non-JSON arguments (e.g. a no-arg tool call serialized as "") instead of
         // throwing — a single poisoned history item would otherwise 400 every subsequent turn.
-        let args: Record<string, unknown> = {};
-        const rawArgs = call.arguments?.trim();
-        if (rawArgs) {
-          try {
-            const parsed: unknown = JSON.parse(rawArgs);
-            if (isObj(parsed)) args = parsed;
-          } catch {
-            // One observed serialization corruption loses exactly the JSON object's opening
-            // brace; the closed envelope is tight enough to repair back into a call the
-            // routed model can still see and retry, instead of replaying {} forever.
-            const repaired = repairJsonObjectEnvelope(rawArgs);
-            if (repaired === undefined) {
-              console.warn(`[parser] function_call ${call.call_id} has non-JSON arguments; defaulting to {}`);
-            } else {
-              args = repaired;
-              console.warn(`[parser] function_call ${call.call_id} arguments lost the JSON opening brace; repaired from history`);
-            }
-          }
-        }
+        const args = functionCallArgumentsFromText(call.call_id, call.arguments);
         // Do NOT map Responses item `id` (fc_/ctc_/…) onto `thoughtSignature`. That field is
         // reserved for Gemini/Antigravity opaque thought tokens; forwarding item ids as
         // thoughtSignature 400s Antigravity (Base64 / TYPE_BYTES). Continuity for CCA comes from
@@ -586,6 +599,10 @@ export function parseRequest(
     .map(t => loadedToolNames.has(namespacedToolName(t.namespace, t.name))
       ? { ...t, loadedFromToolSearch: true }
       : t);
+  // #6764: the decoded checkpoint is ordinary user text on this route. Say in the system prompt
+  // what it is, so a retained earlier request that it reports as completed is not repeated.
+  // A compaction request is a summarizer turn and must not be told to continue.
+  if (decodedCheckpoint && !compactionRequest) systemPrompt.push(CHECKPOINT_CONTINUATION_NOTE);
   const context: OcxContext = {
     ...(systemPrompt.length > 0 ? { systemPrompt } : {}),
     messages,

package/src/responses/state.ts

@@ -710,11 +710,6 @@ export function sweepOrphanedResponseSpills(): number {
 type SnapshotWriteOutcome = "stable" | "unstable" | "failed";
 
 async function writeBoundedSnapshot(path: string, attemptLimit: number): Promise<SnapshotWriteOutcome> {
-  // Serialize writers so concurrent flush + debounce cannot race on temps / ACL (#612).
-  const previous = persistGate;
-  let release!: () => void;
-  persistGate = new Promise<void>(resolve => { release = resolve; });
-  await previous;
   try {
     for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
       const revision = stateRevision;
@@ -751,13 +746,19 @@ async function writeBoundedSnapshot(path: string, attemptLimit: number): Promise
         lastSnapshotTarget = resolveWriteTarget(path);
       }
       persistAttemptHookForTests?.();
-      if (revision === stateRevision) return "stable";
+      if (revision === stateRevision) {
+        // Cancel only the timer whose mutation this stable revision includes,
+        // before an awaiting caller's microtask can record a newer revision.
+        if (persistTimer) clearTimeout(persistTimer);
+        persistTimer = null;
+        pendingPersistPath = null;
+        drainPendingSpillUnlinks();
+        return "stable";
+      }
     }
     return "unstable";
   } catch {
     return "failed";
-  } finally {
-    release();
   }
 }
 
@@ -792,21 +793,30 @@ function schedulePersistAt(path: string, replace = false): void {
 }
 
 async function persistNow(path: string, awaitFollowUp = false): Promise<void> {
-  if (persistTimer) {
-    clearTimeout(persistTimer);
-    persistTimer = null;
-  }
-  pendingPersistPath = null;
-  const attemptLimit = awaitFollowUp ? MAX_SNAPSHOT_REWRITE_ATTEMPTS : 1;
-  let outcome = await writeBoundedSnapshot(path, attemptLimit);
-  if (outcome === "unstable" && awaitFollowUp) {
-    if (persistTimer) clearTimeout(persistTimer);
-    persistTimer = null;
+  // Include follow-up scheduling in the gate: releasing after only the disk write
+  // lets shutdown observe an empty timer before the background pass re-arms it.
+  const previous = persistGate;
+  let release!: () => void;
+  persistGate = new Promise<void>(resolve => { release = resolve; });
+  await previous;
+  try {
+    if (persistTimer) {
+      clearTimeout(persistTimer);
+      persistTimer = null;
+    }
     pendingPersistPath = null;
-    outcome = await writeBoundedSnapshot(path, attemptLimit);
+    const attemptLimit = awaitFollowUp ? MAX_SNAPSHOT_REWRITE_ATTEMPTS : 1;
+    let outcome = await writeBoundedSnapshot(path, attemptLimit);
+    if (outcome === "unstable" && awaitFollowUp) {
+      if (persistTimer) clearTimeout(persistTimer);
+      persistTimer = null;
+      pendingPersistPath = null;
+      outcome = await writeBoundedSnapshot(path, attemptLimit);
+    }
+    if (outcome === "unstable" && !awaitFollowUp) schedulePersistAt(path, true);
+  } finally {
+    release();
   }
-  if (outcome === "stable") drainPendingSpillUnlinks();
-  else if (outcome === "unstable" && !awaitFollowUp) schedulePersistAt(path, true);
 }
 
 function schedulePersist(): void {
@@ -816,15 +826,8 @@ function schedulePersist(): void {
 }
 
 async function flushResponseSnapshot(): Promise<void> {
-  if (persistTimer) {
-    await persistNow(pendingPersistPath ?? snapshotPath(), true);
-    return;
-  }
-  // No pending timer: still await any in-flight write so shutdown does not race (#612).
+  // The gate includes the background pass's timer decision, not just its write.
   await persistGate;
-  // A bounded background pass may have scheduled its same-path follow-up while
-  // this flush was waiting on the single-flight gate. Shutdown owns one awaited
-  // bounded follow-up rather than returning behind that unref'd timer.
   if (persistTimer) await persistNow(pendingPersistPath ?? snapshotPath(), true);
 }
 

package/src/routing/history/build-file.ts

@@ -0,0 +1,92 @@
+/**
+ * The private build file of a worker rebuild (#6990).
+ *
+ * The worker writes the whole index into `<index>.rebuild-<pid>-<n>` next to
+ * the published file; the serving thread swaps it in only once it is complete,
+ * so no reader ever sees a partial index.
+ */
+
+import { Database } from "bun:sqlite";
+import { readdirSync, rmSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import type { UsageLogRevision } from "../../usage/log";
+import { ingestSourceTail, recordSourceMeta, resetAndCreateSchema, setMeta } from "./ingest";
+import { HISTORY_META_KEYS } from "./schema";
+
+const BUILD_SIDE_FILES = ["", "-journal", "-wal", "-shm"] as const;
+
+export function buildFilePath(indexPath: string, pid: number, sequence: number): string {
+  return `${indexPath}.rebuild-${pid}-${sequence}`;
+}
+
+export function removeBuildFiles(buildPath: string): void {
+  for (const suffix of BUILD_SIDE_FILES) {
+    try { rmSync(`${buildPath}${suffix}`, { force: true }); } catch { /* best effort; swept on a later build */ }
+  }
+}
+
+function processIsAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM: the process exists under another user. Anything but ESRCH is treated as alive.
+    return (error as { code?: unknown } | null)?.code !== "ESRCH";
+  }
+}
+
+/**
+ * Remove build files a crashed process left beside the index. A build of this
+ * process, or of another live process, is left alone; this process removes
+ * its own builds once their worker has exited.
+ */
+export function removeAbandonedBuilds(indexPath: string): number {
+  const prefix = `${basename(indexPath)}.rebuild-`;
+  let names: string[];
+  try {
+    names = readdirSync(dirname(indexPath));
+  } catch {
+    return 0;
+  }
+  const abandoned = new Set<string>();
+  for (const name of names) {
+    if (!name.startsWith(prefix)) continue;
+    const match = /^(\d+)-(\d+)/.exec(name.slice(prefix.length));
+    if (!match) continue;
+    const pid = Number(match[1]);
+    if (pid === process.pid || processIsAlive(pid)) continue;
+    abandoned.add(`${prefix}${match[1]}-${match[2]}`);
+  }
+  for (const name of abandoned) removeBuildFiles(join(dirname(indexPath), name));
+  return abandoned.size;
+}
+
+/**
+ * Build a complete index of the ledger snapshot `revision` describes into
+ * `buildPath`, leave it in WAL mode and close it. Returns the rows ingested.
+ * A private build needs no crash durability: an interrupted one is discarded.
+ */
+export function buildIndexFile(buildPath: string, reason: string, revision: UsageLogRevision): number {
+  removeBuildFiles(buildPath);
+  const build = new Database(buildPath, { create: true, readwrite: true });
+  try {
+    build.exec("PRAGMA journal_mode = MEMORY");
+    build.exec("PRAGMA synchronous = OFF");
+    resetAndCreateSchema(build);
+    setMeta(build, HISTORY_META_KEYS.lastError, reason);
+    recordSourceMeta(build, revision);
+    const inserted = ingestSourceTail(build, revision.path, 0, {
+      expected: revision,
+      endOffset: Number(revision.size),
+    });
+    // The serving thread opens the published file in WAL mode; switching here
+    // keeps that first open from rewriting the database header.
+    build.exec("PRAGMA journal_mode = WAL");
+    build.close();
+    return inserted;
+  } catch (error) {
+    try { build.close(); } catch { /* already unusable */ }
+    removeBuildFiles(buildPath);
+    throw error;
+  }
+}