@bitkyc08/opencodex 2.58.0 → 2.60.0

package/src/responses/legacy-dotted-tool-name-repair.ts

@@ -0,0 +1,134 @@
+import { collectResponsesToolGroups } from "./tool-groups";
+import { isSchemaValidResponsesToolName, wireToolInnerName } from "./tool-name-aliases";
+import { CODE_MODE_HELPER_WIRE_NAMES, dottedToolName, namespacedToolName } from "../types";
+
+/**
+ * The one invented namespace this proxy is known to have emitted. Routed providers flatten a
+ * namespaced call as `ns.name`, and a provider that invents the grouping picks `default`; #3402,
+ * #4176 and #4412 all repaired that spelling on the way IN, each on its own path.
+ *
+ * This is deliberately one literal prefix rather than a rule. "Strip everything before the first
+ * dot" would silently rewrite a legitimate tool name that happens to contain a dot in some other
+ * provider's vocabulary, and a replayed history item is the worst possible place to guess: it names
+ * a call that already happened.
+ */
+const LEGACY_INVENTED_NAMESPACE_PREFIX = "default.";
+
+/** Item types whose `name` the upstream validates against the tool-name pattern. */
+const NAMED_CALL_ITEM_TYPES = new Set(["function_call", "custom_tool_call"]);
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+type DeclarationView = {
+  /** Wire spellings the caller's own catalog declares, including flattened namespace forms. */
+  readonly spellings: ReadonlySet<string>;
+  /** How many declared identities answer to a given inner name. */
+  readonly ownersByInnerName: ReadonlyMap<string, number>;
+};
+
+/**
+ * Read the request's own catalog. Namespace children contribute their inner name and both
+ * flattened spellings, which is what makes a declared `default` namespace distinguishable from an
+ * invented one.
+ */
+function readDeclarations(body: unknown): DeclarationView {
+  const spellings = new Set<string>();
+  const owners = new Map<string, number>();
+  const identities = new Map<string, Set<string>>();
+  const claim = (innerName: string, identity: string): void => {
+    const seen = identities.get(innerName);
+    if (seen === undefined) identities.set(innerName, new Set([identity]));
+    else seen.add(identity);
+  };
+  for (const group of collectResponsesToolGroups(body)) {
+    for (const spec of group) {
+      if (!isPlainObject(spec)) continue;
+      if (spec.type === "namespace" && Array.isArray(spec.tools)) {
+        const namespace = typeof spec.name === "string" ? spec.name : undefined;
+        if (namespace === undefined) continue;
+        for (const child of spec.tools) {
+          const inner = wireToolInnerName(child);
+          if (!inner) continue;
+          spellings.add(namespacedToolName(namespace, inner));
+          spellings.add(dottedToolName(namespace, inner));
+          claim(inner, JSON.stringify([namespace, inner]));
+        }
+        continue;
+      }
+      const name = wireToolInnerName(spec);
+      if (!name) continue;
+      spellings.add(name);
+      claim(name, JSON.stringify([null, name]));
+    }
+  }
+  for (const [inner, owning] of identities) owners.set(inner, owning.size);
+  return { spellings, ownersByInnerName: owners };
+}
+
+/**
+ * Canonical name a legacy dotted alias should be replayed under, or undefined to leave it alone.
+ *
+ * Fail-closed in both directions that matter. A dotted spelling the caller's catalog actually
+ * declares is a real tool identity, so it is never stripped — rewriting it would change which tool
+ * the history says was called. A suffix claimed by two declared identities is ambiguous and is
+ * left as it is. Only two things resolve: a suffix that names exactly one declared tool, and the
+ * bounded set of code-mode helper spellings, which a code-mode catalog never declares because they
+ * exist only as nested `tools.*` helpers inside `exec`.
+ */
+function repairedCallName(name: string, declarations: DeclarationView): string | undefined {
+  if (isSchemaValidResponsesToolName(name)) return undefined;
+  if (!name.startsWith(LEGACY_INVENTED_NAMESPACE_PREFIX)) return undefined;
+  if (declarations.spellings.has(name)) return undefined;
+  const suffix = name.slice(LEGACY_INVENTED_NAMESPACE_PREFIX.length);
+  if (!isSchemaValidResponsesToolName(suffix)) return undefined;
+  const owners = declarations.ownersByInnerName.get(suffix);
+  if (owners === 1) return suffix;
+  if (owners !== undefined) return undefined;
+  return CODE_MODE_HELPER_WIRE_NAMES.has(suffix) ? suffix : undefined;
+}
+
+function repairedItem(item: unknown, declarations: DeclarationView): unknown {
+  if (!isPlainObject(item)) return item;
+  if (typeof item.type !== "string" || !NAMED_CALL_ITEM_TYPES.has(item.type)) return item;
+  // An item carrying an explicit namespace is a namespaced identity whose `name` is the inner
+  // name; it is not the flattened shape this repairs, and its own field is the authority.
+  if (item.namespace !== undefined) return item;
+  if (typeof item.name !== "string") return item;
+  const repaired = repairedCallName(item.name, declarations);
+  return repaired === undefined ? item : { ...item, name: repaired };
+}
+
+/**
+ * Make an already-damaged conversation replayable.
+ *
+ * A `function_call` whose name violates `^[a-zA-Z0-9_-]+$` is not a one-turn failure: Codex stores
+ * the item, and the upstream then refuses EVERY later request that replays it
+ * (`Invalid 'input[N].name': ...`) — including the automatic compaction a long task or a side chat
+ * triggers. The emit-side fix stops new items from being created; without this the conversations
+ * that already contain one stay permanently unusable, which is the part of #5095 that actually
+ * hurts.
+ *
+ * Only replayed input items are touched. The caller's tool catalog is never rewritten, and a body
+ * with nothing to repair is returned by reference.
+ */
+export function repairLegacyDottedToolCallNames(body: unknown): unknown {
+  if (!isPlainObject(body) || !Array.isArray(body.input)) return body;
+  if (!body.input.some(item =>
+    isPlainObject(item)
+    && typeof item.type === "string"
+    && NAMED_CALL_ITEM_TYPES.has(item.type)
+    && typeof item.name === "string"
+    && item.name.startsWith(LEGACY_INVENTED_NAMESPACE_PREFIX)
+  )) return body;
+
+  const declarations = readDeclarations(body);
+  let changed = false;
+  const input = body.input.map(item => {
+    const repaired = repairedItem(item, declarations);
+    if (repaired !== item) changed = true;
+    return repaired;
+  });
+  return changed ? { ...body, input } : body;
+}

package/src/responses/progressive-freeform-input.ts

@@ -0,0 +1,130 @@
+import { unwrapFreeformToolInput } from "./apply-patch-envelope";
+import { JSON_ESCAPES, scanFreeformWrapper } from "./freeform-wrapper-scan";
+
+/**
+ * Whether a body could still grow into one complete outer Markdown fence.
+ *
+ * `stripMarkdownCodeFence` removes such a fence at completion for exactly the two tools that
+ * own the grammar, so a fenced body's streamed bytes and its completed input disagree unless
+ * the stream holds. A buffer that does not open with a fence can never acquire one, so
+ * ordinary bodies are unaffected; a buffer that does keeps its preview suppressed for the
+ * whole call, because a closing fence can still be followed by more text that withdraws it.
+ */
+function mayBecomeFencedBody(text: string, toolName: string): boolean {
+  if (toolName !== "exec" && toolName !== "apply_patch") return false;
+  const head = text.trimStart();
+  if (head === "") return true;
+  return head.startsWith("```") || "```".startsWith(head);
+}
+
+const LOW_SURROGATE_ESCAPE = /^\\u[dD][c-fC-F][0-9a-fA-F]{2}$/;
+
+/**
+ * The decoded prefix of a JSON string body, stopping at the first byte it cannot resolve.
+ *
+ * Every stop is a hold rather than a guess, because `JSON.parse` decides the completed value
+ * and anything invented here would be retracted. Three of them are not obvious:
+ *
+ * An escape JSON does not define makes the whole wrapper unparseable no matter what arrives
+ * next, so completion falls back to the raw text. Returning `null` for that case stops the
+ * preview rather than continuing to decode a value the completed item will never carry.
+ * `\\b` and `\\f` are defined, and were previously decoded to the letters b and f.
+ *
+ * A lone high surrogate is not a character. Emitting one alone puts an unpaired code unit in a
+ * delta the client has to decode by itself, so the pair is emitted together or not at all.
+ */
+function decodeJsonStringPrefix(body: string): string | null {
+  let out = "";
+  for (let i = 0; i < body.length; i++) {
+    const c = body[i];
+    if (c === '"') break; // unescaped closing quote: value complete
+    if (c !== "\\") {
+      // A literal control character is not legal inside a JSON string, so `JSON.parse` will
+      // reject this wrapper and completion will return the raw text. Emitting the decoded value
+      // first is the disagreement this decoder exists to prevent, so stop instead.
+      if (c!.charCodeAt(0) <= 0x1f) return null;
+      out += c;
+      continue;
+    }
+    const n = body[i + 1];
+    if (n === undefined) break; // escape split across chunks: wait for more
+    if (n === "u") {
+      const hex = body.slice(i + 2, i + 6);
+      if (hex.length < 4) break; // split across chunks: wait for more
+      if (!/^[0-9a-fA-F]{4}$/.test(hex)) return null; // never parses
+      const code = parseInt(hex, 16);
+      if (code >= 0xd800 && code <= 0xdbff) {
+        const low = body.slice(i + 6, i + 12);
+        if (!LOW_SURROGATE_ESCAPE.test(low)) break;
+        out += String.fromCharCode(code, parseInt(low.slice(2), 16));
+        i += 11;
+        continue;
+      }
+      out += String.fromCharCode(code);
+      i += 5;
+      continue;
+    }
+    const escaped = JSON_ESCAPES.get(n);
+    if (escaped === undefined) return null; // never parses
+    out += escaped;
+    i += 1;
+  }
+  return out;
+}
+
+/**
+ * The value to stream so far, or `null` to HOLD because nothing can be decided yet.
+ *
+ * `input` is decidable from its prefix: `unwrapFreeformToolInput` returns it whenever the
+ * key is present, whatever else the object carries, so its value can be unescaped
+ * progressively and never retracted.
+ *
+ * One case escapes that claim and is accepted rather than fixed: a duplicate `input` key.
+ * `JSON.parse` keeps the last one, so `{"input":"a","input":"b"}` streams a and completes with
+ * b. Closing it means holding every canonical wrapper until its object parses, which is the
+ * progressive streaming this path exists to provide. The completed item stays authoritative,
+ * and no model emits a duplicate key in practice.
+ *
+ * A wrapper that turns invalid AFTER streaming has committed has the same shape and the same
+ * answer. `{"input":"a` followed by `\\qb"}` has already published a when the undefined escape
+ * arrives, and completion returns the raw text because nothing parses. The preview stops
+ * there: no rewind, and no decoded text the completed item does not contain. Bounding the
+ * damage is what is available without giving up progressive streaming, and the args are
+ * unusable in that case whichever representation wins.
+ *
+ * A fallback key is not decidable. It only unwraps when it is the SINGLE string field, and a
+ * second key can still arrive — so a value emitted early would have to be taken back. That is
+ * the rewind this holds instead: stream nothing until the object closes, then publish the one
+ * repaired body. The routed passthrough in `responses-custom-tool-repair.ts` already holds
+ * any object prefix for the same reason (#5047).
+ *
+ * An object that has not reached a canonical key YET is in exactly that position, and used to
+ * be treated as raw because it did not match the literal `{"input":"`. It holds now: `input`
+ * can still arrive after other properties, or wearing an escaped spelling, and completion would
+ * then unwrap a body whose wrapper syntax had already been published as deltas (#5151). The
+ * cost of holding is preview on an object body that turns out not to be a wrapper; the cost of
+ * not holding was publishing bytes the completed item removes.
+ */
+export function progressiveFreeformInput(args: string, toolName: string): string | null {
+  const scan = scanFreeformWrapper(args);
+  if (scan.kind === "input") {
+    const decoded = decodeJsonStringPrefix(args.slice(scan.valueStart));
+    if (decoded === null) return null;
+    return mayBecomeFencedBody(decoded, toolName) ? null : decoded;
+  }
+  if (scan.kind === "raw") return mayBecomeFencedBody(args, toolName) ? null : args;
+
+  // Undecided: some wrapper may still apply, and only the completed object says which one.
+  // The parse runs exactly where the scan SAW the object close, so it reads a buffer the scan
+  // already walked and its cost is bounded by the same clamp. A hold from either limit stays
+  // held: an incomplete object has nothing to parse, and re-reading a budget-exhausted buffer
+  // on every delta is quadratic work for a delta that would arrive in the same instant as the
+  // authoritative completion behind it.
+  if (!scan.parse) return null;
+  try {
+    JSON.parse(args);
+  } catch {
+    return null;
+  }
+  return unwrapFreeformToolInput(args, toolName);
+}

package/src/responses/reasoning-envelope.ts

@@ -63,6 +63,36 @@ export function encodeReasoningEnvelope(envelope: ReasoningEnvelope, budget?: Tr
   }
 }
 
+/**
+ * Whether a thinking part's `signature` is an attestation some provider issued, rather than
+ * the serialized reasoning item the parser parks in the same field.
+ *
+ * `src/responses/parser.ts` stores `JSON.stringify(reasoningItem)` on an UNSIGNED thinking
+ * part so the opaque item survives a same-provider round trip. Every adapter that forwards a
+ * signature upstream is forwarding an opaque attestation, and a provider asked to verify our
+ * own parser state cannot recognize it. The predicate lives beside the envelope code that owns
+ * this field's representation rather than in one adapter, because a private second copy is how
+ * the side that writes the field and a side that reads it come to disagree about what it holds.
+ *
+ * This is a deny-list for exactly that one shape, not a guess at what an attestation looks
+ * like. `src/adapters/anthropic.ts` applies a stricter allow-list on top of it, because an
+ * Anthropic signature has a known base64 spelling; that shape is a fact about Anthropic's wire
+ * and is not assumed of any other provider's token here.
+ */
+export function isProviderIssuedThinkingSignature(
+  signature: string | undefined,
+): signature is string {
+  if (typeof signature !== "string" || signature.length === 0) return false;
+  if (!signature.startsWith("{")) return true;
+  try {
+    const parsed: unknown = JSON.parse(signature);
+    return !parsed || typeof parsed !== "object" || Array.isArray(parsed)
+      || (parsed as { type?: unknown }).type !== "reasoning";
+  } catch {
+    return true;
+  }
+}
+
 /** Decode an ocxr1 envelope; returns null for native (OpenAI-encrypted) blobs or garbage. */
 export function decodeReasoningEnvelope(encryptedContent: string, budget?: TranslatorBudget): ReasoningEnvelope | null {
   if (!encryptedContent.startsWith(OCX_REASONING_PREFIX)) return null;

package/src/responses/schema.ts

@@ -126,10 +126,17 @@ export const toolSchema = z.object({
 
 const builtinToolSchema = z.object({ type: z.string() }).loose();
 
-const hostedToolType = z.enum([
+/**
+ * Hosted tool types a client may declare on an inbound Responses request. Exported so the
+ * provider-side capability vocabulary in `src/responses/hosted-tool-policy.ts` can be
+ * asserted to cover all of them: a gateway must be able to deny anything it can be sent.
+ */
+export const HOSTED_TOOL_TYPES = [
   "web_search", "web_search_preview", "file_search", "computer_use_preview",
   "code_interpreter", "image_generation", "mcp",
-]);
+] as const;
+
+const hostedToolType = z.enum(HOSTED_TOOL_TYPES);
 
 const allowedToolEntrySchema = z.object({ type: z.string(), name: z.string().optional() });
 

package/src/responses/state.ts

@@ -135,11 +135,10 @@ export type ResidentInput = Omit<ResidentResponseState, "kind" | "sizeBytes">;
 
 export type PreviousResponseReplayFailure = {
   code: "previous_response_not_found";
-  reason: "spill_missing" | "spill_corrupt" | "spill_failed" | "spill_too_large";
+  reason: "spill_missing" | "spill_corrupt" | "spill_failed" | "spill_too_large" | "scope_mismatch";
 };
 
 const states = new Map<string, StoredResponseState>();
-const replayScopeMismatches = new WeakSet<object>();
 let storedResponseBytes = 0;
 let residentResponseBytes = 0;
 let oldestResidentId: string | undefined;
@@ -1044,11 +1043,6 @@ function normalizedClientThreadId(value: unknown): string | undefined {
   return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
 }
 
-function withoutPreviousResponseId(request: Record<string, unknown>): Record<string, unknown> {
-  const { previous_response_id: _previousResponseId, ...freshRequest } = request;
-  return freshRequest;
-}
-
 export function expandPreviousResponseInput(body: unknown, clientThreadId?: string): unknown {
   if (!body || typeof body !== "object" || Array.isArray(body)) return body;
   const request = body as Record<string, unknown>;
@@ -1068,10 +1062,9 @@ export function expandPreviousResponseInput(body: unknown, clientThreadId?: stri
   // A Codex task must never inherit another task's continuation, nor a legacy unscoped entry.
   // Unscoped callers retain backward-compatible replay only with other unscoped entries.
   if (requestThreadId !== storedThreadId) {
-    const freshRequest = withoutPreviousResponseId(request);
-    replayScopeMismatches.add(freshRequest);
+    replayFailures.set(request, { code: "previous_response_not_found", reason: "scope_mismatch" });
     replayScopeMismatchDrops += 1;
-    return freshRequest;
+    return body;
   }
   // The client already replayed this history verbatim. Prepending the stored copy would
   // double it, and the doubled turn is stored again, so the next turn triples (#1412 saw
@@ -1136,9 +1129,9 @@ export function copyPreviousResponseReplayProvenance(source: unknown, target: un
   replayedInputPrefixLengths.set(target, prefixLength);
 }
 
-/** True when a stale or foreign previous_response_id was removed from this exact request body. */
+/** True when this exact request could not replay because its task scope did not match. */
 export function previousResponseScopeMismatch(body: unknown): boolean {
-  return !!body && typeof body === "object" && replayScopeMismatches.has(body as object);
+  return previousResponseReplayFailure(body)?.reason === "scope_mismatch";
 }
 
 export function previousResponseConversationId(responseId: string | undefined): string | undefined {

package/src/responses/tool-name-aliases.ts

@@ -19,6 +19,21 @@ export function dottedAliasIsUnambiguous(namespace: string, name: string): boole
   return !namespace.includes("__") && !name.includes("__");
 }
 
+/**
+ * Charset the Responses API enforces on a tool-call `name`.
+ *
+ * A name outside it is not merely rejected for the turn that carries it: Codex stores the item in
+ * the conversation, and every later request that replays that history is refused before it runs
+ * (`Invalid 'input[N].name': string does not match pattern '^[a-zA-Z0-9_-]+$'`). One relayed item
+ * therefore ends the conversation permanently, which is why the emit boundary treats an invalid
+ * name as a defect to repair rather than as a name to forward. See #5095.
+ */
+const RESPONSES_TOOL_NAME_PATTERN = /^[a-zA-Z0-9_-]+$/;
+
+export function isSchemaValidResponsesToolName(name: string): boolean {
+  return RESPONSES_TOOL_NAME_PATTERN.test(name);
+}
+
 export function wireToolInnerName(tool: unknown): string | undefined {
   if (!isPlainObject(tool)) return undefined;
   const nestedFunction = tool.type === "function" && isPlainObject(tool.function)
@@ -76,4 +91,3 @@ export function collectAmbiguousDottedAliases(specGroups: readonly unknown[]): S
   for (const [alias, owner] of owners) if (owner === null) ambiguous.add(alias);
   return ambiguous;
 }
-