@bitkyc08/opencodex 2.61.0 → 2.63.0-preview.20260923

package/src/codex/inject/provider-table.ts

@@ -0,0 +1,108 @@
+import { isDeepStrictEqual } from "node:util";
+import { OCX_SECTION_MARKER } from "../injected-marker";
+import { splitSourceLines, type SourceLine } from "../toml-source-lines";
+
+function record(value: unknown): Record<string, unknown> | null {
+  return value !== null && typeof value === "object" && !Array.isArray(value)
+    ? value as Record<string, unknown> : null;
+}
+
+function providerValue(text: string): Record<string, unknown> {
+  let parsed: unknown;
+  try { parsed = Bun.TOML.parse(text); }
+  catch { throw new Error("Codex provider table could not be read safely."); }
+  const root = record(parsed);
+  const providers = record(root?.model_providers);
+  const provider = record(providers?.opencodex);
+  if (!provider || Object.keys(root!).length !== 1 || Object.keys(providers!).length !== 1) {
+    throw new Error("Codex provider table could not be read safely.");
+  }
+  return provider;
+}
+
+const KEY = String.raw`(?:[A-Za-z0-9_-]+|"(?:[^"\\]|\\.)*"|'[^']*')`;
+const ASSIGNMENT = new RegExp(`^\\s*(${KEY}(?:\\s*\\.\\s*${KEY})*)\\s*=`);
+
+function unsupportedProviderAssignment(text: string, section: string): boolean {
+  if (section !== "root" && section !== "providers") return false;
+  const match = ASSIGNMENT.exec(text);
+  if (!match) return false;
+  // Parsing only the key path handles quoted/escaped components without reading
+  // unrelated large integer values or allowing strings to impersonate a header.
+  const parsed = record(Bun.TOML.parse(`${match[1]} = {}\n`))!;
+  if (section === "providers") return Object.hasOwn(parsed, "opencodex");
+  const providers = record(parsed.model_providers);
+  return providers !== null && (Object.keys(providers).length === 0 || Object.hasOwn(providers, "opencodex"));
+}
+
+/** Recognize real table headers, including quoted keys, without parsing unrelated values. */
+function headerKind(text: string): "provider" | "providers" | "foreign" {
+  let parsed: Record<string, unknown> | null;
+  try { parsed = record(Bun.TOML.parse(text + "\n")); }
+  catch { throw new Error("Codex table header could not be read safely."); }
+  const providers = record(parsed?.model_providers);
+  if (providers && Object.hasOwn(providers, "opencodex")) {
+    if (!record(providers.opencodex)) throw new Error("Codex provider array tables cannot be retained safely.");
+    return "provider";
+  }
+  return providers && Object.keys(providers).length === 0 ? "providers" : "foreign";
+}
+
+/** Physical spans are shared by capture and removal so neither sees headers inside values. */
+function providerLines(content: string): { bom: string; lines: SourceLine[]; owned: Set<number> } {
+  const bom = content.startsWith("\uFEFF") ? "\uFEFF" : "";
+  const lines = splitSourceLines(content.slice(bom.length));
+  const owned = new Set<number>();
+  let section: "root" | "provider" | "providers" | "foreign" = "root";
+  for (let index = 0; index < lines.length; index++) {
+    const line = lines[index]!;
+    const text = line.text;
+    if (line.structural && /^\s*\[/.test(text)) {
+      section = headerKind(text);
+      if (section === "provider") {
+        const previous = lines[index - 1];
+        if (previous?.structural && previous.text.trim() === OCX_SECTION_MARKER) owned.add(index - 1);
+      }
+    } else if (line.structural && unsupportedProviderAssignment(text, section)) {
+      // These forms have no independently removable table span. Do not append a
+      // competing definition or silently treat a user-owned inline value as absent.
+      throw new Error("Codex provider retention requires an explicit provider table.");
+    }
+    if (section === "provider") owned.add(index);
+  }
+  return { bom, lines, owned };
+}
+
+export function hasOcxProviderTable(content: string): boolean {
+  return providerLines(content).owned.size > 0;
+}
+
+/** Preserve value bytes, including multiline newlines and separated child tables. */
+export function extractOcxProviderTableBlock(content: string): string | null {
+  const { lines, owned } = providerLines(content);
+  if (owned.size === 0) return null;
+  const captured = lines.filter((_, index) => owned.has(index));
+  while (captured.length && captured.at(-1)!.structural && !captured.at(-1)!.text.trim()) captured.pop();
+  const text = captured.map(line => line.text + line.eol).join("");
+  providerValue(text); // Ambiguous or incomplete owned spans never become a retained block.
+  return text.endsWith("\n") ? text : text + "\n";
+}
+
+export function removeOcxSection(content: string): string {
+  const { bom, lines, owned } = providerLines(content);
+  if (owned.size) providerValue(lines.filter((_, index) => owned.has(index)).map(line => line.text + line.eol).join(""));
+  return bom + lines.filter((_, index) => !owned.has(index)).map(line => line.text + line.eol).join("");
+}
+
+/** Compare decoded values; cosmetic layout never rewrites the existing file. */
+export function appendOcxProviderTableBlock(content: string, block: string): string {
+  const captured = providerValue(block);
+  const existing = extractOcxProviderTableBlock(content);
+  if (existing !== null) {
+    if (!isDeepStrictEqual(providerValue(existing), captured)) {
+      throw new Error("Codex restore refused: the native config already defines a different [model_providers.opencodex] table.");
+    }
+    return content;
+  }
+  return `${content}${content.endsWith("\n") ? "\n" : "\n\n"}${block}${block.endsWith("\n") ? "" : "\n"}`;
+}

package/src/codex/inject/remove.ts

@@ -1,7 +1,6 @@
 import { existsSync, readFileSync, unlinkSync } from "node:fs";
 import { atomicWriteFile } from "../../config";
 import {
-  OCX_SECTION_MARKER,
   REALTIME_WS_BASE_URL_KEY,
   hasInjectedOpenaiBaseUrl,
   rootTomlString,
@@ -23,119 +22,8 @@ import {
   stripRootRoutedModel,
 } from "./config-toml";
 
-/**
- * Sub-table headers like `[model_providers.opencodex.env_http_headers]` appear when a Codex app
- * config rewrite re-serializes the provider's inline `env_http_headers` table. They define the
- * same `model_providers.opencodex` provider, so cleanup must remove them too — otherwise the
- * provider survives with no `name` and Codex rejects the whole config
- * ("provider name must not be empty"). The dot terminator keeps a user's
- * `[model_providers.opencodex_backup]`-style tables out of scope.
- */
-function isOcxProviderHeaderLine(trimmedLine: string): boolean {
-  // Root form matched by regex, not equality: TOML v1.0 allows a trailing comment
-  // (`[model_providers.opencodex] # comment`), and an exact compare would miss that form.
-  // The sub-table prefix check already tolerates trailing comments by construction.
-  return (
-    /^\[model_providers\.opencodex\]\s*(?:#.*)?$/.test(trimmedLine) ||
-    trimmedLine.startsWith("[model_providers.opencodex.")
-  );
-}
-
-export function hasOcxProviderTable(content: string): boolean {
-  return content
-    .split("\n")
-    .some((line) => isOcxProviderHeaderLine(line.trim()));
-}
-
-export function removeOcxSection(content: string): string {
-  const lines = content.split("\n");
-  const filtered: string[] = [];
-  let inOcxSection = false;
-  for (const line of lines) {
-    if (
-      line.includes(OCX_SECTION_MARKER) ||
-      isOcxProviderHeaderLine(line.trim())
-    ) {
-      inOcxSection = true;
-      continue;
-    }
-    if (inOcxSection) {
-      // End the injected section at the next table header that ISN'T our own. Exact match on the
-      // provider name (plus our own sub-tables) so a user's
-      // "[model_providers.opencodex_backup]" (or similar) is preserved, not swallowed.
-      if (/^\s*\[/.test(line) && !isOcxProviderHeaderLine(line.trim())) {
-        inOcxSection = false;
-        filtered.push(line);
-      }
-      continue;
-    }
-    filtered.push(line);
-  }
-  return (
-    filtered
-      .join("\n")
-      .replace(/\n{3,}/g, "\n\n")
-      .trimEnd() + "\n"
-  );
-}
-
-/**
- * Capture `[model_providers.opencodex]` verbatim so it can survive a restore that only
- * takes routing down (#4812).
- *
- * This is deliberately NOT a mirror of `removeOcxSection`'s scan. That one opens a
- * section on any line containing `OCX_SECTION_MARKER`, which is safe there only because
- * `stripInjectedOpenaiBaseUrl` has already consumed the identical marker that annotates
- * the root `openai_base_url`. Capture runs against the untouched file, so the same rule
- * would collect that marker and the routing line under it — and re-appending the result
- * would restore the exact base-url override the caller just removed.
- *
- * So the anchor is the provider header itself, via the shared `isOcxProviderHeaderLine`,
- * with an immediately preceding marker line pulled in as its comment. Sharing that
- * predicate is what keeps capture and removal from disagreeing about what our table is.
- */
-export function extractOcxProviderTableBlock(content: string): string | null {
-  const lines = content.split("\n");
-  const collected: string[] = [];
-  let capturing = false;
-  for (let index = 0; index < lines.length; index++) {
-    const line = lines[index]!;
-    if (isOcxProviderHeaderLine(line.trim())) {
-      if (!capturing) {
-        const previous = lines[index - 1];
-        if (previous !== undefined && previous.includes(OCX_SECTION_MARKER)) collected.push(previous);
-        capturing = true;
-      }
-      collected.push(line);
-      continue;
-    }
-    if (!capturing) continue;
-    // A foreign table header closes ours, exactly as in `removeOcxSection`. A later
-    // `[model_providers.opencodex.*]` sub-table reopens capture on the next iteration,
-    // which is why the two are separate passes over the same predicate.
-    if (/^\s*\[/.test(line)) {
-      capturing = false;
-      continue;
-    }
-    collected.push(line);
-  }
-  if (collected.length === 0) return null;
-  return collected.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
-}
-
-/**
- * Append a captured provider table to stripped content, as one buffer.
- *
- * Pure on purpose. Upstream resolves `model_provider` against the merged provider map and
- * fails the WHOLE config load on a miss — not the one thread — so a config carrying root
- * `model_provider = "opencodex"` without this table breaks every `codex` invocation. The
- * strip and the re-append therefore have to reach disk in a single write, which they can
- * only do if the append is a transform rather than a second file operation.
- */
-export function appendOcxProviderTableBlock(content: string, block: string): string {
-  if (hasOcxProviderTable(content)) return content;
-  return `${content.replace(/\n+$/, "")}\n\n${block.replace(/\n+$/, "")}\n`;
-}
+import { appendOcxProviderTableBlock, extractOcxProviderTableBlock, hasOcxProviderTable, removeOcxSection } from "./provider-table";
+export { appendOcxProviderTableBlock, extractOcxProviderTableBlock, hasOcxProviderTable, removeOcxSection } from "./provider-table";
 
 /** Read the provider table straight off disk, before anything has transformed it. */
 export function readOcxProviderTableBlock(): string | null {

package/src/codex/model-entitlements.ts

@@ -20,7 +20,7 @@ import {
 } from "./catalog/native-models";
 import { loadPersistedCodexRuntime } from "./runtime";
 import { codexRuntimeStateEpoch } from "./runtime";
-import upstreamModelsSnapshot from "./data/upstream-models.json";
+import { pinnedNativeModelRows } from "./catalog/pinned-models";
 import { codexCredentialMutationEpoch } from "./credential-mutation-epoch";
 import {
   clearObservedCodexModelDenial,
@@ -90,9 +90,10 @@ export function deriveGatedClientVersionFloor(
 /**
  * Lowest `client_version` MEASURED to actually return the account-gated rows.
  *
- * The bundled snapshot is not sufficient on its own. It records `0.142.2` for the gpt-5.6
- * rows, and `0.142.2` is a version upstream answers with 200 and five models, none of them
- * gpt-5.6; `0.144.0` and above answer with the gated rows present
+ * The bundled snapshot was not sufficient on its own. Until the 2026-09-23 re-pin it recorded
+ * `0.142.2` for the gpt-5.6 rows (upstream now records `0.144.0`), and `0.142.2` is a version
+ * upstream answers with 200 and five models, none of them gpt-5.6; `0.144.0` and above answer
+ * with the gated rows present
  * (devlog/_fin/260817_native_gpt56_1m_context/001_measurement_evidence.md, independently
  * reproduced by the #2886 and #3022 reporters). So a floor derived from the snapshot alone
  * asks a question whose honest answer is an empty gated set — and the fail-closed gate then
@@ -107,9 +108,9 @@ const MEASURED_GATED_CLIENT_VERSION_MINIMUM = "0.144.0";
 /**
  * Lowest versions measured to return each account-gated model when the account owns it.
  *
- * This is deliberately independent of the bundled upstream snapshot. The snapshot still
- * records 0.142.2 for sol/terra/luna, while live measurements show that upstream omits them
- * below 0.144.0. Daybreak has no snapshot row or independent minimum, so its omission remains
+ * This is deliberately independent of the bundled upstream snapshot, which can lag or lead the
+ * live roster: live measurements show that upstream omits sol/terra/luna below 0.144.0.
+ * Daybreak's shipped row records 0.142.2 and has no independent measured minimum, so its omission remains
  * authoritative instead of inheriting a guessed floor from another model.
  */
 export const ACCOUNT_GATED_NATIVE_MODEL_MINIMUM_CLIENT_VERSIONS: ReadonlyMap<string, string> = new Map([
@@ -121,8 +122,8 @@ export const ACCOUNT_GATED_NATIVE_MODEL_MINIMUM_CLIENT_VERSIONS: ReadonlyMap<str
 /**
  * Fallback when the snapshot records no usable gated floor.
  *
- * Not every gated slug carries a `minimal_client_version` — `gpt-daybreak-blue-latest` has no
- * row in the current snapshot at all — so the derivation can legitimately come back empty as the
+ * Not every gated slug is guaranteed a `minimal_client_version` — `gpt-daybreak-blue-latest` had
+ * no row until the 2026-09-23 re-pin — so the derivation can legitimately come back empty as the
  * gated set changes. This is the last resort behind it, and it is still a version upstream can
  * filter on rather than the placeholder that caused #2886.
  */
@@ -147,8 +148,11 @@ function composeGatedClientVersionFloor(
     : MEASURED_GATED_CLIENT_VERSION_MINIMUM;
 }
 
+// Reads every pinned row (codex-rs snapshot plus roster-captured rows), so a gated slug whose row
+// arrives through roster-pinned-models.json still contributes its floor. None does today:
+// gpt-6-sol/luna are ungated, and gpt-6-astra-minor has no row and no measured minimum.
 export const GATED_MODEL_CLIENT_VERSION_FLOOR: string = composeGatedClientVersionFloor(
-  (upstreamModelsSnapshot as { models?: Array<Record<string, unknown>> }).models ?? [],
+  pinnedNativeModelRows(),
 );
 
 /** Test-only seam: the composition on synthetic rows, so both directions can be proven. */

package/src/codex/subagent-defaults.ts

@@ -1,3 +1,5 @@
+import { splitSourceLines, type SourceLine } from "./toml-source-lines";
+
 /**
  * Pure, ownership-aware edits for Codex's native `[agents]` defaults.
  *
@@ -40,12 +42,6 @@ export type ManagedSubagentDefaultsTransformResult =
       error: string;
     };
 
-interface SourceLine {
-  text: string;
-  eol: "\r\n" | "\n" | "";
-  /** False when this physical line began inside a TOML multiline string. */
-  structural: boolean;
-}
 
 interface TargetDefinition {
   key: ManagedSubagentDefaultKey;
@@ -65,109 +61,6 @@ const TARGET_KEYS: readonly ManagedSubagentDefaultKey[] = [
   "default_subagent_reasoning_effort",
 ];
 
-function splitSourceLines(content: string): SourceLine[] {
-  const lines: SourceLine[] = [];
-  let offset = 0;
-  while (offset < content.length) {
-    const lf = content.indexOf("\n", offset);
-    if (lf === -1) {
-      lines.push({ text: content.slice(offset), eol: "", structural: true });
-      break;
-    }
-    const crlf = lf > offset && content[lf - 1] === "\r";
-    lines.push({
-      text: content.slice(offset, crlf ? lf - 1 : lf),
-      eol: crlf ? "\r\n" : "\n",
-      structural: true,
-    });
-    offset = lf + 1;
-  }
-  markStructuralLines(lines);
-  return lines;
-}
-
-type MultilineStringKind = "basic" | "literal" | null;
-
-/**
- * TOML table-looking text inside a multiline string is data, not syntax. Keep
- * a deliberately small lexical state machine so the format-preserving editor
- * never treats those physical lines as headers, keys, or ownership markers.
- */
-function markStructuralLines(lines: SourceLine[]): void {
-  let multiline: MultilineStringKind = null;
-  let squareDepth = 0;
-  let curlyDepth = 0;
-
-  for (const line of lines) {
-    line.structural = multiline === null && squareDepth === 0 && curlyDepth === 0;
-    let single: "basic" | "literal" | null = null;
-
-    for (let index = 0; index < line.text.length;) {
-      if (multiline === "basic") {
-        if (line.text.startsWith('"""', index)) {
-          multiline = null;
-          index += 3;
-        } else if (line.text[index] === "\\") {
-          index += 2;
-        } else {
-          index += 1;
-        }
-        continue;
-      }
-      if (multiline === "literal") {
-        if (line.text.startsWith("'''", index)) {
-          multiline = null;
-          index += 3;
-        } else {
-          index += 1;
-        }
-        continue;
-      }
-      if (single === "basic") {
-        if (line.text[index] === "\\") index += 2;
-        else if (line.text[index] === '"') {
-          single = null;
-          index += 1;
-        } else index += 1;
-        continue;
-      }
-      if (single === "literal") {
-        if (line.text[index] === "'") single = null;
-        index += 1;
-        continue;
-      }
-
-      if (line.text[index] === "#") break;
-      if (line.text.startsWith('"""', index)) {
-        multiline = "basic";
-        index += 3;
-      } else if (line.text.startsWith("'''", index)) {
-        multiline = "literal";
-        index += 3;
-      } else if (line.text[index] === '"') {
-        single = "basic";
-        index += 1;
-      } else if (line.text[index] === "'") {
-        single = "literal";
-        index += 1;
-      } else if (line.text[index] === "[") {
-        squareDepth += 1;
-        index += 1;
-      } else if (line.text[index] === "]") {
-        squareDepth = Math.max(0, squareDepth - 1);
-        index += 1;
-      } else if (line.text[index] === "{") {
-        curlyDepth += 1;
-        index += 1;
-      } else if (line.text[index] === "}") {
-        curlyDepth = Math.max(0, curlyDepth - 1);
-        index += 1;
-      } else {
-        index += 1;
-      }
-    }
-  }
-}
 
 function joinSourceLines(lines: readonly SourceLine[]): string {
   return lines.map(line => `${line.text}${line.eol}`).join("");

package/src/codex/toml-source-lines.ts

@@ -0,0 +1,112 @@
+/** Lossless physical TOML lines with lexical structural boundaries. */
+export interface SourceLine {
+  text: string;
+  eol: "\r\n" | "\n" | "";
+  /** False when this physical line began inside a TOML multiline string. */
+  structural: boolean;
+}
+
+
+export function splitSourceLines(content: string): SourceLine[] {
+  const lines: SourceLine[] = [];
+  let offset = 0;
+  while (offset < content.length) {
+    const lf = content.indexOf("\n", offset);
+    if (lf === -1) {
+      lines.push({ text: content.slice(offset), eol: "", structural: true });
+      break;
+    }
+    const crlf = lf > offset && content[lf - 1] === "\r";
+    lines.push({
+      text: content.slice(offset, crlf ? lf - 1 : lf),
+      eol: crlf ? "\r\n" : "\n",
+      structural: true,
+    });
+    offset = lf + 1;
+  }
+  markStructuralLines(lines);
+  return lines;
+}
+
+type MultilineStringKind = "basic" | "literal" | null;
+
+/**
+ * TOML table-looking text inside a multiline string is data, not syntax. Keep
+ * a deliberately small lexical state machine so the format-preserving editor
+ * never treats those physical lines as headers, keys, or ownership markers.
+ */
+function markStructuralLines(lines: SourceLine[]): void {
+  let multiline: MultilineStringKind = null;
+  let squareDepth = 0;
+  let curlyDepth = 0;
+
+  for (const line of lines) {
+    line.structural = multiline === null && squareDepth === 0 && curlyDepth === 0;
+    let single: "basic" | "literal" | null = null;
+
+    for (let index = 0; index < line.text.length;) {
+      if (multiline === "basic") {
+        if (line.text.startsWith('"""', index)) {
+          multiline = null;
+          index += 3;
+        } else if (line.text[index] === "\\") {
+          index += 2;
+        } else {
+          index += 1;
+        }
+        continue;
+      }
+      if (multiline === "literal") {
+        if (line.text.startsWith("'''", index)) {
+          multiline = null;
+          index += 3;
+        } else {
+          index += 1;
+        }
+        continue;
+      }
+      if (single === "basic") {
+        if (line.text[index] === "\\") index += 2;
+        else if (line.text[index] === '"') {
+          single = null;
+          index += 1;
+        } else index += 1;
+        continue;
+      }
+      if (single === "literal") {
+        if (line.text[index] === "'") single = null;
+        index += 1;
+        continue;
+      }
+
+      if (line.text[index] === "#") break;
+      if (line.text.startsWith('"""', index)) {
+        multiline = "basic";
+        index += 3;
+      } else if (line.text.startsWith("'''", index)) {
+        multiline = "literal";
+        index += 3;
+      } else if (line.text[index] === '"') {
+        single = "basic";
+        index += 1;
+      } else if (line.text[index] === "'") {
+        single = "literal";
+        index += 1;
+      } else if (line.text[index] === "[") {
+        squareDepth += 1;
+        index += 1;
+      } else if (line.text[index] === "]") {
+        squareDepth = Math.max(0, squareDepth - 1);
+        index += 1;
+      } else if (line.text[index] === "{") {
+        curlyDepth += 1;
+        index += 1;
+      } else if (line.text[index] === "}") {
+        curlyDepth = Math.max(0, curlyDepth - 1);
+        index += 1;
+      } else {
+        index += 1;
+      }
+    }
+  }
+}