@iislee/opencodex 2.11.0 → 2.31.0

package/src/codex/injected-marker.ts

@@ -7,6 +7,8 @@
  * them here breaks that cycle. `inject.ts` imports them back and re-exports the
  * two public predicates, so external callers see no change.
  */
+import { parseTomlString } from "./paths";
+
 export const OCX_SECTION_MARKER = "# Auto-injected by opencodex";
 
 export function isRootOpenaiBaseUrlLine(line: string): boolean {
@@ -16,7 +18,11 @@ export function isRootOpenaiBaseUrlLine(line: string): boolean {
 export function tomlStringPattern(key: string): RegExp {
   const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
   const keyToken = `(?:${escaped}|"${escaped}"|'${escaped}')`;
-  return new RegExp(`^\\s*${keyToken}\\s*=\\s*["']([^"']+)["']\\s*(?:#.*)?$`);
+  // The quoted value is captured WITH its quotes so callers can decode it as TOML.
+  // A basic string escapes backslashes, so a Windows path is stored doubled; reading
+  // the raw bytes back returned a path that matched nothing on disk and made the
+  // journal's recorded catalog path un-restorable (#1798).
+  return new RegExp(`^\\s*${keyToken}\\s*=\\s*("(?:\\\\.|[^"])*"|'[^']*')\\s*(?:#.*)?$`);
 }
 
 export function rootTomlString(content: string, key: string): string | null {
@@ -26,7 +32,7 @@ export function rootTomlString(content: string, key: string): string | null {
   const pattern = tomlStringPattern(key);
   for (const line of rootLines) {
     const match = pattern.exec(line);
-    if (match?.[1]) return match[1].trim();
+    if (match?.[1]) return parseTomlString(match[1]).trim();
   }
   return null;
 }
@@ -45,11 +51,39 @@ export function providerTableString(content: string, provider: string, key: stri
   const pattern = tomlStringPattern(key);
   for (let index = start + 1; index < lines.length && !/^\s*\[/.test(lines[index]); index += 1) {
     const match = pattern.exec(lines[index]);
-    if (match?.[1]) return match[1].trim();
+    if (match?.[1]) return parseTomlString(match[1]).trim();
   }
   return null;
 }
 
+/**
+ * Drop a root `openai_base_url` whose VALUE is the one a recorded injection wrote.
+ *
+ * #1798: the marker-adjacency rule below is formatting evidence, and the Codex app
+ * reserializes the file -- values kept, comments dropped. This rule is value evidence
+ * instead, so it still recognizes our URL after that rewrite. It is deliberately an
+ * EXACT value match against what we recorded writing: a user gateway we never wrote
+ * cannot match, so restore can never delete a URL that was not ours.
+ */
+export function stripJournaledOpenaiBaseUrl(content: string, injectedUrl: string | null): string {
+  if (!injectedUrl) return content;
+  const lines = content.split(String.fromCharCode(10));
+  const firstTable = lines.findIndex(l => /^\s*\[/.test(l));
+  const rootEnd = firstTable === -1 ? lines.length : firstTable;
+  const drop = new Set<number>();
+  for (let i = 0; i < rootEnd; i++) {
+    const line = lines[i]!;
+    if (!isRootOpenaiBaseUrlLine(line)) continue;
+    if (rootTomlString(line, "openai_base_url") !== injectedUrl) continue;
+    drop.add(i);
+    // Take an ownership marker directly above it too, so repeated cycles cannot
+    // accumulate orphaned comments.
+    if (i > 0 && lines[i - 1]!.includes(OCX_SECTION_MARKER)) drop.add(i - 1);
+  }
+  if (drop.size === 0) return content;
+  return lines.filter((_, i) => !drop.has(i)).join(String.fromCharCode(10));
+}
+
 export function hasInjectedOpenaiBaseUrl(content: string): boolean {
   const lines = content.split("\n");
   const firstTable = lines.findIndex(l => /^\s*\[/.test(l));

package/src/codex/integration-record.ts

@@ -0,0 +1,266 @@
+/**
+ * The sole reader/writer for integrations/codex.json.
+ *
+ * Provenance once disappeared when a corrupt record was treated as an empty one
+ * (005_disable_leaves_a_broken_file.md). Reads therefore validate the complete
+ * known shape, and updates refuse malformed bytes instead of manufacturing a new
+ * baseline over evidence we can no longer trust.
+ */
+import { mkdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { atomicWriteFile, getConfigDir, withConfigMutationLockSync } from "../config";
+import type {
+  CodexArtifactId,
+  CodexIntegrationRecord,
+  CodexProvenanceEntry,
+  CodexProvenanceLedger,
+} from "./convergence-types";
+
+const RECORD_FILENAME = "codex.json";
+const LEGACY_TRANSITION_KEYS = new Set([
+  "nativeGeneration",
+  "currentTxId",
+  "generation",
+  "history",
+  "historySchedule",
+]);
+
+function recordPath(): string {
+  return join(getConfigDir(), "integrations", RECORD_FILENAME);
+}
+
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isMissingPathError(error: unknown): boolean {
+  return error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "ENOENT";
+}
+
+function invalid(message: string) {
+  return { kind: "invalid" as const, message };
+}
+
+function validateArtifact(value: unknown): value is CodexArtifactId {
+  if (!isPlainRecord(value) || typeof value.kind !== "string") return false;
+  switch (value.kind) {
+    case "config":
+    case "generated-profile":
+    case "models-cache":
+    case "injection-journal":
+      return true;
+    case "active-catalog":
+    case "history-manifest":
+    case "history-rollout":
+      return typeof value.canonicalPath === "string"
+        && (value.kind === "active-catalog" || typeof value.stateDbId === "string");
+    case "catalog-backup":
+      return (value.form === "hashed" || value.form === "legacy")
+        && typeof value.canonicalPath === "string";
+    case "history-row":
+    case "history-manifest-entry":
+      return typeof value.stateDbId === "string" && typeof value.threadId === "string";
+    default:
+      return false;
+  }
+}
+
+function validateBaseline(value: unknown): boolean {
+  if (!isPlainRecord(value)) return false;
+  if (value.kind === "absent") return true;
+  return value.kind === "present"
+    && typeof value.sha256 === "string"
+    && typeof value.bytesBase64 === "string";
+}
+
+function validateEntry(value: unknown): value is CodexProvenanceEntry {
+  return isPlainRecord(value)
+    && validateArtifact(value.artifact)
+    && validateBaseline(value.baseline)
+    && (typeof value.postImage === "string" || value.postImage === null)
+    && typeof value.txId === "string"
+    && typeof value.at === "string";
+}
+
+function validateLedger(value: unknown): value is CodexProvenanceLedger {
+  return isPlainRecord(value)
+    && Array.isArray(value.entries)
+    && value.entries.every(validateEntry);
+}
+
+function validateRecord(value: unknown): value is CodexIntegrationRecord {
+  return isPlainRecord(value)
+    && value.version === 1
+    && !Object.keys(value).some(key => LEGACY_TRANSITION_KEYS.has(key))
+    && (value.provenance === undefined || validateLedger(value.provenance));
+}
+
+function readIntegrationRecordUnlocked() {
+  let raw: string;
+  try {
+    raw = readFileSync(recordPath(), "utf8");
+  } catch (error) {
+    if (isMissingPathError(error)) return { kind: "missing" as const, record: null };
+    return invalid(`Codex integration record is unreadable: ${error instanceof Error ? error.message : String(error)}`);
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw.replace(/^\uFEFF/, ""));
+  } catch {
+    return invalid("Codex integration record contains invalid JSON");
+  }
+  if (!validateRecord(parsed)) {
+    return invalid("Codex integration record has an unsupported or malformed v1 shape");
+  }
+  return { kind: "ready" as const, record: parsed };
+}
+
+export const readIntegrationRecord = readIntegrationRecordUnlocked;
+
+function copyUnknown(
+  previous: Record<string, unknown>,
+  next: Record<string, unknown>,
+  knownKeys: ReadonlySet<string>,
+): Record<string, unknown> {
+  const merged: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(previous)) {
+    if (!knownKeys.has(key)) merged[key] = value;
+  }
+  return { ...merged, ...next };
+}
+
+function artifactKnownKeys(artifact: CodexArtifactId): ReadonlySet<string> {
+  switch (artifact.kind) {
+    case "config":
+    case "generated-profile":
+    case "models-cache":
+    case "injection-journal":
+      return new Set(["kind"]);
+    case "active-catalog":
+      return new Set(["kind", "canonicalPath"]);
+    case "catalog-backup":
+      return new Set(["kind", "form", "canonicalPath"]);
+    case "history-row":
+    case "history-manifest-entry":
+      return new Set(["kind", "stateDbId", "threadId"]);
+    case "history-manifest":
+    case "history-rollout":
+      return new Set(["kind", "stateDbId", "canonicalPath"]);
+  }
+}
+
+function mergeArtifact(previous: CodexArtifactId, next: CodexArtifactId): CodexArtifactId {
+  return copyUnknown(
+    previous as CodexArtifactId & Record<string, unknown>,
+    next as CodexArtifactId & Record<string, unknown>,
+    artifactKnownKeys(previous),
+  ) as CodexArtifactId;
+}
+
+function mergeBaseline(
+  previous: CodexProvenanceEntry["baseline"],
+  next: CodexProvenanceEntry["baseline"],
+): CodexProvenanceEntry["baseline"] {
+  const known = previous.kind === "present"
+    ? new Set(["kind", "sha256", "bytesBase64"])
+    : new Set(["kind"]);
+  return copyUnknown(previous, next, known) as CodexProvenanceEntry["baseline"];
+}
+
+function knownArtifactIdentity(artifact: CodexArtifactId): string {
+  switch (artifact.kind) {
+    case "config":
+    case "generated-profile":
+    case "models-cache":
+    case "injection-journal":
+      return artifact.kind;
+    case "active-catalog":
+      return `${artifact.kind}\0${artifact.canonicalPath}`;
+    case "catalog-backup":
+      return `${artifact.kind}\0${artifact.form}\0${artifact.canonicalPath}`;
+    case "history-row":
+    case "history-manifest-entry":
+      return `${artifact.kind}\0${artifact.stateDbId}\0${artifact.threadId}`;
+    case "history-manifest":
+    case "history-rollout":
+      return `${artifact.kind}\0${artifact.stateDbId}\0${artifact.canonicalPath}`;
+  }
+}
+
+function mergeEntry(previous: CodexProvenanceEntry, next: CodexProvenanceEntry): CodexProvenanceEntry {
+  const merged = copyUnknown(
+    previous,
+    next,
+    new Set(["artifact", "baseline", "postImage", "txId", "at"]),
+  );
+  return {
+    ...merged,
+    artifact: mergeArtifact(previous.artifact, next.artifact),
+    baseline: mergeBaseline(previous.baseline, next.baseline),
+  } as CodexProvenanceEntry;
+}
+
+function mergeLedger(previous: CodexProvenanceLedger, next: CodexProvenanceLedger): CodexProvenanceLedger {
+  const unused = new Set(previous.entries.map((_, index) => index));
+  const entries = next.entries.map((entry, nextIndex) => {
+    let previousIndex = previous.entries.findIndex((candidate, index) =>
+      unused.has(index)
+      && candidate.txId === entry.txId
+      && candidate.at === entry.at
+      && knownArtifactIdentity(candidate.artifact) === knownArtifactIdentity(entry.artifact));
+    if (previousIndex < 0 && unused.has(nextIndex)) previousIndex = nextIndex;
+    if (previousIndex < 0) return entry;
+    unused.delete(previousIndex);
+    return mergeEntry(previous.entries[previousIndex]!, entry);
+  });
+  return {
+    ...copyUnknown(previous, next, new Set(["entries"])),
+    entries,
+  } as CodexProvenanceLedger;
+}
+
+function preserveExtensions(
+  previous: CodexIntegrationRecord,
+  next: CodexIntegrationRecord,
+): CodexIntegrationRecord {
+  const merged = copyUnknown(previous, next, new Set(["version", "provenance"]));
+  if (previous.provenance && next.provenance) {
+    merged.provenance = mergeLedger(previous.provenance, next.provenance);
+  }
+  return merged as CodexIntegrationRecord;
+}
+
+/**
+ * Keep read, extension merge, and atomic replacement inside the config mutation
+ * coordinator. WP12 must call this function instead of recreating the incident's
+ * stale read/merge/write sequence at its own call site.
+ */
+export const updateIntegrationRecord = (
+  mutate: (record: CodexIntegrationRecord) => CodexIntegrationRecord,
+) => {
+  try {
+    return withConfigMutationLockSync(() => {
+      const read = readIntegrationRecordUnlocked();
+      if (read.kind === "invalid") return read;
+      const previous: CodexIntegrationRecord = read.kind === "ready" ? read.record : { version: 1 };
+      const proposed = mutate(previous);
+      if (!validateRecord(proposed)) {
+        return { kind: "invalid", message: "Codex integration record update produced a malformed v1 shape" };
+      }
+      const record = preserveExtensions(previous, proposed);
+      if (!validateRecord(record)) {
+        return { kind: "invalid", message: "Codex integration record extension merge produced a malformed v1 shape" };
+      }
+      const path = recordPath();
+      mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+      atomicWriteFile(path, `${JSON.stringify(record, null, 2)}\n`);
+      return { kind: "updated", record };
+    });
+  } catch (error) {
+    return {
+      kind: "invalid",
+      message: `Codex integration record update failed: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+};

package/src/codex/internal/catalog-writer.ts

@@ -0,0 +1,203 @@
+import { chmodSync, linkSync, mkdirSync, truncateSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+
+import {
+  AtomicWriteResidualTempError,
+  AtomicWriteSecretResidualError,
+  atomicWriteFile,
+  resolveWriteTarget,
+  type AtomicWriteIO,
+} from "../../config";
+import {
+  assertCatalogWritePermit,
+  type CatalogWritePermit,
+} from "../catalog-write-serialization";
+import {
+  forgetEphemeralSecretPath,
+  hardenSecretPath,
+} from "../../lib/windows-secret-acl";
+
+export interface PreparedCatalogFileWrite {
+  readonly path: string;
+  readonly content: string;
+}
+
+export type CatalogBackupPublication = "written" | "preserved";
+
+export interface CatalogBackupWriteIO {
+  readonly resolveTarget: (path: string) => string;
+  readonly write: (path: string, content: string) => void;
+  readonly harden: (path: string) => void;
+  /** Must fail with EEXIST rather than replacing an existing destination. */
+  readonly publishNoReplace: (source: string, destination: string) => void;
+  readonly truncate: (path: string) => void;
+  readonly unlink: (path: string) => void;
+}
+
+let backupTempSequence = 0;
+
+function isMissingPathError(error: unknown): boolean {
+  return (error as NodeJS.ErrnoException | undefined)?.code === "ENOENT";
+}
+
+function isExistingPathError(error: unknown): boolean {
+  return (error as NodeJS.ErrnoException | undefined)?.code === "EEXIST";
+}
+
+function defaultBackupWriteIO(destinationPath: string): CatalogBackupWriteIO {
+  return {
+    resolveTarget: resolveWriteTarget,
+    write: (path, content) => writeFileSync(path, content, { encoding: "utf8", mode: 0o600 }),
+    harden: path => {
+      try { chmodSync(path, 0o600); } catch { /* platform may ignore chmod */ }
+      if (process.platform === "win32") {
+        hardenSecretPath(path, { required: true, timeoutMemoKey: destinationPath });
+      }
+    },
+    publishNoReplace: linkSync,
+    truncate: path => truncateSync(path, 0),
+    unlink: unlinkSync,
+  };
+}
+
+function removePublishedTemp(tempPath: string, io: CatalogBackupWriteIO): void {
+  try {
+    io.unlink(tempPath);
+    forgetEphemeralSecretPath(tempPath);
+    return;
+  } catch (firstError) {
+    if (isMissingPathError(firstError)) {
+      forgetEphemeralSecretPath(tempPath);
+      return;
+    }
+    try {
+      io.unlink(tempPath);
+      forgetEphemeralSecretPath(tempPath);
+      return;
+    } catch (retryError) {
+      if (isMissingPathError(retryError)) {
+        forgetEphemeralSecretPath(tempPath);
+        return;
+      }
+      throw new AtomicWriteResidualTempError(tempPath, true, { cause: retryError });
+    }
+  }
+}
+
+function scrubAndRemoveUnpublishedTemp(
+  tempPath: string,
+  hardened: boolean,
+  io: CatalogBackupWriteIO,
+  cause: unknown,
+): void {
+  let scrubbed = false;
+  try {
+    io.truncate(tempPath);
+    scrubbed = true;
+  } catch (error) {
+    if (isMissingPathError(error)) scrubbed = true;
+    else {
+      try {
+        io.write(tempPath, "");
+        scrubbed = true;
+      } catch { /* removal may still succeed */ }
+    }
+  }
+
+  let removed = false;
+  try {
+    io.unlink(tempPath);
+    removed = true;
+  } catch (error) {
+    if (isMissingPathError(error)) removed = true;
+    else {
+      try {
+        io.unlink(tempPath);
+        removed = true;
+      } catch (retryError) {
+        if (isMissingPathError(retryError)) removed = true;
+      }
+    }
+  }
+
+  if (!removed && !scrubbed) {
+    throw new AtomicWriteSecretResidualError(tempPath, { cause });
+  }
+  if (!removed && !hardened) {
+    try {
+      io.harden(tempPath);
+      hardened = true;
+    } catch { /* zero-byte residual is reported honestly */ }
+  }
+  if (removed) forgetEphemeralSecretPath(tempPath);
+  if (!removed) throw new AtomicWriteResidualTempError(tempPath, hardened, { cause });
+}
+
+function publishCatalogBackup(
+  prepared: PreparedCatalogFileWrite,
+  suppliedIo?: CatalogBackupWriteIO,
+): CatalogBackupPublication {
+  const io = suppliedIo ?? defaultBackupWriteIO(prepared.path);
+  const target = io.resolveTarget(prepared.path);
+  if (!suppliedIo) mkdirSync(dirname(target), { recursive: true, mode: 0o700 });
+  const tempPath = `${target}.ocx.${process.pid}.backup.${++backupTempSequence}.tmp`;
+  let hardened = false;
+
+  try {
+    io.write(tempPath, prepared.content);
+    io.harden(tempPath);
+    hardened = true;
+    io.publishNoReplace(tempPath, target);
+  } catch (error) {
+    scrubAndRemoveUnpublishedTemp(tempPath, hardened, io, error);
+    if (isExistingPathError(error)) return "preserved";
+    throw error;
+  }
+
+  removePublishedTemp(tempPath, io);
+  return "written";
+}
+
+/** Replace the active catalog with the caller's already-prepared bytes. */
+export function replaceActiveCodexCatalog(
+  permit: CatalogWritePermit,
+  owningCodexHome: string,
+  prepared: PreparedCatalogFileWrite,
+  io?: AtomicWriteIO,
+): void {
+  assertCatalogWritePermit(permit, owningCodexHome);
+  atomicWriteFile(prepared.path, prepared.content, io);
+}
+
+/** Atomically publish the catalog-path-keyed immutable backup without clobbering. */
+export function publishHashedCodexCatalogBackup(
+  permit: CatalogWritePermit,
+  owningCodexHome: string,
+  prepared: PreparedCatalogFileWrite,
+  io?: CatalogBackupWriteIO,
+): CatalogBackupPublication {
+  assertCatalogWritePermit(permit, owningCodexHome);
+  return publishCatalogBackup(prepared, io);
+}
+
+/** Atomically publish the legacy immutable backup without clobbering. */
+export function publishLegacyCodexCatalogBackup(
+  permit: CatalogWritePermit,
+  owningCodexHome: string,
+  prepared: PreparedCatalogFileWrite,
+  io?: CatalogBackupWriteIO,
+): CatalogBackupPublication {
+  assertCatalogWritePermit(permit, owningCodexHome);
+  return publishCatalogBackup(prepared, io);
+}
+
+/** Replace Codex's models cache with the caller's already-prepared bytes. */
+export function replaceCodexModelsCache(
+  permit: CatalogWritePermit,
+  owningCodexHome: string,
+  prepared: PreparedCatalogFileWrite,
+  io?: AtomicWriteIO,
+): void {
+  assertCatalogWritePermit(permit, owningCodexHome);
+  atomicWriteFile(prepared.path, prepared.content, io);
+}

package/src/codex/internal/history-writer.ts

@@ -0,0 +1,80 @@
+/**
+ * The history mutation boundary. Every byte Codex history owns is written here,
+ * and only while H is held.
+ *
+ * This module exists because the mutations are spread across three surfaces that
+ * do not share a transaction: SQLite rows, the backup manifest, and the rollout
+ * files. `syncCodexHistoryProvider` writes the manifest BEFORE its database
+ * transaction and patches rollouts inside it; restore writes rollouts, then the
+ * database, then the manifest, then ejects again
+ * (`src/codex/history-provider.ts:606-648,656-698`). A SQLite busy timeout
+ * serializes exactly one of those three, which is why an opposite-direction
+ * process could overtake through the other two.
+ *
+ * The permit argument is not decoration and not a type-level claim. Each entry
+ * point asks the lock owner at RUNTIME whether the permit it was handed is still
+ * live for the state database about to be written, because a permit leaked past
+ * its callback type-checks perfectly. A writer reached without H therefore fails
+ * closed rather than racing.
+ *
+ * Reachability: `src/codex/history-worker.ts` is the only permitted production
+ * root. Readers and probes stay in `history-provider.ts`; nothing in the CLI, the
+ * server, the guardian, `inject.ts` or `sync.ts` may reach these symbols.
+ *
+ * Design record: devlog/_fin/260804_codex_write_substrate/020_history_isolation.md.
+ */
+import { assertHistoryWritePermit, type HistoryWritePermit } from "../history-lock";
+import {
+  restoreLegacyOpenaiHistory,
+  syncCodexHistoryProvider,
+  type CodexHistoryProvider,
+  type CodexHistorySyncResult,
+} from "../history-provider";
+
+/**
+ * Everything a history mutation needs, with no ambient state.
+ *
+ * The paths are explicit because a Worker is a separate process: it does not
+ * inherit the module-load `CODEX_HOME` that `history-provider.ts` resolves at
+ * import time (`:16`, `:22`), so a request that relied on those constants would
+ * silently address the wrong home.
+ */
+export interface HistoryWriteTarget {
+  /** Canonical, absolute; the same identity H was acquired for. */
+  readonly canonicalStateDbPath: string;
+  /** Canonical, absolute path of the backup manifest for that database. */
+  readonly canonicalBackupPath: string;
+}
+
+/**
+ * Apply opencodex routing to resumable history, or restore it to native.
+ *
+ * `provider` is the DURABLE operation's direction, resolved by the caller from
+ * the coordinator row — never a caller-supplied preference. The Worker passes
+ * what the row said, which is what stops a request from turning a restore into
+ * an apply.
+ */
+export function writeHistoryProviderTransition(
+  permit: HistoryWritePermit,
+  target: HistoryWriteTarget,
+  provider: CodexHistoryProvider,
+): CodexHistorySyncResult {
+  assertHistoryWritePermit(permit, target.canonicalStateDbPath);
+  return syncCodexHistoryProvider(provider, target.canonicalStateDbPath, target.canonicalBackupPath);
+}
+
+/**
+ * Manifest-independent legacy ejection.
+ *
+ * Distinct from the generic restore above: it never reads, consumes, deletes or
+ * replaces the backup manifest. It does patch rollout metadata and returns a
+ * `files` count, so calling it "DB-only" would be wrong — the name says what it
+ * actually avoids.
+ */
+export function writeLegacyOpenaiHistoryRecovery(
+  permit: HistoryWritePermit,
+  target: HistoryWriteTarget,
+): CodexHistorySyncResult {
+  assertHistoryWritePermit(permit, target.canonicalStateDbPath);
+  return restoreLegacyOpenaiHistory(target.canonicalStateDbPath);
+}