@iislee/opencodex 2.11.0 → 2.31.0

package/src/codex/generation.ts

@@ -0,0 +1,202 @@
+/**
+ * Durable generation ownership for OpenCodex config bytes.
+ *
+ * The WP8b C-phase review found that config ABA and moved configuration could
+ * not be detected because cooperating saves had no durable counter. This module
+ * owns the singleton schema and conditional increment in the existing config
+ * mutation database; callers that already hold its transaction reuse that
+ * Database handle so bytes and generation remain one cooperating commit.
+ *
+ * A hand edit from a non-cooperating writer deliberately does not increment this
+ * counter. The convergence contract detects that case with its post-commit file
+ * observation instead of pretending SQLite can coordinate an external editor.
+ */
+import { chmodSync, statSync } from "node:fs";
+
+import { Database } from "bun:sqlite";
+
+import type {
+  ConfigGeneration,
+  ConfigGenerationBump,
+  ConfigGenerationRead,
+} from "./convergence-types";
+
+const CREATE_CONFIG_GENERATION = `
+  CREATE TABLE IF NOT EXISTS config_generation (
+    singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
+    value INTEGER NOT NULL CHECK (value >= 0)
+  )`;
+const INITIALIZE_CONFIG_GENERATION = `
+  INSERT OR IGNORE INTO config_generation (singleton, value) VALUES (1, 0)`;
+const SELECT_CONFIG_GENERATION = `
+  SELECT value FROM config_generation WHERE singleton = 1`;
+const BUMP_CONFIG_GENERATION = `
+  UPDATE config_generation
+     SET value = value + 1
+   WHERE singleton = 1 AND value = ?`;
+
+interface ConfigGenerationRow {
+  value: unknown;
+}
+
+interface SchemaVersionRow {
+  schema_version: unknown;
+}
+
+/**
+ * Observation adds exactly one variant to `ConfigGenerationRead`: `absent`.
+ *
+ * This used to be deliberately absent itself, on the reasoning that a caller
+ * who may only LOOK must not be handed something it could mistake for a
+ * known-good baseline of zero. That reasoning still holds, and `absent` does
+ * not violate it — because `absent` is not a baseline. It authorizes nothing on
+ * its own. A caller may only promote it after taking the config transaction and
+ * reading a real zero THERE (`readConfigGenerationInCurrentMutationTransaction`),
+ * at which point the zero is observed rather than assumed. A caller with no
+ * transaction to open, such as catalog gather, must keep refusing it.
+ *
+ * What forced the distinction: refusing on absence meant refusing every Codex
+ * write on any home whose config predates this database — a permanent refusal
+ * for existing users, not a fixture problem.
+ *
+ * `absent` is returned ONLY for ENOENT on the initial `statSync`. A file that
+ * exists but cannot be read, a directory in its place, a bad schema version, or
+ * corrupt SQLite all stay `unavailable`, because those are reasons to stop, and
+ * collapsing them into absence is how a corrupt coordinator would become a
+ * licence to write.
+ */
+export type ConfigGenerationObservation =
+  | ConfigGenerationRead
+  | { kind: "absent" };
+
+function errorCode(error: unknown): string {
+  return error && typeof error === "object" && "code" in error
+    ? String((error as { code?: unknown }).code)
+    : "";
+}
+
+function isBusy(error: unknown): boolean {
+  const code = errorCode(error);
+  const message = error instanceof Error ? error.message : "";
+  return code === "SQLITE_BUSY"
+    || code === "SQLITE_LOCKED"
+    || /database (?:is|table is) locked/i.test(message);
+}
+
+function unavailable(error: unknown): Extract<ConfigGenerationRead, { kind: "unavailable" }> {
+  return { kind: "unavailable", reason: isBusy(error) ? "busy" : "database" };
+}
+
+export function initializeConfigGeneration(database: Database): void {
+  database.exec(CREATE_CONFIG_GENERATION);
+  database.exec(INITIALIZE_CONFIG_GENERATION);
+}
+
+export function readConfigGenerationInTransaction(database: Database): ConfigGeneration {
+  const row = database.query<ConfigGenerationRow, []>(SELECT_CONFIG_GENERATION).get();
+  if (!row || !Number.isSafeInteger(row.value) || Number(row.value) < 0) {
+    throw new Error("The config generation singleton is missing or invalid.");
+  }
+  return { value: Number(row.value) };
+}
+
+export function bumpConfigGenerationInTransaction(
+  database: Database,
+  expected: ConfigGeneration,
+): ConfigGenerationBump {
+  const result = database.query(BUMP_CONFIG_GENERATION).run(expected.value);
+  if (result.changes === 1) {
+    return { kind: "updated", generation: { value: expected.value + 1 } };
+  }
+  return { kind: "conflict", current: readConfigGenerationInTransaction(database) };
+}
+
+export function bumpCurrentConfigGeneration(database: Database): ConfigGeneration {
+  const current = readConfigGenerationInTransaction(database);
+  const result = bumpConfigGenerationInTransaction(database, current);
+  if (result.kind !== "updated") {
+    throw new Error("The config generation changed inside its owning transaction.");
+  }
+  return result.generation;
+}
+
+function runGenerationTransaction<T>(databasePath: string, operation: (database: Database) => T): T {
+  let database: Database | undefined;
+  let transactionOpen = false;
+  try {
+    database = new Database(databasePath, { create: true });
+    try { chmodSync(databasePath, 0o600); } catch { /* platform may ignore chmod */ }
+    database.exec("PRAGMA busy_timeout = 0; BEGIN IMMEDIATE");
+    transactionOpen = true;
+    initializeConfigGeneration(database);
+    const result = operation(database);
+    database.exec("COMMIT");
+    transactionOpen = false;
+    return result;
+  } catch (error) {
+    if (transactionOpen) {
+      try { database?.exec("ROLLBACK"); } catch { /* close releases the transaction */ }
+    }
+    throw error;
+  } finally {
+    try { database?.close(); } catch { /* operation already completed */ }
+  }
+}
+
+export function readConfigGenerationAtPath(databasePath: string): ConfigGenerationRead {
+  try {
+    return {
+      kind: "ready",
+      generation: runGenerationTransaction(databasePath, readConfigGenerationInTransaction),
+    };
+  } catch (error) {
+    return unavailable(error);
+  }
+}
+
+/**
+ * Observe generation state without preparing the mutation database in any way.
+ * Missing storage is a first-class state: only cooperating config writes have
+ * authority to create and initialize the generation singleton.
+ */
+export function observeConfigGenerationAtPath(
+  databasePath: string,
+): ConfigGenerationObservation {
+  try {
+    statSync(databasePath);
+  } catch (error) {
+    // ENOENT alone means absent. Everything else — EACCES, ENOTDIR, EIO — is a
+    // reason the question could not be answered, which is not the same answer.
+    return errorCode(error) === "ENOENT" ? { kind: "absent" } : unavailable(error);
+  }
+
+  let database: Database | undefined;
+  try {
+    database = new Database(databasePath, { readonly: true });
+    const schema = database.query<SchemaVersionRow, []>("PRAGMA schema_version").get();
+    if (!schema || !Number.isSafeInteger(schema.schema_version)) {
+      throw new Error("The config generation schema version is invalid.");
+    }
+    return {
+      kind: "ready",
+      generation: readConfigGenerationInTransaction(database),
+    };
+  } catch (error) {
+    return unavailable(error);
+  } finally {
+    try { database?.close(); } catch { /* observation already completed */ }
+  }
+}
+
+export function bumpConfigGenerationAtPath(
+  databasePath: string,
+  expected: ConfigGeneration,
+): ConfigGenerationBump {
+  try {
+    return runGenerationTransaction(databasePath, database => (
+      bumpConfigGenerationInTransaction(database, expected)
+    ));
+  } catch (error) {
+    return unavailable(error);
+  }
+}

package/src/codex/history-job.ts

@@ -0,0 +1,407 @@
+/**
+ * The parent half of the history unit: derive the operation, dispatch the
+ * Worker, and never let its failure become the caller's stall.
+ *
+ * The operation is DERIVED here from what the caller already decided — its
+ * config and the direction its native mutation just took — and then handed down
+ * as a fixed value. It is not a request field the Worker trusts, because the
+ * distinctions are real: `syncResumeHistory: false` means leave history alone,
+ * apply targets opencodex only in legacy mode, and legacy recovery must not
+ * touch the manifest that generic restore consumes.
+ *
+ * Every exit is typed. A Worker that errors, dies, or overruns its watchdog
+ * produces an outcome the caller can record, because the alternative — an
+ * exception crossing back into a route that already persisted its mutation — is
+ * how a successful change gets reported as a 500.
+ *
+ * Design record: devlog/_fin/260804_codex_write_substrate/020_history_isolation.md.
+ */
+import { randomUUID } from "node:crypto";
+import { homedir } from "node:os";
+
+import type {
+  CodexHistoryWorkerOperation,
+  HistoryWorkerResult,
+} from "./history-worker";
+import { historyBackupPathFor } from "./history-provider";
+import type { CodexHistoryFailureReason, CodexHistoryVerifiedNoopProof } from "./history-provider";
+import { getCodexHome, resolveCodexStateDbPath } from "./paths";
+
+/**
+ * Resolve the paths a history job needs, at CALL time.
+ *
+ * The SQLite root can differ from CODEX_HOME and both environment/config inputs
+ * can change between invocations. The parent resolves one exact target and hands
+ * those canonical paths to the Worker rather than asking the Worker to infer a
+ * possibly different environment.
+ */
+export function resolveCodexHistoryJobTarget(): {
+  readonly canonicalCodexHome: string;
+  readonly canonicalStateDbPath: string;
+  readonly canonicalBackupPath: string;
+} {
+  const home = getCodexHome();
+  const stateDb = resolveCodexStateDbPath({ codexHome: home });
+  return {
+    canonicalCodexHome: home,
+    canonicalStateDbPath: stateDb,
+    // Derived by the provider's own rule rather than guessed: the manifest lives
+    // in the config directory under a hash of the state database, so a
+    // hand-built path would address a different file entirely.
+    canonicalBackupPath: historyBackupPathFor(stateDb),
+  };
+}
+
+/** How long a history unit may run before the parent stops waiting on it. */
+const WORKER_TIMEOUT_MS = 30_000;
+/** A terminated Worker must close before its caller can cross the file boundary. */
+const WORKER_CLOSE_TIMEOUT_MS = 5_000;
+
+function historyWorkerOsJoinSettleMs(platform = process.platform): number {
+  if (platform === "win32") return 1_500;
+  if (platform === "darwin" || platform === "linux") return 250;
+  return 0;
+}
+
+async function terminateAndJoinHistoryWorker(
+  worker: Worker,
+  closed: Promise<void>,
+): Promise<boolean> {
+  let timedOut = false;
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  const closeTimeout = new Promise<void>(resolve => {
+    timeout = setTimeout(() => {
+      timedOut = true;
+      resolve();
+    }, WORKER_CLOSE_TIMEOUT_MS);
+  });
+
+  try {
+    worker.terminate();
+  } catch {
+    // A Worker that already closed still resolves through the close listener.
+  }
+
+  try {
+    await Promise.race([closed, closeTimeout]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+
+  const settleMs = historyWorkerOsJoinSettleMs();
+  if (settleMs > 0) {
+    await Bun.sleep(0);
+    await Bun.sleep(settleMs);
+  }
+  return !timedOut;
+}
+
+export interface CodexHistoryJobRequest {
+  readonly canonicalCodexHome: string;
+  readonly canonicalStateDbPath: string;
+  readonly canonicalBackupPath: string;
+  readonly operation: CodexHistoryWorkerOperation;
+  readonly expectedDesiredEnabled?: boolean;
+}
+
+export type CodexHistoryJobOutcome =
+  | { readonly kind: "converged"; readonly rows: number; readonly files: number; readonly proof?: CodexHistoryVerifiedNoopProof }
+  | { readonly kind: "skipped" }
+  | { readonly kind: "blocked"; readonly reason: "busy" | "database" | "unsafe-path" | "desired_disabled" | "desired_enabled" }
+  | { readonly kind: "failed"; readonly reason: "worker-error" | "worker-died" | "timeout";
+      readonly message: string; readonly historyFailureReason?: CodexHistoryFailureReason };
+
+/**
+ * Derive the durable history operation from admitted intent.
+ *
+ * `resumeHistory === false` is the user's explicit opt-out and outranks the
+ * direction entirely — an apply that quietly migrated history anyway would be
+ * the setting failing silently. `legacyMode` is the only case that routes
+ * history TO opencodex; the ordinary apply migrates to native so a later restore
+ * has nothing to undo.
+ */
+/**
+ * Test seam: the boundary suite exercises the validator without standing up a
+ * Worker whose malformed message it cannot easily emit from inside a test.
+ */
+export function isPlausibleWorkerResultForTests(
+  message: Record<string, unknown>,
+  requestId: string,
+  jobId: string,
+  target?: Pick<CodexHistoryJobRequest, "canonicalStateDbPath" | "canonicalBackupPath" | "operation">,
+): boolean {
+  return isPlausibleWorkerResult(message, requestId, jobId, target);
+}
+
+/**
+ * Reject a message that names a recognized type but lacks its payload.
+ *
+ * Without this, `{requestId, type:"done"}` reached an unchecked cast and read
+ * as `converged` with undefined fields — a success report for work that may not
+ * have happened. Each type is checked for the fields it actually carries, and
+ * the ids are matched against the request in flight, not merely present.
+ */
+function isPlausibleWorkerResult(
+  message: Record<string, unknown>,
+  requestId: string,
+  jobId: string,
+  target?: Pick<CodexHistoryJobRequest, "canonicalStateDbPath" | "canonicalBackupPath" | "operation">,
+): boolean {
+  if (message.requestId !== requestId || message.jobId !== jobId) return false;
+  switch (message.type) {
+    case "done":
+      if (!((message.outcome === "converged" || message.outcome === "skipped")
+        && typeof message.rows === "number"
+        && typeof message.files === "number")) return false;
+      if (message.proof === undefined) return true;
+      if (!target || !message.proof || typeof message.proof !== "object" || Array.isArray(message.proof)) return false;
+      {
+        const proof = message.proof as Record<string, unknown>;
+        return target.operation === "migrate-openai"
+          && message.outcome === "converged"
+          && message.rows === 0
+          && message.files === 0
+          && proof.kind === "verified-noop"
+          && proof.pendingRows === 0
+          && proof.backupEntries === 0
+          && proof.canonicalStateDbPath === target.canonicalStateDbPath
+          && proof.stateDbPresent === true
+          && proof.canonicalBackupPath === target.canonicalBackupPath
+          && typeof proof.backupPresent === "boolean";
+      }
+    case "blocked":
+      return message.reason === "busy" || message.reason === "database" || message.reason === "unsafe-path"
+        || message.reason === "desired_disabled" || message.reason === "desired_enabled";
+    case "error":
+      return typeof message.message === "string"
+        && (message.reason === undefined || message.reason === "busy" || message.reason === "permission");
+    default:
+      return false;
+  }
+}
+
+export function deriveCodexHistoryOperation(intent: {
+  readonly direction: "apply" | "restore";
+  readonly resumeHistory: boolean;
+  readonly legacyMode: boolean;
+}): CodexHistoryWorkerOperation {
+  if (!intent.resumeHistory) return "skip";
+  if (intent.direction === "restore") return "restore-openai";
+  return intent.legacyMode ? "apply-opencodex" : "migrate-openai";
+}
+
+/**
+ * The honest failure clause for one history job outcome.
+ *
+ * The caller adds its own framing ("sync SKIPPED", "could NOT be restored").
+ * The point of the surface argument is that a genuine lock keeps today's
+ * actionable wording, while every other reason stops blaming the Codex app:
+ * an unsafe-path refusal, an unavailable coordinator database, a permission
+ * denial, or a dead worker is a different problem with a different remedy.
+ */
+export function describeHistoryJobFailure(
+  outcome: CodexHistoryJobOutcome,
+  surface: "apply" | "restore" | "recover-legacy",
+  legacyMode = false,
+): string {
+  // Callers only invoke this after observing a failure flag, but that flag is
+  // derived from "not converged", which also covers "skipped". Naming those
+  // two kinds keeps a widened or miscast call site from printing `undefined`.
+  if (outcome.kind === "skipped") {
+    return "the history operation was skipped; no failure was recorded.";
+  }
+  if (outcome.kind === "converged") {
+    return "the history job reported no failure; run 'ocx doctor' if this is unexpected.";
+  }
+  // A busy database reaches here two ways: the lock itself was contended
+  // (blocked/busy), or the lock was acquired and the worker then found SQLite
+  // busy (failed with historyFailureReason "busy"). Both are the same user
+  // situation and deserve the same surface-specific guidance.
+  const busyText = surface === "apply"
+    ? legacyMode
+      ? "the history DB is locked (Codex app/IDE open?). Close it and rerun 'ocx start'."
+      : "the history DB is locked (Codex app/IDE open?). It is retried automatically (while the proxy runs and on every 'ocx start'); to force it now, close the Codex app and run 'ocx sync'."
+    : surface === "recover-legacy"
+      ? "the Codex history DB is locked (Codex app/IDE open?). Close it and rerun this command."
+      : "the Codex app appears to be holding the history database. Close Codex and run `ocx restore` again.";
+  if (outcome.kind === "blocked") {
+    if (outcome.reason === "busy") return busyText;
+    switch (outcome.reason) {
+      case "unsafe-path":
+        return "opencodex refused its history lock path (unsafe coordinator namespace); this is not a Codex app lock. Run 'ocx doctor' and check the opencodex runtime directory.";
+      case "database":
+        return "the history coordinator database is unavailable; this is not a Codex app lock. Run 'ocx doctor'.";
+      case "desired_disabled":
+        return "Codex integration is disabled, so the history operation was skipped.";
+      case "desired_enabled":
+        return "Codex integration is enabled, so the history operation was skipped.";
+    }
+  }
+  if (outcome.historyFailureReason === "busy") return busyText;
+  if (outcome.historyFailureReason === "permission") {
+    return "permission was denied while writing Codex history; this is not a Codex app lock. Run 'ocx doctor'.";
+  }
+  switch (outcome.reason) {
+    case "worker-error":
+      return `the history worker failed (${outcome.message}). Run 'ocx doctor'.`;
+    case "worker-died":
+      return "the history worker exited unexpectedly; this is not a Codex app lock. Run 'ocx doctor'.";
+    case "timeout":
+      return "the history worker timed out; this is not a Codex app lock. Run 'ocx doctor'.";
+  }
+}
+
+/**
+ * Worker exceptions travel into user-facing CLI output, and a raw filesystem
+ * error carries absolute paths — on every platform that includes the account
+ * name (`/Users/x`, `/home/x`, `C:\Users\x`). Folding the home directory to
+ * `~` keeps the diagnostic value and drops the identifier.
+ */
+function redactWorkerMessage(message: string): string {
+  const home = homedir();
+  if (home.length <= 1) return message;
+  // Windows spellings vary in case and separator; an exact match would leave
+  // the account name in the message.
+  if (process.platform === "win32") {
+    const escaped = home.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\\/g, "[\\\\/]");
+    return message.replace(new RegExp(escaped, "gi"), "~");
+  }
+  return message.split(home).join("~");
+}
+
+function classifyWorkerResult(result: HistoryWorkerResult): CodexHistoryJobOutcome {
+  if (result.type === "blocked") return { kind: "blocked", reason: result.reason };
+  if (result.type === "error") {
+    return {
+      kind: "failed",
+      reason: "worker-error",
+      message: redactWorkerMessage(result.message),
+      ...(result.reason ? { historyFailureReason: result.reason } : {}),
+    };
+  }
+  return result.outcome === "skipped"
+    ? { kind: "skipped" }
+    : { kind: "converged", rows: result.rows, files: result.files, ...(result.proof ? { proof: result.proof } : {}) };
+}
+
+/**
+ * Run one history unit in a Worker and join it before returning.
+ *
+ * The join is not optional politeness: returning while the thread may still be
+ * mutating CODEX_HOME would let a caller's next step observe a half-applied
+ * transition, and would let a test suite reach its next file with a worker still
+ * exiting behind it.
+ */
+export async function runCodexHistoryJob(
+  request: CodexHistoryJobRequest,
+  options: { readonly timeoutMs?: number } = {},
+): Promise<CodexHistoryJobOutcome> {
+  const requestId = randomUUID();
+  const jobId = randomUUID();
+  const timeoutMs = options.timeoutMs ?? WORKER_TIMEOUT_MS;
+
+  // `skip` writes nothing, so spawning a thread to decide that would be pure
+  // cost. It still returns a recorded outcome rather than silence.
+  if (request.operation === "skip") return { kind: "skipped" };
+
+  let worker: Worker;
+  try {
+    worker = new Worker(new URL("./history-worker.ts", import.meta.url).href);
+  } catch (error) {
+    return {
+      kind: "failed",
+      reason: "worker-died",
+      message: error instanceof Error ? error.message : "history_worker_spawn_failed",
+    };
+  }
+
+  return new Promise<CodexHistoryJobOutcome>(resolve => {
+    let settled = false;
+    let resolveWorkerClosed!: () => void;
+    const workerClosed = new Promise<void>(closed => {
+      resolveWorkerClosed = closed;
+    });
+    const finish = (outcome: CodexHistoryJobOutcome) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      // Bun 1.3.14 returns void from terminate(); resolving on that return lets
+      // the Worker outlive this job and, under `bun test --isolate`, its file.
+      void terminateAndJoinHistoryWorker(worker, workerClosed).then(joined => {
+        resolve(joined
+          ? outcome
+          : { kind: "failed", reason: "worker-died", message: "history_worker_close_timeout" });
+      }, () => {
+        resolve({ kind: "failed", reason: "worker-died", message: "history_worker_join_failed" });
+      });
+    };
+
+    const timer = setTimeout(() => {
+      finish({ kind: "failed", reason: "timeout", message: "history_worker_timeout" });
+    }, timeoutMs);
+
+    const died = (detail: string): void => {
+      finish({ kind: "failed", reason: "worker-died", message: detail });
+    };
+
+    worker.onmessage = (event: MessageEvent<unknown>) => {
+      const data = event.data;
+      if (!data || typeof data !== "object") {
+        // Not the shape at all: this is a death signal, not silence. Ignoring it
+        // would let the watchdog call a dead Worker a timeout.
+        died("history_worker_malformed_message");
+        return;
+      }
+      const message = data as Record<string, unknown>;
+      // A reply for a different request is somebody else's; ignoring it is not
+      // the same as accepting it.
+      if (message.requestId !== requestId) return;
+      if (message.type !== "done" && message.type !== "blocked" && message.type !== "error") {
+        died("history_worker_unknown_message_type");
+        return;
+      }
+      if (!isPlausibleWorkerResult(message, requestId, jobId, request)) {
+        // A recognized type with a missing payload read as `converged` with
+        // undefined fields once — success for work that may not have happened.
+        died("history_worker_malformed_payload");
+        return;
+      }
+      finish(classifyWorkerResult(message as unknown as HistoryWorkerResult));
+    };
+
+    /*
+     * A Worker that exits early — without erroring — is not an error event, so
+     * without these it surfaced as `timeout` after the full wait. Both are death
+     * signals. Neither can overturn a settled success: `finish` is idempotent,
+     * and the Worker always closes after posting its result.
+     */
+    worker.addEventListener("messageerror", () => died("history_worker_unserializable_message"));
+    worker.addEventListener("close", () => {
+      resolveWorkerClosed();
+      died("history_worker_closed_early");
+    }, { once: true });
+
+    worker.onerror = (event: ErrorEvent) => {
+      finish({
+        kind: "failed",
+        reason: "worker-died",
+        message: event.message || "history_worker_failed",
+      });
+    };
+
+    worker.postMessage({
+      type: "run",
+      requestId,
+      jobId,
+      operation: request.operation,
+      canonicalCodexHome: request.canonicalCodexHome,
+      canonicalStateDbPath: request.canonicalStateDbPath,
+      canonicalBackupPath: request.canonicalBackupPath,
+      ...(request.expectedDesiredEnabled === undefined ? {} : { expectedDesiredEnabled: request.expectedDesiredEnabled }),
+      env: {
+        ...(process.env.CODEX_HOME ? { CODEX_HOME: process.env.CODEX_HOME } : {}),
+        ...(process.env.OPENCODEX_HOME ? { OPENCODEX_HOME: process.env.OPENCODEX_HOME } : {}),
+      },
+    });
+  });
+}