@iislee/opencodex 2.11.0 → 2.35.0

package/src/codex/history-lock.ts

@@ -0,0 +1,242 @@
+/**
+ * H — the cross-process history serialization primitive.
+ *
+ * Today a native apply or restore performs SQLite rows, the backup manifest and
+ * every rollout file on the caller thread, and the manifest and rollout writes
+ * sit OUTSIDE the provider's SQLite transaction
+ * (`src/codex/history-provider.ts:606-648,656-698`). A SQLite busy timeout
+ * therefore serializes one third of a state transition and explains one stall.
+ * H is held across the whole unit — DB, manifest and rollouts together — so an
+ * opposite-direction process cannot overtake through the unguarded files.
+ *
+ * Two ways H differs from K, both forced by review:
+ *
+ * `H -> N` is a real edge, not a violation. The Worker reads the coordinator row
+ * and writes its terminal state while holding H, and both take BEGIN IMMEDIATE
+ * on N. That is why H must be its own database: sharing N's would make the
+ * Worker contend with itself. `N -> H`, `K -> H` and `C -> H` are forbidden, and
+ * the resulting `H -> N -> K -> C` order was checked acyclic in review round 3.
+ *
+ * H is keyed by the canonical state database as well as the canonical home,
+ * because one CODEX_HOME can name a different `state_5.sqlite` and those are not
+ * the same exclusion (`src/codex/user-identity.ts`).
+ *
+ * The permit is a runtime registration, not a type. An opaque TypeScript brand
+ * proves a permit-bearing call path exists; it cannot prove the callback still
+ * holds the lock, and a permit leaked past its callback type-checks perfectly.
+ * So every history mutator asks this module at runtime whether the permit it was
+ * handed is still live for the state database it is about to write.
+ *
+ * Design record: devlog/_fin/260804_codex_write_substrate/005_contract.md §6.
+ */
+import { chmodSync, lstatSync, realpathSync } from "node:fs";
+
+import { Database } from "bun:sqlite";
+
+import {
+  CodexUserIdentityRefusal,
+  resolveCodexHistorySerializationDatabasePath,
+  resolveEffectiveUserIdentity,
+  samePathIdentity,
+} from "./user-identity";
+
+/**
+ * Authorization to perform history mutations inside ONE H acquisition.
+ *
+ * Deliberately carries no usable field. Holding this object is necessary and
+ * never sufficient: `assertHistoryWritePermit` decides, by looking the object up
+ * in a registry only this module can write. A forged cast, a prototype copy or a
+ * symbol clone produces a value of this type that every writer refuses.
+ */
+export interface HistoryWritePermit {
+  readonly [historyWritePermitBrand]: true;
+}
+
+declare const historyWritePermitBrand: unique symbol;
+
+export type HistorySerializationOutcome<T> =
+  | { kind: "completed"; value: T }
+  | { kind: "unavailable"; reason: "busy" | "database" | "unsafe-path" };
+
+export class HistoryWritePermitRefusal extends Error {
+  readonly code = "CODEX_HISTORY_WRITE_PERMIT_REFUSED";
+
+  constructor(message: string) {
+    super(message);
+    this.name = "HistoryWritePermitRefusal";
+  }
+}
+
+interface PermitRegistration {
+  readonly canonicalStateDbPath: string;
+  readonly transactionId: string;
+  live: boolean;
+}
+
+/**
+ * Registrations are marked dead, never deleted.
+ *
+ * Deleting would make a leaked permit indistinguishable from a forged one, and
+ * those are different bugs: "used after its acquisition released" points at a
+ * caller that kept a permit past its callback, while "never minted here" points
+ * at a cast or a clone. Both are refused; only the diagnosis differs. The map is
+ * weak, so a dead entry costs nothing once the permit is unreachable.
+ */
+const activePermits = new WeakMap<object, PermitRegistration>();
+let acquisitionCounter = 0;
+
+function isBusy(error: unknown): boolean {
+  const code = error && typeof error === "object" && "code" in error
+    ? String((error as { code?: unknown }).code)
+    : "";
+  const message = error instanceof Error ? error.message : String(error);
+  return code === "SQLITE_BUSY" || code === "SQLITE_LOCKED"
+    || /database (?:is|table is) locked/i.test(message);
+}
+
+/**
+ * Refuse unless this exact permit is live for this exact state database.
+ *
+ * Every history mutator calls this BEFORE touching the database, the manifest or
+ * a rollout file. The state database is named by the caller rather than derived,
+ * because a rollout path can point anywhere the manifest recorded.
+ */
+export function assertHistoryWritePermit(
+  permit: HistoryWritePermit,
+  canonicalStateDbPath: string,
+): void {
+  const registration = activePermits.get(permit as unknown as object);
+  if (!registration) {
+    throw new HistoryWritePermitRefusal(
+      "The history write permit was not minted by the serialization owner.",
+    );
+  }
+  if (!registration.live) {
+    throw new HistoryWritePermitRefusal(
+      "The history write permit belongs to a released acquisition.",
+    );
+  }
+  if (registration.canonicalStateDbPath !== canonicalStateDbPath) {
+    throw new HistoryWritePermitRefusal(
+      "The history write permit authorizes a different Codex state database.",
+    );
+  }
+}
+
+/**
+ * Acquire H for one canonical state database and run `work` while it is held.
+ *
+ * The callback may enter N — that is the `H -> N` edge the Worker needs to read
+ * the coordinator row and publish its terminal state. It must never re-enter H.
+ *
+ * `busy_timeout = 0` with `BEGIN IMMEDIATE` makes contention fail fast and
+ * typed. Blocking here would hold the acquisition across an unbounded wait,
+ * which is the stall this phase exists to remove.
+ */
+export function withHistoryWriteSerialization<T>(
+  canonicalCodexHome: string,
+  canonicalStateDbPath: string,
+  work: (permit: HistoryWritePermit) => T,
+): HistorySerializationOutcome<T> {
+  let databasePath: string;
+  try {
+    databasePath = resolveCodexHistorySerializationDatabasePath(
+      resolveEffectiveUserIdentity(),
+      canonicalCodexHome,
+      canonicalStateDbPath,
+    );
+  } catch (error) {
+    if (error instanceof CodexUserIdentityRefusal) {
+      return { kind: "unavailable", reason: "unsafe-path" };
+    }
+    return { kind: "unavailable", reason: "database" };
+  }
+
+  let database: Database | undefined;
+  let transactionOpen = false;
+  let registration: PermitRegistration | undefined;
+  let permit: HistoryWritePermit | undefined;
+
+  try {
+    let databaseWasAbsent = false;
+    try {
+      const before = lstatSync(databasePath);
+      if (before.isSymbolicLink() || !before.isFile()) {
+        return { kind: "unavailable", reason: "unsafe-path" };
+      }
+      if (process.platform !== "win32") {
+        const uid = process.getuid?.();
+        if (uid === undefined || before.uid !== uid || (before.mode & 0o777) !== 0o600) {
+          return { kind: "unavailable", reason: "unsafe-path" };
+        }
+      }
+    } catch (cause) {
+      const code = cause && typeof cause === "object" && "code" in cause
+        ? String((cause as { code?: unknown }).code)
+        : "";
+      if (code !== "ENOENT") throw cause;
+      databaseWasAbsent = true;
+    }
+
+    database = new Database(databasePath, { create: true });
+    if (databaseWasAbsent) {
+      try { chmodSync(databasePath, 0o600); } catch { /* Windows applies ACLs in WP11. */ }
+    }
+    const opened = lstatSync(databasePath);
+    if (opened.isSymbolicLink() || !opened.isFile()
+      || !samePathIdentity(realpathSync.native(databasePath), databasePath)) {
+      return { kind: "unavailable", reason: "unsafe-path" };
+    }
+
+    database.exec("PRAGMA busy_timeout = 0; BEGIN IMMEDIATE");
+    transactionOpen = true;
+
+    acquisitionCounter += 1;
+    registration = {
+      canonicalStateDbPath,
+      transactionId: `${process.pid}:${acquisitionCounter}`,
+      live: true,
+    };
+    // A bare object: nothing about it is guessable or reconstructable, because
+    // the authority lives in the registry entry rather than in the value.
+    permit = {} as HistoryWritePermit;
+    activePermits.set(permit as unknown as object, registration);
+
+    let value: T;
+    try {
+      value = work(permit);
+    } finally {
+      // Revoke BEFORE releasing the transaction, so a mutator racing the release
+      // can never find a live permit without a live lock. This runs on the
+      // throwing path too, which is what a `finally`-less version gets wrong.
+      registration.live = false;
+    }
+
+    database.exec("COMMIT");
+    transactionOpen = false;
+    return { kind: "completed", value };
+  } catch (error) {
+    if (transactionOpen) {
+      try { database?.exec("ROLLBACK"); } catch { /* close releases the transaction */ }
+      transactionOpen = false;
+    }
+    if (registration?.live) {
+      registration.live = false;
+    }
+    if (error instanceof CodexUserIdentityRefusal) {
+      return { kind: "unavailable", reason: "unsafe-path" };
+    }
+    if (isBusy(error)) return { kind: "unavailable", reason: "busy" };
+    // A callback failure is the caller's error, not a lock outcome: H acquired
+    // fine. Reporting it as `unavailable` would tell the caller to retry
+    // something that fails identically.
+    throw error;
+  } finally {
+    try { database?.close(); } catch { /* acquisition already finished */ }
+  }
+}
+
+/** Test-only: prove a leaked permit is dead without reaching into the registry. */
+export function isHistoryWritePermitLive(permit: HistoryWritePermit): boolean {
+  return activePermits.get(permit as unknown as object)?.live === true;
+}

package/src/codex/history-manifest.ts

@@ -0,0 +1,112 @@
+import { createHash } from "node:crypto";
+import { isAbsolute, resolve } from "node:path";
+
+/** Sources whose native OpenAI provenance can be restored exactly. */
+export const CODEX_HISTORY_RESUMABLE_SOURCES = ["cli", "vscode"] as const;
+
+export interface CodexHistoryBackupEntry {
+  id: string;
+  rolloutPath: string;
+  modelProvider: string;
+  source: string;
+  hasUserEvent: 0 | 1;
+}
+
+export interface CodexHistoryBackupManifest {
+  version: 1;
+  stateDbPath: string;
+  entries: Record<string, CodexHistoryBackupEntry>;
+}
+
+export type CodexHistoryManifestValidation =
+  | { readonly ok: true; readonly manifest: CodexHistoryBackupManifest }
+  | { readonly ok: false; readonly reason: "foreign-database" }
+  | {
+      readonly ok: false;
+      readonly reason: "schema";
+      readonly scope: "manifest" | "entry-shape" | "entry-provenance";
+    };
+
+function codexHistoryPathIdentity(path: string): string {
+  const canonical = resolve(path);
+  return process.platform === "win32" ? canonical.toLowerCase() : canonical;
+}
+
+/**
+ * Canonical identity used by both backup naming and manifest ownership checks.
+ * Windows paths are case-insensitive for this contract; other platforms keep
+ * the resolved path's case.
+ */
+export function codexHistoryStateDbIdentity(path: string): string {
+  return codexHistoryPathIdentity(path);
+}
+
+/** Stable, non-secret file-name component for one state database. */
+export function codexHistoryBackupId(stateDbPath: string): string {
+  return createHash("sha256")
+    .update(codexHistoryStateDbIdentity(stateDbPath))
+    .digest("hex")
+    .slice(0, 16);
+}
+
+/** Platform-aware identity comparison shared by database and rollout checks. */
+export function sameCodexHistoryPath(left: string, right: string): boolean {
+  return codexHistoryPathIdentity(left) === codexHistoryPathIdentity(right);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function hasAllowedProvenance(entry: Record<string, unknown>): boolean {
+  return (entry.modelProvider === "openai"
+      && CODEX_HISTORY_RESUMABLE_SOURCES.includes(
+        entry.source as (typeof CODEX_HISTORY_RESUMABLE_SOURCES)[number],
+      ))
+    || (entry.modelProvider === "opencodex" && entry.source === "exec");
+}
+
+/**
+ * Validate only the versioned data contract and database ownership identity.
+ * Filesystem type checks, reads, fingerprints, rollout inspection, SQLite, and
+ * mutation deliberately remain with the callers.
+ */
+export function validateCodexHistoryBackupManifest(
+  raw: unknown,
+  expectedStateDbPath: string,
+): CodexHistoryManifestValidation {
+  if (!isRecord(raw)
+    || raw.version !== 1
+    || typeof raw.stateDbPath !== "string"
+    || !raw.stateDbPath.trim()
+    || !isAbsolute(raw.stateDbPath)
+    || !isRecord(raw.entries)) {
+    return { ok: false, reason: "schema", scope: "manifest" };
+  }
+  if (!sameCodexHistoryPath(raw.stateDbPath, expectedStateDbPath)) {
+    return { ok: false, reason: "foreign-database" };
+  }
+
+  for (const [id, value] of Object.entries(raw.entries)) {
+    if (!isRecord(value)) {
+      return { ok: false, reason: "schema", scope: "entry-shape" };
+    }
+    if (!id
+      || value.id !== id
+      || typeof value.rolloutPath !== "string"
+      || !value.rolloutPath.trim()
+      || !isAbsolute(value.rolloutPath)
+      || typeof value.modelProvider !== "string"
+      || !value.modelProvider.trim()
+      || typeof value.source !== "string"
+      || !value.source.trim()
+      || typeof value.hasUserEvent !== "number"
+      || !Number.isSafeInteger(value.hasUserEvent)
+      || (value.hasUserEvent !== 0 && value.hasUserEvent !== 1)
+      || !hasAllowedProvenance(value)) {
+      return { ok: false, reason: "schema", scope: "entry-provenance" };
+    }
+  }
+
+  return { ok: true, manifest: raw as unknown as CodexHistoryBackupManifest };
+}

package/src/codex/history-migration-guardian.ts

@@ -1,11 +1,12 @@
-import { countPendingOpencodexHistory, migrateHistoryToOpenai } from "./history-provider";
+import { migrateHistoryToOpenai } from "./history-provider";
+import { resolveCodexHistoryJobTarget, runCodexHistoryJob } from "./history-job";
 
 /**
- * Daemon-side retry for the one-time Design-B history migration.
+ * Daemon-side retry for the one-time Design-B history metadata restoration.
  *
  * Most upgrades run `ocx start` while the Codex app still holds `state_5.sqlite`,
- * so the inject-time migration often fails on the FIRST start — exactly the moment
- * every legacy thread is still tagged `opencodex` and invisible to the app. Instead
+ * so the inject-time restore can fail on the FIRST start while manifest-backed original
+ * metadata is still pending. Instead
  * of asking the user to close the app and rerun start, this guardian keeps retrying
  * in the background until the migration lands.
  *
@@ -22,8 +23,8 @@ export interface HistoryMigrationGuardianHandle {
 }
 
 export interface HistoryMigrationGuardianDeps {
-  countFn?: typeof countPendingOpencodexHistory;
-  migrateFn?: () => ReturnType<typeof migrateHistoryToOpenai>;
+  migrateFn?: () => ReturnType<typeof migrateHistoryToOpenai>
+    | Promise<ReturnType<typeof migrateHistoryToOpenai>>;
   log?: Pick<Console, "log">;
   tickMs?: number;
   maxTicks?: number;
@@ -41,8 +42,18 @@ function defaultSchedule(fn: () => void, ms: number): { cancel(): void } {
 }
 
 export function startHistoryMigrationGuardian(deps: HistoryMigrationGuardianDeps = {}): HistoryMigrationGuardianHandle {
-  const countFn = deps.countFn ?? countPendingOpencodexHistory;
-  const migrateFn = deps.migrateFn ?? (() => migrateHistoryToOpenai(undefined, undefined, { attempts: 1 }));
+  // The default migration goes through the history job, so the guardian's timer
+  // thread never performs the transition itself. A background repair that races
+  // an apply or a restore is exactly what H exists to order.
+  const migrateFn = deps.migrateFn ?? (async () => {
+    const outcome = await runCodexHistoryJob({
+      ...resolveCodexHistoryJobTarget(),
+      operation: "migrate-openai",
+    });
+    return outcome.kind === "converged"
+      ? { rows: outcome.rows, files: outcome.files, verifiedNoop: outcome.proof?.kind === "verified-noop" }
+      : { rows: 0, files: 0, failed: true as const };
+  });
   const log = deps.log ?? console;
   const tickMs = deps.tickMs ?? DEFAULT_TICK_MS;
   const maxTicks = deps.maxTicks ?? DEFAULT_MAX_TICKS;
@@ -56,27 +67,22 @@ export function startHistoryMigrationGuardian(deps: HistoryMigrationGuardianDeps
     pending = (deps.scheduleFn ?? defaultSchedule)(tick, tickMs);
   };
 
-  const tick = () => {
+  const tick = async () => {
     if (stopped) return;
     ticks++;
     try {
-      const count = countFn();
-      if (!count.failed && count.pendingRows === 0 && count.backupEntries === 0) {
-        stopped = true; // nothing left to migrate — normal steady state, no log noise
-        return;
-      }
-      // Locked probe or pending work: attempt one migration pass.
-      const result = migrateFn();
+      // The worker re-derives state under H on every pass; no pre-lock probe is
+      // allowed to stop or suppress this attempt.
+      const result = await migrateFn();
       if (!result.failed) {
-        const moved = result.rows + (result.ejectedRows ?? 0);
+        const moved = result.rows + ((result as { ejectedRows?: number }).ejectedRows ?? 0);
         if (moved > 0) {
-          log.log(`🩹 history-migration: ${moved} legacy opencodex thread(s) migrated back to openai.`);
+          log.log(`🩹 history-migration: restored original provider metadata for ${moved} manifest-backed thread(s).`);
         }
-        // A "successful" zero-row migration can also mean the DB does not exist YET while a
-        // backup manifest still holds restore work (fresh reinstall race). Only stop when a
-        // re-count proves nothing is pending; otherwise keep ticking within the budget.
-        const after = countFn();
-        if (moved > 0 || (!after.failed && after.pendingRows === 0 && after.backupEntries === 0)) {
+        // Zero mutations are authoritative only when the worker verified the
+        // exact DB/manifest state while H was held.
+        const verifiedNoop = (result as { verifiedNoop?: boolean }).verifiedNoop === true;
+        if (moved > 0 || verifiedNoop) {
           stopped = true;
           return;
         }
@@ -86,7 +92,7 @@ export function startHistoryMigrationGuardian(deps: HistoryMigrationGuardianDeps
     }
     if (ticks >= maxTicks) {
       stopped = true;
-      log.log("⚠️ history-migration: Codex history DB stayed locked; legacy threads not yet migrated. Close the Codex app and run 'ocx sync' (or check 'ocx doctor').");
+      log.log("⚠️ history-migration: Could not verify that backed-up provider metadata was restored; the history database may be busy, unavailable, or not yet ready. Run 'ocx sync' (or check 'ocx doctor').");
       return;
     }
     schedule();