@iislee/opencodex 2.31.0 → 2.35.0

package/src/codex/history-worker.ts

@@ -4,8 +4,8 @@
  * Everything mutable about history happens here, behind H: the SQLite rows, the
  * backup manifest, and every rollout patch. Those three do not share a
  * transaction — sync writes the manifest before its database transaction, and
- * restore writes rollouts, then the database, then the manifest
- * (`src/codex/history-provider.ts:606-648,656-698`) — so a busy timeout only
+ * restore preflights every rollout, applies database CAS before rollout changes inside one
+ * SQLite transaction, then consumes the manifest after exact readback — so a busy timeout only
  * ever serialized a third of a state transition. Holding H across the whole unit
  * is what stops an opposite-direction process overtaking through the other two.
  *
@@ -74,7 +74,8 @@ export type HistoryWorkerResult =
   | { readonly type: "blocked"; readonly requestId: string; readonly jobId: string;
       readonly reason: "busy" | "database" | "unsafe-path" | "desired_disabled" | "desired_enabled" }
   | { readonly type: "error"; readonly requestId: string; readonly jobId: string;
-      readonly message: string; readonly reason?: CodexHistoryFailureReason };
+      readonly message: string; readonly reason?: CodexHistoryFailureReason;
+      readonly rows?: number; readonly files?: number };
 
 const OPERATIONS: ReadonlySet<string> = new Set<CodexHistoryWorkerOperation>([
   "skip",
@@ -142,8 +143,9 @@ export function runHistoryUnitUnderLock(
         const proof = snapshotCodexHistoryNoop(message.canonicalStateDbPath, message.canonicalBackupPath);
         if (proof.kind === "verified-noop") return { verifiedNoop: proof } as const;
       }
-      // apply-opencodex routes history to opencodex; migrate/restore return it to
-      // native. The provider is derived from the operation, never from a caller.
+      // apply-opencodex routes history to opencodex; migrate/restore recover only
+      // manifest-backed original metadata. The provider is derived from the operation,
+      // never from a caller; only recover-legacy-openai force-labels bare routed rows.
       const provider = operation === "apply-opencodex" ? "opencodex" : "openai";
       return writeHistoryProviderTransition(permit, target, provider);
     },
@@ -179,6 +181,7 @@ export function runHistoryUnitUnderLock(
       jobId,
       message: "history_transition_failed",
       ...(result.failureReason ? { reason: result.failureReason } : {}),
+      ...(result.rows > 0 || result.files > 0 ? { rows: result.rows, files: result.files } : {}),
     };
   }
   return {

package/src/codex/inject-coordination.ts

@@ -11,7 +11,12 @@ import { atomicWriteFile } from "../config";
 import type { CodexWriteLockResult } from "./codex-write-lock";
 import { inspectCodexCoordinatorPath } from "./coordinator-doctor";
 import { JOURNAL_PATH } from "./journal";
+import { updateIntegrationRecord } from "./integration-record";
 import { CODEX_CONFIG_PATH, CODEX_PROFILE_PATH } from "./paths";
+import type {
+  CodexArtifactId,
+  CodexProvenanceEntry,
+} from "./convergence-types";
 import {
   codexWriteCoordination,
   type CodexWriteCandidate,
@@ -41,6 +46,7 @@ export const DEFAULT_INJECT_LOCK_TIMEOUT_MS = 5_000;
  */
 export type CodexWriteCoordinationEligibility =
   | { kind: "coordinated" }
+  | { kind: "adopt" }
   | { kind: "legacy-uncoordinated"; reason: string }
   | { kind: "refused"; reason: string };
 
@@ -97,6 +103,7 @@ export function codexWriteCoordinationEligibility(deps: {
 
   const residue = deps.residue();
   if (residue.kind === "clean") return { kind: "coordinated" };
+  if (residue.kind === "residue" && !coordinatorIsStableZeroByte) return { kind: "adopt" };
   /*
    * Everything else keeps the path it has always had.
    *
@@ -176,6 +183,82 @@ export function captureCodexPreImages(): CodexPreImages {
   };
 }
 
+/**
+ * How many transactions of evidence the ledger keeps.
+ *
+ * Each transaction appends three entries, and a `present` baseline embeds the artifact's exact
+ * bytes as base64 — a 25 KB `config.toml` is ~34 KB per entry, so roughly 100 KB per transaction.
+ * A machine that syncs on every start would grow this file without limit, and it is re-read and
+ * re-serialized on every append, so the cost is quadratic rather than merely large.
+ *
+ * A ledger is evidence, not an archive. The most recent transactions are the ones anyone
+ * diagnoses against, so the window keeps those and drops the oldest.
+ */
+export const CODEX_PROVENANCE_MAX_TRANSACTIONS = 16;
+
+function provenanceBaseline(bytes: string | null): CodexProvenanceEntry["baseline"] {
+  if (bytes === null) return { kind: "absent" };
+  return {
+    kind: "present",
+    sha256: createHash("sha256").update(bytes).digest("hex"),
+    bytesBase64: Buffer.from(bytes).toString("base64"),
+  };
+}
+
+function provenancePostImage(path: string): string | null {
+  try {
+    return createHash("sha256").update(readFileSync(path)).digest("hex");
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Keep the newest `CODEX_PROVENANCE_MAX_TRANSACTIONS` transactions, whole.
+ *
+ * Trimming by ENTRY count would cut a transaction in half and leave evidence that says a
+ * transaction touched two artifacts when it touched three — worse than dropping it outright,
+ * because a partial record still reads as complete. Order is preserved; only whole leading
+ * transactions are removed.
+ */
+export function boundProvenanceEntries(
+  entries: readonly CodexProvenanceEntry[],
+  maxTransactions = CODEX_PROVENANCE_MAX_TRANSACTIONS,
+): readonly CodexProvenanceEntry[] {
+  const order: string[] = [];
+  for (const entry of entries) if (!order.includes(entry.txId)) order.push(entry.txId);
+  if (order.length <= maxTransactions) return entries;
+  const keep = new Set(order.slice(order.length - maxTransactions));
+  return entries.filter(entry => keep.has(entry.txId));
+}
+
+/** Append evidence for an already-committed native transaction, best-effort. */
+export function recordCodexNativeTransactionProvenance(
+  preImages: CodexPreImages,
+  txId: string,
+) {
+  const at = new Date().toISOString();
+  const surfaces: readonly [CodexArtifactId, string, string | null][] = [
+    [{ kind: "config" }, CODEX_CONFIG_PATH, preImages.config],
+    [{ kind: "generated-profile" }, CODEX_PROFILE_PATH, preImages.profile],
+    [{ kind: "injection-journal" }, JOURNAL_PATH, preImages.journal],
+  ];
+  const entries: readonly CodexProvenanceEntry[] = surfaces.map(([artifact, path, baseline]) => ({
+    artifact,
+    baseline: provenanceBaseline(baseline),
+    postImage: provenancePostImage(path),
+    txId,
+    at,
+  }));
+  return updateIntegrationRecord(record => ({
+    ...record,
+    provenance: {
+      ...record.provenance,
+      entries: boundProvenanceEntries([...(record.provenance?.entries ?? []), ...entries]),
+    },
+  }));
+}
+
 /**
  * Put back exactly what was there, and report honestly when that fails.
  *

package/src/codex/inject.ts

@@ -19,6 +19,7 @@ import {
   CodexWriteConflictError,
   DEFAULT_INJECT_LOCK_TIMEOUT_MS,
   recomputeInjectWitness,
+  recordCodexNativeTransactionProvenance,
   restoreCodexPreImages,
 } from "./inject-coordination";
 import { readIntegrationRecord } from "./integration-record";
@@ -953,6 +954,7 @@ export async function injectCodexConfig(
     const coordinated = await withCodexWriteLock(
       {
         timeoutMs: options.lockTimeoutMs ?? DEFAULT_INJECT_LOCK_TIMEOUT_MS,
+        ...(eligibility.kind === "adopt" ? { adoption: { direction: "apply" as const } } : {}),
         admitted: { authoritySnapshotId: witness.comparisonId },
         readAdmissionUnderLock: () => ({
           authoritySnapshotId: recomputeInjectWitness({
@@ -1019,6 +1021,7 @@ export async function injectCodexConfig(
         }
         return {
           kind: "applied" as const,
+          preImages,
           /*
            * The receipt the terminal update matches on. The transition commits
            * when the callback returns, so this pair is what the post-job
@@ -1036,6 +1039,10 @@ export async function injectCodexConfig(
     if (coordinated.status !== "acquired") {
       return codexInjectLockOutcome(coordinated);
     }
+    recordCodexNativeTransactionProvenance(
+      coordinated.value.preImages,
+      coordinated.value.receipt.currentTxId,
+    );
     transitionReceipt = coordinated.value.receipt;
   }
   // Legacy mode still forward-tags history so re-tagged threads stay listable. Design B needs
@@ -1092,8 +1099,8 @@ export async function injectCodexConfig(
         : legacyMode
           ? `  Codex resume history: ${history.rows} thread(s) made visible for opencodex; originals backed up for restore.\n`
           : migratedRows > 0
-            ? `  Codex resume history: ${migratedRows} legacy opencodex-tagged thread(s) migrated back to openai (one-time).\n`
-            : `  Codex resume history: untouched (threads keep their native openai tag).\n`;
+            ? `  Codex resume history: restored original provider metadata for ${migratedRows} manifest-backed thread(s) (one-time).\n`
+            : `  Codex resume history: no backed-up metadata pending; untracked routed history left unchanged.\n`;
   // A user-owned root openai_base_url means we did NOT install routing — say so honestly
   // instead of claiming the proxy route is active (catalog/fast_mode were still written).
   if (keptUserBaseUrl) {
@@ -1332,18 +1339,33 @@ export interface CodexNativeRestoreResult {
   };
 }
 
-function failedHistoryRestore(reason?: CodexHistoryFailureReason, detail?: string): CodexRestoreHistoryResult {
+function failedHistoryRestore(
+  reason?: CodexHistoryFailureReason,
+  detail?: string,
+  progress: { rows?: number; files?: number } = {},
+): CodexRestoreHistoryResult {
+  const rows = progress.rows ?? 0;
+  const files = progress.files ?? 0;
+  const changed = rows > 0 || files > 0;
   return {
     state: "failed",
-    changed: false,
+    changed,
     ...(reason ? { reason } : {}),
-    rows: 0,
-    files: 0,
+    rows,
+    files,
     ejectedRows: 0,
     message: reason === "permission"
-      ? "Codex resume history could NOT be restored because permission was denied."
+      ? changed
+        ? "Codex resume history changed but did NOT converge because permission was denied while finalizing the backup manifest; the manifest was retained for review and safe retry."
+        : "Codex resume history could NOT be restored because permission was denied."
       : reason === "busy"
-        ? "Codex resume history could NOT be restored — the Codex app appears to be holding the history database."
+        ? changed
+          ? "Codex resume history changed but did NOT converge because backup-manifest finalization remained busy; the manifest was retained for review and safe retry."
+          : detail ?? "Codex resume history could NOT be restored — the Codex app appears to be holding the history database."
+        : reason === "integrity"
+          ? changed
+            ? "Codex resume history changed but did NOT converge because the backup or target changed; the manifest was retained for review and safe retry."
+            : "Codex resume history could NOT be restored because the backup or restore target failed integrity checks; unverified provider metadata was left unchanged."
         : detail
           ? `Codex resume history could NOT be restored: ${detail}`
           : "Codex resume history could NOT be restored; the reason was not recorded. Run 'ocx doctor'.",
@@ -1365,9 +1387,18 @@ export function failedHistoryRestoreFromOutcome(
   outcome: Extract<CodexHistoryJobOutcome, { kind: "blocked" | "failed" }>,
 ): CodexRestoreHistoryResult {
   if (outcome.kind === "blocked" && outcome.reason === "busy") return failedHistoryRestore("busy");
-  if (outcome.kind === "failed" && outcome.historyFailureReason === "busy") return failedHistoryRestore("busy");
+  if (outcome.kind === "failed" && outcome.historyFailureReason === "busy") {
+    return failedHistoryRestore(
+      "busy",
+      describeHistoryJobFailure(outcome, "restore"),
+      { rows: outcome.rows, files: outcome.files },
+    );
+  }
   if (outcome.kind === "failed" && outcome.historyFailureReason === "permission") {
-    return failedHistoryRestore("permission");
+    return failedHistoryRestore("permission", undefined, { rows: outcome.rows, files: outcome.files });
+  }
+  if (outcome.kind === "failed" && outcome.historyFailureReason === "integrity") {
+    return failedHistoryRestore("integrity", undefined, { rows: outcome.rows, files: outcome.files });
   }
   return failedHistoryRestore(undefined, describeHistoryJobFailure(outcome, "restore"));
 }
@@ -1523,13 +1554,14 @@ export async function restoreNativeCodexAsync(
   let config: CodexRestoreConfigResult;
   let transitionReceipt: { nativeGeneration: number; currentTxId: string } | undefined;
 
-  if (eligibility.kind === "coordinated") {
+  if (eligibility.kind === "coordinated" || eligibility.kind === "adopt") {
     // The restore has no candidate bytes to witness; freshness comes from the
     // filesystem reads and the desired-state re-read performed under the lock.
     const witness = { authoritySnapshotId: "codex-native-restore" };
     const coordinated = await withCodexWriteLock(
       {
         timeoutMs: DEFAULT_INJECT_LOCK_TIMEOUT_MS,
+        ...(eligibility.kind === "adopt" ? { adoption: { direction: "remove" as const } } : {}),
         admitted: witness,
         readAdmissionUnderLock: () => witness,
       },
@@ -1565,6 +1597,7 @@ export async function restoreNativeCodexAsync(
         }
         return {
           config: restored,
+          preImages,
           receipt: {
             nativeGeneration: ctx.expectation.nativeAfter,
             currentTxId: ctx.expectation.txId,
@@ -1583,6 +1616,10 @@ export async function restoreNativeCodexAsync(
           : `Codex configuration was not restored: ${coordinated.message}`,
       };
     } else {
+      recordCodexNativeTransactionProvenance(
+        coordinated.value.preImages,
+        coordinated.value.receipt.currentTxId,
+      );
       config = coordinated.value.config;
       transitionReceipt = coordinated.value.receipt;
     }
@@ -1607,10 +1644,10 @@ export async function restoreNativeCodexAsync(
   }
   const history: CodexRestoreHistoryResult = outcome.kind === "converged"
     ? {
-        state: "ok", changed: outcome.rows > 0, rows: outcome.rows, files: outcome.files, ejectedRows: 0,
+        state: "ok", changed: outcome.rows > 0 || outcome.files > 0, rows: outcome.rows, files: outcome.files, ejectedRows: 0,
         message: outcome.rows > 0
-          ? `Resume history restored from opencodex backup (${outcome.rows} thread(s)).`
-          : "Codex resume history was already native.",
+          ? `Resume history metadata restored from opencodex backup (${outcome.rows} thread(s)); original providers preserved.`
+          : "No backed-up resume-history metadata was pending; untracked routed history was left unchanged.",
       }
     : outcome.kind === "skipped"
       ? { state: "skipped", changed: false, rows: 0, files: 0, ejectedRows: 0, message: "Codex resume history was skipped." }
@@ -1672,16 +1709,16 @@ export function restoreNativeCodex(options: { skipHistory?: boolean; revalidateD
   const history: CodexRestoreHistoryResult = options.skipHistory
     ? { state: "skipped", changed: false, rows: 0, files: 0, ejectedRows: 0, message: "History restoration runs asynchronously." }
     : rawHistory.failed
-      ? failedHistoryRestore(rawHistory.failureReason)
+      ? failedHistoryRestore(rawHistory.failureReason, undefined, rawHistory)
       : {
           state: "ok",
-          changed: rawHistory.rows > 0 || (rawHistory.ejectedRows ?? 0) > 0,
+          changed: rawHistory.rows > 0 || rawHistory.files > 0 || (rawHistory.ejectedRows ?? 0) > 0,
           rows: rawHistory.rows,
           files: rawHistory.files,
           ejectedRows: rawHistory.ejectedRows ?? 0,
           message: rawHistory.rows > 0
-            ? `Resume history restored from opencodex backup (${rawHistory.rows} thread(s)).`
-            : "Codex resume history was already native.",
+            ? `Resume history metadata restored from opencodex backup (${rawHistory.rows} thread(s)); original providers preserved.`
+            : "No backed-up resume-history metadata was pending; untracked routed history was left unchanged.",
         };
   const message = catalog.removed > 0
     ? `${config.message} Catalog restored to ${catalog.kept} native model(s) (dropped ${catalog.removed} proxy-routed).`
@@ -1711,10 +1748,14 @@ export function formatApplyHistoryFailure(outcome: CodexHistoryJobOutcome, legac
   const busy =
     (outcome.kind === "blocked" && outcome.reason === "busy") ||
     (outcome.kind === "failed" && outcome.historyFailureReason === "busy");
-  const headline = legacyMode
-    ? "Codex resume history sync SKIPPED"
-    : busy
-      ? "Codex resume history migration deferred"
+  const partiallyChanged = outcome.kind === "failed"
+    && ((outcome.rows ?? 0) > 0 || (outcome.files ?? 0) > 0);
+  const headline = partiallyChanged
+    ? "Codex resume history changed but did not converge"
+    : legacyMode
+      ? "Codex resume history sync SKIPPED"
+      : busy
+      ? "Codex resume history metadata restore deferred"
       : "Codex resume history NOT changed";
   return `  ⚠️ ${headline}: ${describeHistoryJobFailure(outcome, "apply", legacyMode)}\n`;
 }

package/src/codex/injected-marker.ts

@@ -22,7 +22,7 @@ export function tomlStringPattern(key: string): RegExp {
   // A basic string escapes backslashes, so a Windows path is stored doubled; reading
   // the raw bytes back returned a path that matched nothing on disk and made the
   // journal's recorded catalog path un-restorable (#1798).
-  return new RegExp(`^\\s*${keyToken}\\s*=\\s*("(?:\\\\.|[^"])*"|'[^']*')\\s*(?:#.*)?$`);
+  return new RegExp(`^\\s*${keyToken}\\s*=\\s*("(?:\\\\.|[^"\\\\])*"|'[^']*')\\s*(?:#.*)?$`);
 }
 
 export function rootTomlString(content: string, key: string): string | null {

package/src/codex/internal/history-writer.ts

@@ -5,9 +5,10 @@
  * This module exists because the mutations are spread across three surfaces that
  * do not share a transaction: SQLite rows, the backup manifest, and the rollout
  * files. `syncCodexHistoryProvider` writes the manifest BEFORE its database
- * transaction and patches rollouts inside it; restore writes rollouts, then the
- * database, then the manifest, then ejects again
- * (`src/codex/history-provider.ts:606-648,656-698`). A SQLite busy timeout
+ * transaction and patches rollouts inside it; restore preflights every rollout,
+ * applies database CAS before file changes in one SQLite transaction, then consumes the
+ * manifest after exact readback (`src/codex/history-provider.ts`). A
+ * SQLite busy timeout
  * serializes exactly one of those three, which is why an opposite-direction
  * process could overtake through the other two.
  *

package/src/codex/log-guard/inspect.ts

@@ -11,6 +11,9 @@ import {
 
 const IMMUTABLE_READONLY_FLAGS = constants.SQLITE_OPEN_READONLY | constants.SQLITE_OPEN_URI;
 const KNOWN_LOG_LEVELS = new Set(["TRACE", "DEBUG", "INFO", "WARN", "ERROR"]);
+// The issue reporter measured a ~1 GB database taking 17.3s for GROUP BY level
+// alone; skipping all row aggregates above 64 MiB reduced /api/storage to 628ms.
+const MAX_SYNCHRONOUS_METRICS_DATABASE_BYTES = 64 * 1024 * 1024;
 
 interface CurrentLogColumn {
   name: string;
@@ -115,6 +118,10 @@ export interface CodexLogGuardInspection {
     reclaim: CodexLogGuardCapability;
   };
   metrics: CodexLogGuardMetrics | null;
+  metricsSkipped: null | {
+    reason: "database_too_large";
+    thresholdBytes: number;
+  };
 }
 
 interface ColumnRow {
@@ -167,9 +174,8 @@ function fileSize(path: string): number {
  * repeated stalls without ever serving stale numbers: any write changes the WAL
  * and invalidates the entry.
  *
- * This bounds the repeat cost, not the first one. A cold inspection of a huge
- * database still blocks; moving that work off-thread needs a Worker and is
- * tracked separately.
+ * Memoization bounds repeat cost. The database-size gate below separately bounds
+ * cold request-thread work by omitting these aggregates for large databases.
  */
 type InspectionCacheEntry = {
   key: string;
@@ -233,6 +239,7 @@ function unavailableInspection(): CodexLogGuardInspection {
       reclaim: unavailable,
     },
     metrics: null,
+    metricsSkipped: null,
   };
 }
 
@@ -425,6 +432,7 @@ function inspectCodexLogsUncached(deps: CodexSqliteHomeDeps = {}): CodexLogGuard
       ...common,
       schema,
       metrics: null,
+      metricsSkipped: null,
       capabilities: {
         inspection: { state: "supported" },
         protection: mutation,
@@ -440,6 +448,7 @@ function inspectCodexLogsUncached(deps: CodexSqliteHomeDeps = {}): CodexLogGuard
       ...common,
       schema,
       metrics: null,
+      metricsSkipped: null,
       capabilities: {
         inspection: { state: "supported" },
         protection: mutation,
@@ -458,6 +467,7 @@ function inspectCodexLogsUncached(deps: CodexSqliteHomeDeps = {}): CodexLogGuard
       ...common,
       schema,
       metrics: null,
+      metricsSkipped: null,
       capabilities: {
         inspection: { state: "supported" },
         protection: mutation,
@@ -476,10 +486,17 @@ function inspectCodexLogsUncached(deps: CodexSqliteHomeDeps = {}): CodexLogGuard
         ? { state: "compatible" }
         : { state: "unsupported", reason: "unknown_schema" };
       const mutation = capabilityFor(schema);
+      const metricsSkipped = files.databaseBytes > MAX_SYNCHRONOUS_METRICS_DATABASE_BYTES
+        ? {
+            reason: "database_too_large" as const,
+            thresholdBytes: MAX_SYNCHRONOUS_METRICS_DATABASE_BYTES,
+          }
+        : null;
       return {
         ...common,
         schema,
-        metrics: readMetrics(db, columns),
+        metrics: metricsSkipped === null ? readMetrics(db, columns) : null,
+        metricsSkipped,
         capabilities: {
           inspection: { state: "supported" },
           protection: mutation,
@@ -496,6 +513,7 @@ function inspectCodexLogsUncached(deps: CodexSqliteHomeDeps = {}): CodexLogGuard
       ...common,
       schema,
       metrics: null,
+      metricsSkipped: null,
       capabilities: {
         inspection: { state: "supported" },
         protection: mutation,

package/src/codex/model-entitlements.ts

@@ -40,6 +40,10 @@ export interface CodexModelEntitlementResolveOptions {
   readonly now?: number;
   /** Test-only credential seam; production callers enumerate local main + Pool credentials. */
   readonly credentials?: readonly CodexModelEntitlementCredentialSnapshot[];
+  /** Test-only seam for proving lifecycle exclusions happen before credential reads. */
+  readonly credentialSnapshot?: typeof accountCredentialSnapshot;
+  /** Accounts whose credentials must not be read while another lifecycle owns them. */
+  readonly excludeAccountIds?: ReadonlySet<string>;
 }
 
 const accountModelsCache = new Map<string, CachedAccountModels>();
@@ -252,9 +256,12 @@ export async function resolveCodexModelEntitlements(
 ): Promise<CodexModelEntitlementSnapshot> {
   const now = options.now ?? Date.now();
   const fetcher = options.fetcher ?? fetch;
+  const allowedAccountIds = candidateAccountIds(config)
+    .filter(accountId => !options.excludeAccountIds?.has(accountId));
+  const credentialSnapshot = options.credentialSnapshot ?? accountCredentialSnapshot;
   const credentials = options.credentials
-    ? [...options.credentials]
-    : (await Promise.all(candidateAccountIds(config).map(accountCredentialSnapshot)))
+    ? [...options.credentials].filter(credential => !options.excludeAccountIds?.has(credential.accountId))
+    : (await Promise.all(allowedAccountIds.map(credentialSnapshot)))
       .filter((value): value is CodexModelEntitlementCredentialSnapshot => value !== null);
   const results = await Promise.all(credentials.map(async credential => ({
     credential,