@iislee/opencodex 2.11.0

package/src/storage/restore-job.ts

@@ -0,0 +1,299 @@
+/**
+ * Single-flight controller for trash restore runs.
+ *
+ * Heavy work (file moves, SQLite reconcile) runs in a Bun Worker so the proxy
+ * event loop stays responsive while the management API awaits the outcome.
+ *
+ * Restore shares the CODEX_HOME storage-mutation coordinator with manual cleanup
+ * and (Phase 3) policy-driven cleanup. Concurrent callers receive
+ * `storage_mutation_busy` (409) instead of queueing.
+ */
+import { resolveCodexHomeDir } from "../codex/home";
+import { restoreTrashEntry, type RestoreResult, type RestoreTestHooks } from "./cleanup";
+import {
+  resetStorageMutationCoordinatorForTests,
+  setStorageMutationCoordinatorTestHooks,
+  withStorageMutationSlot,
+  type StorageMutationCoordinatorTestHooks,
+} from "./storage-mutation-coordinator";
+import {
+  cancelQueuedStorageWorkerSpawns,
+  drainStorageWorkers,
+  StorageWorkerAdmissionBusyError,
+  terminateStorageWorker,
+  tryReserveStorageWorker,
+  withStorageWorkerSpawnGate,
+} from "./worker-lifecycle";
+
+export interface RestoreJobTestHooks extends StorageMutationCoordinatorTestHooks {
+  /**
+   * When true, run on the main thread via dynamic import + optional sleep.
+   * Responsiveness tests must leave this unset so work stays in a Worker.
+   */
+  runInProcess?: boolean;
+  /** Expose GET /api/storage/trash/restore/test-stream for responsiveness tests. */
+  enableTestStream?: boolean;
+  /** Forwarded to restoreTrashEntry _test hooks. */
+  restoreTest?: RestoreTestHooks;
+}
+
+const WORKER_TIMEOUT_MS = 10 * 60 * 1000;
+
+const WORKER_REJECTION_CODES: Record<string, RestoreResult["error"]> = {
+  restore_worker_timeout: "restore_worker_timeout",
+  aborted: "restore_worker_aborted",
+  worker_failed: "restore_worker_failed",
+};
+
+/** Map a rejected worker promise to a precise restore outcome (exported for tests). */
+export function restoreResultFromWorkerRejection(err: unknown, trashId: string): RestoreResult {
+  const raw = err instanceof Error ? err.message : String(err);
+  const error = WORKER_REJECTION_CODES[raw] ?? "restore_worker_failed";
+  const detail =
+    error === "restore_worker_failed" && raw !== "worker_failed" && raw.trim().length > 0
+      ? raw
+      : undefined;
+  console.error(
+    `[storage] trash restore worker failed (${error}) trashId=${trashId}${detail ? `: ${detail}` : ""}`,
+  );
+  return {
+    ok: false,
+    count: 0,
+    bytes: 0,
+    restoredPaths: [],
+    error,
+    ...(detail ? { message: detail } : {}),
+  };
+}
+
+let activeWorker: Worker | null = null;
+let testHooks: RestoreJobTestHooks | null = null;
+let cancelActiveRun: (() => void) | null = null;
+
+export function setRestoreTrashJobTestHooks(hooks: RestoreJobTestHooks | null): void {
+  testHooks = hooks;
+  setStorageMutationCoordinatorTestHooks(hooks);
+}
+
+/**
+ * Fire-and-forget reset. Prefer {@link resetRestoreTrashJobForTestsAsync} from
+ * test beforeEach/afterEach under `bun test --isolate` on Windows.
+ */
+export function resetRestoreTrashJobForTests(): void {
+  cancelQueuedStorageWorkerSpawns();
+  if (activeWorker) {
+    void terminateStorageWorker(activeWorker);
+    activeWorker = null;
+  }
+  cancelActiveRun?.();
+  cancelActiveRun = null;
+  testHooks = null;
+  resetStorageMutationCoordinatorForTests();
+}
+
+/**
+ * Await-able reset for test teardown; see policy-job's equivalent for why.
+ * Awaits the active worker directly so drain cannot race a fire-and-forget
+ * deregister from the sync reset.
+ */
+export async function resetRestoreTrashJobForTestsAsync(): Promise<void> {
+  cancelQueuedStorageWorkerSpawns();
+  const worker = activeWorker;
+  activeWorker = null;
+  cancelActiveRun?.();
+  cancelActiveRun = null;
+  testHooks = null;
+  // Join the worker before clearing the mutation coordinator so a concurrent
+  // run cannot acquire CODEX_HOME while the aborted thread is still mutating.
+  try {
+    if (worker) await terminateStorageWorker(worker);
+    await drainStorageWorkers();
+  } finally {
+    resetStorageMutationCoordinatorForTests();
+  }
+}
+
+/** Terminate an in-flight worker during process shutdown. */
+export function abortRestoreTrashJob(): void {
+  cancelQueuedStorageWorkerSpawns();
+  if (activeWorker) {
+    void terminateStorageWorker(activeWorker);
+    activeWorker = null;
+  }
+  cancelActiveRun?.();
+  cancelActiveRun = null;
+}
+
+/** Await-able shutdown sibling; joins the restore worker thread before returning. */
+export async function abortRestoreTrashJobAsync(): Promise<void> {
+  cancelQueuedStorageWorkerSpawns();
+  const worker = activeWorker;
+  activeWorker = null;
+  cancelActiveRun?.();
+  cancelActiveRun = null;
+  if (worker) await terminateStorageWorker(worker);
+  await drainStorageWorkers();
+}
+
+/** Test-only SSE/text stream served from the proxy while a worker is blocked. */
+export function getRestoreTrashTestStreamResponse(): Response | null {
+  if (!testHooks?.enableTestStream) return null;
+  const encoder = new TextEncoder();
+  return new Response(
+    new ReadableStream({
+      async start(controller) {
+        for (let i = 0; i < 8; i++) {
+          controller.enqueue(encoder.encode(`chunk-${i}\n`));
+          await Bun.sleep(50);
+        }
+        controller.close();
+      },
+    }),
+    { headers: { "content-type": "text/plain; charset=utf-8" } },
+  );
+}
+
+function runInWorker(opts: {
+  trashId: string;
+  codexHome?: string;
+  busyTimeoutMs?: number;
+  blockMs?: number;
+  restoreTest?: RestoreTestHooks;
+}): Promise<RestoreResult> {
+  const reservation = tryReserveStorageWorker();
+  if (!reservation) return Promise.reject(new StorageWorkerAdmissionBusyError());
+  return withStorageWorkerSpawnGate(() => new Promise<RestoreResult>((resolve, reject) => {
+    const requestId = crypto.randomUUID();
+    let settled = false;
+    let worker: Worker;
+    try {
+      worker = new Worker(new URL("./restore-worker.ts", import.meta.url).href);
+      reservation.bind(worker);
+    } catch (error) {
+      reservation.release();
+      reject(error);
+      return;
+    }
+    activeWorker = worker;
+
+    const finish = (fn: () => void) => {
+      if (settled) return;
+      settled = true;
+      cancelActiveRun = null;
+      clearTimeout(timer);
+      if (activeWorker === worker) activeWorker = null;
+      // Join before settle so mutation coordination inside the restore worker
+      // cannot overlap a follow-on run after a watchdog timeout.
+      void terminateStorageWorker(worker).then(fn, fn);
+    };
+
+    const timer = setTimeout(() => {
+      finish(() => reject(new Error("restore_worker_timeout")));
+    }, WORKER_TIMEOUT_MS);
+
+    cancelActiveRun = () => {
+      finish(() => reject(new Error("aborted")));
+    };
+
+    worker.onmessage = (event: MessageEvent<unknown>) => {
+      const data = event.data;
+      if (!data || typeof data !== "object") return;
+      const msg = data as Record<string, unknown>;
+      if (msg.requestId !== requestId) return;
+      if (msg.type === "done" && msg.result && typeof msg.result === "object") {
+        finish(() => resolve(msg.result as RestoreResult));
+        return;
+      }
+      if (msg.type === "error") {
+        const message = typeof msg.message === "string" ? msg.message : "worker_failed";
+        finish(() => reject(new Error(message)));
+      }
+    };
+
+    worker.onerror = (err: ErrorEvent) => {
+      finish(() => reject(err.error instanceof Error ? err.error : new Error(err.message || "worker_failed")));
+    };
+
+    worker.postMessage({
+      type: "run",
+      requestId,
+      trashId: opts.trashId,
+      ...(opts.codexHome ? { codexHome: opts.codexHome } : {}),
+      ...(opts.busyTimeoutMs !== undefined ? { busyTimeoutMs: opts.busyTimeoutMs } : {}),
+      ...(opts.blockMs !== undefined ? { blockMs: opts.blockMs } : {}),
+      ...(opts.restoreTest ? { restoreTest: opts.restoreTest } : {}),
+      env: {
+        ...(process.env.CODEX_HOME ? { CODEX_HOME: process.env.CODEX_HOME } : {}),
+        ...(process.env.OPENCODEX_HOME ? { OPENCODEX_HOME: process.env.OPENCODEX_HOME } : {}),
+      },
+    });
+  })).catch(error => {
+    reservation.release();
+    throw error;
+  });
+}
+
+async function executeRestore(opts: {
+  trashId: string;
+  codexHome?: string;
+  busyTimeoutMs?: number;
+  _test?: RestoreTestHooks;
+}): Promise<RestoreResult> {
+  const blockMs = testHooks?.blockMs;
+  const restoreTest = opts._test ?? testHooks?.restoreTest;
+
+  if (testHooks?.runInProcess) {
+    if (typeof blockMs === "number" && blockMs > 0) await Bun.sleep(blockMs);
+    return restoreTrashEntry(opts.trashId, {
+      ...(opts.codexHome ? { codexHome: opts.codexHome } : {}),
+      ...(opts.busyTimeoutMs !== undefined ? { busyTimeoutMs: opts.busyTimeoutMs } : {}),
+      ...(restoreTest ? { _test: restoreTest } : {}),
+    });
+  }
+
+  try {
+    return await runInWorker({
+      trashId: opts.trashId,
+      ...(opts.codexHome ? { codexHome: opts.codexHome } : {}),
+      ...(opts.busyTimeoutMs !== undefined ? { busyTimeoutMs: opts.busyTimeoutMs } : {}),
+      ...(typeof blockMs === "number" && blockMs > 0 ? { blockMs } : {}),
+      ...(restoreTest ? { restoreTest } : {}),
+    });
+  } catch (err) {
+    if (err instanceof StorageWorkerAdmissionBusyError) return busyRestoreResult();
+    return restoreResultFromWorkerRejection(err, opts.trashId);
+  }
+}
+
+function busyRestoreResult(): RestoreResult {
+  return {
+    ok: false,
+    count: 0,
+    bytes: 0,
+    restoredPaths: [],
+    error: "storage_mutation_busy",
+  };
+}
+
+/**
+ * Run trash restore off the event loop under the shared storage-mutation gate.
+ * Returns `storage_mutation_busy` when cleanup or another restore is in flight.
+ */
+export async function runRestoreTrashEntryJob(
+  trashId: string,
+  options?: {
+    codexHome?: string;
+    busyTimeoutMs?: number;
+    _test?: RestoreTestHooks;
+  },
+): Promise<RestoreResult> {
+  const codexHome = options?.codexHome ?? resolveCodexHomeDir();
+  const result = await withStorageMutationSlot("restore", codexHome, () =>
+    executeRestore({ trashId, ...options }),
+  );
+  if (result && typeof result === "object" && "ok" in result && result.ok === false
+    && "error" in result && result.error === "storage_mutation_busy") {
+    return busyRestoreResult();
+  }
+  return result as RestoreResult;
+}

package/src/storage/restore-worker.ts

@@ -0,0 +1,58 @@
+/**
+ * Worker-thread entry for trash restore runs.
+ * Keeps file moves and SQLite reconcile off the proxy event loop.
+ */
+import { restoreTrashEntry, type RestoreResult, type RestoreTestHooks } from "./cleanup";
+
+interface RunMessage {
+  type: "run";
+  requestId: string;
+  trashId: string;
+  codexHome?: string;
+  busyTimeoutMs?: number;
+  /** Test-only: block before restore so responsiveness tests can probe /healthz. */
+  blockMs?: number;
+  restoreTest?: RestoreTestHooks;
+  /** Env snapshot — Workers may not see parent mutations on all platforms. */
+  env?: { CODEX_HOME?: string; OPENCODEX_HOME?: string };
+}
+
+function isRunMessage(data: unknown): data is RunMessage {
+  if (!data || typeof data !== "object" || Array.isArray(data)) return false;
+  const o = data as Record<string, unknown>;
+  return o.type === "run" && typeof o.requestId === "string" && typeof o.trashId === "string";
+}
+
+declare const self: Worker;
+
+self.onmessage = async (event: MessageEvent<unknown>) => {
+  if (!isRunMessage(event.data)) return;
+  const { requestId, trashId, codexHome, busyTimeoutMs, blockMs, restoreTest, env } = event.data;
+  try {
+    if (env?.CODEX_HOME) process.env.CODEX_HOME = env.CODEX_HOME;
+    if (env?.OPENCODEX_HOME) process.env.OPENCODEX_HOME = env.OPENCODEX_HOME;
+    if (typeof blockMs === "number" && Number.isFinite(blockMs) && blockMs > 0) {
+      await Bun.sleep(Math.floor(blockMs));
+    }
+    const result = restoreTrashEntry(trashId, {
+      ...(codexHome ? { codexHome } : {}),
+      ...(busyTimeoutMs !== undefined ? { busyTimeoutMs } : {}),
+      ...(restoreTest ? { _test: restoreTest } : {}),
+    });
+    self.postMessage({ type: "done", requestId, result });
+  } catch (err) {
+    self.postMessage({
+      type: "error",
+      requestId,
+      message: err instanceof Error ? err.message : "worker_failed",
+    });
+  } finally {
+    // Close from inside the worker so the thread begins exiting before the
+    // parent’s terminate() races isolate realm reclaim on Windows.
+    try {
+      (self as unknown as { close?: () => void }).close?.();
+    } catch { /* already closing */ }
+  }
+};
+
+export type { RestoreResult };

package/src/storage/scanner.ts

@@ -0,0 +1,238 @@
+import { readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { Database, constants } from "bun:sqlite";
+import { resolveCodexHomeDir } from "../codex/home";
+import { TRASH_DIR } from "./cleanup";
+
+// SQLITE_OPEN_READONLY alone is not filesystem-read-only for a WAL-mode DB: Bun's
+// `{ readonly: true }` can still materialize *.sqlite-wal/-shm sidecars the first time a
+// checkpointed WAL database (no live sidecars yet) is opened and queried. `immutable=1`
+// (via a file: URI, which requires the SQLITE_OPEN_URI flag) tells SQLite the file will
+// never change for this connection's lifetime, so it skips WAL/shm entirely — the
+// tradeoff is reading the last-checkpointed snapshot instead of blocking on a live writer,
+// which is the right tradeoff for a passive diagnostics scan that must never write.
+const IMMUTABLE_READONLY_FLAGS = constants.SQLITE_OPEN_READONLY | constants.SQLITE_OPEN_URI;
+
+/**
+ * Read-only CODEX_HOME storage scanner — Phase 1 of the Storage page epic
+ * (devlog/_plan/500_storage-page-session-cleanup). Pure measurement: sizes via
+ * fs.stat walks, DB row counts via immutable readonly opens that degrade to
+ * null on lock/corruption. Performs zero writes under CODEX_HOME.
+ */
+
+export type StorageBucketKey =
+  | "sessions"
+  | "archived_sessions"
+  | "logs_db"
+  | "state_db"
+  | "attachments"
+  | "deletion_manifests"
+  | "other";
+
+export interface StorageLargestEntry {
+  /** Path relative to CODEX_HOME, forward-slash separated on every platform. */
+  path: string;
+  bytes: number;
+}
+
+export interface StorageBucket {
+  key: StorageBucketKey;
+  label: string;
+  bytes: number;
+  fileCount: number;
+  /** Epoch ms of the oldest/newest file mtime; absent for empty buckets. */
+  oldest?: number;
+  newest?: number;
+  largest?: StorageLargestEntry[];
+  /** sqlite buckets only: row count from the newest versioned DB, null when locked/unreadable. */
+  rows?: number | null;
+}
+
+export interface StorageReport {
+  codexHome: string;
+  generatedAt: number;
+  total: { bytes: number; fileCount: number };
+  buckets: StorageBucket[];
+}
+
+const LARGEST_CAP = 5;
+
+const BUCKET_LABELS: Record<StorageBucketKey, string> = {
+  sessions: "Active sessions",
+  archived_sessions: "Archived sessions",
+  logs_db: "Logs database",
+  state_db: "State database",
+  attachments: "Attachments",
+  deletion_manifests: "Deletion manifests",
+  other: "Other",
+};
+
+/** Dirs under CODEX_HOME that map to a dedicated bucket; anything else is "other". */
+const DIR_BUCKETS: Record<string, StorageBucketKey> = {
+  sessions: "sessions",
+  archived_sessions: "archived_sessions",
+  attachments: "attachments",
+  deletion_manifests: "deletion_manifests",
+};
+
+// state_5.sqlite / logs_2.sqlite carry a version suffix and live WAL/SHM siblings.
+const STATE_DB_FILE = /^state_(\d+)\.sqlite(-wal|-shm)?$/;
+const LOGS_DB_FILE = /^logs_(\d+)\.sqlite(-wal|-shm)?$/;
+
+interface FileEntry {
+  relPath: string;
+  bytes: number;
+  mtimeMs: number;
+}
+
+/** Recursive fs.stat walk. Unreadable entries (races, broken symlinks) are skipped, never fatal. */
+function walkFiles(dir: string, relPrefix: string, out: FileEntry[]): void {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    const relPath = relPrefix ? `${relPrefix}/${entry.name}` : entry.name;
+    try {
+      if (entry.isDirectory()) {
+        walkFiles(full, relPath, out);
+      } else if (entry.isFile()) {
+        const stat = statSync(full);
+        out.push({ relPath, bytes: stat.size, mtimeMs: stat.mtimeMs });
+      }
+    } catch {
+      /* entry vanished mid-scan — diagnostics tolerate racy trees */
+    }
+  }
+}
+
+function buildBucket(key: StorageBucketKey, files: FileEntry[]): StorageBucket {
+  const bucket: StorageBucket = {
+    key,
+    label: BUCKET_LABELS[key],
+    bytes: 0,
+    fileCount: files.length,
+  };
+  for (const file of files) {
+    bucket.bytes += file.bytes;
+    if (bucket.oldest === undefined || file.mtimeMs < bucket.oldest) bucket.oldest = file.mtimeMs;
+    if (bucket.newest === undefined || file.mtimeMs > bucket.newest) bucket.newest = file.mtimeMs;
+  }
+  if (files.length > 0) {
+    bucket.largest = [...files]
+      .sort((a, b) => b.bytes - a.bytes)
+      .slice(0, LARGEST_CAP)
+      .map(f => ({ path: f.relPath, bytes: f.bytes }));
+  }
+  return bucket;
+}
+
+/**
+ * Row count via an immutable readonly open — guarantees zero writes under CODEX_HOME even
+ * for a checkpointed WAL-mode DB with no sidecars yet. Any error (corruption, a file that
+ * vanished mid-scan, a future schema change) degrades to null — "unknown", never a crash.
+ */
+function countRowsReadonly(dbPath: string, table: string): number | null {
+  try {
+    // pathToFileURL percent-encodes reserved characters (space, #, ?, %) that a naive
+    // `file:${dbPath}` concatenation would misparse as a URI fragment/query/escape.
+    const uri = `${pathToFileURL(dbPath).href}?immutable=1`;
+    const db = new Database(uri, IMMUTABLE_READONLY_FLAGS);
+    try {
+      const row = db.query<{ n: number }, []>(`SELECT count(*) AS n FROM "${table}"`).get();
+      return row?.n ?? null;
+    } finally {
+      db.close();
+    }
+  } catch {
+    return null;
+  }
+}
+
+/** Newest versioned DB main file (e.g. state_5.sqlite over state_4.sqlite), or null. */
+function newestVersionedDb(names: string[], pattern: RegExp): string | null {
+  let best: string | null = null;
+  let bestVersion = -1;
+  for (const name of names) {
+    const match = name.match(pattern);
+    if (!match || match[2]) continue; // -wal/-shm siblings never win
+    const version = Number(match[1]);
+    if (version > bestVersion) {
+      bestVersion = version;
+      best = name;
+    }
+  }
+  return best;
+}
+
+export function scanStorage(codexHome: string = resolveCodexHomeDir()): StorageReport {
+  const files: Record<StorageBucketKey, FileEntry[]> = {
+    sessions: [],
+    archived_sessions: [],
+    logs_db: [],
+    state_db: [],
+    attachments: [],
+    deletion_manifests: [],
+    other: [],
+  };
+
+  let rootNames: string[] = [];
+  try {
+    rootNames = readdirSync(codexHome);
+  } catch (error) {
+    // A missing home is a normal fresh-machine state — report zeros. Anything else
+    // (e.g. ENOTDIR: CODEX_HOME points at a file) is a broken setup the caller
+    // must surface as a scan failure, not silently render as an empty home.
+    const code = (error as NodeJS.ErrnoException).code;
+    if (code !== "ENOENT") throw error;
+  }
+
+  for (const name of rootNames) {
+    const full = join(codexHome, name);
+    let stat;
+    try {
+      stat = statSync(full);
+    } catch {
+      continue;
+    }
+    if (stat.isDirectory()) {
+      // Quarantine trash (Phase 2) must not inflate "other" or totals.
+      if (name === TRASH_DIR) continue;
+      walkFiles(full, name, files[DIR_BUCKETS[name] ?? "other"]);
+    } else if (stat.isFile()) {
+      const key: StorageBucketKey = STATE_DB_FILE.test(name) ? "state_db" : LOGS_DB_FILE.test(name) ? "logs_db" : "other";
+      files[key].push({ relPath: name, bytes: stat.size, mtimeMs: stat.mtimeMs });
+    }
+  }
+
+  const buckets = (Object.keys(files) as StorageBucketKey[]).map(key => buildBucket(key, files[key]));
+
+  const stateDbName = newestVersionedDb(rootNames, STATE_DB_FILE);
+  const stateBucket = buckets.find(b => b.key === "state_db");
+  if (stateBucket && stateBucket.fileCount > 0) {
+    stateBucket.rows = stateDbName ? countRowsReadonly(join(codexHome, stateDbName), "threads") : null;
+  }
+  const logsDbName = newestVersionedDb(rootNames, LOGS_DB_FILE);
+  const logsBucket = buckets.find(b => b.key === "logs_db");
+  if (logsBucket && logsBucket.fileCount > 0) {
+    logsBucket.rows = logsDbName ? countRowsReadonly(join(codexHome, logsDbName), "logs") : null;
+  }
+
+  let totalBytes = 0;
+  let totalFiles = 0;
+  for (const bucket of buckets) {
+    totalBytes += bucket.bytes;
+    totalFiles += bucket.fileCount;
+  }
+
+  return {
+    codexHome,
+    generatedAt: Date.now(),
+    total: { bytes: totalBytes, fileCount: totalFiles },
+    buckets,
+  };
+}