@bitkyc08/opencodex 2.8.2-preview.20260731 → 2.9.0

package/src/storage/policy-job.ts

@@ -8,9 +8,9 @@
 import type { CleanupMode, CleanupResult } from "./cleanup";
 import { resolveCodexHomeDir } from "../codex/home";
 import {
-  endStorageMutation,
   tryBeginStorageMutation,
 } from "./storage-mutation-coordinator";
+import type { AdmissionLease } from "../lib/admission";
 import {
   isPolicyDue,
   readStorageCleanupPolicyFromConfig,
@@ -19,9 +19,12 @@ import {
   type PolicySkipReason,
 } from "./policy";
 import {
+  cancelQueuedStorageWorkerSpawns,
   drainStorageWorkers,
-  registerStorageWorker,
+  StorageWorkerAdmissionBusyError,
   terminateStorageWorker,
+  tryReserveStorageWorker,
+  withStorageWorkerSpawnGate,
 } from "./worker-lifecycle";
 
 export type PolicyJobStatus = "idle" | "running";
@@ -82,12 +85,11 @@ let cancelActiveRun: (() => void) | null = null;
 /** Bumped on abort/reset so a late worker completion cannot clobber newer job state. */
 let runGeneration = 0;
 /** CODEX_HOME whose mutation slot this job holds (parent thread only). */
-let heldMutationHome: string | undefined;
+let heldMutationLease: AdmissionLease | undefined;
 
 function releaseHeldMutationSlot(): void {
-  if (heldMutationHome === undefined) return;
-  endStorageMutation(heldMutationHome);
-  heldMutationHome = undefined;
+  heldMutationLease?.release();
+  heldMutationLease = undefined;
 }
 
 export function setStorageCleanupPolicyJobLiveApply(
@@ -129,8 +131,16 @@ function disownActiveRun(): void {
   cancel?.();
 }
 
+/**
+ * Fire-and-forget reset. Prefer {@link resetStorageCleanupPolicyJobForTestsAsync}
+ * from test beforeEach/afterEach — sync terminate races Windows
+ * `bun test --isolate` reclaim when the next case spawns immediately.
+ */
 export function resetStorageCleanupPolicyJobForTests(): void {
   disownActiveRun();
+  // Invalidate spawnGate callbacks that have not created a Worker yet — otherwise
+  // they can still spawn after this sync path releases the mutation slot.
+  cancelQueuedStorageWorkerSpawns();
   if (activeWorker) {
     void terminateStorageWorker(activeWorker);
     activeWorker = null;
@@ -148,19 +158,39 @@ export function resetStorageCleanupPolicyJobForTests(): void {
  * worker still exiting at that moment trips a Bun-internal assertion on Windows
  * and takes the whole run down, so a suite that spawns workers must be able to
  * wait for them rather than fire-and-forget.
+ *
+ * Do not route through the sync reset's `void terminate(...)`: that used to
+ * deregister the worker before the thread exited, so a follow-up drain saw an
+ * empty registry and returned while Windows was still reclaiming the thread.
  */
 export async function resetStorageCleanupPolicyJobForTestsAsync(): Promise<void> {
-  resetStorageCleanupPolicyJobForTests();
-  await drainStorageWorkers();
+  disownActiveRun();
+  cancelQueuedStorageWorkerSpawns();
+  const worker = activeWorker;
+  activeWorker = null;
+  inflight = null;
+  testHooks = null;
+  state = { status: "idle" };
+  // Join the worker before releasing the mutation slot so a concurrent run
+  // cannot acquire CODEX_HOME while the aborted thread is still mutating.
+  try {
+    if (worker) await terminateStorageWorker(worker);
+    await drainStorageWorkers();
+  } finally {
+    releaseHeldMutationSlot();
+  }
 }
 
 /** Terminate an in-flight worker during process shutdown. */
 export function abortStorageCleanupPolicyJob(): void {
   disownActiveRun();
+  cancelQueuedStorageWorkerSpawns();
   if (activeWorker) {
     void terminateStorageWorker(activeWorker);
     activeWorker = null;
   }
+  // Sync path cannot join; prefer abortStorageCleanupPolicyJobAsync when the
+  // caller can await so the mutation slot stays held until the thread exits.
   releaseHeldMutationSlot();
   if (state.status === "running") {
     state = {
@@ -177,6 +207,37 @@ export function abortStorageCleanupPolicyJob(): void {
   inflight = null;
 }
 
+/**
+ * Shutdown path that joins worker threads. Prefer this over the sync abort when
+ * the caller can await (server drain, test teardown) so Windows isolate reclaim
+ * does not race a still-exiting storage worker.
+ */
+export async function abortStorageCleanupPolicyJobAsync(): Promise<void> {
+  disownActiveRun();
+  cancelQueuedStorageWorkerSpawns();
+  const worker = activeWorker;
+  activeWorker = null;
+  if (state.status === "running") {
+    state = {
+      ...state,
+      status: "idle",
+      finishedAt: Date.now(),
+      lastError: "aborted",
+      lastOutcome: {
+        ok: false,
+        error: "evaluation_failed",
+      },
+    };
+  }
+  inflight = null;
+  try {
+    if (worker) await terminateStorageWorker(worker);
+    await drainStorageWorkers();
+  } finally {
+    releaseHeldMutationSlot();
+  }
+}
+
 function outcomeFromResult(result: PolicyRunResult): PolicyJobOutcome {
   return {
     ok: result.ok,
@@ -232,22 +293,22 @@ function applyMutationBusy(): void {
 }
 
 function runInWorker(opts: RequestPolicyRunOptions & { blockMs?: number }): Promise<PolicyRunResult> {
-  return new Promise((resolve, reject) => {
+  const reservation = tryReserveStorageWorker();
+  if (!reservation) return Promise.reject(new StorageWorkerAdmissionBusyError());
+  return withStorageWorkerSpawnGate(() => new Promise<PolicyRunResult>((resolve, reject) => {
     const requestId = crypto.randomUUID();
     let settled = false;
-    const worker = new Worker(new URL("./policy-worker.ts", import.meta.url).href);
-    registerStorageWorker(worker);
+    let worker: Worker;
+    try {
+      worker = new Worker(new URL("./policy-worker.ts", import.meta.url).href);
+      reservation.bind(worker);
+    } catch (error) {
+      reservation.release();
+      reject(error);
+      return;
+    }
     activeWorker = worker;
 
-    const timer = setTimeout(() => {
-      if (settled) return;
-      settled = true;
-      cancelActiveRun = null;
-      void terminateStorageWorker(worker);
-      if (activeWorker === worker) activeWorker = null;
-      reject(new Error("storage_cleanup_worker_timeout"));
-    }, WORKER_TIMEOUT_MS);
-
     const finish = (fn: () => void) => {
       if (settled) return;
       settled = true;
@@ -256,10 +317,15 @@ function runInWorker(opts: RequestPolicyRunOptions & { blockMs?: number }): Prom
       if (activeWorker === worker) activeWorker = null;
       // Settle the caller only after the thread is actually gone, so a suite
       // that awaits its request cannot reach the next test file with a worker
-      // still exiting behind it.
+      // still exiting behind it. Also keeps executeJob's finally from releasing
+      // the mutation slot while the thread may still mutate CODEX_HOME.
       void terminateStorageWorker(worker).then(fn, fn);
     };
 
+    const timer = setTimeout(() => {
+      finish(() => reject(new Error("storage_cleanup_worker_timeout")));
+    }, WORKER_TIMEOUT_MS);
+
     cancelActiveRun = () => {
       finish(() => reject(new Error("aborted")));
     };
@@ -296,6 +362,9 @@ function runInWorker(opts: RequestPolicyRunOptions & { blockMs?: number }): Prom
         ...(process.env.OPENCODEX_HOME ? { OPENCODEX_HOME: process.env.OPENCODEX_HOME } : {}),
       },
     });
+  })).catch(error => {
+    reservation.release();
+    throw error;
   });
 }
 
@@ -309,7 +378,7 @@ async function executeJob(opts: RequestPolicyRunOptions): Promise<void> {
     }
     return;
   }
-  heldMutationHome = codexHome;
+  heldMutationLease = gate.lease;
   try {
     const blockMs = testHooks?.blockMs;
     let result: PolicyRunResult;
@@ -336,7 +405,8 @@ async function executeJob(opts: RequestPolicyRunOptions): Promise<void> {
     applyFinished(result);
   } catch (err) {
     if (generation !== runGeneration) return;
-    applyFailed(err instanceof Error ? err.message : "worker_failed");
+    if (err instanceof StorageWorkerAdmissionBusyError) applyMutationBusy();
+    else applyFailed(err instanceof Error ? err.message : "worker_failed");
   } finally {
     releaseHeldMutationSlot();
   }

package/src/storage/policy-worker.ts

@@ -49,5 +49,11 @@ self.onmessage = (event: MessageEvent<unknown>) => {
       requestId,
       message: err instanceof Error ? err.message : "worker_failed",
     });
+  } finally {
+    // Close from inside the worker so the thread begins exiting before the
+    // parent’s terminate() races isolate realm reclaim on Windows.
+    try {
+      (self as unknown as { close?: () => void }).close?.();
+    } catch { /* already closing */ }
   }
 };

package/src/storage/restore-job.ts

@@ -17,9 +17,12 @@ import {
   type StorageMutationCoordinatorTestHooks,
 } from "./storage-mutation-coordinator";
 import {
+  cancelQueuedStorageWorkerSpawns,
   drainStorageWorkers,
-  registerStorageWorker,
+  StorageWorkerAdmissionBusyError,
   terminateStorageWorker,
+  tryReserveStorageWorker,
+  withStorageWorkerSpawnGate,
 } from "./worker-lifecycle";
 
 export interface RestoreJobTestHooks extends StorageMutationCoordinatorTestHooks {
@@ -72,7 +75,12 @@ export function setRestoreTrashJobTestHooks(hooks: RestoreJobTestHooks | null):
   setStorageMutationCoordinatorTestHooks(hooks);
 }
 
+/**
+ * Fire-and-forget reset. Prefer {@link resetRestoreTrashJobForTestsAsync} from
+ * test beforeEach/afterEach under `bun test --isolate` on Windows.
+ */
 export function resetRestoreTrashJobForTests(): void {
+  cancelQueuedStorageWorkerSpawns();
   if (activeWorker) {
     void terminateStorageWorker(activeWorker);
     activeWorker = null;
@@ -83,14 +91,31 @@ export function resetRestoreTrashJobForTests(): void {
   resetStorageMutationCoordinatorForTests();
 }
 
-/** Await-able reset for test teardown; see policy-job's equivalent for why. */
+/**
+ * Await-able reset for test teardown; see policy-job's equivalent for why.
+ * Awaits the active worker directly so drain cannot race a fire-and-forget
+ * deregister from the sync reset.
+ */
 export async function resetRestoreTrashJobForTestsAsync(): Promise<void> {
-  resetRestoreTrashJobForTests();
-  await drainStorageWorkers();
+  cancelQueuedStorageWorkerSpawns();
+  const worker = activeWorker;
+  activeWorker = null;
+  cancelActiveRun?.();
+  cancelActiveRun = null;
+  testHooks = null;
+  // Join the worker before clearing the mutation coordinator so a concurrent
+  // run cannot acquire CODEX_HOME while the aborted thread is still mutating.
+  try {
+    if (worker) await terminateStorageWorker(worker);
+    await drainStorageWorkers();
+  } finally {
+    resetStorageMutationCoordinatorForTests();
+  }
 }
 
 /** Terminate an in-flight worker during process shutdown. */
 export function abortRestoreTrashJob(): void {
+  cancelQueuedStorageWorkerSpawns();
   if (activeWorker) {
     void terminateStorageWorker(activeWorker);
     activeWorker = null;
@@ -99,6 +124,17 @@ export function abortRestoreTrashJob(): void {
   cancelActiveRun = null;
 }
 
+/** Await-able shutdown sibling; joins the restore worker thread before returning. */
+export async function abortRestoreTrashJobAsync(): Promise<void> {
+  cancelQueuedStorageWorkerSpawns();
+  const worker = activeWorker;
+  activeWorker = null;
+  cancelActiveRun?.();
+  cancelActiveRun = null;
+  if (worker) await terminateStorageWorker(worker);
+  await drainStorageWorkers();
+}
+
 /** Test-only SSE/text stream served from the proxy while a worker is blocked. */
 export function getRestoreTrashTestStreamResponse(): Response | null {
   if (!testHooks?.enableTestStream) return null;
@@ -124,31 +160,37 @@ function runInWorker(opts: {
   blockMs?: number;
   restoreTest?: RestoreTestHooks;
 }): Promise<RestoreResult> {
-  return new Promise((resolve, reject) => {
+  const reservation = tryReserveStorageWorker();
+  if (!reservation) return Promise.reject(new StorageWorkerAdmissionBusyError());
+  return withStorageWorkerSpawnGate(() => new Promise<RestoreResult>((resolve, reject) => {
     const requestId = crypto.randomUUID();
     let settled = false;
-    const worker = new Worker(new URL("./restore-worker.ts", import.meta.url).href);
-    registerStorageWorker(worker);
+    let worker: Worker;
+    try {
+      worker = new Worker(new URL("./restore-worker.ts", import.meta.url).href);
+      reservation.bind(worker);
+    } catch (error) {
+      reservation.release();
+      reject(error);
+      return;
+    }
     activeWorker = worker;
 
-    const timer = setTimeout(() => {
-      if (settled) return;
-      settled = true;
-      cancelActiveRun = null;
-      void terminateStorageWorker(worker);
-      if (activeWorker === worker) activeWorker = null;
-      reject(new Error("restore_worker_timeout"));
-    }, WORKER_TIMEOUT_MS);
-
     const finish = (fn: () => void) => {
       if (settled) return;
       settled = true;
       cancelActiveRun = null;
       clearTimeout(timer);
       if (activeWorker === worker) activeWorker = null;
+      // Join before settle so mutation coordination inside the restore worker
+      // cannot overlap a follow-on run after a watchdog timeout.
       void terminateStorageWorker(worker).then(fn, fn);
     };
 
+    const timer = setTimeout(() => {
+      finish(() => reject(new Error("restore_worker_timeout")));
+    }, WORKER_TIMEOUT_MS);
+
     cancelActiveRun = () => {
       finish(() => reject(new Error("aborted")));
     };
@@ -185,6 +227,9 @@ function runInWorker(opts: {
         ...(process.env.OPENCODEX_HOME ? { OPENCODEX_HOME: process.env.OPENCODEX_HOME } : {}),
       },
     });
+  })).catch(error => {
+    reservation.release();
+    throw error;
   });
 }
 
@@ -215,6 +260,7 @@ async function executeRestore(opts: {
       ...(restoreTest ? { restoreTest } : {}),
     });
   } catch (err) {
+    if (err instanceof StorageWorkerAdmissionBusyError) return busyRestoreResult();
     return restoreResultFromWorkerRejection(err, opts.trashId);
   }
 }

package/src/storage/restore-worker.ts

@@ -46,6 +46,12 @@ self.onmessage = async (event: MessageEvent<unknown>) => {
       requestId,
       message: err instanceof Error ? err.message : "worker_failed",
     });
+  } finally {
+    // Close from inside the worker so the thread begins exiting before the
+    // parent’s terminate() races isolate realm reclaim on Windows.
+    try {
+      (self as unknown as { close?: () => void }).close?.();
+    } catch { /* already closing */ }
   }
 };
 

package/src/storage/storage-mutation-coordinator.ts

@@ -7,6 +7,7 @@
  */
 import { resolve } from "node:path";
 import { resolveCodexHomeDir } from "../codex/home";
+import { createAdmissionGate, type AdmissionLease, type AdmissionMetrics } from "../lib/admission";
 
 export type StorageMutationKind = "cleanup" | "restore" | "policy";
 
@@ -20,9 +21,13 @@ export interface StorageMutationCoordinatorTestHooks {
 interface ActiveSlot {
   kind: StorageMutationKind;
   startedAt: number;
+  lease: AdmissionLease;
 }
 
+export const MAX_ACTIVE_STORAGE_HOME_SLOTS = 32;
 const slots = new Map<string, ActiveSlot>();
+const slotGate = createAdmissionGate("storage_home_slots", MAX_ACTIVE_STORAGE_HOME_SLOTS);
+let releaseMisses = 0;
 let testHooks: StorageMutationCoordinatorTestHooks | null = null;
 
 function slotKey(codexHome?: string): string {
@@ -37,9 +42,15 @@ export function setStorageMutationCoordinatorTestHooks(
 
 export function resetStorageMutationCoordinatorForTests(): void {
   testHooks = null;
+  for (const slot of slots.values()) slot.lease.release();
   slots.clear();
 }
 
+export function storageMutationAdmissionMetrics(): AdmissionMetrics {
+  const metrics = slotGate.metrics();
+  return { ...metrics, releaseMisses: metrics.releaseMisses + releaseMisses };
+}
+
 export function getActiveStorageMutation(
   codexHome?: string,
 ): { kind: StorageMutationKind; startedAt: number } | null {
@@ -50,17 +61,36 @@ export function getActiveStorageMutation(
 export function tryBeginStorageMutation(
   kind: StorageMutationKind,
   codexHome?: string,
-): { acquired: true } | { acquired: false; error: StorageMutationBusyError } {
+): { acquired: true; lease: AdmissionLease } | { acquired: false; error: StorageMutationBusyError } {
   const key = slotKey(codexHome);
   if (slots.has(key)) {
     return { acquired: false, error: "storage_mutation_busy" };
   }
-  slots.set(key, { kind, startedAt: Date.now() });
-  return { acquired: true };
+  const lease = slotGate.tryAcquire();
+  if (!lease) return { acquired: false, error: "storage_mutation_busy" };
+  let active = true;
+  const ownerLease: AdmissionLease = {
+    release() {
+      if (!active) return;
+      active = false;
+      const owner = slots.get(key);
+      if (owner?.lease === ownerLease) slots.delete(key);
+      lease.release();
+    },
+  };
+  slots.set(key, { kind, startedAt: Date.now(), lease: ownerLease });
+  return { acquired: true, lease: ownerLease };
 }
 
 export function endStorageMutation(codexHome?: string): void {
-  slots.delete(slotKey(codexHome));
+  const key = slotKey(codexHome);
+  const slot = slots.get(key);
+  if (!slot) {
+    releaseMisses += 1;
+    return;
+  }
+  slots.delete(key);
+  slot.lease.release();
 }
 
 async function applyCoordinatorBlock(): Promise<void> {
@@ -87,7 +117,7 @@ export async function runPolicyStorageMutation<T>(
     await applyCoordinatorBlock();
     return await work();
   } finally {
-    endStorageMutation(codexHome);
+    gate.lease.release();
   }
 }
 
@@ -104,6 +134,6 @@ export async function withStorageMutationSlot<T>(
     await applyCoordinatorBlock();
     return await work();
   } finally {
-    endStorageMutation(codexHome);
+    gate.lease.release();
   }
 }