@bastani/atomic 0.9.42 → 0.9.43-alpha.1

package/dist/builtin/workflows/src/extension/index.bundle.mjs

@@ -6491,8 +6491,14 @@ function registerSqlPoolStorage(journal, storage) {
 function sqlPoolStorage(journal) {
   return poolStorage.get(journal);
 }
-var JOURNAL_READ_PAGE_SIZE = 256, versionReaders, poolStorage;
+var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
 var init_internal = __esm(() => {
+  init_types();
+  NON_SNAPSHOT_EVENT_TYPES = [
+    RUN_EVENT_TYPES.checkpointRecorded,
+    RUN_EVENT_TYPES.messageSettled,
+    RUN_EVENT_TYPES.messageDelivered
+  ];
   versionReaders = new WeakMap;
   poolStorage = new WeakMap;
 });
@@ -6800,6 +6806,7 @@ class JournalDurableBackend {
   projection = createJournalProjection();
   sequences = new Map;
   foldedAt = new Map;
+  summarized = new Map;
   catalogToken;
   leases = new Map;
   executions = new Map;
@@ -6825,8 +6832,8 @@ class JournalDurableBackend {
     let backend;
     try {
       backend = new JournalDurableBackend(journal, undefined, options.ownsJournal ?? true);
-      for (const run of await journal.listRuns())
-        for await (const entry of journal.read(run.runId))
+      for (const runId of options.runs ?? (await journal.listRuns()).map((run) => run.runId))
+        for await (const entry of journal.read(runId))
           backend.fold(entry);
       return backend;
     } catch (error) {
@@ -6899,6 +6906,18 @@ class JournalDurableBackend {
     this.sequences.set(entry.runId, entry.seq);
     this.foldedAt.set(entry.runId, entry.at);
   }
+  summarize(runId, tail) {
+    const event = tail.snapshot.event;
+    const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
+    this.projection.deleteWorkflow(runId);
+    this.prompts.delete(runId);
+    this.projection.applyWorkflowSnapshot({
+      ...handle,
+      completedCheckpoints: handle.completedCheckpoints + tail.laterCheckpoints,
+      ...tail.lastCheckpointAt === undefined ? {} : { updatedAt: tail.lastCheckpointAt }
+    });
+    this.summarized.set(runId, tail.version);
+  }
   *entries(runId, fromSeq) {
     return yield* invoke(async () => {
       const entries = [];
@@ -6912,6 +6931,7 @@ class JournalDurableBackend {
     this.projection.deleteWorkflow(runId);
     this.sequences.delete(runId);
     this.foldedAt.delete(runId);
+    this.summarized.delete(runId);
     this.prompts.delete(runId);
     for (const entry of entries)
       this.fold(entry);
@@ -7233,6 +7253,9 @@ class JournalDurableBackend {
   async hydrateWorkflow(runId) {
     await this.serialize(runId, () => this.rebuild(runId), false);
   }
+  async catchUpWorkflow(runId) {
+    await this.serialize(runId, () => this.extend(runId), false);
+  }
   hydrateResumableWorkflows() {
     return this.serialize(this.catalogQueue, () => invoke(async () => {
       const journal = runJournalVersions(this.journal);
@@ -7246,16 +7269,33 @@ class JournalDurableBackend {
       if (token === this.catalogToken)
         return;
       const versions = await journal.runVersions();
-      for (const runId of new Set([...this.sequences.keys(), ...versions.keys()]))
-        await this.serialize(runId, () => this.catchUp(runId, versions.get(runId)), false);
+      for (const runId of new Set([...this.sequences.keys(), ...this.summarized.keys(), ...versions.keys()]))
+        await this.serialize(runId, () => this.catchUp(runId, versions.get(runId), journal), false);
       this.catalogToken = token;
     }));
   }
-  *catchUp(runId, version) {
+  *catchUp(runId, version, journal) {
     const folded = this.sequences.get(runId) ?? 0;
+    if (folded === 0 && version !== undefined) {
+      const summarized = this.summarized.get(runId);
+      if (summarized !== undefined && summarized.seq === version.seq && summarized.at === version.at)
+        return;
+      const tail = yield* invoke(() => journal.snapshotTail(runId));
+      if (tail !== undefined)
+        return this.summarize(runId, tail);
+      if (summarized === undefined && version.seq === 0)
+        return;
+      return yield* this.rebuild(runId);
+    }
     if (version !== undefined && version.seq === folded && version.at === (this.foldedAt.get(runId) ?? 0))
       return;
-    if (version !== undefined && folded > 0 && version.seq > folded) {
+    if (version !== undefined && folded > 0 && version.seq > folded)
+      return yield* this.extend(runId);
+    yield* this.rebuild(runId);
+  }
+  *extend(runId) {
+    const folded = this.sequences.get(runId) ?? 0;
+    if (folded > 0) {
       const entries = yield* this.entries(runId, folded);
       if (entries[0]?.seq === folded && entries[0].at === this.foldedAt.get(runId)) {
         for (const entry of entries)
@@ -7283,6 +7323,7 @@ class JournalDurableBackend {
     this.leases.delete(runId);
     this.sequences.delete(runId);
     this.foldedAt.delete(runId);
+    this.summarized.delete(runId);
     this.prompts.delete(runId);
     this.projection.deleteWorkflow(runId);
   }
@@ -7647,7 +7688,8 @@ class SqlRunJournal {
     this.legacy = legacy;
     registerRunJournalVersions(this, {
       changeToken: () => this.changeToken(),
-      runVersions: () => this.runVersions()
+      runVersions: () => this.runVersions(),
+      snapshotTail: (runId) => this.snapshotTail(runId)
     });
     registerSqlPoolStorage(this, {
       transaction: (body) => this.operation(() => this.database.transaction(body)),
@@ -7773,6 +7815,28 @@ class SqlRunJournal {
       ]));
     });
   }
+  async snapshotTail(runId) {
+    this.assertOpen();
+    validateText("runId", runId);
+    return this.operation(async () => {
+      const events = await this.isUnconvertedLegacyRun(runId) ? JOURNAL_TABLES.events : this.tables.events;
+      const later = `FROM ${events} AS later WHERE later.${JOURNAL_COLUMNS.runId} = snapshot.${JOURNAL_COLUMNS.runId} AND later.${JOURNAL_COLUMNS.seq} > snapshot.${JOURNAL_COLUMNS.seq} AND later.${JOURNAL_COLUMNS.type} = ?`;
+      const row = await this.database.get(`SELECT ${EVENT_COLUMNS.map((column) => `snapshot.${column}`).join(", ")},
+					latest.${JOURNAL_COLUMNS.seq} AS latest_seq, latest.${JOURNAL_COLUMNS.at} AS latest_at,
+					(SELECT count(*) ${later}) AS later_checkpoints,
+					(SELECT json_extract(later.${JOURNAL_COLUMNS.payload}, '$.completedAt') ${later} ORDER BY later.${JOURNAL_COLUMNS.seq} DESC LIMIT 1) AS last_checkpoint_at
+				FROM (SELECT ${JOURNAL_COLUMNS.seq}, ${JOURNAL_COLUMNS.at} FROM ${events} WHERE ${JOURNAL_COLUMNS.runId} = ? ORDER BY ${JOURNAL_COLUMNS.seq} DESC LIMIT 1) AS latest,
+					(SELECT ${EVENT_COLUMNS.join(", ")} FROM ${events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.type} NOT IN (${NON_SNAPSHOT_EVENT_TYPES.map(() => "?").join(", ")}) ORDER BY ${JOURNAL_COLUMNS.seq} DESC LIMIT 1) AS snapshot`, RUN_EVENT_TYPES.checkpointRecorded, RUN_EVENT_TYPES.checkpointRecorded, runId, runId, ...NON_SNAPSHOT_EVENT_TYPES);
+      if (row === undefined)
+        return;
+      return {
+        version: { seq: journalCounter(runId, "seq", row.latest_seq), at: Number(row.latest_at) },
+        snapshot: entryFromRow(row),
+        laterCheckpoints: Number(row.later_checkpoints),
+        ...row.last_checkpoint_at === null ? {} : { lastCheckpointAt: Number(row.last_checkpoint_at) }
+      };
+    });
+  }
   async requireRun(tx, runId) {
     if (await tx.get(`SELECT ${JOURNAL_COLUMNS.runId} FROM ${this.tables.runs} WHERE ${JOURNAL_COLUMNS.runId} = ?`, runId))
       return;
@@ -7990,13 +8054,14 @@ class SqlRunJournal {
     return this.closePromise;
   }
 }
-var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
+var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
 var init_sql_run_journal = __esm(() => {
   init_internal();
   init_legacy_journal_tables();
   init_types();
   init_values();
   init_internal();
+  EVENT_COLUMNS = [JOURNAL_COLUMNS.runId, JOURNAL_COLUMNS.seq, JOURNAL_COLUMNS.epoch, JOURNAL_COLUMNS.key, JOURNAL_COLUMNS.type, JOURNAL_COLUMNS.payload, JOURNAL_COLUMNS.at];
 });
 
 // dist/builtin/workflows/src/durable/journal/resolution.ts
@@ -8107,7 +8172,13 @@ async function initializeDurableBackend(options = {}) {
   const selected = owner.selectedBackends;
   const existing = selected.get(key);
   const entry = existing ?? {
-    ready: openSelectedRunJournal(selection, agentDir).then((journal) => JournalDurableBackend.open(journal, { ownsJournal })).then((opened) => {
+    ready: openSelectedRunJournal(selection, agentDir).then((journal) => JournalDurableBackend.open(journal, { ownsJournal, runs: [] })).then(async (opened) => {
+      try {
+        await opened.hydrateResumableWorkflows();
+      } catch (error) {
+        await opened.close();
+        throw error;
+      }
       if (ownsJournal)
         opened.closeOnProcessExit();
       entry.close = () => opened.close();
@@ -63754,6 +63825,57 @@ var init_types2 = __esm(() => {
   });
 });
 
+// dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
+function isInfrastructureError(value2) {
+  if (value2 instanceof WorkflowInfrastructureError)
+    return true;
+  if (typeof value2 !== "object" || value2 === null)
+    return false;
+  const candidate = value2;
+  return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
+}
+function isRetryableWorkflowError(error) {
+  let current = error;
+  for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined && current !== null; depth++) {
+    if (isInfrastructureError(current))
+      return true;
+    current = current instanceof Error ? current.cause : undefined;
+  }
+  return false;
+}
+function workflowInfrastructureFault(message) {
+  return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
+}
+function asWorkflowInfrastructureError(error) {
+  if (isInfrastructureError(error))
+    return error;
+  if (!(error instanceof Error))
+    return;
+  const code = workflowInfrastructureFault(error.message);
+  return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
+}
+var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
+var init_workflow_infrastructure_error = __esm(() => {
+  FAULT_PATTERNS = [
+    [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
+    [/Local worker retry limit/, "worker_retry_limit"],
+    [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
+    [/Worker exited before delivery acknowledgement/, "worker_lost"],
+    [/Stale work lease/, "stale_work_lease"],
+    [/Intercom runtime no longer active/, "intercom_unavailable"],
+    [/Storage unavailable/, "storage_unavailable"]
+  ];
+  WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
+    name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
+    retryable = true;
+    code;
+    constructor(code, message, options) {
+      super(message, options);
+      this.code = code;
+    }
+  };
+});
+
 // dist/builtin/workflows/src/workflows/registry.ts
 function makeRegistry(store) {
   return {
@@ -63825,11 +63947,13 @@ __export(exports_sdk_surface, {
   StaleLeaseError: () => StaleLeaseError,
   StorageUnavailableError: () => StorageUnavailableError,
   WORKFLOW_HEARTBEAT_CUSTOM_TYPE: () => WORKFLOW_HEARTBEAT_CUSTOM_TYPE,
+  WorkflowInfrastructureError: () => WorkflowInfrastructureError,
   cancellationRegistry: () => cancellationRegistry,
   createAgentSessionAdapter: () => createAgentSessionAdapter,
   createCancellationRegistry: () => createCancellationRegistry,
   createRegistry: () => createRegistry,
   createStore: () => createStore,
+  isRetryableWorkflowError: () => isRetryableWorkflowError,
   keepContext: () => keepContext,
   loadWorkerWorkflow: () => loadWorkerWorkflow,
   normalizeWorkflowName: () => normalizeWorkflowName,
@@ -63856,6 +63980,7 @@ var init_sdk_surface = __esm(() => {
   init_worktree();
   init_store();
   init_types2();
+  init_workflow_infrastructure_error();
   init_definitions();
   init_registry();
   init_types();
@@ -64308,7 +64433,7 @@ class SqlPoolQueue {
       });
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch || row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       assertSession();
     }, action);
   }
@@ -64317,7 +64442,7 @@ class SqlPoolQueue {
       const { now } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch || row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
     });
   }
   async enqueue(lease, descriptor, boundary) {
@@ -64397,7 +64522,7 @@ class SqlPoolQueue {
   async current(tx, lease, token, allowCompleted = false) {
     const row = await tx.get(`SELECT * FROM ${this.work} WHERE id = ? AND root = ?`, token.id, lease.runId);
     if (row?.state !== "leased" && !(allowCompleted && row?.state === "completed") || row.epoch !== token.epoch || row.generation !== token.generation || row.worker !== token.worker)
-      throw new Error("Stale work lease");
+      throw new Error(STALE_WORK_LEASE);
     return row;
   }
   async renew(lease, token, operationId, durationMs) {
@@ -64410,7 +64535,7 @@ class SqlPoolQueue {
       const { now } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch || row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
       if (previous) {
         if (previous.request !== request)
@@ -64438,7 +64563,7 @@ class SqlPoolQueue {
       const { now, status } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
       if (previous) {
         if (previous.request !== request)
@@ -64446,7 +64571,7 @@ class SqlPoolQueue {
         return JSON.parse(previous.receipt);
       }
       if (row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       if (!await this.admitsAttemptWrites(tx, lease, status))
         throw new Error("Pool root is not running");
       const checkpoint = mutation.checkpoint;
@@ -64486,7 +64611,7 @@ class SqlPoolQueue {
       const { now, status } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token, true);
       if (row.epoch !== lease.epoch)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       if (row.state === "completed") {
         const previous = await tx.get(`SELECT result FROM ${this.work}_results WHERE work = ?`, token.id);
         if (previous?.result !== payload)
@@ -64494,7 +64619,7 @@ class SqlPoolQueue {
         return;
       }
       if (row.expires_at < now || !await this.admitsAttemptWrites(tx, lease, status))
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       if (JSON.parse(payload).runId !== fromRow(row).childRunId)
         throw new Error("Pool result identity mismatch");
       await tx.run(`INSERT INTO ${this.work}_results VALUES (?, ?)`, token.id, payload);
@@ -64519,6 +64644,40 @@ class SqlPoolQueue {
       return true;
     });
   }
+  async extend(lease, token, durationMs) {
+    positive(durationMs);
+    lease = snapshotLease(lease);
+    token = { ...token };
+    return this.transaction(async (tx) => {
+      const { now } = await this.authority(tx, lease);
+      const row = await this.current(tx, lease, token);
+      if (row.epoch !== lease.epoch)
+        throw new Error(STALE_WORK_LEASE);
+      const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
+      await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
+      return { ...fromRow(row), expiresAt, renewedAt: now };
+    });
+  }
+  async releaseExited(lease, token) {
+    lease = snapshotLease(lease);
+    token = { ...token };
+    return this.transaction(async (tx) => {
+      await this.authority(tx, lease);
+      const row = await tx.get(`SELECT * FROM ${this.work} WHERE id = ? AND root = ?`, token.id, lease.runId);
+      if (row?.state !== "leased" || row.generation !== token.generation || row.worker !== token.worker || row.epoch !== token.epoch)
+        return false;
+      await tx.run(`UPDATE ${this.work} SET state = 'ready', worker = NULL, expires_at = 0 WHERE id = ?`, row.id);
+      return true;
+    });
+  }
+  async releasePreviousIncarnations(lease) {
+    lease = snapshotLease(lease);
+    return this.transaction(async (tx) => {
+      await this.authority(tx, lease);
+      const released = await tx.run(`UPDATE ${this.work} SET state = 'ready', worker = NULL, expires_at = 0 WHERE root = ? AND state = 'leased' AND epoch < ?`, lease.runId, lease.epoch);
+      return released.changes;
+    });
+  }
   async inspect(root) {
     return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
   }
@@ -64529,7 +64688,7 @@ function openSqlPoolQueue(journal) {
     return Promise.reject(new Error("Worker pool requires a SQL run journal"));
   return SqlPoolQueue.open(storage);
 }
-var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000;
+var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
 var init_sql_pool_queue = __esm(() => {
   init_serializable();
   init_boundary_topology();
@@ -64617,6 +64776,9 @@ var init_host_options = __esm(() => {
   hostWorkerBindings = Symbol("host-worker-bindings");
 });
 
+// dist/builtin/workflows/src/workers/protocol.ts
+var WORKER_HEARTBEAT_INTERVAL_MS = 1000;
+
 // dist/builtin/workflows/src/runs/foreground/pending-stage-delivery.ts
 function isWorkflowPendingStageDeliveryFailure(error) {
   if (error instanceof WorkflowPendingStageDeliveryFailedError)
@@ -65156,20 +65318,33 @@ var init_stage_handoff = __esm(() => {
 // dist/builtin/workflows/src/workers/local-pool.ts
 import { fork } from "node:child_process";
 import { randomUUID as randomUUID17 } from "node:crypto";
-import { resolve as resolve13 } from "node:path";
+import { createWriteStream } from "node:fs";
+import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
+import { dirname as dirname14, join as join29, resolve as resolve13 } from "node:path";
 import { setTimeout as delay2 } from "node:timers/promises";
 import { fileURLToPath as fileURLToPath6 } from "node:url";
 import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
+function workerLogDirectory(journalPath, root) {
+  return join29(dirname14(resolve13(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
+}
 
 class LocalWorkerPool {
   backend;
   options;
   children = new Set;
   incarnations = new Map;
+  readyWorkers = new Set;
+  logs = new Map;
+  logDirectory;
   cleanups = new Set;
   recovering;
   nextRecoveryAt = 0;
   controlAcks = new Map;
+  lastHeard = new Map;
+  lastTick = performance.now();
+  rootStalledMs = 0;
+  ticker;
+  leaseMs;
   desiredControl;
   root;
   store;
@@ -65193,9 +65368,15 @@ class LocalWorkerPool {
     this.options = options;
     if (!Number.isSafeInteger(options.workers) || options.workers < 1)
       throw new Error("Invalid local worker count");
+    if (options.startupTimeoutMs !== undefined && (!Number.isSafeInteger(options.startupTimeoutMs) || options.startupTimeoutMs < 1 || options.startupTimeoutMs > MAX_TIMER_DELAY_MS))
+      throw new Error("Invalid local worker startup timeout");
+    if (options.leaseMs !== undefined && (!Number.isSafeInteger(options.leaseMs) || options.leaseMs < MINIMUM_WORK_LEASE_MS || options.leaseMs > MAX_TIMER_DELAY_MS))
+      throw new Error(`Invalid local worker lease: must be an integer from ${MINIMUM_WORK_LEASE_MS} to ${MAX_TIMER_DELAY_MS} ms`);
+    this.leaseMs = options.leaseMs ?? DEFAULT_WORK_LEASE_MS;
     this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
     if (options.admission && options.rootId === undefined)
       workflowResourceAdmission.set(backend, options.admission);
+    this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
   }
   get workerPids() {
     return [...this.children].flatMap((child) => child.pid === undefined ? [] : [child.pid]);
@@ -65221,6 +65402,7 @@ class LocalWorkerPool {
     this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
     const queue = await openSqlPoolQueue(this.backend.journal);
     await queue.initialize(this.lease, COORDINATOR_LEASE_MS);
+    await queue.releasePreviousIncarnations(this.lease);
     for (;; ) {
       const intended = this.desiredControl ?? "running";
       await queue.setControl(this.lease, intended);
@@ -65235,8 +65417,8 @@ class LocalWorkerPool {
       if (!this.queue || !this.lease)
         return;
       const control = await this.queue.setControl(this.lease, state);
-      const deadline = Date.now() + WORK_LEASE_MS;
-      while ([...this.incarnations.keys()].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
+      const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
+      while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
         if (this.desiredControl !== state)
           return;
         if (Date.now() >= deadline) {
@@ -65306,7 +65488,16 @@ class LocalWorkerPool {
         root,
         holder: "coordinator"
       });
-    await Promise.all(Array.from({ length: this.options.workers }, () => this.spawn(root)));
+    this.logDirectory = workerLogDirectory(this.options.journalPath, root);
+    await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
+      return;
+    });
+    const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
+    try {
+      await Promise.any(workers);
+    } catch (error) {
+      throw error.errors[0];
+    }
   }
   spawn(root) {
     if (this.closed)
@@ -65327,26 +65518,56 @@ class LocalWorkerPool {
     let serial = Promise.resolve();
     let work;
     const deliveries = new Map;
+    const logPath = join29(this.logDirectory, `${worker}.log`);
+    this.logs.set(child, logPath);
+    let logFailure;
+    const log = createWriteStream(logPath, { flags: "a" });
+    log.on("error", (error) => {
+      logFailure = error.message;
+    });
     let stderr = "";
     child.stderr?.on("data", (chunk) => {
-      stderr = (stderr + chunk.toString()).slice(-8192);
+      stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
+    });
+    child.stderr?.on("error", (error) => {
+      logFailure ??= error.message;
+      log.end();
     });
+    child.stderr?.pipe(log);
+    const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
+    const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
+    let timedOut = false;
+    let terminated = false;
+    const terminate = () => {
+      if (terminated)
+        return;
+      terminated = true;
+      child.kill("SIGKILL");
+    };
+    const replied = (error) => {
+      if (error)
+        terminate();
+    };
     return new Promise((resolve, reject) => {
       const startup = setTimeout(() => {
-        this.failure = new Error("Local worker startup timed out");
-        reject(this.failure);
-        child.kill("SIGKILL");
-      }, 30000);
-      child.once("error", (error) => {
+        timedOut = true;
+        terminate();
+      }, startupTimeoutMs);
+      child.on("error", (error) => {
         clearTimeout(startup);
         reject(error);
+        terminate();
       });
+      const closed = new Promise((resolve) => child.once("close", () => resolve()));
       child.once("exit", () => {
+        Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
         this.children.delete(child);
+        this.logs.delete(child);
         clearTimeout(startup);
         this.incarnations.delete(worker);
-        reject(new Error(`Local worker exited before readiness: ${stderr}`));
+        this.readyWorkers.delete(worker);
         this.controlAcks.delete(worker);
+        this.lastHeard.delete(worker);
         if (work)
           this.handoff?.fence(work);
         for (const delivery of deliveries.values())
@@ -65369,9 +65590,11 @@ class LocalWorkerPool {
           }
           if (this.closed)
             return;
+          if (work)
+            await this.queue.releaseExited(this.lease, work);
           const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
           if (exhausted)
-            this.failure = new Error(`Local worker retry limit: ${stderr}`);
+            this.failure = diagnosed("Local worker retry limit");
           else
             await this.spawn(root);
         }).catch((error) => {
@@ -65381,6 +65604,8 @@ class LocalWorkerPool {
         cleanup.finally(() => this.cleanups.delete(cleanup));
       });
       child.on("message", (message) => {
+        this.observeResponsiveness();
+        this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
         if (message.kind === "delivery-ack") {
           const pending = deliveries.get(message.deliveryId);
           deliveries.delete(message.deliveryId);
@@ -65392,7 +65617,10 @@ class LocalWorkerPool {
         }
         const response = serial.then(async () => {
           if (message.kind === "ready") {
+            if (terminated)
+              return;
             ready = true;
+            this.readyWorkers.add(worker);
             clearTimeout(startup);
             resolve();
             return;
@@ -65419,11 +65647,13 @@ class LocalWorkerPool {
             work = await queue.claim(lease, worker, this.sources.map((source) => ({
               definition: `${source.path}#${source.exportName}`,
               digest: source.digest
-            })), WORK_LEASE_MS);
+            })), this.leaseMs);
             return work;
           }
           if (!work)
             throw new Error("Worker has no active attempt");
+          if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
+            work = await queue.extend(lease, work, this.leaseMs);
           if (message.kind === "activity")
             return this.handoff?.publish(work, message.runs);
           if (message.kind === "route")
@@ -65444,10 +65674,8 @@ class LocalWorkerPool {
               });
             }));
           }
-          if (message.kind === "renew") {
-            work = await queue.renew(lease, work, randomUUID17(), WORK_LEASE_MS);
+          if (message.kind === "renew")
             return;
-          }
           if (message.kind === "commit") {
             const receipt = await queue.commit(lease, work, message.operation, message.mutation);
             this.usageTotal = await this.usageLedger.total(root);
@@ -65472,13 +65700,7 @@ class LocalWorkerPool {
         });
         if (message.kind !== "delivery" && message.kind !== "delivery-ready")
           serial = response.then(() => {}, () => {});
-        response.then((value2) => {
-          if (child.connected)
-            child.send({ id: message.id, value: value2 });
-        }, (error) => {
-          if (child.connected)
-            child.send({ id: message.id, error: error.message });
-        });
+        response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
       });
     });
   }
@@ -65488,14 +65710,22 @@ class LocalWorkerPool {
     if (this.closed || performance.now() < this.nextRecoveryAt)
       return Promise.resolve();
     this.recovering = (async () => {
+      await new Promise((resolve) => setImmediate(resolve));
       await this.admission?.reconcile();
       const expiredBefore = Date.now();
       for (const work of await this.queue.inspect(root)) {
         if (work.state !== "leased" || work.expiresAt >= expiredBefore)
           continue;
         const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
-        if (child)
-          await this.stop(child);
+        if (child && work.worker !== null && this.isLive(work.worker, child)) {
+          await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
+            if (error.message !== STALE_WORK_LEASE)
+              throw error;
+          });
+          continue;
+        }
+        if (child && !await this.stop(child).then(() => true, () => false))
+          continue;
         await this.queue.expire(this.lease, work);
       }
     })().finally(() => {
@@ -65504,6 +65734,18 @@ class LocalWorkerPool {
     });
     return this.recovering;
   }
+  observeResponsiveness() {
+    const now = performance.now();
+    const gap = now - this.lastTick;
+    if (gap > ROOT_STALL_THRESHOLD_MS)
+      this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
+    this.lastTick = now;
+  }
+  isLive(worker, child) {
+    this.observeResponsiveness();
+    const heard = this.lastHeard.get(worker);
+    return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.rootStalledMs - heard.rootStalled) < this.leaseMs;
+  }
   stop(child) {
     if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
       return Promise.resolve();
@@ -65520,9 +65762,13 @@ class LocalWorkerPool {
     try {
       return await this.dispatch(definition, inputs, options);
     } catch (error) {
-      if (this.desiredControl !== "quit" || !findWorkflowGracefulQuit(error))
-        await this.close();
-      throw error;
+      try {
+        if (this.desiredControl !== "quit" || !findWorkflowGracefulQuit(error))
+          await this.close();
+      } catch (closeError) {
+        throw asWorkflowInfrastructureError(closeError) ?? closeError;
+      }
+      throw asWorkflowInfrastructureError(error) ?? error;
     }
   }
   async dispatch(definition, inputs, options) {
@@ -65566,6 +65812,8 @@ class LocalWorkerPool {
     for (;; ) {
       options.signal?.throwIfAborted();
       if (this.closed) {
+        if (this.failure)
+          throw this.failure;
         if (this.desiredControl === "quit")
           throw new WorkflowGracefulQuitError(root, "worker pool teardown");
         throw new Error("Local worker pool is closed");
@@ -65592,19 +65840,28 @@ class LocalWorkerPool {
   }
   async dispose() {
     this.closed = true;
+    clearInterval(this.ticker);
     this.releaseBinding();
     if (this.options.rootId === undefined)
       workflowResourceAdmission.delete(this.backend);
     let failure;
+    let stopping = [];
     try {
       await this.recovering?.catch(() => {});
-      await Promise.all([...this.children].map((child) => this.stop(child)));
+      stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
+      await Promise.all(stopping.map(({ child }) => this.stop(child)));
       await Promise.all([...this.cleanups]);
       if (this.cleanupFailure)
         throw this.cleanupFailure;
     } catch (error) {
       failure = error;
     }
+    if (failure === undefined) {
+      const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
+      await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
+      if (this.logDirectory !== undefined)
+        await rmdir(this.logDirectory).catch(() => {});
+    }
     try {
       await this.releaseRoutes?.();
     } catch (error) {
@@ -65614,7 +65871,7 @@ class LocalWorkerPool {
       throw failure;
   }
 }
-var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", WORK_LEASE_MS = 5000, RECOVERY_INTERVAL_MS = 500, COORDINATOR_LEASE_MS;
+var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, ROOT_TICK_INTERVAL_MS = 100, ROOT_STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000;
 var init_local_pool = __esm(() => {
   init_internal();
   init_sql_pool_queue();
@@ -65622,12 +65879,14 @@ var init_local_pool = __esm(() => {
   init_sql_run_journal();
   init_workflow_tool_abort();
   init_sdk_intercom();
+  init_workflow_infrastructure_error();
   init_admission_binding();
   init_definitions();
   init_dispatch();
   init_host_options();
   init_resource_admission();
   init_stage_handoff();
+  MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
   COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
 });
 
@@ -65679,6 +65938,8 @@ async function activatePublicPool(backend, options) {
   const cwd = options.cwd ?? process.cwd();
   const localWorkers = options.localWorkers && {
     ...options.localWorkers,
+    startupTimeoutMs: undefined,
+    leaseMs: undefined,
     definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve14(cwd, source.path) }))
   };
   const admission = options.admission && {
@@ -65707,7 +65968,9 @@ async function activatePublicPool(backend, options) {
     journalPath: backend.journal.path,
     workers: localWorkers.count,
     definitions,
-    admission: resources.admission ?? {}
+    admission: resources.admission ?? {},
+    startupTimeoutMs: options.localWorkers?.startupTimeoutMs,
+    leaseMs: options.localWorkers?.leaseMs
   });
   try {
     pool.validateOptions(options);
@@ -66625,12 +66888,12 @@ function createChildWorkflowRunner(input) {
       const childExitStatus = isWorkflowExitStatus2(childRun.status) ? childRun.status : "blocked";
       if (!isWorkflowExitStatus2(childRun.status) && !childBudgetExceeded || childRun.status === "failed" && childRun.exited !== true) {
         const failedChildStage = childRun.stages.find((stage) => stage.failureKind !== undefined);
-        throw new Error(`atomic-workflows: child workflow "${childName}" (${child.name}) failed with status ${childRun.status}${childRun.error !== undefined ? `: ${childRun.error}` : ""}`, {
-          cause: {
-            ...failedChildStage?.failureKind !== undefined ? { code: failedChildStage.failureKind } : {},
-            ...failedChildStage?.failureMessage !== undefined ? { message: failedChildStage.failureMessage } : {}
-          }
-        });
+        const message = `atomic-workflows: child workflow "${childName}" (${child.name}) failed with status ${childRun.status}${childRun.error !== undefined ? `: ${childRun.error}` : ""}`;
+        const cause = {
+          ...failedChildStage?.failureKind !== undefined ? { code: failedChildStage.failureKind } : {},
+          ...failedChildStage?.failureMessage !== undefined ? { message: failedChildStage.failureMessage } : {}
+        };
+        throw new Error(message, { cause });
       }
       const outputs = selectWorkflowOutputs(child, childRun.result);
       const childExited = childBudgetExceeded || childRun.exited === true || childRun.status !== "completed";
@@ -66691,7 +66954,7 @@ function describeErrorWithCauses(error) {
   const messages = [];
   const visited = new Set;
   let current = error;
-  for (let depth = 0;current !== undefined && depth < MAX_CAUSE_DEPTH && !visited.has(current); depth++) {
+  for (let depth = 0;current !== undefined && depth < MAX_CAUSE_DEPTH2 && !visited.has(current); depth++) {
     visited.add(current);
     const message = current instanceof Error ? current.message : String(current);
     if (!messages.includes(message))
@@ -66700,7 +66963,7 @@ function describeErrorWithCauses(error) {
   }
   return messages.join(": ");
 }
-var MAX_CAUSE_DEPTH = 5;
+var MAX_CAUSE_DEPTH2 = 5;
 
 // dist/builtin/workflows/src/engine/run-durable-finalize.ts
 async function finalizeDurableTerminalStatus(input) {
@@ -67220,7 +67483,7 @@ function recordedOwners(boundary, reason) {
   const pending = [{ value: reason, depth: 0 }];
   while (pending.length > 0) {
     const { value: value2, depth } = pending.pop();
-    if (depth > MAX_CAUSE_DEPTH2)
+    if (depth > MAX_CAUSE_DEPTH3)
       continue;
     if (isPrimitive(value2)) {
       if (depth === 0 || value2 !== undefined && value2 !== null) {
@@ -67542,7 +67805,7 @@ function containUiRejections(ui, scope) {
     custom: (factory, options) => scope.watch(ui.custom(factory, options))
   };
 }
-var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections", MAX_CAUSE_DEPTH2 = 8, MAX_PRIMITIVES_PER_RUN = 64, MAX_STOP_STRINGS = 64, ENDED_RUN_GRACE_MS = 30000, NATIVE_REPORTING_MODES, WATCHED_STAGE_METHODS;
+var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections", MAX_CAUSE_DEPTH3 = 8, MAX_PRIMITIVES_PER_RUN = 64, MAX_STOP_STRINGS = 64, ENDED_RUN_GRACE_MS = 30000, NATIVE_REPORTING_MODES, WATCHED_STAGE_METHODS;
 var init_run_rejection_boundary = __esm(() => {
   init_prompt_callsite();
   init_workflow_source();
@@ -72290,8 +72553,8 @@ var init_stage_runner_controller = __esm(() => {
 
 // dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
 import { createHash as createHash4 } from "node:crypto";
-import { mkdir as mkdir6, writeFile as writeFile10 } from "node:fs/promises";
-import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
+import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
+import { basename as basename5, dirname as dirname15, isAbsolute as isAbsolute11, join as join30, resolve as resolve15 } from "node:path";
 function normalizeMaxOutput2(maxOutput) {
   return {
     bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
@@ -72354,7 +72617,7 @@ function formatByteSize(bytes) {
 }
 function transcriptPath(runId, outputPath) {
   const digest = createHash4("sha256").update(outputPath).digest("hex").slice(0, 16);
-  return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
+  return join30(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
 }
 function appendContent(lines, content) {
   if (typeof content === "string") {
@@ -72474,7 +72737,7 @@ async function finalizePromptOutput(fullOutput, outputOptions, runtimeCwd, runId
   const transcriptFile = transcriptPath(runId, outputPath);
   const transcript = renderTranscript(messages, fullOutput);
   try {
-    await mkdir6(dirname14(outputPath), { recursive: true });
+    await mkdir7(dirname15(outputPath), { recursive: true });
     await writeFile10(outputPath, fullOutput, "utf8");
   } catch (err) {
     return `${displayOutput}
@@ -72486,7 +72749,7 @@ ${err instanceof Error ? err.message : String(err)}`;
   let transcriptError;
   try {
     await ensureWorkflowArtifactRunDirectory(runId);
-    await mkdir6(dirname14(transcriptFile), { recursive: true });
+    await mkdir7(dirname15(transcriptFile), { recursive: true });
     await writeFile10(transcriptFile, transcript, "utf8");
     transcriptAvailable = true;
   } catch (err) {
@@ -73395,7 +73658,7 @@ function reportsClosedSession(error) {
 }
 function isSessionClosedError(error) {
   let current = error;
-  for (let depth = 0;depth < MAX_CAUSE_DEPTH3 && current !== undefined; depth++) {
+  for (let depth = 0;depth < MAX_CAUSE_DEPTH4 && current !== undefined; depth++) {
     try {
       if (reportsClosedSession(current))
         return true;
@@ -73406,7 +73669,7 @@ function isSessionClosedError(error) {
   }
   return false;
 }
-var MAX_CAUSE_DEPTH3 = 8;
+var MAX_CAUSE_DEPTH4 = 8;
 var init_session_closed_error = () => {};
 
 // dist/builtin/workflows/src/engine/workflow-activity.ts
@@ -88486,7 +88749,7 @@ function installCompactionHook(api, store) {
   });
 }
 // dist/builtin/workflows/src/extension/config-loader.ts
-import { isAbsolute as isAbsolute12, join as join30 } from "node:path";
+import { isAbsolute as isAbsolute12, join as join31 } from "node:path";
 import { CONFIG_DIR_NAME as CONFIG_DIR_NAME2, getAgentDir as getAgentDir6 } from "@bastani/atomic";
 // dist/builtin/workflows/src/extension/lifecycle-notifications.ts
 init_returned_run_status();
@@ -89403,7 +89666,7 @@ function resolveWorkflowPaths(workflows, baseDir) {
   }
   return Object.fromEntries(Object.entries(workflows).map(([name, entry]) => [
     name,
-    isAbsolute12(entry.path) ? entry.path : join30(baseDir, entry.path)
+    isAbsolute12(entry.path) ? entry.path : join31(baseDir, entry.path)
   ]));
 }
 function workflowAgentDirs(opts) {
@@ -89411,12 +89674,12 @@ function workflowAgentDirs(opts) {
     return opts.agentDirs;
   if (opts.homeDir !== undefined) {
     const homeDir = opts.homeDir;
-    return [join30(homeDir, CONFIG_DIR_NAME2, "agent")];
+    return [join31(homeDir, CONFIG_DIR_NAME2, "agent")];
   }
   return [getAgentDir6()];
 }
 function toScopedDiscoveryConfig(globalConfig, projectConfig, opts) {
-  const globalBase = opts.agentDir ?? (opts.homeDir === undefined ? getAgentDir6() : join30(opts.homeDir, CONFIG_DIR_NAME2, "agent"));
+  const globalBase = opts.agentDir ?? (opts.homeDir === undefined ? getAgentDir6() : join31(opts.homeDir, CONFIG_DIR_NAME2, "agent"));
   const projectBase = opts.projectRoot;
   const result = {};
   const projectWorkflows = resolveWorkflowPaths(projectConfig?.workflows, projectBase);
@@ -89426,7 +89689,7 @@ function toScopedDiscoveryConfig(globalConfig, projectConfig, opts) {
   if (hasWorkflows(globalConfig)) {
     const projectKeys = new Set(Object.keys(projectWorkflows ?? {}));
     const globalEntries = Object.entries(globalConfig.workflows).filter(([name]) => !projectKeys.has(name)).map(([name, entry]) => {
-      return [name, isAbsolute12(entry.path) ? entry.path : join30(globalBase, entry.path)];
+      return [name, isAbsolute12(entry.path) ? entry.path : join31(globalBase, entry.path)];
     });
     if (globalEntries.length > 0) {
       result.globalWorkflows = Object.fromEntries(globalEntries);
@@ -89437,8 +89700,8 @@ function toScopedDiscoveryConfig(globalConfig, projectConfig, opts) {
 async function loadWorkflowConfig(opts = {}) {
   const projectRoot = opts.projectRoot ?? process.cwd();
   const diagnostics = [];
-  const globalCandidates = workflowAgentDirs(opts).map((agentDir) => join30(agentDir, "extensions", "workflow", "config.json"));
-  const projectCandidates = [join30(projectRoot, CONFIG_DIR_NAME2, "extensions", "workflow", "config.json")];
+  const globalCandidates = workflowAgentDirs(opts).map((agentDir) => join31(agentDir, "extensions", "workflow", "config.json"));
+  const projectCandidates = [join31(projectRoot, CONFIG_DIR_NAME2, "extensions", "workflow", "config.json")];
   let globalConfig = null;
   for (let i = globalCandidates.length - 1;i >= 0; i--) {
     const globalPath = globalCandidates[i];
@@ -91094,7 +91357,7 @@ init_possible_stages();
 import { registerExtensionContextRetirementEffect as registerExtensionContextRetirementEffect2 } from "@bastani/atomic";
 
 // dist/builtin/workflows/src/extension/discovery.ts
-import { join as join32 } from "node:path";
+import { join as join33 } from "node:path";
 import { CONFIG_DIR_NAME as CONFIG_DIR_NAME3, getAgentDir as getAgentDir7 } from "@bastani/atomic";
 
 // dist/builtin/workflows/builtin/index.ts
@@ -91115,12 +91378,12 @@ init_registry();
 // dist/builtin/workflows/src/extension/discovery-loaders.ts
 init_workflow_module_loader();
 import { readdir as readdir2, stat as stat2 } from "node:fs/promises";
-import { extname, isAbsolute as isAbsolute13, join as join31, resolve as resolve17 } from "node:path";
+import { extname, isAbsolute as isAbsolute13, join as join32, resolve as resolve17 } from "node:path";
 async function scanWorkflowDir(dir) {
   try {
     const entries = await readdir2(dir, { withFileTypes: true });
     const WORKFLOW_EXTS = new Set([".ts", ".js", ".mjs", ".cjs"]);
-    return entries.filter((e) => e.isFile() && WORKFLOW_EXTS.has(extname(e.name))).map((e) => join31(dir, e.name)).sort();
+    return entries.filter((e) => e.isFile() && WORKFLOW_EXTS.has(extname(e.name))).map((e) => join32(dir, e.name)).sort();
   } catch {
     return null;
   }
@@ -91190,7 +91453,7 @@ function workflowAgentDirs2(options) {
     return options.agentDirs;
   if (options?.homeDir !== undefined) {
     const homeDir = options.homeDir;
-    return [join32(homeDir, CONFIG_DIR_NAME3, "agent")];
+    return [join33(homeDir, CONFIG_DIR_NAME3, "agent")];
   }
   return [getAgentDir7()];
 }
@@ -91346,7 +91609,7 @@ async function discoverWorkflows(options) {
       registry = await applyBatch(candidates, registry, sources, diagnostics);
     }
   }
-  for (const dir of [join32(cwd, CONFIG_DIR_NAME3, "workflows")]) {
+  for (const dir of [join33(cwd, CONFIG_DIR_NAME3, "workflows")]) {
     const candidates = await loadFromDir(dir, "project-local", diagnostics, getWorkflowHostModules);
     registry = await applyBatch(candidates, registry, sources, diagnostics);
   }
@@ -91358,7 +91621,7 @@ async function discoverWorkflows(options) {
       registry = await applyBatch(candidates, registry, sources, diagnostics);
     }
   }
-  for (const dir of agentDirs.map((agentDir) => join32(agentDir, "workflows")).reverse()) {
+  for (const dir of agentDirs.map((agentDir) => join33(agentDir, "workflows")).reverse()) {
     const candidates = await loadFromDir(dir, "user-global", diagnostics, getWorkflowHostModules);
     registry = await applyBatch(candidates, registry, sources, diagnostics);
   }
@@ -91959,6 +92222,8 @@ function createDurableResumeRuntime(deps) {
         try {
           await backend.hydrateResumableWorkflows();
           await hydrateStoredWorkflowCandidates(backend);
+          for (const entry of backend.listCompletedWorkflows())
+            await backend.hydrateWorkflow(entry.workflowId);
           return await listOpenableCompletedWorkflows(backend, deps.adapters?.sessionStore);
         } finally {
           purgeSuppressedWorkflowRuns(backend, deps.store, deps.toolControlRegistry);
@@ -91969,8 +92234,11 @@ function createDurableResumeRuntime(deps) {
       const finish = workflowObservationRuntime(deps.store).beginRecovery();
       try {
         const backend = selectedBackend(workflowId);
-        const entry = resolveCatalogEntry(workflowId, catalog ?? []);
-        const handle = backend.getWorkflow(entry?.workflowId ?? workflowId);
+        const entry = resolveCatalogEntry(workflowId, catalog ?? backend.listCompletedWorkflows());
+        const id = entry?.workflowId ?? workflowId;
+        if (entry !== undefined || isFullRunId(workflowId))
+          await backend.hydrateWorkflow(id);
+        const handle = backend.getWorkflow(id);
         return await openCompletedDurableWorkflow(workflowId, {
           durableBackend: backend,
           store: deps.store,
@@ -92227,19 +92495,19 @@ function createExtensionRuntime(opts = {}) {
 }
 
 // dist/builtin/workflows/src/extension/status-writer.ts
-import { mkdir as mkdir7, rename as rename2, writeFile as writeFile11 } from "node:fs/promises";
-import { dirname as dirname15, join as join33 } from "node:path";
+import { mkdir as mkdir8, rename as rename2, writeFile as writeFile11 } from "node:fs/promises";
+import { dirname as dirname16, join as join34 } from "node:path";
 import { CONFIG_DIR_NAME as CONFIG_DIR_NAME4 } from "@bastani/atomic";
-var DEFAULT_STATUS_SUBPATH = join33(CONFIG_DIR_NAME4, "workflows", "status.json");
+var DEFAULT_STATUS_SUBPATH = join34(CONFIG_DIR_NAME4, "workflows", "status.json");
 function resolveStatusFilePath(config, opts = {}) {
   if (config.statusFilePath)
     return config.statusFilePath;
   const root = opts.projectRoot ?? process.cwd();
-  return join33(root, DEFAULT_STATUS_SUBPATH);
+  return join34(root, DEFAULT_STATUS_SUBPATH);
 }
 async function atomicWriteJson(path, content) {
-  const dir = dirname15(path);
-  await mkdir7(dir, { recursive: true });
+  const dir = dirname16(path);
+  await mkdir8(dir, { recursive: true });
   const tmpPath = `${path}.tmp`;
   await writeFile11(tmpPath, content, "utf8");
   await rename2(tmpPath, path);
@@ -92524,7 +92792,7 @@ function createWorkflowHeartbeatDelivery(options) {
 }
 
 // dist/builtin/workflows/src/extension/workflow-heartbeat-scheduler.ts
-var MAX_TIMER_DELAY_MS = 2147483647;
+var MAX_TIMER_DELAY_MS2 = 2147483647;
 var MS_PER_MINUTE = 60000;
 function createWorkflowHeartbeatSchedulerState() {
   return {
@@ -92753,7 +93021,7 @@ function installWorkflowHeartbeatScheduler(options) {
     if (next === undefined)
       return;
     const nextAt = next.scheduledAt;
-    const delay = Math.min(Math.max(1, Math.ceil(nextAt - now())), MAX_TIMER_DELAY_MS);
+    const delay = Math.min(Math.max(1, Math.ceil(nextAt - now())), MAX_TIMER_DELAY_MS2);
     const handle = timers.setTimeout(() => {
       timerHandle = undefined;
       processDue();
@@ -93410,6 +93678,11 @@ function createWorkflowExtensionRuntimeState(pi, adapters, options = {}) {
         return targeted.call(runtimeRef.current, workflowIds);
       return runtimeRef.current.prepareDurableResumable();
     },
+    get prepareDurableCatalog() {
+      const runtime = runtimeRef.current;
+      const catalog = runtime.prepareDurableCatalog;
+      return catalog === undefined ? undefined : () => catalog.call(runtime);
+    },
     prepareCompletedDurable() {
       return runtimeRef.current.prepareCompletedDurable?.() ?? Promise.resolve([]);
     },