@bastani/atomic 0.9.42 → 0.9.43-alpha.1

package/dist/builtin/workflows/src/index.js

@@ -989,8 +989,14 @@ function registerSqlPoolStorage(journal, storage) {
 function sqlPoolStorage(journal) {
   return poolStorage.get(journal);
 }
-var JOURNAL_READ_PAGE_SIZE = 256, versionReaders, poolStorage;
+var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
 var init_internal = __esm(() => {
+  init_types();
+  NON_SNAPSHOT_EVENT_TYPES = [
+    RUN_EVENT_TYPES.checkpointRecorded,
+    RUN_EVENT_TYPES.messageSettled,
+    RUN_EVENT_TYPES.messageDelivered
+  ];
   versionReaders = new WeakMap;
   poolStorage = new WeakMap;
 });
@@ -1372,7 +1378,8 @@ class SqlRunJournal {
     this.legacy = legacy;
     registerRunJournalVersions(this, {
       changeToken: () => this.changeToken(),
-      runVersions: () => this.runVersions()
+      runVersions: () => this.runVersions(),
+      snapshotTail: (runId) => this.snapshotTail(runId)
     });
     registerSqlPoolStorage(this, {
       transaction: (body) => this.operation(() => this.database.transaction(body)),
@@ -1498,6 +1505,28 @@ class SqlRunJournal {
       ]));
     });
   }
+  async snapshotTail(runId) {
+    this.assertOpen();
+    validateText("runId", runId);
+    return this.operation(async () => {
+      const events = await this.isUnconvertedLegacyRun(runId) ? JOURNAL_TABLES.events : this.tables.events;
+      const later = `FROM ${events} AS later WHERE later.${JOURNAL_COLUMNS.runId} = snapshot.${JOURNAL_COLUMNS.runId} AND later.${JOURNAL_COLUMNS.seq} > snapshot.${JOURNAL_COLUMNS.seq} AND later.${JOURNAL_COLUMNS.type} = ?`;
+      const row = await this.database.get(`SELECT ${EVENT_COLUMNS.map((column) => `snapshot.${column}`).join(", ")},
+					latest.${JOURNAL_COLUMNS.seq} AS latest_seq, latest.${JOURNAL_COLUMNS.at} AS latest_at,
+					(SELECT count(*) ${later}) AS later_checkpoints,
+					(SELECT json_extract(later.${JOURNAL_COLUMNS.payload}, '$.completedAt') ${later} ORDER BY later.${JOURNAL_COLUMNS.seq} DESC LIMIT 1) AS last_checkpoint_at
+				FROM (SELECT ${JOURNAL_COLUMNS.seq}, ${JOURNAL_COLUMNS.at} FROM ${events} WHERE ${JOURNAL_COLUMNS.runId} = ? ORDER BY ${JOURNAL_COLUMNS.seq} DESC LIMIT 1) AS latest,
+					(SELECT ${EVENT_COLUMNS.join(", ")} FROM ${events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.type} NOT IN (${NON_SNAPSHOT_EVENT_TYPES.map(() => "?").join(", ")}) ORDER BY ${JOURNAL_COLUMNS.seq} DESC LIMIT 1) AS snapshot`, RUN_EVENT_TYPES.checkpointRecorded, RUN_EVENT_TYPES.checkpointRecorded, runId, runId, ...NON_SNAPSHOT_EVENT_TYPES);
+      if (row === undefined)
+        return;
+      return {
+        version: { seq: journalCounter(runId, "seq", row.latest_seq), at: Number(row.latest_at) },
+        snapshot: entryFromRow(row),
+        laterCheckpoints: Number(row.later_checkpoints),
+        ...row.last_checkpoint_at === null ? {} : { lastCheckpointAt: Number(row.last_checkpoint_at) }
+      };
+    });
+  }
   async requireRun(tx, runId) {
     if (await tx.get(`SELECT ${JOURNAL_COLUMNS.runId} FROM ${this.tables.runs} WHERE ${JOURNAL_COLUMNS.runId} = ?`, runId))
       return;
@@ -1715,13 +1744,14 @@ class SqlRunJournal {
     return this.closePromise;
   }
 }
-var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
+var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
 var init_sql_run_journal = __esm(() => {
   init_internal();
   init_legacy_journal_tables();
   init_types();
   init_values();
   init_internal();
+  EVENT_COLUMNS = [JOURNAL_COLUMNS.runId, JOURNAL_COLUMNS.seq, JOURNAL_COLUMNS.epoch, JOURNAL_COLUMNS.key, JOURNAL_COLUMNS.type, JOURNAL_COLUMNS.payload, JOURNAL_COLUMNS.at];
 });
 
 // dist/builtin/workflows/src/durable/journal/resolution.ts
@@ -9853,6 +9883,7 @@ class JournalDurableBackend {
   projection = createJournalProjection();
   sequences = new Map;
   foldedAt = new Map;
+  summarized = new Map;
   catalogToken;
   leases = new Map;
   executions = new Map;
@@ -9878,8 +9909,8 @@ class JournalDurableBackend {
     let backend;
     try {
       backend = new JournalDurableBackend(journal, undefined, options.ownsJournal ?? true);
-      for (const run of await journal.listRuns())
-        for await (const entry of journal.read(run.runId))
+      for (const runId of options.runs ?? (await journal.listRuns()).map((run) => run.runId))
+        for await (const entry of journal.read(runId))
           backend.fold(entry);
       return backend;
     } catch (error) {
@@ -9952,6 +9983,18 @@ class JournalDurableBackend {
     this.sequences.set(entry.runId, entry.seq);
     this.foldedAt.set(entry.runId, entry.at);
   }
+  summarize(runId, tail) {
+    const event = tail.snapshot.event;
+    const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
+    this.projection.deleteWorkflow(runId);
+    this.prompts.delete(runId);
+    this.projection.applyWorkflowSnapshot({
+      ...handle,
+      completedCheckpoints: handle.completedCheckpoints + tail.laterCheckpoints,
+      ...tail.lastCheckpointAt === undefined ? {} : { updatedAt: tail.lastCheckpointAt }
+    });
+    this.summarized.set(runId, tail.version);
+  }
   *entries(runId, fromSeq) {
     return yield* invoke(async () => {
       const entries = [];
@@ -9965,6 +10008,7 @@ class JournalDurableBackend {
     this.projection.deleteWorkflow(runId);
     this.sequences.delete(runId);
     this.foldedAt.delete(runId);
+    this.summarized.delete(runId);
     this.prompts.delete(runId);
     for (const entry of entries)
       this.fold(entry);
@@ -10286,6 +10330,9 @@ class JournalDurableBackend {
   async hydrateWorkflow(runId) {
     await this.serialize(runId, () => this.rebuild(runId), false);
   }
+  async catchUpWorkflow(runId) {
+    await this.serialize(runId, () => this.extend(runId), false);
+  }
   hydrateResumableWorkflows() {
     return this.serialize(this.catalogQueue, () => invoke(async () => {
       const journal = runJournalVersions(this.journal);
@@ -10299,16 +10346,33 @@ class JournalDurableBackend {
       if (token === this.catalogToken)
         return;
       const versions = await journal.runVersions();
-      for (const runId of new Set([...this.sequences.keys(), ...versions.keys()]))
-        await this.serialize(runId, () => this.catchUp(runId, versions.get(runId)), false);
+      for (const runId of new Set([...this.sequences.keys(), ...this.summarized.keys(), ...versions.keys()]))
+        await this.serialize(runId, () => this.catchUp(runId, versions.get(runId), journal), false);
       this.catalogToken = token;
     }));
   }
-  *catchUp(runId, version) {
+  *catchUp(runId, version, journal) {
     const folded = this.sequences.get(runId) ?? 0;
+    if (folded === 0 && version !== undefined) {
+      const summarized = this.summarized.get(runId);
+      if (summarized !== undefined && summarized.seq === version.seq && summarized.at === version.at)
+        return;
+      const tail = yield* invoke(() => journal.snapshotTail(runId));
+      if (tail !== undefined)
+        return this.summarize(runId, tail);
+      if (summarized === undefined && version.seq === 0)
+        return;
+      return yield* this.rebuild(runId);
+    }
     if (version !== undefined && version.seq === folded && version.at === (this.foldedAt.get(runId) ?? 0))
       return;
-    if (version !== undefined && folded > 0 && version.seq > folded) {
+    if (version !== undefined && folded > 0 && version.seq > folded)
+      return yield* this.extend(runId);
+    yield* this.rebuild(runId);
+  }
+  *extend(runId) {
+    const folded = this.sequences.get(runId) ?? 0;
+    if (folded > 0) {
       const entries = yield* this.entries(runId, folded);
       if (entries[0]?.seq === folded && entries[0].at === this.foldedAt.get(runId)) {
         for (const entry of entries)
@@ -10336,6 +10400,7 @@ class JournalDurableBackend {
     this.leases.delete(runId);
     this.sequences.delete(runId);
     this.foldedAt.delete(runId);
+    this.summarized.delete(runId);
     this.prompts.delete(runId);
     this.projection.deleteWorkflow(runId);
   }
@@ -10600,7 +10665,13 @@ async function initializeDurableBackend(options = {}) {
   const selected = owner.selectedBackends;
   const existing = selected.get(key);
   const entry = existing ?? {
-    ready: openSelectedRunJournal(selection, agentDir).then((journal) => JournalDurableBackend.open(journal, { ownsJournal })).then((opened) => {
+    ready: openSelectedRunJournal(selection, agentDir).then((journal) => JournalDurableBackend.open(journal, { ownsJournal, runs: [] })).then(async (opened) => {
+      try {
+        await opened.hydrateResumableWorkflows();
+      } catch (error) {
+        await opened.close();
+        throw error;
+      }
       if (ownsJournal)
         opened.closeOnProcessExit();
       entry.close = () => opened.close();
@@ -57495,7 +57566,7 @@ class SqlPoolQueue {
       });
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch || row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       assertSession();
     }, action);
   }
@@ -57504,7 +57575,7 @@ class SqlPoolQueue {
       const { now } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch || row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
     });
   }
   async enqueue(lease, descriptor, boundary) {
@@ -57584,7 +57655,7 @@ class SqlPoolQueue {
   async current(tx, lease, token, allowCompleted = false) {
     const row = await tx.get(`SELECT * FROM ${this.work} WHERE id = ? AND root = ?`, token.id, lease.runId);
     if (row?.state !== "leased" && !(allowCompleted && row?.state === "completed") || row.epoch !== token.epoch || row.generation !== token.generation || row.worker !== token.worker)
-      throw new Error("Stale work lease");
+      throw new Error(STALE_WORK_LEASE);
     return row;
   }
   async renew(lease, token, operationId, durationMs) {
@@ -57597,7 +57668,7 @@ class SqlPoolQueue {
       const { now } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch || row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
       if (previous) {
         if (previous.request !== request)
@@ -57625,7 +57696,7 @@ class SqlPoolQueue {
       const { now, status } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token);
       if (row.epoch !== lease.epoch)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
       if (previous) {
         if (previous.request !== request)
@@ -57633,7 +57704,7 @@ class SqlPoolQueue {
         return JSON.parse(previous.receipt);
       }
       if (row.expires_at < now)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       if (!await this.admitsAttemptWrites(tx, lease, status))
         throw new Error("Pool root is not running");
       const checkpoint = mutation.checkpoint;
@@ -57673,7 +57744,7 @@ class SqlPoolQueue {
       const { now, status } = await this.authority(tx, lease);
       const row = await this.current(tx, lease, token, true);
       if (row.epoch !== lease.epoch)
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       if (row.state === "completed") {
         const previous = await tx.get(`SELECT result FROM ${this.work}_results WHERE work = ?`, token.id);
         if (previous?.result !== payload)
@@ -57681,7 +57752,7 @@ class SqlPoolQueue {
         return;
       }
       if (row.expires_at < now || !await this.admitsAttemptWrites(tx, lease, status))
-        throw new Error("Stale work lease");
+        throw new Error(STALE_WORK_LEASE);
       if (JSON.parse(payload).runId !== fromRow(row).childRunId)
         throw new Error("Pool result identity mismatch");
       await tx.run(`INSERT INTO ${this.work}_results VALUES (?, ?)`, token.id, payload);
@@ -57706,6 +57777,40 @@ class SqlPoolQueue {
       return true;
     });
   }
+  async extend(lease, token, durationMs) {
+    positive(durationMs);
+    lease = snapshotLease(lease);
+    token = { ...token };
+    return this.transaction(async (tx) => {
+      const { now } = await this.authority(tx, lease);
+      const row = await this.current(tx, lease, token);
+      if (row.epoch !== lease.epoch)
+        throw new Error(STALE_WORK_LEASE);
+      const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
+      await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
+      return { ...fromRow(row), expiresAt, renewedAt: now };
+    });
+  }
+  async releaseExited(lease, token) {
+    lease = snapshotLease(lease);
+    token = { ...token };
+    return this.transaction(async (tx) => {
+      await this.authority(tx, lease);
+      const row = await tx.get(`SELECT * FROM ${this.work} WHERE id = ? AND root = ?`, token.id, lease.runId);
+      if (row?.state !== "leased" || row.generation !== token.generation || row.worker !== token.worker || row.epoch !== token.epoch)
+        return false;
+      await tx.run(`UPDATE ${this.work} SET state = 'ready', worker = NULL, expires_at = 0 WHERE id = ?`, row.id);
+      return true;
+    });
+  }
+  async releasePreviousIncarnations(lease) {
+    lease = snapshotLease(lease);
+    return this.transaction(async (tx) => {
+      await this.authority(tx, lease);
+      const released = await tx.run(`UPDATE ${this.work} SET state = 'ready', worker = NULL, expires_at = 0 WHERE root = ? AND state = 'leased' AND epoch < ?`, lease.runId, lease.epoch);
+      return released.changes;
+    });
+  }
   async inspect(root) {
     return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
   }
@@ -57716,7 +57821,7 @@ function openSqlPoolQueue(journal) {
     return Promise.reject(new Error("Worker pool requires a SQL run journal"));
   return SqlPoolQueue.open(storage);
 }
-var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000;
+var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
 var init_sql_pool_queue = __esm(() => {
   init_serializable();
   init_boundary_topology();
@@ -61157,12 +61262,66 @@ var init_sdk_intercom = __esm(() => {
   owners2 = new WeakMap;
 });
 
+// dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
+function isInfrastructureError(value2) {
+  if (value2 instanceof WorkflowInfrastructureError)
+    return true;
+  if (typeof value2 !== "object" || value2 === null)
+    return false;
+  const candidate = value2;
+  return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
+}
+function isRetryableWorkflowError(error) {
+  let current = error;
+  for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined && current !== null; depth++) {
+    if (isInfrastructureError(current))
+      return true;
+    current = current instanceof Error ? current.cause : undefined;
+  }
+  return false;
+}
+function workflowInfrastructureFault(message) {
+  return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
+}
+function asWorkflowInfrastructureError(error) {
+  if (isInfrastructureError(error))
+    return error;
+  if (!(error instanceof Error))
+    return;
+  const code = workflowInfrastructureFault(error.message);
+  return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
+}
+var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
+var init_workflow_infrastructure_error = __esm(() => {
+  FAULT_PATTERNS = [
+    [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
+    [/Local worker retry limit/, "worker_retry_limit"],
+    [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
+    [/Worker exited before delivery acknowledgement/, "worker_lost"],
+    [/Stale work lease/, "stale_work_lease"],
+    [/Intercom runtime no longer active/, "intercom_unavailable"],
+    [/Storage unavailable/, "storage_unavailable"]
+  ];
+  WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
+    name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
+    retryable = true;
+    code;
+    constructor(code, message, options) {
+      super(message, options);
+      this.code = code;
+    }
+  };
+});
+
 // dist/builtin/workflows/src/workers/host-options.ts
 var hostWorkerBindings;
 var init_host_options = __esm(() => {
   hostWorkerBindings = Symbol("host-worker-bindings");
 });
 
+// dist/builtin/workflows/src/workers/protocol.ts
+var WORKER_HEARTBEAT_INTERVAL_MS = 1000;
+
 // dist/builtin/workflows/src/runs/foreground/pending-stage-delivery.ts
 function isWorkflowPendingStageDeliveryFailure(error) {
   if (error instanceof WorkflowPendingStageDeliveryFailedError)
@@ -61702,20 +61861,33 @@ var init_stage_handoff = __esm(() => {
 // dist/builtin/workflows/src/workers/local-pool.ts
 import { fork } from "node:child_process";
 import { randomUUID as randomUUID15 } from "node:crypto";
-import { resolve as resolve11 } from "node:path";
+import { createWriteStream } from "node:fs";
+import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
+import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
 import { setTimeout as delay2 } from "node:timers/promises";
 import { fileURLToPath as fileURLToPath5 } from "node:url";
 import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
+function workerLogDirectory(journalPath, root) {
+  return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
+}
 
 class LocalWorkerPool {
   backend;
   options;
   children = new Set;
   incarnations = new Map;
+  readyWorkers = new Set;
+  logs = new Map;
+  logDirectory;
   cleanups = new Set;
   recovering;
   nextRecoveryAt = 0;
   controlAcks = new Map;
+  lastHeard = new Map;
+  lastTick = performance.now();
+  rootStalledMs = 0;
+  ticker;
+  leaseMs;
   desiredControl;
   root;
   store;
@@ -61739,9 +61911,15 @@ class LocalWorkerPool {
     this.options = options;
     if (!Number.isSafeInteger(options.workers) || options.workers < 1)
       throw new Error("Invalid local worker count");
+    if (options.startupTimeoutMs !== undefined && (!Number.isSafeInteger(options.startupTimeoutMs) || options.startupTimeoutMs < 1 || options.startupTimeoutMs > MAX_TIMER_DELAY_MS))
+      throw new Error("Invalid local worker startup timeout");
+    if (options.leaseMs !== undefined && (!Number.isSafeInteger(options.leaseMs) || options.leaseMs < MINIMUM_WORK_LEASE_MS || options.leaseMs > MAX_TIMER_DELAY_MS))
+      throw new Error(`Invalid local worker lease: must be an integer from ${MINIMUM_WORK_LEASE_MS} to ${MAX_TIMER_DELAY_MS} ms`);
+    this.leaseMs = options.leaseMs ?? DEFAULT_WORK_LEASE_MS;
     this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
     if (options.admission && options.rootId === undefined)
       workflowResourceAdmission.set(backend, options.admission);
+    this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
   }
   get workerPids() {
     return [...this.children].flatMap((child) => child.pid === undefined ? [] : [child.pid]);
@@ -61767,6 +61945,7 @@ class LocalWorkerPool {
     this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
     const queue = await openSqlPoolQueue(this.backend.journal);
     await queue.initialize(this.lease, COORDINATOR_LEASE_MS);
+    await queue.releasePreviousIncarnations(this.lease);
     for (;; ) {
       const intended = this.desiredControl ?? "running";
       await queue.setControl(this.lease, intended);
@@ -61781,8 +61960,8 @@ class LocalWorkerPool {
       if (!this.queue || !this.lease)
         return;
       const control = await this.queue.setControl(this.lease, state);
-      const deadline = Date.now() + WORK_LEASE_MS;
-      while ([...this.incarnations.keys()].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
+      const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
+      while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
         if (this.desiredControl !== state)
           return;
         if (Date.now() >= deadline) {
@@ -61852,7 +62031,16 @@ class LocalWorkerPool {
         root,
         holder: "coordinator"
       });
-    await Promise.all(Array.from({ length: this.options.workers }, () => this.spawn(root)));
+    this.logDirectory = workerLogDirectory(this.options.journalPath, root);
+    await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
+      return;
+    });
+    const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
+    try {
+      await Promise.any(workers);
+    } catch (error) {
+      throw error.errors[0];
+    }
   }
   spawn(root) {
     if (this.closed)
@@ -61873,26 +62061,56 @@ class LocalWorkerPool {
     let serial = Promise.resolve();
     let work;
     const deliveries = new Map;
+    const logPath = join28(this.logDirectory, `${worker}.log`);
+    this.logs.set(child, logPath);
+    let logFailure;
+    const log = createWriteStream(logPath, { flags: "a" });
+    log.on("error", (error) => {
+      logFailure = error.message;
+    });
     let stderr = "";
     child.stderr?.on("data", (chunk) => {
-      stderr = (stderr + chunk.toString()).slice(-8192);
+      stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
     });
+    child.stderr?.on("error", (error) => {
+      logFailure ??= error.message;
+      log.end();
+    });
+    child.stderr?.pipe(log);
+    const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
+    const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
+    let timedOut = false;
+    let terminated = false;
+    const terminate = () => {
+      if (terminated)
+        return;
+      terminated = true;
+      child.kill("SIGKILL");
+    };
+    const replied = (error) => {
+      if (error)
+        terminate();
+    };
     return new Promise((resolve, reject) => {
       const startup = setTimeout(() => {
-        this.failure = new Error("Local worker startup timed out");
-        reject(this.failure);
-        child.kill("SIGKILL");
-      }, 30000);
-      child.once("error", (error) => {
+        timedOut = true;
+        terminate();
+      }, startupTimeoutMs);
+      child.on("error", (error) => {
         clearTimeout(startup);
         reject(error);
+        terminate();
       });
+      const closed = new Promise((resolve) => child.once("close", () => resolve()));
       child.once("exit", () => {
+        Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
         this.children.delete(child);
+        this.logs.delete(child);
         clearTimeout(startup);
         this.incarnations.delete(worker);
-        reject(new Error(`Local worker exited before readiness: ${stderr}`));
+        this.readyWorkers.delete(worker);
         this.controlAcks.delete(worker);
+        this.lastHeard.delete(worker);
         if (work)
           this.handoff?.fence(work);
         for (const delivery of deliveries.values())
@@ -61915,9 +62133,11 @@ class LocalWorkerPool {
           }
           if (this.closed)
             return;
+          if (work)
+            await this.queue.releaseExited(this.lease, work);
           const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
           if (exhausted)
-            this.failure = new Error(`Local worker retry limit: ${stderr}`);
+            this.failure = diagnosed("Local worker retry limit");
           else
             await this.spawn(root);
         }).catch((error) => {
@@ -61927,6 +62147,8 @@ class LocalWorkerPool {
         cleanup.finally(() => this.cleanups.delete(cleanup));
       });
       child.on("message", (message) => {
+        this.observeResponsiveness();
+        this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
         if (message.kind === "delivery-ack") {
           const pending = deliveries.get(message.deliveryId);
           deliveries.delete(message.deliveryId);
@@ -61938,7 +62160,10 @@ class LocalWorkerPool {
         }
         const response = serial.then(async () => {
           if (message.kind === "ready") {
+            if (terminated)
+              return;
             ready = true;
+            this.readyWorkers.add(worker);
             clearTimeout(startup);
             resolve();
             return;
@@ -61965,11 +62190,13 @@ class LocalWorkerPool {
             work = await queue.claim(lease, worker, this.sources.map((source) => ({
               definition: `${source.path}#${source.exportName}`,
               digest: source.digest
-            })), WORK_LEASE_MS);
+            })), this.leaseMs);
             return work;
           }
           if (!work)
             throw new Error("Worker has no active attempt");
+          if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
+            work = await queue.extend(lease, work, this.leaseMs);
           if (message.kind === "activity")
             return this.handoff?.publish(work, message.runs);
           if (message.kind === "route")
@@ -61990,10 +62217,8 @@ class LocalWorkerPool {
               });
             }));
           }
-          if (message.kind === "renew") {
-            work = await queue.renew(lease, work, randomUUID15(), WORK_LEASE_MS);
+          if (message.kind === "renew")
             return;
-          }
           if (message.kind === "commit") {
             const receipt = await queue.commit(lease, work, message.operation, message.mutation);
             this.usageTotal = await this.usageLedger.total(root);
@@ -62018,13 +62243,7 @@ class LocalWorkerPool {
         });
         if (message.kind !== "delivery" && message.kind !== "delivery-ready")
           serial = response.then(() => {}, () => {});
-        response.then((value2) => {
-          if (child.connected)
-            child.send({ id: message.id, value: value2 });
-        }, (error) => {
-          if (child.connected)
-            child.send({ id: message.id, error: error.message });
-        });
+        response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
       });
     });
   }
@@ -62034,14 +62253,22 @@ class LocalWorkerPool {
     if (this.closed || performance.now() < this.nextRecoveryAt)
       return Promise.resolve();
     this.recovering = (async () => {
+      await new Promise((resolve) => setImmediate(resolve));
       await this.admission?.reconcile();
       const expiredBefore = Date.now();
       for (const work of await this.queue.inspect(root)) {
         if (work.state !== "leased" || work.expiresAt >= expiredBefore)
           continue;
         const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
-        if (child)
-          await this.stop(child);
+        if (child && work.worker !== null && this.isLive(work.worker, child)) {
+          await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
+            if (error.message !== STALE_WORK_LEASE)
+              throw error;
+          });
+          continue;
+        }
+        if (child && !await this.stop(child).then(() => true, () => false))
+          continue;
         await this.queue.expire(this.lease, work);
       }
     })().finally(() => {
@@ -62050,6 +62277,18 @@ class LocalWorkerPool {
     });
     return this.recovering;
   }
+  observeResponsiveness() {
+    const now = performance.now();
+    const gap = now - this.lastTick;
+    if (gap > ROOT_STALL_THRESHOLD_MS)
+      this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
+    this.lastTick = now;
+  }
+  isLive(worker, child) {
+    this.observeResponsiveness();
+    const heard = this.lastHeard.get(worker);
+    return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.rootStalledMs - heard.rootStalled) < this.leaseMs;
+  }
   stop(child) {
     if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
       return Promise.resolve();
@@ -62066,9 +62305,13 @@ class LocalWorkerPool {
     try {
       return await this.dispatch(definition, inputs, options);
     } catch (error) {
-      if (this.desiredControl !== "quit" || !findWorkflowGracefulQuit(error))
-        await this.close();
-      throw error;
+      try {
+        if (this.desiredControl !== "quit" || !findWorkflowGracefulQuit(error))
+          await this.close();
+      } catch (closeError) {
+        throw asWorkflowInfrastructureError(closeError) ?? closeError;
+      }
+      throw asWorkflowInfrastructureError(error) ?? error;
     }
   }
   async dispatch(definition, inputs, options) {
@@ -62112,6 +62355,8 @@ class LocalWorkerPool {
     for (;; ) {
       options.signal?.throwIfAborted();
       if (this.closed) {
+        if (this.failure)
+          throw this.failure;
         if (this.desiredControl === "quit")
           throw new WorkflowGracefulQuitError(root, "worker pool teardown");
         throw new Error("Local worker pool is closed");
@@ -62138,19 +62383,28 @@ class LocalWorkerPool {
   }
   async dispose() {
     this.closed = true;
+    clearInterval(this.ticker);
     this.releaseBinding();
     if (this.options.rootId === undefined)
       workflowResourceAdmission.delete(this.backend);
     let failure;
+    let stopping = [];
     try {
       await this.recovering?.catch(() => {});
-      await Promise.all([...this.children].map((child) => this.stop(child)));
+      stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
+      await Promise.all(stopping.map(({ child }) => this.stop(child)));
       await Promise.all([...this.cleanups]);
       if (this.cleanupFailure)
         throw this.cleanupFailure;
     } catch (error) {
       failure = error;
     }
+    if (failure === undefined) {
+      const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
+      await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
+      if (this.logDirectory !== undefined)
+        await rmdir(this.logDirectory).catch(() => {});
+    }
     try {
       await this.releaseRoutes?.();
     } catch (error) {
@@ -62160,7 +62414,7 @@ class LocalWorkerPool {
       throw failure;
   }
 }
-var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", WORK_LEASE_MS = 5000, RECOVERY_INTERVAL_MS = 500, COORDINATOR_LEASE_MS;
+var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, ROOT_TICK_INTERVAL_MS = 100, ROOT_STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000;
 var init_local_pool = __esm(() => {
   init_internal();
   init_sql_pool_queue();
@@ -62168,12 +62422,14 @@ var init_local_pool = __esm(() => {
   init_sql_run_journal();
   init_workflow_tool_abort();
   init_sdk_intercom();
+  init_workflow_infrastructure_error();
   init_admission_binding();
   init_definitions();
   init_dispatch();
   init_host_options();
   init_resource_admission();
   init_stage_handoff();
+  MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
   COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
 });
 
@@ -62215,6 +62471,8 @@ async function activatePublicPool(backend, options) {
   const cwd = options.cwd ?? process.cwd();
   const localWorkers = options.localWorkers && {
     ...options.localWorkers,
+    startupTimeoutMs: undefined,
+    leaseMs: undefined,
     definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve12(cwd, source.path) }))
   };
   const admission = options.admission && {
@@ -62243,7 +62501,9 @@ async function activatePublicPool(backend, options) {
     journalPath: backend.journal.path,
     workers: localWorkers.count,
     definitions,
-    admission: resources.admission ?? {}
+    admission: resources.admission ?? {},
+    startupTimeoutMs: options.localWorkers?.startupTimeoutMs,
+    leaseMs: options.localWorkers?.leaseMs
   });
   try {
     pool.validateOptions(options);
@@ -63161,12 +63421,12 @@ function createChildWorkflowRunner(input) {
       const childExitStatus = isWorkflowExitStatus(childRun.status) ? childRun.status : "blocked";
       if (!isWorkflowExitStatus(childRun.status) && !childBudgetExceeded || childRun.status === "failed" && childRun.exited !== true) {
         const failedChildStage = childRun.stages.find((stage) => stage.failureKind !== undefined);
-        throw new Error(`atomic-workflows: child workflow "${childName}" (${child.name}) failed with status ${childRun.status}${childRun.error !== undefined ? `: ${childRun.error}` : ""}`, {
-          cause: {
-            ...failedChildStage?.failureKind !== undefined ? { code: failedChildStage.failureKind } : {},
-            ...failedChildStage?.failureMessage !== undefined ? { message: failedChildStage.failureMessage } : {}
-          }
-        });
+        const message = `atomic-workflows: child workflow "${childName}" (${child.name}) failed with status ${childRun.status}${childRun.error !== undefined ? `: ${childRun.error}` : ""}`;
+        const cause = {
+          ...failedChildStage?.failureKind !== undefined ? { code: failedChildStage.failureKind } : {},
+          ...failedChildStage?.failureMessage !== undefined ? { message: failedChildStage.failureMessage } : {}
+        };
+        throw new Error(message, { cause });
       }
       const outputs = selectWorkflowOutputs(child, childRun.result);
       const childExited = childBudgetExceeded || childRun.exited === true || childRun.status !== "completed";
@@ -63227,7 +63487,7 @@ function describeErrorWithCauses(error) {
   const messages = [];
   const visited = new Set;
   let current = error;
-  for (let depth = 0;current !== undefined && depth < MAX_CAUSE_DEPTH && !visited.has(current); depth++) {
+  for (let depth = 0;current !== undefined && depth < MAX_CAUSE_DEPTH2 && !visited.has(current); depth++) {
     visited.add(current);
     const message = current instanceof Error ? current.message : String(current);
     if (!messages.includes(message))
@@ -63236,7 +63496,7 @@ function describeErrorWithCauses(error) {
   }
   return messages.join(": ");
 }
-var MAX_CAUSE_DEPTH = 5;
+var MAX_CAUSE_DEPTH2 = 5;
 
 // dist/builtin/workflows/src/shared/returned-run-status.ts
 function normalizeReturnedWorkflowStatus(status) {
@@ -63563,7 +63823,7 @@ function hasUsableContentBlock(block) {
 
 // dist/builtin/workflows/src/shared/stage-session-store.ts
 import { randomUUID as randomUUID17 } from "node:crypto";
-import { dirname as dirname12, resolve as resolve13 } from "node:path";
+import { dirname as dirname13, resolve as resolve13 } from "node:path";
 import {
   JsonlSessionStore,
   resolveSessionStore,
@@ -63600,7 +63860,7 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
   if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
     return;
   const location = resolve13(sessionFile);
-  const legacy = await JsonlSessionStore.open(dirname12(location), { exactDirectory: true, scanExisting: false });
+  const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
   try {
     for await (const file of legacy.scanFiles()) {
       if (resolve13(file.location) !== location || file.header.id !== sessionId)
@@ -64748,7 +65008,7 @@ function recordedOwners(boundary, reason) {
   const pending = [{ value: reason, depth: 0 }];
   while (pending.length > 0) {
     const { value: value2, depth } = pending.pop();
-    if (depth > MAX_CAUSE_DEPTH2)
+    if (depth > MAX_CAUSE_DEPTH3)
       continue;
     if (isPrimitive(value2)) {
       if (depth === 0 || value2 !== undefined && value2 !== null) {
@@ -65070,7 +65330,7 @@ function containUiRejections(ui, scope) {
     custom: (factory, options) => scope.watch(ui.custom(factory, options))
   };
 }
-var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections", MAX_CAUSE_DEPTH2 = 8, MAX_PRIMITIVES_PER_RUN = 64, MAX_STOP_STRINGS = 64, ENDED_RUN_GRACE_MS = 30000, NATIVE_REPORTING_MODES, WATCHED_STAGE_METHODS;
+var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections", MAX_CAUSE_DEPTH3 = 8, MAX_PRIMITIVES_PER_RUN = 64, MAX_STOP_STRINGS = 64, ENDED_RUN_GRACE_MS = 30000, NATIVE_REPORTING_MODES, WATCHED_STAGE_METHODS;
 var init_run_rejection_boundary = __esm(() => {
   init_prompt_callsite();
   init_workflow_source();
@@ -69863,8 +70123,8 @@ var init_stage_runner_controller = __esm(() => {
 
 // dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
 import { createHash as createHash4 } from "node:crypto";
-import { mkdir as mkdir6, writeFile as writeFile10 } from "node:fs/promises";
-import { basename as basename5, dirname as dirname13, isAbsolute as isAbsolute11, join as join28, resolve as resolve14 } from "node:path";
+import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
+import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve14 } from "node:path";
 function normalizeMaxOutput2(maxOutput) {
   return {
     bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
@@ -69927,7 +70187,7 @@ function formatByteSize(bytes) {
 }
 function transcriptPath(runId, outputPath) {
   const digest = createHash4("sha256").update(outputPath).digest("hex").slice(0, 16);
-  return join28(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
+  return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
 }
 function appendContent(lines, content) {
   if (typeof content === "string") {
@@ -70047,7 +70307,7 @@ async function finalizePromptOutput(fullOutput, outputOptions, runtimeCwd, runId
   const transcriptFile = transcriptPath(runId, outputPath);
   const transcript = renderTranscript(messages, fullOutput);
   try {
-    await mkdir6(dirname13(outputPath), { recursive: true });
+    await mkdir7(dirname14(outputPath), { recursive: true });
     await writeFile10(outputPath, fullOutput, "utf8");
   } catch (err) {
     return `${displayOutput}
@@ -70059,7 +70319,7 @@ ${err instanceof Error ? err.message : String(err)}`;
   let transcriptError;
   try {
     await ensureWorkflowArtifactRunDirectory(runId);
-    await mkdir6(dirname13(transcriptFile), { recursive: true });
+    await mkdir7(dirname14(transcriptFile), { recursive: true });
     await writeFile10(transcriptFile, transcript, "utf8");
     transcriptAvailable = true;
   } catch (err) {
@@ -70968,7 +71228,7 @@ function reportsClosedSession(error) {
 }
 function isSessionClosedError(error) {
   let current = error;
-  for (let depth = 0;depth < MAX_CAUSE_DEPTH3 && current !== undefined; depth++) {
+  for (let depth = 0;depth < MAX_CAUSE_DEPTH4 && current !== undefined; depth++) {
     try {
       if (reportsClosedSession(current))
         return true;
@@ -70979,7 +71239,7 @@ function isSessionClosedError(error) {
   }
   return false;
 }
-var MAX_CAUSE_DEPTH3 = 8;
+var MAX_CAUSE_DEPTH4 = 8;
 var init_session_closed_error = () => {};
 
 // dist/builtin/workflows/src/engine/workflow-activity.ts
@@ -72220,11 +72480,13 @@ __export(exports_sdk_surface, {
   StaleLeaseError: () => StaleLeaseError,
   StorageUnavailableError: () => StorageUnavailableError,
   WORKFLOW_HEARTBEAT_CUSTOM_TYPE: () => WORKFLOW_HEARTBEAT_CUSTOM_TYPE,
+  WorkflowInfrastructureError: () => WorkflowInfrastructureError,
   cancellationRegistry: () => cancellationRegistry,
   createAgentSessionAdapter: () => createAgentSessionAdapter,
   createCancellationRegistry: () => createCancellationRegistry,
   createRegistry: () => createRegistry,
   createStore: () => createStore,
+  isRetryableWorkflowError: () => isRetryableWorkflowError,
   keepContext: () => keepContext,
   loadWorkerWorkflow: () => loadWorkerWorkflow,
   normalizeWorkflowName: () => normalizeWorkflowName,
@@ -72251,6 +72513,7 @@ var init_sdk_surface = __esm(() => {
   init_worktree();
   init_store();
   init_types2();
+  init_workflow_infrastructure_error();
   init_definitions();
   init_registry();
   init_types();
@@ -72284,11 +72547,13 @@ export {
   StaleLeaseError,
   StorageUnavailableError,
   WORKFLOW_HEARTBEAT_CUSTOM_TYPE,
+  WorkflowInfrastructureError,
   cancellationRegistry,
   createAgentSessionAdapter,
   createCancellationRegistry,
   createRegistry,
   createStore,
+  isRetryableWorkflowError,
   keepContext,
   loadWorkerWorkflow,
   normalizeWorkflowName,