@bastani/atomic 0.9.57 → 0.9.58-alpha.1

package/dist/builtin/workflows/src/index.js

@@ -2413,6 +2413,75 @@ function redactSensitiveText(value2) {
   return value2.replace(/(sk-[A-Za-z0-9_-]{8})[A-Za-z0-9_-]+/g, redactedSecretReplacement("$1")).replace(/\b(authorization\s*:\s*bearer\s+)[^\s,;]+/gi, "$1[redacted]").replace(/\b(bearer\s+)[A-Za-z0-9._~+/-]{8,}=*/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|credential|secret)\s*[:=]\s*)[^\s,;]+/gi, "$1[redacted]");
 }
 
+// dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
+function isInfrastructureError(value2) {
+  if (value2 instanceof WorkflowInfrastructureError)
+    return true;
+  if (typeof value2 !== "object" || value2 === null)
+    return false;
+  const candidate = value2;
+  return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
+}
+function isRetryableWorkflowError(error) {
+  let current = error;
+  for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined && current !== null; depth++) {
+    if (isInfrastructureError(current))
+      return true;
+    current = current instanceof Error ? current.cause : undefined;
+  }
+  return false;
+}
+function workflowInfrastructureFault(message) {
+  return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
+}
+function asWorkflowInfrastructureError(error) {
+  if (isInfrastructureError(error))
+    return error;
+  if (!(error instanceof Error))
+    return;
+  const code = workflowInfrastructureFault(error.message);
+  return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
+}
+function failedOnInfrastructureFault(result) {
+  if (result.status !== "failed")
+    return false;
+  const names = (value2) => typeof value2 === "string" && (WORKFLOW_INFRASTRUCTURE_FAULTS.has(value2) || workflowInfrastructureFault(value2) !== undefined);
+  const failure = result.failure;
+  if (names(failure?.code) || names(result.error))
+    return true;
+  return Array.isArray(result.stages) && result.stages.some((stage) => stage?.status === "failed" && (names(stage.failureCode) || names(stage.error)));
+}
+var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", WORKFLOW_INFRASTRUCTURE_FAULTS, MAX_CAUSE_DEPTH = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
+var init_workflow_infrastructure_error = __esm(() => {
+  WORKFLOW_INFRASTRUCTURE_FAULTS = new Set([
+    "worker_lost",
+    "worker_cleanup_unconfirmed",
+    "worker_retry_limit",
+    "worker_startup_failed",
+    "stale_work_lease",
+    "intercom_unavailable",
+    "storage_unavailable"
+  ]);
+  FAULT_PATTERNS = [
+    [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
+    [/Local worker retry limit/, "worker_retry_limit"],
+    [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
+    [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
+    [/Stale work lease|Stale resource admission attempt/, "stale_work_lease"],
+    [/Intercom runtime no longer active/, "intercom_unavailable"],
+    [/Storage unavailable/, "storage_unavailable"]
+  ];
+  WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
+    name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
+    retryable = true;
+    code;
+    constructor(code, message, options) {
+      super(message, options);
+      this.code = code;
+    }
+  };
+});
+
 // dist/builtin/workflows/src/shared/workflow-failures-decisions.ts
 function authDecision(code) {
   if (code === "invalid_api_key") {
@@ -2501,6 +2570,16 @@ function cancelledDecision() {
     disposition: "terminal_killed"
   };
 }
+function infrastructureFaultDecision(code) {
+  return {
+    kind: "unknown",
+    code,
+    retryable: true,
+    resumable: true,
+    recoverability: "recoverable",
+    disposition: "terminal_failed"
+  };
+}
 function unknownDecision() {
   return {
     kind: "unknown",
@@ -2528,6 +2607,8 @@ function strongDecisionFromNormalizedCode(normalized, retryAfterMs) {
       disposition: "active_blocked"
     };
   }
+  if (WORKFLOW_INFRASTRUCTURE_FAULTS.has(normalized))
+    return infrastructureFaultDecision(normalized);
   if (CANCELLED_CODES.has(normalized))
     return cancelledDecision();
   if (INVALID_API_KEY_CODES.has(normalized))
@@ -2640,6 +2721,9 @@ function classificationFromNormalizedCode(normalized, retryAfterMs, source, mess
   return weak !== undefined ? { weak: classificationForDecision(weak, source, message, "weak_signal") } : {};
 }
 function fallbackDecisionFromMessage(message, name) {
+  const fault = workflowInfrastructureFault(message);
+  if (fault !== undefined)
+    return infrastructureFaultDecision(fault);
   const tokens = tokenize(message);
   const decision = decisionFromMessageTokens(tokens, name);
   const providerAuthDecision = classifyFallbackProviderAuthMessage(message, tokens);
@@ -2654,6 +2738,7 @@ function fallbackDecisionFromMessage(message, name) {
 var INVALID_API_KEY_CODES, LOGIN_REQUIRED_CODES, MISSING_API_KEY_CODES, RATE_LIMIT_CODES, QUOTA_LIMIT_CODES, CANCELLED_CODES, PROVIDER_UNAVAILABLE_CODES, UNKNOWN_MODEL_CODES, FORBIDDEN_CONFIG_CODES, REQUEST_AUTH_TIMEOUT_PHRASES, LOGIN_REQUIRED_PHRASES, LOCAL_LOGIN_REQUIRED_PHRASES, PROVIDER_AUTH_FALLBACK_PHRASES, MISSING_API_KEY_PHRASES, INVALID_API_KEY_PHRASES, INVALID_API_KEY_CONTEXT, HTTP_RATE_LIMIT_PHRASES, RATE_LIMIT_PHRASES, QUOTA_LIMIT_PHRASES, CANCELLED_PHRASES, UNKNOWN_MODEL_PHRASES, FORBIDDEN_CONFIG_PHRASES, PROVIDER_UNAVAILABLE_PHRASES, AUTH_CONTEXT, MODEL_PROVIDER_CONTEXT, PROVIDER_CONTEXT, STATUS_MESSAGE_REFINEMENT_CODES, BROAD_AUTH_MESSAGE_REFINEMENT_CODES, STATUS_RELATED_MESSAGE_REFINEMENT_CODES;
 var init_workflow_failures_decisions = __esm(() => {
   init_workflow_failures_contract();
+  init_workflow_infrastructure_error();
   INVALID_API_KEY_CODES = new Set([
     "401",
     "invalid_api_key",
@@ -2841,6 +2926,13 @@ function decisionOfRecordedCode(code, retryAfterMs) {
     case "cancelled":
       return cancelledDecision();
     case "pending_stage_delivery_failed":
+    case "worker_lost":
+    case "worker_cleanup_unconfirmed":
+    case "worker_retry_limit":
+    case "worker_startup_failed":
+    case "stale_work_lease":
+    case "intercom_unavailable":
+    case "storage_unavailable":
       return strongDecisionFromNormalizedCode(code, retryAfterMs);
     case "unknown":
       return;
@@ -11809,7 +11901,7 @@ class JournalDurableBackend {
   timer;
   state = "open";
   closing;
-  renewing;
+  renewing = new Map;
   mutationContext = new AsyncLocalStorage3;
   processExitCleanup;
   unregisterExecutionStorages = registerExecutionStorages(this.executionContext, this.mutationContext);
@@ -12530,27 +12622,21 @@ class JournalDurableBackend {
       yield this.enqueue(runId, () => this.deleteOwned(runId));
   }
   renewLeases() {
-    if (this.renewing !== undefined)
-      return this.renewing;
     if (this.state !== "open")
       return Promise.resolve();
-    this.renewing = this.renewOwnedLeases().finally(() => {
-      this.renewing = undefined;
-    });
-    return this.renewing;
-  }
-  async renewOwnedLeases() {
     for (const [runId, lease] of this.leases) {
-      if (this.state !== "open")
-        return;
-      try {
-        await this.serialize(runId, () => this.renewLease(runId, lease), false);
-      } catch (error) {
-        if (error instanceof Error)
-          this.lost.set(runId, error);
+      if (this.renewing.has(runId))
+        continue;
+      const renewal = this.serialize(runId, () => this.renewLease(runId, lease), false).catch((error) => {
+        this.lost.set(runId, error);
         this.dropLease(runId);
-      }
+      }).finally(() => {
+        if (this.renewing.get(runId) === renewal)
+          this.renewing.delete(runId);
+      });
+      this.renewing.set(runId, renewal);
     }
+    return Promise.all(this.renewing.values()).then(() => {});
   }
   *renewLease(runId, lease) {
     if (this.leases.get(runId) !== lease || this.lost.has(runId))
@@ -12558,9 +12644,10 @@ class JournalDurableBackend {
     try {
       yield* invoke(() => this.journal.renew(lease));
     } catch (error) {
+      if (!(error instanceof StaleLeaseError))
+        return;
       if (this.leases.get(runId) === lease) {
-        if (error instanceof Error)
-          this.lost.set(runId, yield* this.mapOwnership(error, runId));
+        this.lost.set(runId, yield* this.mapOwnership(error, runId));
         this.dropLease(runId);
       }
     }
@@ -12654,7 +12741,7 @@ class JournalDurableBackend {
   }
   async drainAndClose() {
     try {
-      await this.renewing;
+      await Promise.all(this.renewing.values());
       while (this.pending.size > 0)
         await Promise.all(this.pending.values());
       for (const runId of this.leases.keys())
@@ -17029,7 +17116,7 @@ class SqlResourceAdmission {
       for (const value2 of Object.values(limits ?? {}))
         if (!Number.isSafeInteger(value2) || value2 < 1)
           throw new Error("Invalid resource limit");
-    for (const value2 of Object.values(options.host ?? {}))
+    for (const value2 of [...Object.values(options.host ?? {}), options.stageStartMaxLoadPerCpu ?? 0])
       if (!Number.isFinite(value2) || value2 < 0)
         throw new Error("Invalid host admission threshold");
     const admission = new SqlResourceAdmission(storage, {
@@ -17047,6 +17134,8 @@ class SqlResourceAdmission {
       for (const column of ["owner", "attempt", "process_identity"])
         if (!columns.some((entry) => entry.name === column))
           await tx.exec(`ALTER TABLE ${admission.permits} ADD COLUMN ${column} TEXT`);
+      if (!columns.some((entry) => entry.name === "granted_at"))
+        await tx.exec(`ALTER TABLE ${admission.permits} ADD COLUMN granted_at INTEGER`);
       await tx.exec(`CREATE INDEX IF NOT EXISTS ${admission.permits}_owner ON ${admission.permits}(owner, state);
 				CREATE INDEX IF NOT EXISTS ${admission.permits}_state ON ${admission.permits}(state)`);
       if (!(await tx.all(`PRAGMA table_info(${admission.owners})`)).some((entry) => entry.name === "boot"))
@@ -17076,14 +17165,24 @@ class SqlResourceAdmission {
     await admission.reconcile();
     return admission;
   }
+  sampleHost() {
+    return this.options.sampleHost?.() ?? {
+      freeMemoryBytes: freemem(),
+      load: process.platform === "win32" ? undefined : loadavg()[0] / cpus().length
+    };
+  }
+  stageStartPressure(kind, delegated) {
+    const threshold = this.options.stageStartMaxLoadPerCpu;
+    if (kind !== "session" || delegated === true || threshold === undefined || threshold === 0)
+      return false;
+    const load = this.sampleHost().load;
+    return load !== undefined && load > threshold;
+  }
   hostReason() {
     const host = this.options.host;
     if (!host)
       return;
-    const sample = this.options.sampleHost?.() ?? {
-      freeMemoryBytes: freemem(),
-      load: process.platform === "win32" ? undefined : loadavg()[0] / cpus().length
-    };
+    const sample = this.sampleHost();
     if (host.minFreeMemoryBytes !== undefined) {
       if (sample.freeMemoryBytes === undefined)
         return "host memory unsupported";
@@ -17097,21 +17196,32 @@ class SqlResourceAdmission {
         return "host load";
     }
   }
-  async assertCurrentAttempt(tx) {
+  async attemptStanding(tx) {
     const { root, attempt } = this.options;
     if (!attempt)
-      return;
-    const work = await tx.get(`SELECT 1 AS valid FROM ${this.storage.tables.runs}_work w JOIN ${this.storage.tables.leases} l ON l.run_id = w.root
+      return "current";
+    const work = await tx.get(`SELECT (w.expires_at < ${NOW_MS} OR l.renewed_at + p.timeout < ${NOW_MS}) AS late
+			FROM ${this.storage.tables.runs}_work w JOIN ${this.storage.tables.leases} l ON l.run_id = w.root
 			JOIN ${this.storage.tables.runs}_pool p ON p.root = w.root JOIN ${this.storage.tables.runs} r ON r.run_id = w.root
 			WHERE w.id = ? AND w.root = ? AND w.generation = ? AND w.epoch = ? AND w.state = 'leased' AND r.status IN ('running', 'paused')
-			AND w.expires_at >= CAST(unixepoch('subsec') * 1000 AS INTEGER)
-			AND l.epoch = w.epoch AND l.renewed_at + p.timeout >= CAST(unixepoch('subsec') * 1000 AS INTEGER)`, attempt.id, root, attempt.generation, attempt.epoch);
+			AND l.epoch = w.epoch`, attempt.id, root, attempt.generation, attempt.epoch);
     if (!work)
+      return "stale";
+    return Number(work.late) === 1 ? "late" : "current";
+  }
+  async assertCurrentAttempt(tx) {
+    const standing = await this.attemptStanding(tx);
+    if (standing === "stale")
       throw new Error("Stale resource admission attempt");
+    return standing === "current";
   }
   async validate(signal) {
-    signal?.throwIfAborted();
-    await this.storage.transaction((tx) => this.assertCurrentAttempt(tx));
+    for (;; ) {
+      signal?.throwIfAborted();
+      if (await this.storage.transaction((tx) => this.assertCurrentAttempt(tx)))
+        return;
+      await delay(LATE_ATTEMPT_POLL_MS, undefined, { signal });
+    }
   }
   async acquire(kind, signal, request) {
     signal?.throwIfAborted();
@@ -17145,11 +17255,19 @@ class SqlResourceAdmission {
           this.nextReconciliation = performance.now() + 1000;
         }
         const hostReason = this.hostReason();
+        const pressure = this.stageStartPressure(kind, request?.delegated);
         const granted = await this.storage.transaction(async (tx) => {
           await tx.run(`UPDATE ${this.owners} SET expires_at = CAST(unixepoch('subsec') * 1000 AS INTEGER) + ? WHERE id = ?`, OWNER_LEASE_MS, this.owner);
           const { root } = this.options;
-          await this.assertCurrentAttempt(tx);
+          if (!await this.assertCurrentAttempt(tx)) {
+            const waiting = await tx.run(`UPDATE ${this.permits} SET reason = ? WHERE operation = ? AND state = 'waiting'`, LATE_ATTEMPT_REASON, operation);
+            if (waiting.changes !== 1)
+              throw new Error("Resource admission holder was stopped");
+            return false;
+          }
           let reason = hostReason;
+          if (pressure && (await tx.get(`SELECT 1 AS recent FROM ${this.permits} WHERE root = ? AND kind = 'session' AND granted_at >= ${NOW_MS} - ? LIMIT 1`, root, STAGE_START_SPACING_MS) || await tx.get(`SELECT 1 AS older FROM ${this.permits} WHERE root = ? AND kind = 'session' AND state = 'waiting' AND reason = ? AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, root, STAGE_START_PRESSURE_REASON, operation)))
+            reason ??= STAGE_START_PRESSURE_REASON;
           const controls = `${this.storage.tables.runs}_pool_control`;
           if (await tx.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", controls)) {
             const control = await tx.get(`SELECT state FROM ${controls} WHERE root = ?`, root);
@@ -17164,13 +17282,14 @@ class SqlResourceAdmission {
             const rootClause = scope === "user" ? "" : "AND root = ?";
             const args = scope === "user" ? [kind] : [kind, root];
             const count = await tx.get(`SELECT COUNT(*) AS count FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'granted'`, ...args);
-            const older = await tx.get(`SELECT 1 AS waiting FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'waiting' AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, ...args, operation);
+            const older = await tx.get(`SELECT 1 AS waiting FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'waiting' AND reason IS NOT ? AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, ...args, STAGE_START_PRESSURE_REASON, operation);
             if (Number(count?.count) >= limit || older)
               reason ??= `${scope} ${kind}`;
           }
           if (reason && request?.delegated)
             throw new Error(`Delegated session cannot be admitted without capacity headroom: ${reason}`);
-          const updated = await tx.run(`UPDATE ${this.permits} SET state = ?, reason = ? WHERE operation = ? AND state = 'waiting'`, reason ? "waiting" : "granted", reason ?? null, operation);
+          const stageStart = kind === "session" && request?.delegated !== true;
+          const updated = await tx.run(`UPDATE ${this.permits} SET state = ?, reason = ?, granted_at = CASE WHEN ? IS NULL AND ? THEN ${NOW_MS} ELSE NULL END WHERE operation = ? AND state = 'waiting'`, reason ? "waiting" : "granted", reason ?? null, reason ?? null, stageStart ? 1 : 0, operation);
           if (updated.changes !== 1)
             throw new Error("Resource admission holder was stopped");
           return !reason;
@@ -17330,7 +17449,7 @@ async function reapProcessGroups(journal, root, scope) {
     return { survivors: [], unprobeable: [] };
   return settleProcessGroups(storage, root, scope, true);
 }
-var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
+var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", LATE_ATTEMPT_POLL_MS = 100, LATE_ATTEMPT_REASON = "waiting for its attempt lease", STAGE_START_SPACING_MS = 5000, STAGE_START_PRESSURE_REASON = "paced for host load";
 var init_resource_admission = __esm(() => {
   init_internal();
   init_sql_run_journal();
@@ -59903,14 +60022,23 @@ class SqlPoolQueue {
     }
     await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
   }
+  leaseRow(tx, lease) {
+    return tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
+  }
   async authority(tx, lease, timeout) {
-    const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
+    const row = await this.leaseRow(tx, lease);
     if (row?.timeout === null && timeout === undefined)
       throw new Error("Pool not initialized");
     if (!row || row.epoch !== lease.epoch || row.holder_id !== lease.holderId || isWorkflowLeaseExpired(row.renewed_at, row.timeout ?? timeout ?? 0, row.now))
       throw new StaleLeaseError(lease.runId, lease.epoch, Number(row?.epoch ?? 0));
     return row;
   }
+  async rootLease(lease, timeout) {
+    const row = await this.storage.read((executor) => this.leaseRow(executor, lease));
+    if (!row || row.epoch !== lease.epoch || row.holder_id !== lease.holderId)
+      return "lost";
+    return isWorkflowLeaseExpired(row.renewed_at, row.timeout ?? timeout, row.now) ? "late" : "current";
+  }
   async initialize(lease, coordinatorTimeoutMs) {
     positive(coordinatorTimeoutMs);
     lease = snapshotLease(lease);
@@ -59922,6 +60050,9 @@ class SqlPoolQueue {
         throw new Error("Pool configuration is immutable");
     });
   }
+  async initialized(root) {
+    return await this.storage.read((executor) => executor.get(`SELECT 1 AS held FROM ${this.roots} WHERE root = ?`, root)) !== undefined;
+  }
   async control(lease) {
     return this.transaction(async (tx) => {
       await this.authority(tx, lease);
@@ -60316,6 +60447,21 @@ class SqlPoolQueue {
       return row ? JSON.parse(String(row.result)) : undefined;
     });
   }
+  async reopenCompleted(lease, id, rerun) {
+    lease = snapshotLease(lease);
+    return this.transaction(async (tx) => {
+      await this.authority(tx, lease);
+      const row = await tx.get(`SELECT * FROM ${this.work} WHERE id = ? AND root = ?`, id, lease.runId);
+      if (row?.state !== "completed" || row.epoch >= lease.epoch)
+        return;
+      const stored = await tx.get(`SELECT result FROM ${this.work}_results WHERE work = ?`, id);
+      if (!stored || !rerun(JSON.parse(String(stored.result))))
+        return;
+      await tx.run(`DELETE FROM ${this.work}_results WHERE work = ?`, id);
+      await tx.run(`UPDATE ${this.work} SET state = 'ready', worker = NULL, expires_at = 0, epoch = ? WHERE id = ?`, lease.epoch, id);
+      return fromRow({ ...row, state: "ready", worker: null, expires_at: 0, epoch: lease.epoch });
+    });
+  }
   async expire(lease, token) {
     lease = snapshotLease(lease);
     token = { ...token };
@@ -60437,7 +60583,7 @@ function reportsClosedSession(error) {
 }
 function isSessionClosedError(error) {
   let current = error;
-  for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined; depth++) {
+  for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined; depth++) {
     try {
       if (reportsClosedSession(current))
         return true;
@@ -60448,7 +60594,7 @@ function isSessionClosedError(error) {
   }
   return false;
 }
-var MAX_CAUSE_DEPTH = 8;
+var MAX_CAUSE_DEPTH2 = 8;
 var init_session_closed_error = () => {};
 
 // dist/builtin/workflows/src/shared/external-run-routes.ts
@@ -64772,57 +64918,6 @@ var init_sdk_intercom = __esm(() => {
   owners2 = new WeakMap;
 });
 
-// dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
-function isInfrastructureError(value2) {
-  if (value2 instanceof WorkflowInfrastructureError)
-    return true;
-  if (typeof value2 !== "object" || value2 === null)
-    return false;
-  const candidate = value2;
-  return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
-}
-function isRetryableWorkflowError(error) {
-  let current = error;
-  for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined && current !== null; depth++) {
-    if (isInfrastructureError(current))
-      return true;
-    current = current instanceof Error ? current.cause : undefined;
-  }
-  return false;
-}
-function workflowInfrastructureFault(message) {
-  return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
-}
-function asWorkflowInfrastructureError(error) {
-  if (isInfrastructureError(error))
-    return error;
-  if (!(error instanceof Error))
-    return;
-  const code = workflowInfrastructureFault(error.message);
-  return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
-}
-var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH2 = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
-var init_workflow_infrastructure_error = __esm(() => {
-  FAULT_PATTERNS = [
-    [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
-    [/Local worker retry limit/, "worker_retry_limit"],
-    [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
-    [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
-    [/Stale work lease/, "stale_work_lease"],
-    [/Intercom runtime no longer active/, "intercom_unavailable"],
-    [/Storage unavailable/, "storage_unavailable"]
-  ];
-  WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
-    name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
-    retryable = true;
-    code;
-    constructor(code, message, options) {
-      super(message, options);
-      this.code = code;
-    }
-  };
-});
-
 // dist/builtin/workflows/src/workers/coordinator-protocol.ts
 var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
 
@@ -65106,6 +65201,16 @@ class RateLimitObservationOutbox {
 }
 var RATE_LIMIT_SHARE_INTERVAL_MS = 250;
 
+// dist/builtin/workflows/src/workers/stage-session-commands.ts
+function runsWhilePaused(command) {
+  if (command.kind === "abort-bash" || command.kind === "abort-compaction")
+    return true;
+  if (command.kind !== "task")
+    return false;
+  const action = command.action.kind;
+  return action === "cancel" || action === "output" || action === "detail" || action === "transcript";
+}
+
 // dist/builtin/workflows/src/workers/stage-watch-coalesce.ts
 function streamedDelta(event) {
   if (event.type !== "message_update")
@@ -65432,6 +65537,7 @@ class PoolCoordinator {
   closing;
   replacements = 0;
   rootLeaseLost = false;
+  lateRoot;
   attemptBase = new Map;
   constructor(port) {
     this.port = port;
@@ -65445,6 +65551,47 @@ class PoolCoordinator {
     await this.journal?.close().catch(() => {});
     await this.durable?.close().catch(() => {});
   }
+  async whileRootLate(queue, action, resume) {
+    for (let waited = false;; ) {
+      try {
+        if (waited)
+          await resume?.();
+        return await action();
+      } catch (error) {
+        if (!(error instanceof Error) || !isRootLeaseLost(error) || !await this.rootRenewal(queue))
+          throw error;
+        waited = true;
+      }
+    }
+  }
+  rootRenewal(queue) {
+    this.lateRoot ??= this.awaitRootRenewal(queue).finally(() => {
+      this.lateRoot = undefined;
+    });
+    return this.lateRoot;
+  }
+  async awaitRootRenewal(queue) {
+    const lease = this.lease;
+    const since = performance.now();
+    let late = false;
+    try {
+      for (;; ) {
+        if (this.closed)
+          return false;
+        const lost = await this.remote?.fresh(lease).then(() => false, () => true);
+        if (lost)
+          return false;
+        const standing = await queue.rootLease(lease, COORDINATOR_LEASE_MS);
+        if (standing !== "late")
+          return standing === "current";
+        late = true;
+        await delay2(LATE_ROOT_POLL_MS);
+      }
+    } finally {
+      if (late)
+        this.stalledMs += performance.now() - since;
+    }
+  }
   receive(message) {
     if (message.kind === "request-reply") {
       const pending = this.requests.get(message.id);
@@ -65513,6 +65660,8 @@ class PoolCoordinator {
         return this.stageAnswer(command.runId, command.stageId, command.answer);
       case "stage-pause":
         return this.stageControl(command.watch, { op: "pause" });
+      case "stage-session":
+        return this.stageSession(command.watch, command.command);
       case "stage-resume":
         return this.stageControl(command.watch, {
           op: "resume",
@@ -65542,8 +65691,8 @@ class PoolCoordinator {
     this.usageTotal = await this.usageLedger.total(root);
     this.lease = lease;
     const queue = await openSqlPoolQueue(this.journal, this.remote);
-    await queue.initialize(lease, COORDINATOR_LEASE_MS);
-    await queue.drainOutbox(lease);
+    await this.whileRootLate(queue, () => queue.initialize(lease, COORDINATOR_LEASE_MS));
+    await this.whileRootLate(queue, () => queue.drainOutbox(lease));
     if (this.remote !== undefined) {
       const remote = this.remote;
       this.mirrorTimer = setInterval(() => void remote.fresh(lease, 0).catch(() => {}), MIRROR_INTERVAL_MS).unref();
@@ -65552,10 +65701,10 @@ class PoolCoordinator {
     for (const work of await queue.inspect(root))
       if (work.state === "leased" && work.epoch < lease.epoch && (await unconfirmedProcessGroups(this.journal, root, { attempt: work })).length > 0)
         retained.push(work);
-    await queue.releasePreviousIncarnations(lease, retained.map((work) => work.id));
+    await this.whileRootLate(queue, () => queue.releasePreviousIncarnations(lease, retained.map((work) => work.id)));
     for (;; ) {
       const intended = this.desiredControl ?? "running";
-      await queue.setControl(lease, intended);
+      await this.whileRootLate(queue, () => queue.setControl(lease, intended));
       if (intended === (this.desiredControl ?? "running"))
         break;
     }
@@ -65591,7 +65740,7 @@ class PoolCoordinator {
     return this.retainedTransition(async () => {
       const record = this.retained.get(work.id);
       this.retained.delete(work.id);
-      await this.queue.releaseRetained(this.lease, work, record?.enqueued === true).catch((error) => {
+      await this.whileRootLate(this.queue, () => this.queue.releaseRetained(this.lease, work, record?.enqueued === true)).catch((error) => {
         if (error.message !== STALE_WORK_LEASE)
           throw error;
       });
@@ -65605,20 +65754,29 @@ class PoolCoordinator {
   async control(state) {
     this.desiredControl = state;
     if (!this.queue || !this.lease)
-      return;
-    const control = await this.queue.setControl(this.lease, state);
-    const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
-    while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
+      return 0;
+    const queue = this.queue;
+    const lease = this.lease;
+    const control = await this.whileRootLate(queue, () => queue.setControl(lease, state));
+    const unacknowledged = () => [...this.readyWorkers].filter((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision).length;
+    let pending = unacknowledged();
+    let progressAt = performance.now();
+    let stalledAt = this.stalledMs;
+    while (pending > 0) {
       if (this.desiredControl !== state)
-        return;
-      if (Date.now() >= deadline) {
-        if (state === "quit")
-          break;
-        throw new Error("Worker control acknowledgement unresolved");
-      }
+        return pending;
       await delay2(25);
+      this.observeResponsiveness();
+      const now = unacknowledged();
+      if (now < pending) {
+        pending = now;
+        progressAt = performance.now();
+        stalledAt = this.stalledMs;
+      } else if (performance.now() - progressAt - (this.stalledMs - stalledAt) >= CONTROL_ACKNOWLEDGEMENT_STALL_MS)
+        break;
     }
     this.flush();
+    return pending;
   }
   async start(sources, cwd, publish, rootRun) {
     if (this.closed)
@@ -65646,7 +65804,11 @@ class PoolCoordinator {
       return;
     });
     this.recoveryTimer = setInterval(() => {
-      this.recoverExpired().catch((error) => this.fail(error));
+      this.recoverExpired().catch(async (error) => {
+        if (this.queue && isRootLeaseLost(error) && await this.rootRenewal(this.queue).catch(() => false))
+          return;
+        this.fail(error);
+      });
     }, RECOVERY_INTERVAL_MS).unref();
     const workers = Array.from({ length: this.options.workers }, () => this.spawn());
     try {
@@ -65663,7 +65825,9 @@ class PoolCoordinator {
     const queue = this.queue;
     const lease = this.lease;
     const work = await this.retainedTransition(async () => {
-      const enqueued = await queue.enqueue(lease, command.descriptor, command.boundary);
+      let enqueued = await this.whileRootLate(queue, () => queue.enqueue(lease, command.descriptor, command.boundary));
+      if (enqueued.state === "completed" && enqueued.epoch < lease.epoch)
+        enqueued = await this.whileRootLate(queue, () => queue.reopenCompleted(lease, enqueued.id, failedOnInfrastructureFault)) ?? enqueued;
       const retained = this.retained.get(enqueued.id);
       if (retained)
         retained.enqueued = true;
@@ -65676,7 +65840,9 @@ class PoolCoordinator {
   }
   async transport(request, payload) {
     const target = this.deliveryRequests.get(request);
-    if (target === undefined || !this.incarnations.has(target.slot.worker))
+    if (target === undefined)
+      throw new Error("Pooled stage delivery request already answered");
+    if (!this.incarnations.has(target.slot.worker))
       throw new Error("Worker exited before delivery acknowledgement");
     const { slot, workerRequest } = target;
     await new Promise((resolve, reject) => {
@@ -65904,6 +66070,34 @@ class PoolCoordinator {
       return { ok: false, refusal: "StaleAttempt" };
     return { ok: true, ...result.action === undefined ? {} : { action: result.action } };
   }
+  async stageSession(id, command) {
+    const watch = this.watches.get(id);
+    if (watch === undefined || !watch.open)
+      return {
+        ok: false,
+        refusal: this.endedWatches.get(id) === "stage_finished" ? "StageNotLive" : "StaleAttempt"
+      };
+    const { slot, attempt } = watch;
+    if (slot.work?.id !== attempt.workId || slot.work.generation !== attempt.generation)
+      return { ok: false, refusal: "StaleAttempt" };
+    if (this.desiredControl !== undefined && this.desiredControl !== "running" && !runsWhilePaused(command))
+      return { ok: false, refusal: "PoolPausedOrQuitting" };
+    const result = await this.stageCommand(slot, { attempt, op: "session", watch: id, command });
+    if (result === undefined)
+      return { ok: false, refusal: "StaleAttempt" };
+    if (result.error !== undefined)
+      throw new Error(result.error);
+    const refusal = result.refusal;
+    if (refusal === "StageNotLive" || refusal === "PoolPausedOrQuitting")
+      return { ok: false, refusal };
+    if (refusal !== undefined)
+      return { ok: false, refusal: "StaleAttempt" };
+    return {
+      ok: true,
+      ...result.bash === undefined ? {} : { bash: result.bash },
+      ...result.task === undefined ? {} : { task: result.task }
+    };
+  }
   stageCommand(slot, command) {
     if (!this.incarnations.has(slot.worker)) {
       tracePooledWatch("coordinator.command.no-worker", { op: command.op, worker: slot.worker });
@@ -66019,7 +66213,8 @@ class PoolCoordinator {
         root,
         definitions: this.sources,
         admission: options.admission,
-        ...this.promptNodes ? { promptNodes: true } : {}
+        ...this.promptNodes ? { promptNodes: true } : {},
+        ...options.stageStartMaxLoadPerCpu === undefined ? {} : { stageStartMaxLoadPerCpu: options.stageStartMaxLoadPerCpu }
       })
     ], {
       stdio: ["ignore", "ignore", "pipe", "ipc"],
@@ -66140,7 +66335,10 @@ class PoolCoordinator {
           if (this.closed)
             return;
           const exhausted = work !== undefined && this.attempts(work) >= MAX_CHILD_ATTEMPTS;
-          const settled = work !== undefined && (exhausted ? await this.queue.failExited(this.lease, work) : await this.queue.releaseExited(this.lease, work));
+          const queue = this.queue;
+          const lease = this.lease;
+          const attempt = work;
+          const settled = attempt !== undefined && await this.whileRootLate(queue, () => exhausted ? queue.failExited(lease, attempt) : queue.releaseExited(lease, attempt));
           const superseded = work !== undefined && !settled && await this.superseded(work);
           if (work !== undefined && exhausted && !superseded)
             this.failWork(work, diagnosed(childRetryLimit(work, "lost its worker")));
@@ -66202,21 +66400,40 @@ class PoolCoordinator {
           await this.starting;
           const queue = this.queue;
           const lease = this.lease;
+          const renewAttempt = async () => {
+            const attempt = slot.work;
+            if (attempt === undefined)
+              return;
+            const renewed = await queue.extend(lease, attempt, options.leaseMs);
+            if (slot.work === attempt)
+              slot.work = renewed;
+          };
+          const current = async (action) => {
+            try {
+              return await action();
+            } catch (error) {
+              if (!(error instanceof Error) || error.message !== STALE_WORK_LEASE || slot.work === undefined)
+                throw error;
+              await renewAttempt();
+              return action();
+            }
+          };
+          const leased = (action) => this.whileRootLate(queue, () => current(action), renewAttempt);
           if (message.kind === "control")
-            return queue.control(lease);
+            return leased(() => queue.control(lease));
           if (message.kind === "fenced") {
             const fenced = slot.work;
             slot.work = undefined;
             if (fenced && !await this.superseded(fenced)) {
               if (this.attempts(fenced) < MAX_CHILD_ATTEMPTS)
-                await queue.releaseExited(lease, fenced);
-              else if (await queue.failExited(lease, fenced))
+                await leased(() => queue.releaseExited(lease, fenced));
+              else if (await leased(() => queue.failExited(lease, fenced)))
                 this.failWork(fenced, new Error(childRetryLimit(fenced, "was refused by the root as stale on every attempt")));
             }
             return;
           }
           if (message.kind === "controlled") {
-            const control = await queue.control(lease);
+            const control = await leased(() => queue.control(lease));
             if (message.revision > control.revision)
               throw new Error("Unknown control revision");
             this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
@@ -66227,21 +66444,21 @@ class PoolCoordinator {
               throw new Error("Worker already holds an attempt");
             if (this.budgetBlocked)
               return;
-            await this.recoverExpired();
-            slot.runs.clear();
-            slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
-              definition: `${source.path}#${source.exportName}`,
-              digest: source.digest
-            })), options.leaseMs);
+            slot.work = await leased(async () => {
+              await this.recoverExpired();
+              slot.runs.clear();
+              return queue.claim(lease, worker, this.sources.map((source) => ({
+                definition: `${source.path}#${source.exportName}`,
+                digest: source.digest
+              })), options.leaseMs);
+            });
             return slot.work;
           }
-          let work = slot.work;
-          if (!work)
+          const held = slot.work;
+          if (!held)
             throw new Error("Worker has no active attempt");
-          if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
-            work = await queue.extend(lease, work, options.leaseMs);
-            slot.work = work;
-          }
+          const work = message.kind === "renew" || held.expiresAt - Date.now() < options.leaseMs / 2 ? await leased(() => queue.extend(lease, held, options.leaseMs)) : held;
+          slot.work = work;
           if (message.kind === "activity") {
             for (const run of message.runs)
               slot.runs.add(run.id);
@@ -66301,24 +66518,24 @@ class PoolCoordinator {
           if (message.kind === "renew")
             return;
           if (message.kind === "commit") {
-            const receipt = await queue.commit(lease, work, message.operation, message.mutation);
+            const receipt = await leased(() => queue.commit(lease, work, message.operation, message.mutation));
             if (message.mutation.kind !== "session-append")
               this.noteUsage(await this.usageLedger.total(root));
             return receipt;
           }
           if (message.kind === "usage") {
-            this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
+            this.noteUsage(await leased(() => this.usageLedger.report(lease, work, message.reports)));
             return;
           }
           if (message.kind === "budget") {
-            await queue.assertCurrent(lease, work);
+            await leased(() => queue.assertCurrent(lease, work));
             if (this.budgetBlocked)
               throw new Error("Root workflow budget exhausted");
-            return queue.control(lease);
+            return leased(() => queue.control(lease));
           }
           if (message.kind !== "complete")
             throw new Error("Invalid worker operation");
-          await queue.complete(lease, work, message.result);
+          await leased(() => queue.complete(lease, work, message.result));
           this.endStageWatches((watch) => watch.slot === slot, "stage_finished");
           this.replacements = 0;
           slot.work = undefined;
@@ -66357,7 +66574,7 @@ class PoolCoordinator {
     if (this.recovering)
       return this.recovering;
     const root = this.root;
-    if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
+    if (this.closed || root === undefined || !this.queue || this.lateRoot !== undefined || performance.now() < this.nextRecoveryAt)
       return Promise.resolve();
     const queue = this.queue;
     const lease = this.lease;
@@ -66366,9 +66583,12 @@ class PoolCoordinator {
       await new Promise((resolve) => setImmediate(resolve));
       await this.admission?.reconcile();
       const expiredBefore = Date.now();
-      for (const work of await queue.inspect(root)) {
-        if (work.state !== "leased" || work.expiresAt >= expiredBefore)
-          continue;
+      const expired = (await queue.inspect(root)).filter((work) => work.state === "leased" && work.expiresAt < expiredBefore);
+      if (expired.length > 0 && await queue.rootLease(lease, COORDINATOR_LEASE_MS) === "late") {
+        this.rootRenewal(queue).catch(() => {});
+        return;
+      }
+      for (const work of expired) {
         if (this.quarantined.has(work.id))
           continue;
         if (this.held.has(work.id)) {
@@ -66484,7 +66704,7 @@ function childRetryLimit(work, reason) {
   const exportName = work.definition.slice(work.definition.lastIndexOf("#") + 1);
   return `Local worker retry limit: child ${work.childRunId} (${exportName}) ${reason} ${MAX_CHILD_ATTEMPTS} times`;
 }
-var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
+var CONTROL_ACKNOWLEDGEMENT_STALL_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, LATE_ROOT_POLL_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
 var init_pool_coordinator = __esm(() => {
   init_internal();
   init_pool_remote_journal();
@@ -66498,12 +66718,14 @@ var init_pool_coordinator = __esm(() => {
   init_pending_stage_route_capability();
   init_pending_stage_route_readiness();
   init_store();
+  init_workflow_infrastructure_error();
   init_pool_journal();
   init_pooled_watch_trace();
   init_resource_admission();
   init_stage_watch_coalesce();
   init_startup_timeout();
   COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
+  LATE_ROOT_POLL_MS = RECOVERY_INTERVAL_MS;
   STAGE_WATCH_BACKLOG_BYTES = 8 * 1024 * 1024;
 });
 
@@ -66594,7 +66816,8 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
   const boundedDrain = (drain, sessionId) => {
     const waiting = new Promise((resolve, reject) => {
       drain.then(resolve, reject);
-      watchStall(drain, sessionId, resolve);
+      if (options.holdUntilDrained !== true)
+        watchStall(drain, sessionId, resolve);
     });
     waiting.catch(() => {});
     return waiting;
@@ -66976,10 +67199,14 @@ class PoolStageHandoff {
       if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
         continue;
       if (run.id === work.childRunId) {
-        const execution = run.stages.find((stage) => stage.workerExecution !== undefined)?.workerExecution;
+        const executions = new Map(run.stages.map((stage) => [stage.id, stage.workerExecution]));
+        const attempt = run.stages.flatMap((stage) => stage.workerExecution === undefined ? [] : [stage.workerExecution]).reduce((latest, execution) => latest === undefined || execution.renewedAt > latest.renewedAt ? execution : latest, undefined);
         this.store.recordRunStart({
           ...run,
-          stages: result.stages.map((stage) => execution === undefined ? stage : { ...stage, workerExecution: execution }),
+          stages: result.stages.map((stage) => {
+            const execution = executions.get(stage.id) ?? attempt;
+            return execution === undefined ? stage : { ...stage, workerExecution: execution };
+          }),
           toolNodes: result.toolNodes
         });
         this.store.recordRunEnd(run.id, result.status, result.result, result.error);
@@ -67033,7 +67260,7 @@ class PoolStageHandoff {
       pendingStageMessages: [
         ...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
       ],
-      stages: withReconnectingStages(known.get(run.id), run, run.stages.map((stage) => ({ ...stage, workerExecution: execution })), work.generation)
+      stages: withReconnectingStages(known.get(run.id), run, run.stages.map((stage) => ({ ...stage, workerExecution: endedUnder(known.get(run.id), stage, execution) })), work.generation)
     }));
     const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
     for (const run of published)
@@ -67164,7 +67391,7 @@ class PoolStageHandoff {
           throw new Error(STALE);
         if (attempt.sessions.get(key) !== sessionId)
           throw new Error(`Stage session ${sessionId} was replaced before it acknowledged delivery`);
-      }, action));
+      }, action), { holdUntilDrained: true });
       attempt.deliveries.set(key, delivery);
     }
     return delivery;
@@ -67176,6 +67403,12 @@ function outside2(runId) {
 function isLive(stage) {
   return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
 }
+function endedUnder(known, stage, execution) {
+  if (!isTerminalStageStatus(stage.status))
+    return execution;
+  const previous = known?.stages.find((candidate) => candidate.id === stage.id);
+  return previous !== undefined && isTerminalStageStatus(previous.status) && previous.workerExecution?.generation === execution.generation && previous.workerExecution.worker === execution.worker ? previous.workerExecution : execution;
+}
 function withReconnectingStages(previous, run, stages, generation) {
   if (previous === undefined || run.endedAt !== undefined || isTerminalRunStatus(run.status))
     return stages;
@@ -67312,6 +67545,116 @@ class StageToolExecutionBuffer {
   }
 }
 
+// dist/builtin/workflows/src/workers/pooled-stage-task-source.ts
+function failure2(error) {
+  return { ok: false, error: { message: typeof error === "string" ? error : error.message } };
+}
+
+class PooledStageTaskSource {
+  run;
+  state = EMPTY;
+  records = [];
+  backgroundRecords = [];
+  settlements = new Map;
+  activity = new Map;
+  statuses = new Map;
+  stdin = new Set;
+  listeners = new Set;
+  constructor(run) {
+    this.run = run;
+  }
+  apply(state) {
+    this.state = state;
+    this.records = [...state.tasks];
+    const background = new Set(state.background);
+    this.backgroundRecords = this.records.filter((task) => background.has(task.ref.taskId));
+    this.settlements = new Map(state.settlement.map(([id, sequence]) => [id, BigInt(sequence)]));
+    this.activity = new Map(state.activity.map((entry) => [entry.id, entry]));
+    this.statuses = new Map(state.statuses.map((entry) => [entry.id, new Map(entry.entries)]));
+    this.stdin = new Set(state.stdin);
+    for (const listener of [...this.listeners])
+      listener();
+  }
+  get empty() {
+    return this.state.tasks.length === 0;
+  }
+  get tasks() {
+    return this.records;
+  }
+  get backgroundTasks() {
+    return this.backgroundRecords;
+  }
+  get settlementOrder() {
+    return this.settlements;
+  }
+  recentActivity(id) {
+    return [...this.activity.get(id)?.entries ?? []];
+  }
+  activityOmitted(id) {
+    return this.activity.get(id)?.omitted === true;
+  }
+  subscribe(listener) {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+  async foregroundTask(id) {
+    const reply = await this.request(id, { kind: "foreground" });
+    return reply.ok ? reply.value.kind === "foreground" ? reply.value.result : unexpected() : reply;
+  }
+  async cancelTask(id) {
+    const reply = await this.request(id, { kind: "cancel" });
+    if (!reply.ok)
+      return reply;
+    return reply.value.kind === "cancel" ? reply.value.result ?? { ok: true, value: undefined } : unexpected();
+  }
+  stdinAvailable(id) {
+    return this.stdin.has(id);
+  }
+  async writeTaskInput(id, text) {
+    const reply = await this.request(id, { kind: "input", text });
+    return reply.ok ? reply.value.kind === "input" ? reply.value.result : unexpected() : reply;
+  }
+  async readTaskOutput(id, range) {
+    const reply = await this.request(id, { kind: "output", start: range.start, maximumBytes: range.maximumBytes });
+    return reply.ok ? reply.value.kind === "output" ? reply.value.result : unexpected() : reply;
+  }
+  async taskDetailText(id) {
+    const reply = await this.request(id, { kind: "detail" });
+    return reply.ok && reply.value.kind === "detail" ? reply.value.detail : {};
+  }
+  async readTaskTranscript(id, cursor) {
+    const reply = await this.request(id, { kind: "transcript", ...cursor === undefined ? {} : { cursor } });
+    return reply.ok ? reply.value.kind === "transcript" ? reply.value.result : unexpected() : reply;
+  }
+  followTaskTranscript(id) {
+    return this.records.some((task) => task.ref.taskId === id && task.kind === "agent") ? { historyOnly: true } : undefined;
+  }
+  extensionStatuses(id) {
+    return this.statuses.get(id) ?? NO_STATUSES;
+  }
+  watchExtensionStatuses(_id, listener) {
+    return this.subscribe(listener);
+  }
+  async request(id, action) {
+    try {
+      const reply = await this.run(id, action);
+      return reply === undefined ? failure2("Task unavailable") : { ok: true, value: reply };
+    } catch (error) {
+      return failure2(error instanceof Error ? error : String(error));
+    }
+  }
+}
+function unexpected() {
+  return failure2("Unexpected reply from the local worker");
+}
+var EMPTY, NO_STATUSES;
+var init_pooled_stage_task_source = __esm(() => {
+  EMPTY = { tasks: [], background: [], settlement: [], activity: [], stdin: [], statuses: [] };
+  NO_STATUSES = new Map;
+});
+
 // dist/builtin/workflows/src/workers/pooled-stage-handle.ts
 import { randomUUID as randomUUID18 } from "node:crypto";
 
@@ -67342,6 +67685,11 @@ class PooledStageHandle {
   undelivered;
   disposed = false;
   session;
+  commands;
+  sessionStarted = true;
+  started = Promise.withResolvers();
+  bashRuns = new Map;
+  taskView = new PooledStageTaskSource(async (taskId, action) => (await this.sessionCommand({ kind: "task", taskId, action })).task);
   pumping;
   pumpAgain = false;
   watchGone = Promise.withResolvers();
@@ -67404,6 +67752,43 @@ class PooledStageHandle {
   streamingMessage() {
     return this.streaming;
   }
+  commandCatalog() {
+    return this.commands;
+  }
+  taskSource() {
+    return this.taskView;
+  }
+  get isSessionStarting() {
+    return !this.sessionStarted && this.ended === undefined;
+  }
+  get isBashRunning() {
+    return this.bashRuns.size > 0;
+  }
+  async executeBash(command, onChunk, options) {
+    const id = randomUUID18();
+    this.bashRuns.set(id, onChunk);
+    try {
+      const result = await this.sessionCommand({
+        kind: "bash",
+        id,
+        command,
+        ...options?.excludeFromContext === undefined ? {} : { excludeFromContext: options.excludeFromContext }
+      });
+      return result.bash ?? { output: "", exitCode: undefined, cancelled: true, truncated: false };
+    } finally {
+      this.bashRuns.delete(id);
+    }
+  }
+  abortBash() {
+    for (const id of this.bashRuns.keys())
+      this.sessionCommand({ kind: "abort-bash", id }).catch(() => {});
+  }
+  async compact() {
+    await this.sessionCommand({ kind: "compact" });
+  }
+  abortCompaction() {
+    this.sessionCommand({ kind: "abort-compaction" }).catch(() => {});
+  }
   pendingToolExecutionEvents() {
     return this.toolExecutions.replayEvents();
   }
@@ -67551,6 +67936,17 @@ class PooledStageHandle {
     if (this.disposed)
       throw new Error(`Stage chat "${this.stageName}" is closed.`);
   }
+  async sessionCommand(command) {
+    this.throwIfClosed();
+    if (this.ended !== undefined)
+      throw new PooledStageDeliveryError("StageNotLive", this.stageName);
+    if (this.transport.session === undefined)
+      throw new Error(`Stage "${this.stageName}" runs on a local worker that cannot run this command.`);
+    const result = await this.transport.session(this.watchId, command);
+    if (!result.ok)
+      throw new PooledStageDeliveryError(result.refusal, this.stageName);
+    return result;
+  }
   undeliveredCount() {
     if (this.undelivered !== undefined)
       return this.undelivered;
@@ -67616,6 +68012,16 @@ class PooledStageHandle {
       entry.waiter = undefined;
     }
   }
+  markSessionStarted(started) {
+    if (started) {
+      this.sessionStarted = true;
+      this.started.resolve();
+      return;
+    }
+    if (this.sessionStarted)
+      this.started = Promise.withResolvers();
+    this.sessionStarted = false;
+  }
   forget(op) {
     const index = this.outbound.findIndex((entry) => entry.op === op);
     if (index >= 0)
@@ -67672,10 +68078,17 @@ class PooledStageHandle {
           const gone = this.watchGone.promise.then(() => ({ ok: false, refusal: "StaleAttempt" }));
           const sending = this.sendEntry(entry, watch);
           let bound;
+          let stopped = false;
           const unconfirmed = new Promise((resolve) => {
-            bound = setTimeout(() => resolve("unconfirmed"), this.admissionWaitMs);
+            this.started.promise.then(() => {
+              if (!stopped)
+                bound = setTimeout(() => resolve("unconfirmed"), this.admissionWaitMs);
+            });
+          });
+          const outcome = await Promise.race([sending, gone, unconfirmed]).finally(() => {
+            stopped = true;
+            clearTimeout(bound);
           });
-          const outcome = await Promise.race([sending, gone, unconfirmed]).finally(() => clearTimeout(bound));
           if (outcome === "unconfirmed") {
             tracePooledWatch("root.outbox.unconfirmed", { op: entry.op, watch, generation });
             entry.unconfirmedOn = generation;
@@ -67765,6 +68178,10 @@ class PooledStageHandle {
   }
   adopt(snapshot) {
     this.session = snapshot.sessionId;
+    this.commands = snapshot.commands;
+    if (!this.taskView.empty)
+      this.taskView.apply(NO_TASKS);
+    this.markSessionStarted(snapshot.sessionStarted !== false);
     this.settled = [...snapshot.messages];
     this.streaming = snapshot.streamingMessage;
     this.currentStatus = snapshot.status;
@@ -67785,6 +68202,25 @@ class PooledStageHandle {
       this.forget(event.op);
       return false;
     }
+    if (event.type === STAGE_SESSION_STARTED_EVENT) {
+      this.commands = event.commands;
+      this.markSessionStarted(true);
+      const queued = this.queuedUserMessages();
+      const redraw = { type: "queue_update", steering: [...queued.steering], followUp: [...queued.followUp] };
+      for (const listener of [...this.listeners])
+        listener(redraw);
+      return false;
+    }
+    if (event.type === STAGE_TASKS_EVENT) {
+      this.taskView.apply(event.tasks);
+      return false;
+    }
+    if (event.type === STAGE_MESSAGES_EVENT) {
+      this.settled = [...event.messages];
+      return false;
+    }
+    if (event.type === "bash_execution_update" && event.id !== undefined)
+      this.bashRuns.get(event.id)?.(event.delta);
     this.applyEvent(event);
     const turnUnderway = TURN_EVENTS.has(event.type) || (event.type === "message_start" || event.type === "message_update") && event.message.role === "assistant";
     if (turnUnderway && this.awaitingTurn) {
@@ -67867,10 +68303,12 @@ async function reconnectPooledStageHandle(handle, transport, wanted = () => true
   await rebinding;
   return !handle.isReconnecting;
 }
-var DELIVERY_RECEIPT_EVENT = "atomic_pooled_delivery_receipt", POOLED_ADMISSION_WAIT_MS = 15000, RECONNECTABLE, TURN_EVENTS, PooledStageDeliveryError;
+var DELIVERY_RECEIPT_EVENT = "atomic_pooled_delivery_receipt", STAGE_SESSION_STARTED_EVENT = "atomic_pooled_stage_session_started", STAGE_MESSAGES_EVENT = "atomic_pooled_stage_messages", STAGE_TASKS_EVENT = "atomic_pooled_stage_tasks", NO_TASKS, POOLED_ADMISSION_WAIT_MS = 15000, RECONNECTABLE, TURN_EVENTS, PooledStageDeliveryError;
 var init_pooled_stage_handle = __esm(() => {
   init_stage_queued_user_messages();
+  init_pooled_stage_task_source();
   init_pooled_watch_trace();
+  NO_TASKS = { tasks: [], background: [], settlement: [], activity: [], stdin: [], statuses: [] };
   RECONNECTABLE = new Set(["attempt_fenced", "worker_exited", "resync"]);
   TURN_EVENTS = new Set(["agent_start", "turn_start", "tool_execution_start", "tool_execution_update"]);
   PooledStageDeliveryError = class PooledStageDeliveryError extends Error {
@@ -67930,6 +68368,9 @@ class RegisteredPooledStageHandle {
   pendingToolExecutionEvents() {
     return this.current()?.pendingToolExecutionEvents?.() ?? [];
   }
+  commandCatalog() {
+    return this.current()?.commandCatalog?.();
+  }
   update(stage) {
     this.stage = stage;
   }
@@ -68228,6 +68669,7 @@ class LocalWorkerPool {
   rootSignature;
   completing = 0;
   rootFlush;
+  connecting;
   starting;
   failure;
   closed = false;
@@ -68257,6 +68699,8 @@ class LocalWorkerPool {
       throw new Error("Invalid local worker startup timeout");
     if (options.leaseMs !== undefined && (!Number.isSafeInteger(options.leaseMs) || options.leaseMs < MINIMUM_WORK_LEASE_MS || options.leaseMs > MAX_TIMER_DELAY_MS))
       throw new Error(`Invalid local worker lease: must be an integer from ${MINIMUM_WORK_LEASE_MS} to ${MAX_TIMER_DELAY_MS} ms`);
+    if (options.maxLoadPerCpu !== undefined && (!Number.isFinite(options.maxLoadPerCpu) || options.maxLoadPerCpu < 0))
+      throw new Error("Invalid local worker load threshold: maxLoadPerCpu must be a number of at least 0");
     this.leaseMs = options.leaseMs ?? DEFAULT_WORK_LEASE_MS;
     this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
     if (options.admission && options.rootId === undefined)
@@ -68284,23 +68728,35 @@ class LocalWorkerPool {
     if (!state || state.holderId !== this.backend.holderId)
       throw new Error("Local worker pool requires the root owner");
     this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
-    const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
-    this.usageTotal = await this.command({
-      kind: "prepare",
-      root,
-      lease: this.lease,
-      options: {
-        journalPath: this.options.journalPath,
-        ...this.options.durable === undefined ? {} : { durable: this.options.durable },
-        workers: this.options.workers,
-        leaseMs: this.leaseMs,
-        startupTimeoutMs: this.options.startupTimeoutMs ?? defaultStartupTimeoutMs(this.options.workers),
-        ...this.options.admission === undefined ? {} : { admission: this.options.admission },
-        ...workerEntry === undefined ? {} : { workerEntry }
-      },
-      ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
-    });
     this.queue ??= await openSqlPoolQueue(poolJournal, this.options.durable === undefined ? undefined : new PoolRemoteJournal(this.backend.journal, sqlPoolStorage(poolJournal)));
+    if (await this.queue.initialized(root))
+      await this.connectPool();
+  }
+  connectPool() {
+    this.connecting ??= (async () => {
+      const root = this.root;
+      const lease = this.lease;
+      if (root === undefined || lease === undefined)
+        throw new Error("Local worker pool requires root preparation");
+      const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
+      this.usageTotal = await this.command({
+        kind: "prepare",
+        root,
+        lease,
+        options: {
+          journalPath: this.options.journalPath,
+          ...this.options.durable === undefined ? {} : { durable: this.options.durable },
+          workers: this.options.workers,
+          leaseMs: this.leaseMs,
+          startupTimeoutMs: this.options.startupTimeoutMs ?? defaultStartupTimeoutMs(this.options.workers),
+          ...this.options.admission === undefined ? {} : { admission: this.options.admission },
+          ...this.options.maxLoadPerCpu === undefined ? {} : { stageStartMaxLoadPerCpu: this.options.maxLoadPerCpu },
+          ...workerEntry === undefined ? {} : { workerEntry }
+        },
+        ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
+      });
+    })();
+    return this.connecting;
   }
   async poolJournal() {
     const journal = this.backend.journal;
@@ -68349,6 +68805,7 @@ class LocalWorkerPool {
   async start(root, promptNodes) {
     if (this.closed)
       throw new Error("Local worker pool is closed");
+    await this.connectPool();
     if (!this.queue || !this.lease || this.lease.runId !== root)
       throw new Error("Local worker pool requires root preparation");
     this.sources = await Promise.all(this.options.definitions.map(async (binding) => {
@@ -68631,6 +69088,11 @@ class LocalWorkerPool {
       ...message === undefined ? {} : { message }
     });
   }
+  async sessionCommandOnPooledStage(watch, command) {
+    if (this.closed || this.coordinatorLost !== undefined)
+      return { ok: false, refusal: "StaleAttempt" };
+    return await this.command({ kind: "stage-session", watch, command });
+  }
   stageTransport(onEnded) {
     return {
       watch: (ref, listener) => this.watchPooledStage(ref, (message) => {
@@ -68641,7 +69103,8 @@ class LocalWorkerPool {
       unwatch: (watch) => this.unwatchPooledStage(watch),
       deliver: (delivery) => this.deliverToPooledStage(delivery),
       pause: (watch) => this.pausePooledStage(watch),
-      resume: (watch, message) => this.resumePooledStage(watch, message)
+      resume: (watch, message) => this.resumePooledStage(watch, message),
+      session: (watch, command) => this.sessionCommandOnPooledStage(watch, command)
     };
   }
   async acquirePooledStage(runId, stageId) {
@@ -69304,6 +69767,9 @@ var init_local_pool = __esm(() => {
   completeInTurns = createTurnSlicer(2, COMPLETION_TURN_BUDGET_MS);
 });
 
+// dist/builtin/workflows/src/workers/public-options.ts
+var DEFAULT_POOLED_STAGE_START_MAX_LOAD_PER_CPU = 3;
+
 // dist/builtin/workflows/src/workers/public-pool.ts
 import { resolve as resolve15 } from "node:path";
 import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
@@ -69354,6 +69820,7 @@ async function activatePublicPool(backend, options) {
     ...options.localWorkers,
     startupTimeoutMs: undefined,
     leaseMs: undefined,
+    maxLoadPerCpu: undefined,
     definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve15(cwd, source.path) }))
   };
   const admission = options.admission && {
@@ -69397,6 +69864,7 @@ async function activatePublicPool(backend, options) {
     admission: resources.admission ?? {},
     startupTimeoutMs: options.localWorkers?.startupTimeoutMs,
     leaseMs: options.localWorkers?.leaseMs,
+    maxLoadPerCpu: options.localWorkers?.maxLoadPerCpu ?? DEFAULT_POOLED_STAGE_START_MAX_LOAD_PER_CPU,
     ...options.stageControlRegistry === undefined || options[hostWorkerBindings] ? {} : { stageControlRegistry: options.stageControlRegistry }
   });
   try {
@@ -75834,7 +76302,7 @@ class StageSessionController {
     return this.modelRoute ? this.checkedSession(await ready) : ready;
   }
   async ensureSessionFromId(sessionId, consumer = "prompt") {
-    const joinsLiveGeneration = !this.sessionShutdownPromise && (this.sessionPromise !== undefined || this.session !== undefined);
+    const joinsLiveGeneration = !this.sessionShutdownPromise && (this.sessionPromise !== undefined || this.session !== undefined || this.activeCreation !== undefined || this.ownedCreationPromise !== undefined);
     if (!this.sessionShutdownPromise && !joinsLiveGeneration)
       this.reattachSessionId = sessionId;
     const session = await this.ensureSession(consumer);