@bastani/atomic 0.9.57 → 0.9.58-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/dist/builtin/intercom/package.json +1 -1
  3. package/dist/builtin/mcp/package.json +1 -1
  4. package/dist/builtin/subagents/CHANGELOG.md +12 -0
  5. package/dist/builtin/subagents/package.json +1 -1
  6. package/dist/builtin/subagents/src/extension/index.bundle.mjs +39 -5
  7. package/dist/builtin/web-access/CHANGELOG.md +12 -0
  8. package/dist/builtin/web-access/index.bundle.mjs +2 -2
  9. package/dist/builtin/web-access/package.json +1 -1
  10. package/dist/builtin/workflows/CHANGELOG.md +42 -0
  11. package/dist/builtin/workflows/package.json +1 -1
  12. package/dist/builtin/workflows/src/extension/index.bundle.mjs +885 -228
  13. package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +933 -175
  14. package/dist/builtin/workflows/src/index.js +630 -162
  15. package/dist/builtin/workflows/src/local-worker-entry.mjs +933 -175
  16. package/dist/builtin/workflows/src/shared/authoring-contract-ui.d.ts +1 -1
  17. package/dist/builtin/workflows/src/shared/workflow-infrastructure-error.d.ts +13 -0
  18. package/dist/builtin/workflows/src/workers/public-options.d.ts +10 -0
  19. package/dist/core/agent-session-persistent-custom-messages.d.ts +5 -0
  20. package/dist/core/agent-session-persistent-custom-messages.d.ts.map +1 -1
  21. package/dist/core/agent-session-persistent-custom-messages.js +16 -0
  22. package/dist/core/agent-session-persistent-custom-messages.js.map +1 -1
  23. package/dist/core/agent-session-tasks.d.ts.map +1 -1
  24. package/dist/core/agent-session-tasks.js +20 -0
  25. package/dist/core/agent-session-tasks.js.map +1 -1
  26. package/dist/core/settings-defaults.d.ts +4 -0
  27. package/dist/core/settings-defaults.d.ts.map +1 -1
  28. package/dist/core/settings-defaults.js +4 -0
  29. package/dist/core/settings-defaults.js.map +1 -1
  30. package/dist/core/settings-manager-resource-accessors.d.ts +2 -0
  31. package/dist/core/settings-manager-resource-accessors.d.ts.map +1 -1
  32. package/dist/core/settings-manager-resource-accessors.js +6 -0
  33. package/dist/core/settings-manager-resource-accessors.js.map +1 -1
  34. package/dist/core/settings-schema.d.ts +9 -0
  35. package/dist/core/settings-schema.d.ts.map +1 -1
  36. package/dist/core/settings-schema.js +11 -0
  37. package/dist/core/settings-schema.js.map +1 -1
  38. package/dist/core/settings-types.d.ts +1 -1
  39. package/dist/core/settings-types.d.ts.map +1 -1
  40. package/dist/core/settings-types.js.map +1 -1
  41. package/dist/core/tasks/agent-adapter.d.ts +10 -1
  42. package/dist/core/tasks/agent-adapter.d.ts.map +1 -1
  43. package/dist/core/tasks/agent-adapter.js +20 -0
  44. package/dist/core/tasks/agent-adapter.js.map +1 -1
  45. package/dist/core/tasks/contracts.d.ts +6 -0
  46. package/dist/core/tasks/contracts.d.ts.map +1 -1
  47. package/dist/core/tasks/contracts.js.map +1 -1
  48. package/dist/core/tasks/heartbeat-notice.d.ts +64 -0
  49. package/dist/core/tasks/heartbeat-notice.d.ts.map +1 -0
  50. package/dist/core/tasks/heartbeat-notice.js +98 -0
  51. package/dist/core/tasks/heartbeat-notice.js.map +1 -0
  52. package/dist/core/tasks/heartbeat.d.ts +75 -0
  53. package/dist/core/tasks/heartbeat.d.ts.map +1 -0
  54. package/dist/core/tasks/heartbeat.js +440 -0
  55. package/dist/core/tasks/heartbeat.js.map +1 -0
  56. package/dist/core/tasks/supervisor.d.ts +2 -0
  57. package/dist/core/tasks/supervisor.d.ts.map +1 -1
  58. package/dist/core/tasks/supervisor.js +32 -4
  59. package/dist/core/tasks/supervisor.js.map +1 -1
  60. package/dist/core/tasks/task-view-source.d.ts +72 -0
  61. package/dist/core/tasks/task-view-source.d.ts.map +1 -0
  62. package/dist/core/tasks/task-view-source.js +155 -0
  63. package/dist/core/tasks/task-view-source.js.map +1 -0
  64. package/dist/core/tools/bash-pty-native.d.ts +2 -0
  65. package/dist/core/tools/bash-pty-native.d.ts.map +1 -1
  66. package/dist/core/tools/bash-pty-native.js +1 -0
  67. package/dist/core/tools/bash-pty-native.js.map +1 -1
  68. package/dist/core/tools/bash.d.ts +3 -0
  69. package/dist/core/tools/bash.d.ts.map +1 -1
  70. package/dist/core/tools/bash.js +20 -2
  71. package/dist/core/tools/bash.js.map +1 -1
  72. package/dist/core/tools/powershell.d.ts +16 -8
  73. package/dist/core/tools/powershell.d.ts.map +1 -1
  74. package/dist/index.d.ts +5 -3
  75. package/dist/index.d.ts.map +1 -1
  76. package/dist/index.js +4 -2
  77. package/dist/index.js.map +1 -1
  78. package/dist/modes/interactive/components/chat-session-host-events.d.ts.map +1 -1
  79. package/dist/modes/interactive/components/chat-session-host-events.js +7 -0
  80. package/dist/modes/interactive/components/chat-session-host-events.js.map +1 -1
  81. package/dist/modes/interactive/components/chat-session-host-state.d.ts +3 -0
  82. package/dist/modes/interactive/components/chat-session-host-state.d.ts.map +1 -1
  83. package/dist/modes/interactive/components/chat-session-host-state.js +4 -0
  84. package/dist/modes/interactive/components/chat-session-host-state.js.map +1 -1
  85. package/dist/modes/interactive/components/chat-session-host-types.d.ts +11 -0
  86. package/dist/modes/interactive/components/chat-session-host-types.d.ts.map +1 -1
  87. package/dist/modes/interactive/components/chat-session-host-types.js.map +1 -1
  88. package/dist/modes/interactive/components/chat-session-host.d.ts +2 -0
  89. package/dist/modes/interactive/components/chat-session-host.d.ts.map +1 -1
  90. package/dist/modes/interactive/components/chat-session-host.js +15 -8
  91. package/dist/modes/interactive/components/chat-session-host.js.map +1 -1
  92. package/dist/modes/interactive/components/custom-message.d.ts.map +1 -1
  93. package/dist/modes/interactive/components/custom-message.js +12 -0
  94. package/dist/modes/interactive/components/custom-message.js.map +1 -1
  95. package/dist/modes/interactive/components/task-detail.d.ts.map +1 -1
  96. package/dist/modes/interactive/components/task-detail.js +5 -0
  97. package/dist/modes/interactive/components/task-detail.js.map +1 -1
  98. package/dist/modes/interactive/components/task-heartbeat-message.d.ts +14 -0
  99. package/dist/modes/interactive/components/task-heartbeat-message.d.ts.map +1 -0
  100. package/dist/modes/interactive/components/task-heartbeat-message.js +76 -0
  101. package/dist/modes/interactive/components/task-heartbeat-message.js.map +1 -0
  102. package/dist/modes/interactive/components/task-inspector.d.ts +6 -3
  103. package/dist/modes/interactive/components/task-inspector.d.ts.map +1 -1
  104. package/dist/modes/interactive/components/task-inspector.js +53 -100
  105. package/dist/modes/interactive/components/task-inspector.js.map +1 -1
  106. package/dist/modes/interactive/skill-command-autocomplete.d.ts +22 -2
  107. package/dist/modes/interactive/skill-command-autocomplete.d.ts.map +1 -1
  108. package/dist/modes/interactive/skill-command-autocomplete.js +28 -9
  109. package/dist/modes/interactive/skill-command-autocomplete.js.map +1 -1
  110. package/docs/background-tasks.md +65 -1
  111. package/docs/settings.md +12 -0
  112. package/docs/subagents.md +2 -0
  113. package/docs/tools.md +2 -1
  114. package/docs/workflows/operations.md +5 -3
  115. package/npm-shrinkwrap.json +32 -32
  116. package/package.json +3 -3
  117. package/schemas/settings.schema.json +24 -0
@@ -8817,6 +8817,75 @@ function redactSensitiveText(value2) {
8817
8817
  return value2.replace(/(sk-[A-Za-z0-9_-]{8})[A-Za-z0-9_-]+/g, redactedSecretReplacement("$1")).replace(/\b(authorization\s*:\s*bearer\s+)[^\s,;]+/gi, "$1[redacted]").replace(/\b(bearer\s+)[A-Za-z0-9._~+/-]{8,}=*/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|credential|secret)\s*[:=]\s*)[^\s,;]+/gi, "$1[redacted]");
8818
8818
  }
8819
8819
 
8820
+ // dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
8821
+ function isInfrastructureError(value2) {
8822
+ if (value2 instanceof WorkflowInfrastructureError)
8823
+ return true;
8824
+ if (typeof value2 !== "object" || value2 === null)
8825
+ return false;
8826
+ const candidate = value2;
8827
+ return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
8828
+ }
8829
+ function isRetryableWorkflowError(error) {
8830
+ let current = error;
8831
+ for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined && current !== null; depth++) {
8832
+ if (isInfrastructureError(current))
8833
+ return true;
8834
+ current = current instanceof Error ? current.cause : undefined;
8835
+ }
8836
+ return false;
8837
+ }
8838
+ function workflowInfrastructureFault(message) {
8839
+ return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
8840
+ }
8841
+ function asWorkflowInfrastructureError(error) {
8842
+ if (isInfrastructureError(error))
8843
+ return error;
8844
+ if (!(error instanceof Error))
8845
+ return;
8846
+ const code = workflowInfrastructureFault(error.message);
8847
+ return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
8848
+ }
8849
+ function failedOnInfrastructureFault(result) {
8850
+ if (result.status !== "failed")
8851
+ return false;
8852
+ const names = (value2) => typeof value2 === "string" && (WORKFLOW_INFRASTRUCTURE_FAULTS.has(value2) || workflowInfrastructureFault(value2) !== undefined);
8853
+ const failure = result.failure;
8854
+ if (names(failure?.code) || names(result.error))
8855
+ return true;
8856
+ return Array.isArray(result.stages) && result.stages.some((stage) => stage?.status === "failed" && (names(stage.failureCode) || names(stage.error)));
8857
+ }
8858
+ var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", WORKFLOW_INFRASTRUCTURE_FAULTS, MAX_CAUSE_DEPTH = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
8859
+ var init_workflow_infrastructure_error = __esm(() => {
8860
+ WORKFLOW_INFRASTRUCTURE_FAULTS = new Set([
8861
+ "worker_lost",
8862
+ "worker_cleanup_unconfirmed",
8863
+ "worker_retry_limit",
8864
+ "worker_startup_failed",
8865
+ "stale_work_lease",
8866
+ "intercom_unavailable",
8867
+ "storage_unavailable"
8868
+ ]);
8869
+ FAULT_PATTERNS = [
8870
+ [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
8871
+ [/Local worker retry limit/, "worker_retry_limit"],
8872
+ [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
8873
+ [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
8874
+ [/Stale work lease|Stale resource admission attempt/, "stale_work_lease"],
8875
+ [/Intercom runtime no longer active/, "intercom_unavailable"],
8876
+ [/Storage unavailable/, "storage_unavailable"]
8877
+ ];
8878
+ WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
8879
+ name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
8880
+ retryable = true;
8881
+ code;
8882
+ constructor(code, message, options) {
8883
+ super(message, options);
8884
+ this.code = code;
8885
+ }
8886
+ };
8887
+ });
8888
+
8820
8889
  // dist/builtin/workflows/src/shared/workflow-failures-decisions.ts
8821
8890
  function authDecision(code) {
8822
8891
  if (code === "invalid_api_key") {
@@ -8905,6 +8974,16 @@ function cancelledDecision() {
8905
8974
  disposition: "terminal_killed"
8906
8975
  };
8907
8976
  }
8977
+ function infrastructureFaultDecision(code) {
8978
+ return {
8979
+ kind: "unknown",
8980
+ code,
8981
+ retryable: true,
8982
+ resumable: true,
8983
+ recoverability: "recoverable",
8984
+ disposition: "terminal_failed"
8985
+ };
8986
+ }
8908
8987
  function unknownDecision() {
8909
8988
  return {
8910
8989
  kind: "unknown",
@@ -8932,6 +9011,8 @@ function strongDecisionFromNormalizedCode(normalized, retryAfterMs) {
8932
9011
  disposition: "active_blocked"
8933
9012
  };
8934
9013
  }
9014
+ if (WORKFLOW_INFRASTRUCTURE_FAULTS.has(normalized))
9015
+ return infrastructureFaultDecision(normalized);
8935
9016
  if (CANCELLED_CODES.has(normalized))
8936
9017
  return cancelledDecision();
8937
9018
  if (INVALID_API_KEY_CODES.has(normalized))
@@ -9044,6 +9125,9 @@ function classificationFromNormalizedCode(normalized, retryAfterMs, source, mess
9044
9125
  return weak !== undefined ? { weak: classificationForDecision(weak, source, message, "weak_signal") } : {};
9045
9126
  }
9046
9127
  function fallbackDecisionFromMessage(message, name) {
9128
+ const fault = workflowInfrastructureFault(message);
9129
+ if (fault !== undefined)
9130
+ return infrastructureFaultDecision(fault);
9047
9131
  const tokens = tokenize(message);
9048
9132
  const decision = decisionFromMessageTokens(tokens, name);
9049
9133
  const providerAuthDecision = classifyFallbackProviderAuthMessage(message, tokens);
@@ -9058,6 +9142,7 @@ function fallbackDecisionFromMessage(message, name) {
9058
9142
  var INVALID_API_KEY_CODES, LOGIN_REQUIRED_CODES, MISSING_API_KEY_CODES, RATE_LIMIT_CODES, QUOTA_LIMIT_CODES, CANCELLED_CODES, PROVIDER_UNAVAILABLE_CODES, UNKNOWN_MODEL_CODES, FORBIDDEN_CONFIG_CODES, REQUEST_AUTH_TIMEOUT_PHRASES, LOGIN_REQUIRED_PHRASES, LOCAL_LOGIN_REQUIRED_PHRASES, PROVIDER_AUTH_FALLBACK_PHRASES, MISSING_API_KEY_PHRASES, INVALID_API_KEY_PHRASES, INVALID_API_KEY_CONTEXT, HTTP_RATE_LIMIT_PHRASES, RATE_LIMIT_PHRASES, QUOTA_LIMIT_PHRASES, CANCELLED_PHRASES, UNKNOWN_MODEL_PHRASES, FORBIDDEN_CONFIG_PHRASES, PROVIDER_UNAVAILABLE_PHRASES, AUTH_CONTEXT, MODEL_PROVIDER_CONTEXT, PROVIDER_CONTEXT, STATUS_MESSAGE_REFINEMENT_CODES, BROAD_AUTH_MESSAGE_REFINEMENT_CODES, STATUS_RELATED_MESSAGE_REFINEMENT_CODES;
9059
9143
  var init_workflow_failures_decisions = __esm(() => {
9060
9144
  init_workflow_failures_contract();
9145
+ init_workflow_infrastructure_error();
9061
9146
  INVALID_API_KEY_CODES = new Set([
9062
9147
  "401",
9063
9148
  "invalid_api_key",
@@ -9245,6 +9330,13 @@ function decisionOfRecordedCode(code, retryAfterMs) {
9245
9330
  case "cancelled":
9246
9331
  return cancelledDecision();
9247
9332
  case "pending_stage_delivery_failed":
9333
+ case "worker_lost":
9334
+ case "worker_cleanup_unconfirmed":
9335
+ case "worker_retry_limit":
9336
+ case "worker_startup_failed":
9337
+ case "stale_work_lease":
9338
+ case "intercom_unavailable":
9339
+ case "storage_unavailable":
9248
9340
  return strongDecisionFromNormalizedCode(code, retryAfterMs);
9249
9341
  case "unknown":
9250
9342
  return;
@@ -13865,7 +13957,7 @@ class JournalDurableBackend {
13865
13957
  timer;
13866
13958
  state = "open";
13867
13959
  closing;
13868
- renewing;
13960
+ renewing = new Map;
13869
13961
  mutationContext = new AsyncLocalStorage3;
13870
13962
  processExitCleanup;
13871
13963
  unregisterExecutionStorages = registerExecutionStorages(this.executionContext, this.mutationContext);
@@ -14586,27 +14678,21 @@ class JournalDurableBackend {
14586
14678
  yield this.enqueue(runId, () => this.deleteOwned(runId));
14587
14679
  }
14588
14680
  renewLeases() {
14589
- if (this.renewing !== undefined)
14590
- return this.renewing;
14591
14681
  if (this.state !== "open")
14592
14682
  return Promise.resolve();
14593
- this.renewing = this.renewOwnedLeases().finally(() => {
14594
- this.renewing = undefined;
14595
- });
14596
- return this.renewing;
14597
- }
14598
- async renewOwnedLeases() {
14599
14683
  for (const [runId, lease] of this.leases) {
14600
- if (this.state !== "open")
14601
- return;
14602
- try {
14603
- await this.serialize(runId, () => this.renewLease(runId, lease), false);
14604
- } catch (error) {
14605
- if (error instanceof Error)
14606
- this.lost.set(runId, error);
14684
+ if (this.renewing.has(runId))
14685
+ continue;
14686
+ const renewal = this.serialize(runId, () => this.renewLease(runId, lease), false).catch((error) => {
14687
+ this.lost.set(runId, error);
14607
14688
  this.dropLease(runId);
14608
- }
14689
+ }).finally(() => {
14690
+ if (this.renewing.get(runId) === renewal)
14691
+ this.renewing.delete(runId);
14692
+ });
14693
+ this.renewing.set(runId, renewal);
14609
14694
  }
14695
+ return Promise.all(this.renewing.values()).then(() => {});
14610
14696
  }
14611
14697
  *renewLease(runId, lease) {
14612
14698
  if (this.leases.get(runId) !== lease || this.lost.has(runId))
@@ -14614,9 +14700,10 @@ class JournalDurableBackend {
14614
14700
  try {
14615
14701
  yield* invoke(() => this.journal.renew(lease));
14616
14702
  } catch (error) {
14703
+ if (!(error instanceof StaleLeaseError))
14704
+ return;
14617
14705
  if (this.leases.get(runId) === lease) {
14618
- if (error instanceof Error)
14619
- this.lost.set(runId, yield* this.mapOwnership(error, runId));
14706
+ this.lost.set(runId, yield* this.mapOwnership(error, runId));
14620
14707
  this.dropLease(runId);
14621
14708
  }
14622
14709
  }
@@ -14710,7 +14797,7 @@ class JournalDurableBackend {
14710
14797
  }
14711
14798
  async drainAndClose() {
14712
14799
  try {
14713
- await this.renewing;
14800
+ await Promise.all(this.renewing.values());
14714
14801
  while (this.pending.size > 0)
14715
14802
  await Promise.all(this.pending.values());
14716
14803
  for (const runId of this.leases.keys())
@@ -20179,7 +20266,7 @@ class SqlResourceAdmission {
20179
20266
  for (const value2 of Object.values(limits ?? {}))
20180
20267
  if (!Number.isSafeInteger(value2) || value2 < 1)
20181
20268
  throw new Error("Invalid resource limit");
20182
- for (const value2 of Object.values(options.host ?? {}))
20269
+ for (const value2 of [...Object.values(options.host ?? {}), options.stageStartMaxLoadPerCpu ?? 0])
20183
20270
  if (!Number.isFinite(value2) || value2 < 0)
20184
20271
  throw new Error("Invalid host admission threshold");
20185
20272
  const admission = new SqlResourceAdmission(storage, {
@@ -20197,6 +20284,8 @@ class SqlResourceAdmission {
20197
20284
  for (const column of ["owner", "attempt", "process_identity"])
20198
20285
  if (!columns.some((entry) => entry.name === column))
20199
20286
  await tx.exec(`ALTER TABLE ${admission.permits} ADD COLUMN ${column} TEXT`);
20287
+ if (!columns.some((entry) => entry.name === "granted_at"))
20288
+ await tx.exec(`ALTER TABLE ${admission.permits} ADD COLUMN granted_at INTEGER`);
20200
20289
  await tx.exec(`CREATE INDEX IF NOT EXISTS ${admission.permits}_owner ON ${admission.permits}(owner, state);
20201
20290
  CREATE INDEX IF NOT EXISTS ${admission.permits}_state ON ${admission.permits}(state)`);
20202
20291
  if (!(await tx.all(`PRAGMA table_info(${admission.owners})`)).some((entry) => entry.name === "boot"))
@@ -20226,14 +20315,24 @@ class SqlResourceAdmission {
20226
20315
  await admission.reconcile();
20227
20316
  return admission;
20228
20317
  }
20318
+ sampleHost() {
20319
+ return this.options.sampleHost?.() ?? {
20320
+ freeMemoryBytes: freemem(),
20321
+ load: process.platform === "win32" ? undefined : loadavg()[0] / cpus().length
20322
+ };
20323
+ }
20324
+ stageStartPressure(kind, delegated) {
20325
+ const threshold = this.options.stageStartMaxLoadPerCpu;
20326
+ if (kind !== "session" || delegated === true || threshold === undefined || threshold === 0)
20327
+ return false;
20328
+ const load = this.sampleHost().load;
20329
+ return load !== undefined && load > threshold;
20330
+ }
20229
20331
  hostReason() {
20230
20332
  const host = this.options.host;
20231
20333
  if (!host)
20232
20334
  return;
20233
- const sample = this.options.sampleHost?.() ?? {
20234
- freeMemoryBytes: freemem(),
20235
- load: process.platform === "win32" ? undefined : loadavg()[0] / cpus().length
20236
- };
20335
+ const sample = this.sampleHost();
20237
20336
  if (host.minFreeMemoryBytes !== undefined) {
20238
20337
  if (sample.freeMemoryBytes === undefined)
20239
20338
  return "host memory unsupported";
@@ -20247,21 +20346,32 @@ class SqlResourceAdmission {
20247
20346
  return "host load";
20248
20347
  }
20249
20348
  }
20250
- async assertCurrentAttempt(tx) {
20349
+ async attemptStanding(tx) {
20251
20350
  const { root, attempt } = this.options;
20252
20351
  if (!attempt)
20253
- return;
20254
- const work = await tx.get(`SELECT 1 AS valid FROM ${this.storage.tables.runs}_work w JOIN ${this.storage.tables.leases} l ON l.run_id = w.root
20352
+ return "current";
20353
+ const work = await tx.get(`SELECT (w.expires_at < ${NOW_MS} OR l.renewed_at + p.timeout < ${NOW_MS}) AS late
20354
+ FROM ${this.storage.tables.runs}_work w JOIN ${this.storage.tables.leases} l ON l.run_id = w.root
20255
20355
  JOIN ${this.storage.tables.runs}_pool p ON p.root = w.root JOIN ${this.storage.tables.runs} r ON r.run_id = w.root
20256
20356
  WHERE w.id = ? AND w.root = ? AND w.generation = ? AND w.epoch = ? AND w.state = 'leased' AND r.status IN ('running', 'paused')
20257
- AND w.expires_at >= CAST(unixepoch('subsec') * 1000 AS INTEGER)
20258
- AND l.epoch = w.epoch AND l.renewed_at + p.timeout >= CAST(unixepoch('subsec') * 1000 AS INTEGER)`, attempt.id, root, attempt.generation, attempt.epoch);
20357
+ AND l.epoch = w.epoch`, attempt.id, root, attempt.generation, attempt.epoch);
20259
20358
  if (!work)
20359
+ return "stale";
20360
+ return Number(work.late) === 1 ? "late" : "current";
20361
+ }
20362
+ async assertCurrentAttempt(tx) {
20363
+ const standing = await this.attemptStanding(tx);
20364
+ if (standing === "stale")
20260
20365
  throw new Error("Stale resource admission attempt");
20366
+ return standing === "current";
20261
20367
  }
20262
20368
  async validate(signal) {
20263
- signal?.throwIfAborted();
20264
- await this.storage.transaction((tx) => this.assertCurrentAttempt(tx));
20369
+ for (;; ) {
20370
+ signal?.throwIfAborted();
20371
+ if (await this.storage.transaction((tx) => this.assertCurrentAttempt(tx)))
20372
+ return;
20373
+ await delay(LATE_ATTEMPT_POLL_MS, undefined, { signal });
20374
+ }
20265
20375
  }
20266
20376
  async acquire(kind, signal, request) {
20267
20377
  signal?.throwIfAborted();
@@ -20295,11 +20405,19 @@ class SqlResourceAdmission {
20295
20405
  this.nextReconciliation = performance.now() + 1000;
20296
20406
  }
20297
20407
  const hostReason = this.hostReason();
20408
+ const pressure = this.stageStartPressure(kind, request?.delegated);
20298
20409
  const granted = await this.storage.transaction(async (tx) => {
20299
20410
  await tx.run(`UPDATE ${this.owners} SET expires_at = CAST(unixepoch('subsec') * 1000 AS INTEGER) + ? WHERE id = ?`, OWNER_LEASE_MS, this.owner);
20300
20411
  const { root } = this.options;
20301
- await this.assertCurrentAttempt(tx);
20412
+ if (!await this.assertCurrentAttempt(tx)) {
20413
+ const waiting = await tx.run(`UPDATE ${this.permits} SET reason = ? WHERE operation = ? AND state = 'waiting'`, LATE_ATTEMPT_REASON, operation);
20414
+ if (waiting.changes !== 1)
20415
+ throw new Error("Resource admission holder was stopped");
20416
+ return false;
20417
+ }
20302
20418
  let reason = hostReason;
20419
+ if (pressure && (await tx.get(`SELECT 1 AS recent FROM ${this.permits} WHERE root = ? AND kind = 'session' AND granted_at >= ${NOW_MS} - ? LIMIT 1`, root, STAGE_START_SPACING_MS) || await tx.get(`SELECT 1 AS older FROM ${this.permits} WHERE root = ? AND kind = 'session' AND state = 'waiting' AND reason = ? AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, root, STAGE_START_PRESSURE_REASON, operation)))
20420
+ reason ??= STAGE_START_PRESSURE_REASON;
20303
20421
  const controls = `${this.storage.tables.runs}_pool_control`;
20304
20422
  if (await tx.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", controls)) {
20305
20423
  const control = await tx.get(`SELECT state FROM ${controls} WHERE root = ?`, root);
@@ -20314,13 +20432,14 @@ class SqlResourceAdmission {
20314
20432
  const rootClause = scope === "user" ? "" : "AND root = ?";
20315
20433
  const args = scope === "user" ? [kind] : [kind, root];
20316
20434
  const count = await tx.get(`SELECT COUNT(*) AS count FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'granted'`, ...args);
20317
- const older = await tx.get(`SELECT 1 AS waiting FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'waiting' AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, ...args, operation);
20435
+ const older = await tx.get(`SELECT 1 AS waiting FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'waiting' AND reason IS NOT ? AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, ...args, STAGE_START_PRESSURE_REASON, operation);
20318
20436
  if (Number(count?.count) >= limit || older)
20319
20437
  reason ??= `${scope} ${kind}`;
20320
20438
  }
20321
20439
  if (reason && request?.delegated)
20322
20440
  throw new Error(`Delegated session cannot be admitted without capacity headroom: ${reason}`);
20323
- const updated = await tx.run(`UPDATE ${this.permits} SET state = ?, reason = ? WHERE operation = ? AND state = 'waiting'`, reason ? "waiting" : "granted", reason ?? null, operation);
20441
+ const stageStart = kind === "session" && request?.delegated !== true;
20442
+ const updated = await tx.run(`UPDATE ${this.permits} SET state = ?, reason = ?, granted_at = CASE WHEN ? IS NULL AND ? THEN ${NOW_MS} ELSE NULL END WHERE operation = ? AND state = 'waiting'`, reason ? "waiting" : "granted", reason ?? null, reason ?? null, stageStart ? 1 : 0, operation);
20324
20443
  if (updated.changes !== 1)
20325
20444
  throw new Error("Resource admission holder was stopped");
20326
20445
  return !reason;
@@ -20480,7 +20599,7 @@ async function reapProcessGroups(journal, root, scope) {
20480
20599
  return { survivors: [], unprobeable: [] };
20481
20600
  return settleProcessGroups(storage, root, scope, true);
20482
20601
  }
20483
- var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
20602
+ var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", LATE_ATTEMPT_POLL_MS = 100, LATE_ATTEMPT_REASON = "waiting for its attempt lease", STAGE_START_SPACING_MS = 5000, STAGE_START_PRESSURE_REASON = "paced for host load";
20484
20603
  var init_resource_admission = __esm(() => {
20485
20604
  init_internal();
20486
20605
  init_sql_run_journal();
@@ -63094,14 +63213,23 @@ class SqlPoolQueue {
63094
63213
  }
63095
63214
  await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
63096
63215
  }
63216
+ leaseRow(tx, lease) {
63217
+ return tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
63218
+ }
63097
63219
  async authority(tx, lease, timeout) {
63098
- const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
63220
+ const row = await this.leaseRow(tx, lease);
63099
63221
  if (row?.timeout === null && timeout === undefined)
63100
63222
  throw new Error("Pool not initialized");
63101
63223
  if (!row || row.epoch !== lease.epoch || row.holder_id !== lease.holderId || isWorkflowLeaseExpired(row.renewed_at, row.timeout ?? timeout ?? 0, row.now))
63102
63224
  throw new StaleLeaseError(lease.runId, lease.epoch, Number(row?.epoch ?? 0));
63103
63225
  return row;
63104
63226
  }
63227
+ async rootLease(lease, timeout) {
63228
+ const row = await this.storage.read((executor) => this.leaseRow(executor, lease));
63229
+ if (!row || row.epoch !== lease.epoch || row.holder_id !== lease.holderId)
63230
+ return "lost";
63231
+ return isWorkflowLeaseExpired(row.renewed_at, row.timeout ?? timeout, row.now) ? "late" : "current";
63232
+ }
63105
63233
  async initialize(lease, coordinatorTimeoutMs) {
63106
63234
  positive(coordinatorTimeoutMs);
63107
63235
  lease = snapshotLease(lease);
@@ -63113,6 +63241,9 @@ class SqlPoolQueue {
63113
63241
  throw new Error("Pool configuration is immutable");
63114
63242
  });
63115
63243
  }
63244
+ async initialized(root) {
63245
+ return await this.storage.read((executor) => executor.get(`SELECT 1 AS held FROM ${this.roots} WHERE root = ?`, root)) !== undefined;
63246
+ }
63116
63247
  async control(lease) {
63117
63248
  return this.transaction(async (tx) => {
63118
63249
  await this.authority(tx, lease);
@@ -63507,6 +63638,21 @@ class SqlPoolQueue {
63507
63638
  return row ? JSON.parse(String(row.result)) : undefined;
63508
63639
  });
63509
63640
  }
63641
+ async reopenCompleted(lease, id, rerun) {
63642
+ lease = snapshotLease(lease);
63643
+ return this.transaction(async (tx) => {
63644
+ await this.authority(tx, lease);
63645
+ const row = await tx.get(`SELECT * FROM ${this.work} WHERE id = ? AND root = ?`, id, lease.runId);
63646
+ if (row?.state !== "completed" || row.epoch >= lease.epoch)
63647
+ return;
63648
+ const stored = await tx.get(`SELECT result FROM ${this.work}_results WHERE work = ?`, id);
63649
+ if (!stored || !rerun(JSON.parse(String(stored.result))))
63650
+ return;
63651
+ await tx.run(`DELETE FROM ${this.work}_results WHERE work = ?`, id);
63652
+ await tx.run(`UPDATE ${this.work} SET state = 'ready', worker = NULL, expires_at = 0, epoch = ? WHERE id = ?`, lease.epoch, id);
63653
+ return fromRow({ ...row, state: "ready", worker: null, expires_at: 0, epoch: lease.epoch });
63654
+ });
63655
+ }
63510
63656
  async expire(lease, token) {
63511
63657
  lease = snapshotLease(lease);
63512
63658
  token = { ...token };
@@ -63628,7 +63774,7 @@ function reportsClosedSession(error) {
63628
63774
  }
63629
63775
  function isSessionClosedError(error) {
63630
63776
  let current = error;
63631
- for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined; depth++) {
63777
+ for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined; depth++) {
63632
63778
  try {
63633
63779
  if (reportsClosedSession(current))
63634
63780
  return true;
@@ -63639,7 +63785,7 @@ function isSessionClosedError(error) {
63639
63785
  }
63640
63786
  return false;
63641
63787
  }
63642
- var MAX_CAUSE_DEPTH = 8;
63788
+ var MAX_CAUSE_DEPTH2 = 8;
63643
63789
  var init_session_closed_error = () => {};
63644
63790
 
63645
63791
  // dist/builtin/workflows/src/shared/external-run-routes.ts
@@ -67741,57 +67887,6 @@ var init_sdk_intercom = __esm(() => {
67741
67887
  owners2 = new WeakMap;
67742
67888
  });
67743
67889
 
67744
- // dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
67745
- function isInfrastructureError(value2) {
67746
- if (value2 instanceof WorkflowInfrastructureError)
67747
- return true;
67748
- if (typeof value2 !== "object" || value2 === null)
67749
- return false;
67750
- const candidate = value2;
67751
- return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
67752
- }
67753
- function isRetryableWorkflowError(error) {
67754
- let current = error;
67755
- for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined && current !== null; depth++) {
67756
- if (isInfrastructureError(current))
67757
- return true;
67758
- current = current instanceof Error ? current.cause : undefined;
67759
- }
67760
- return false;
67761
- }
67762
- function workflowInfrastructureFault(message) {
67763
- return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
67764
- }
67765
- function asWorkflowInfrastructureError(error) {
67766
- if (isInfrastructureError(error))
67767
- return error;
67768
- if (!(error instanceof Error))
67769
- return;
67770
- const code = workflowInfrastructureFault(error.message);
67771
- return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
67772
- }
67773
- var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH2 = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
67774
- var init_workflow_infrastructure_error = __esm(() => {
67775
- FAULT_PATTERNS = [
67776
- [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
67777
- [/Local worker retry limit/, "worker_retry_limit"],
67778
- [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
67779
- [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
67780
- [/Stale work lease/, "stale_work_lease"],
67781
- [/Intercom runtime no longer active/, "intercom_unavailable"],
67782
- [/Storage unavailable/, "storage_unavailable"]
67783
- ];
67784
- WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
67785
- name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
67786
- retryable = true;
67787
- code;
67788
- constructor(code, message, options) {
67789
- super(message, options);
67790
- this.code = code;
67791
- }
67792
- };
67793
- });
67794
-
67795
67890
  // dist/builtin/workflows/src/workers/coordinator-protocol.ts
67796
67891
  var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
67797
67892
 
@@ -68075,6 +68170,16 @@ class RateLimitObservationOutbox {
68075
68170
  }
68076
68171
  var RATE_LIMIT_SHARE_INTERVAL_MS = 250;
68077
68172
 
68173
+ // dist/builtin/workflows/src/workers/stage-session-commands.ts
68174
+ function runsWhilePaused(command) {
68175
+ if (command.kind === "abort-bash" || command.kind === "abort-compaction")
68176
+ return true;
68177
+ if (command.kind !== "task")
68178
+ return false;
68179
+ const action = command.action.kind;
68180
+ return action === "cancel" || action === "output" || action === "detail" || action === "transcript";
68181
+ }
68182
+
68078
68183
  // dist/builtin/workflows/src/workers/stage-watch-coalesce.ts
68079
68184
  function streamedDelta(event) {
68080
68185
  if (event.type !== "message_update")
@@ -68401,6 +68506,7 @@ class PoolCoordinator {
68401
68506
  closing;
68402
68507
  replacements = 0;
68403
68508
  rootLeaseLost = false;
68509
+ lateRoot;
68404
68510
  attemptBase = new Map;
68405
68511
  constructor(port) {
68406
68512
  this.port = port;
@@ -68414,6 +68520,47 @@ class PoolCoordinator {
68414
68520
  await this.journal?.close().catch(() => {});
68415
68521
  await this.durable?.close().catch(() => {});
68416
68522
  }
68523
+ async whileRootLate(queue, action, resume) {
68524
+ for (let waited = false;; ) {
68525
+ try {
68526
+ if (waited)
68527
+ await resume?.();
68528
+ return await action();
68529
+ } catch (error) {
68530
+ if (!(error instanceof Error) || !isRootLeaseLost(error) || !await this.rootRenewal(queue))
68531
+ throw error;
68532
+ waited = true;
68533
+ }
68534
+ }
68535
+ }
68536
+ rootRenewal(queue) {
68537
+ this.lateRoot ??= this.awaitRootRenewal(queue).finally(() => {
68538
+ this.lateRoot = undefined;
68539
+ });
68540
+ return this.lateRoot;
68541
+ }
68542
+ async awaitRootRenewal(queue) {
68543
+ const lease = this.lease;
68544
+ const since = performance.now();
68545
+ let late = false;
68546
+ try {
68547
+ for (;; ) {
68548
+ if (this.closed)
68549
+ return false;
68550
+ const lost = await this.remote?.fresh(lease).then(() => false, () => true);
68551
+ if (lost)
68552
+ return false;
68553
+ const standing = await queue.rootLease(lease, COORDINATOR_LEASE_MS);
68554
+ if (standing !== "late")
68555
+ return standing === "current";
68556
+ late = true;
68557
+ await delay2(LATE_ROOT_POLL_MS);
68558
+ }
68559
+ } finally {
68560
+ if (late)
68561
+ this.stalledMs += performance.now() - since;
68562
+ }
68563
+ }
68417
68564
  receive(message) {
68418
68565
  if (message.kind === "request-reply") {
68419
68566
  const pending = this.requests.get(message.id);
@@ -68482,6 +68629,8 @@ class PoolCoordinator {
68482
68629
  return this.stageAnswer(command.runId, command.stageId, command.answer);
68483
68630
  case "stage-pause":
68484
68631
  return this.stageControl(command.watch, { op: "pause" });
68632
+ case "stage-session":
68633
+ return this.stageSession(command.watch, command.command);
68485
68634
  case "stage-resume":
68486
68635
  return this.stageControl(command.watch, {
68487
68636
  op: "resume",
@@ -68511,8 +68660,8 @@ class PoolCoordinator {
68511
68660
  this.usageTotal = await this.usageLedger.total(root);
68512
68661
  this.lease = lease;
68513
68662
  const queue = await openSqlPoolQueue(this.journal, this.remote);
68514
- await queue.initialize(lease, COORDINATOR_LEASE_MS);
68515
- await queue.drainOutbox(lease);
68663
+ await this.whileRootLate(queue, () => queue.initialize(lease, COORDINATOR_LEASE_MS));
68664
+ await this.whileRootLate(queue, () => queue.drainOutbox(lease));
68516
68665
  if (this.remote !== undefined) {
68517
68666
  const remote = this.remote;
68518
68667
  this.mirrorTimer = setInterval(() => void remote.fresh(lease, 0).catch(() => {}), MIRROR_INTERVAL_MS).unref();
@@ -68521,10 +68670,10 @@ class PoolCoordinator {
68521
68670
  for (const work of await queue.inspect(root))
68522
68671
  if (work.state === "leased" && work.epoch < lease.epoch && (await unconfirmedProcessGroups(this.journal, root, { attempt: work })).length > 0)
68523
68672
  retained.push(work);
68524
- await queue.releasePreviousIncarnations(lease, retained.map((work) => work.id));
68673
+ await this.whileRootLate(queue, () => queue.releasePreviousIncarnations(lease, retained.map((work) => work.id)));
68525
68674
  for (;; ) {
68526
68675
  const intended = this.desiredControl ?? "running";
68527
- await queue.setControl(lease, intended);
68676
+ await this.whileRootLate(queue, () => queue.setControl(lease, intended));
68528
68677
  if (intended === (this.desiredControl ?? "running"))
68529
68678
  break;
68530
68679
  }
@@ -68560,7 +68709,7 @@ class PoolCoordinator {
68560
68709
  return this.retainedTransition(async () => {
68561
68710
  const record = this.retained.get(work.id);
68562
68711
  this.retained.delete(work.id);
68563
- await this.queue.releaseRetained(this.lease, work, record?.enqueued === true).catch((error) => {
68712
+ await this.whileRootLate(this.queue, () => this.queue.releaseRetained(this.lease, work, record?.enqueued === true)).catch((error) => {
68564
68713
  if (error.message !== STALE_WORK_LEASE)
68565
68714
  throw error;
68566
68715
  });
@@ -68574,20 +68723,29 @@ class PoolCoordinator {
68574
68723
  async control(state) {
68575
68724
  this.desiredControl = state;
68576
68725
  if (!this.queue || !this.lease)
68577
- return;
68578
- const control = await this.queue.setControl(this.lease, state);
68579
- const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
68580
- while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
68726
+ return 0;
68727
+ const queue = this.queue;
68728
+ const lease = this.lease;
68729
+ const control = await this.whileRootLate(queue, () => queue.setControl(lease, state));
68730
+ const unacknowledged = () => [...this.readyWorkers].filter((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision).length;
68731
+ let pending = unacknowledged();
68732
+ let progressAt = performance.now();
68733
+ let stalledAt = this.stalledMs;
68734
+ while (pending > 0) {
68581
68735
  if (this.desiredControl !== state)
68582
- return;
68583
- if (Date.now() >= deadline) {
68584
- if (state === "quit")
68585
- break;
68586
- throw new Error("Worker control acknowledgement unresolved");
68587
- }
68736
+ return pending;
68588
68737
  await delay2(25);
68738
+ this.observeResponsiveness();
68739
+ const now = unacknowledged();
68740
+ if (now < pending) {
68741
+ pending = now;
68742
+ progressAt = performance.now();
68743
+ stalledAt = this.stalledMs;
68744
+ } else if (performance.now() - progressAt - (this.stalledMs - stalledAt) >= CONTROL_ACKNOWLEDGEMENT_STALL_MS)
68745
+ break;
68589
68746
  }
68590
68747
  this.flush();
68748
+ return pending;
68591
68749
  }
68592
68750
  async start(sources, cwd, publish, rootRun) {
68593
68751
  if (this.closed)
@@ -68615,7 +68773,11 @@ class PoolCoordinator {
68615
68773
  return;
68616
68774
  });
68617
68775
  this.recoveryTimer = setInterval(() => {
68618
- this.recoverExpired().catch((error) => this.fail(error));
68776
+ this.recoverExpired().catch(async (error) => {
68777
+ if (this.queue && isRootLeaseLost(error) && await this.rootRenewal(this.queue).catch(() => false))
68778
+ return;
68779
+ this.fail(error);
68780
+ });
68619
68781
  }, RECOVERY_INTERVAL_MS).unref();
68620
68782
  const workers = Array.from({ length: this.options.workers }, () => this.spawn());
68621
68783
  try {
@@ -68632,7 +68794,9 @@ class PoolCoordinator {
68632
68794
  const queue = this.queue;
68633
68795
  const lease = this.lease;
68634
68796
  const work = await this.retainedTransition(async () => {
68635
- const enqueued = await queue.enqueue(lease, command.descriptor, command.boundary);
68797
+ let enqueued = await this.whileRootLate(queue, () => queue.enqueue(lease, command.descriptor, command.boundary));
68798
+ if (enqueued.state === "completed" && enqueued.epoch < lease.epoch)
68799
+ enqueued = await this.whileRootLate(queue, () => queue.reopenCompleted(lease, enqueued.id, failedOnInfrastructureFault)) ?? enqueued;
68636
68800
  const retained = this.retained.get(enqueued.id);
68637
68801
  if (retained)
68638
68802
  retained.enqueued = true;
@@ -68645,7 +68809,9 @@ class PoolCoordinator {
68645
68809
  }
68646
68810
  async transport(request, payload) {
68647
68811
  const target = this.deliveryRequests.get(request);
68648
- if (target === undefined || !this.incarnations.has(target.slot.worker))
68812
+ if (target === undefined)
68813
+ throw new Error("Pooled stage delivery request already answered");
68814
+ if (!this.incarnations.has(target.slot.worker))
68649
68815
  throw new Error("Worker exited before delivery acknowledgement");
68650
68816
  const { slot, workerRequest } = target;
68651
68817
  await new Promise((resolve, reject) => {
@@ -68873,6 +69039,34 @@ class PoolCoordinator {
68873
69039
  return { ok: false, refusal: "StaleAttempt" };
68874
69040
  return { ok: true, ...result.action === undefined ? {} : { action: result.action } };
68875
69041
  }
69042
+ async stageSession(id, command) {
69043
+ const watch = this.watches.get(id);
69044
+ if (watch === undefined || !watch.open)
69045
+ return {
69046
+ ok: false,
69047
+ refusal: this.endedWatches.get(id) === "stage_finished" ? "StageNotLive" : "StaleAttempt"
69048
+ };
69049
+ const { slot, attempt } = watch;
69050
+ if (slot.work?.id !== attempt.workId || slot.work.generation !== attempt.generation)
69051
+ return { ok: false, refusal: "StaleAttempt" };
69052
+ if (this.desiredControl !== undefined && this.desiredControl !== "running" && !runsWhilePaused(command))
69053
+ return { ok: false, refusal: "PoolPausedOrQuitting" };
69054
+ const result = await this.stageCommand(slot, { attempt, op: "session", watch: id, command });
69055
+ if (result === undefined)
69056
+ return { ok: false, refusal: "StaleAttempt" };
69057
+ if (result.error !== undefined)
69058
+ throw new Error(result.error);
69059
+ const refusal = result.refusal;
69060
+ if (refusal === "StageNotLive" || refusal === "PoolPausedOrQuitting")
69061
+ return { ok: false, refusal };
69062
+ if (refusal !== undefined)
69063
+ return { ok: false, refusal: "StaleAttempt" };
69064
+ return {
69065
+ ok: true,
69066
+ ...result.bash === undefined ? {} : { bash: result.bash },
69067
+ ...result.task === undefined ? {} : { task: result.task }
69068
+ };
69069
+ }
68876
69070
  stageCommand(slot, command) {
68877
69071
  if (!this.incarnations.has(slot.worker)) {
68878
69072
  tracePooledWatch("coordinator.command.no-worker", { op: command.op, worker: slot.worker });
@@ -68988,7 +69182,8 @@ class PoolCoordinator {
68988
69182
  root,
68989
69183
  definitions: this.sources,
68990
69184
  admission: options.admission,
68991
- ...this.promptNodes ? { promptNodes: true } : {}
69185
+ ...this.promptNodes ? { promptNodes: true } : {},
69186
+ ...options.stageStartMaxLoadPerCpu === undefined ? {} : { stageStartMaxLoadPerCpu: options.stageStartMaxLoadPerCpu }
68992
69187
  })
68993
69188
  ], {
68994
69189
  stdio: ["ignore", "ignore", "pipe", "ipc"],
@@ -69109,7 +69304,10 @@ class PoolCoordinator {
69109
69304
  if (this.closed)
69110
69305
  return;
69111
69306
  const exhausted = work !== undefined && this.attempts(work) >= MAX_CHILD_ATTEMPTS;
69112
- const settled = work !== undefined && (exhausted ? await this.queue.failExited(this.lease, work) : await this.queue.releaseExited(this.lease, work));
69307
+ const queue = this.queue;
69308
+ const lease = this.lease;
69309
+ const attempt = work;
69310
+ const settled = attempt !== undefined && await this.whileRootLate(queue, () => exhausted ? queue.failExited(lease, attempt) : queue.releaseExited(lease, attempt));
69113
69311
  const superseded = work !== undefined && !settled && await this.superseded(work);
69114
69312
  if (work !== undefined && exhausted && !superseded)
69115
69313
  this.failWork(work, diagnosed(childRetryLimit(work, "lost its worker")));
@@ -69171,21 +69369,40 @@ class PoolCoordinator {
69171
69369
  await this.starting;
69172
69370
  const queue = this.queue;
69173
69371
  const lease = this.lease;
69372
+ const renewAttempt = async () => {
69373
+ const attempt = slot.work;
69374
+ if (attempt === undefined)
69375
+ return;
69376
+ const renewed = await queue.extend(lease, attempt, options.leaseMs);
69377
+ if (slot.work === attempt)
69378
+ slot.work = renewed;
69379
+ };
69380
+ const current = async (action) => {
69381
+ try {
69382
+ return await action();
69383
+ } catch (error) {
69384
+ if (!(error instanceof Error) || error.message !== STALE_WORK_LEASE || slot.work === undefined)
69385
+ throw error;
69386
+ await renewAttempt();
69387
+ return action();
69388
+ }
69389
+ };
69390
+ const leased = (action) => this.whileRootLate(queue, () => current(action), renewAttempt);
69174
69391
  if (message.kind === "control")
69175
- return queue.control(lease);
69392
+ return leased(() => queue.control(lease));
69176
69393
  if (message.kind === "fenced") {
69177
69394
  const fenced = slot.work;
69178
69395
  slot.work = undefined;
69179
69396
  if (fenced && !await this.superseded(fenced)) {
69180
69397
  if (this.attempts(fenced) < MAX_CHILD_ATTEMPTS)
69181
- await queue.releaseExited(lease, fenced);
69182
- else if (await queue.failExited(lease, fenced))
69398
+ await leased(() => queue.releaseExited(lease, fenced));
69399
+ else if (await leased(() => queue.failExited(lease, fenced)))
69183
69400
  this.failWork(fenced, new Error(childRetryLimit(fenced, "was refused by the root as stale on every attempt")));
69184
69401
  }
69185
69402
  return;
69186
69403
  }
69187
69404
  if (message.kind === "controlled") {
69188
- const control = await queue.control(lease);
69405
+ const control = await leased(() => queue.control(lease));
69189
69406
  if (message.revision > control.revision)
69190
69407
  throw new Error("Unknown control revision");
69191
69408
  this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
@@ -69196,21 +69413,21 @@ class PoolCoordinator {
69196
69413
  throw new Error("Worker already holds an attempt");
69197
69414
  if (this.budgetBlocked)
69198
69415
  return;
69199
- await this.recoverExpired();
69200
- slot.runs.clear();
69201
- slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
69202
- definition: `${source.path}#${source.exportName}`,
69203
- digest: source.digest
69204
- })), options.leaseMs);
69416
+ slot.work = await leased(async () => {
69417
+ await this.recoverExpired();
69418
+ slot.runs.clear();
69419
+ return queue.claim(lease, worker, this.sources.map((source) => ({
69420
+ definition: `${source.path}#${source.exportName}`,
69421
+ digest: source.digest
69422
+ })), options.leaseMs);
69423
+ });
69205
69424
  return slot.work;
69206
69425
  }
69207
- let work = slot.work;
69208
- if (!work)
69426
+ const held = slot.work;
69427
+ if (!held)
69209
69428
  throw new Error("Worker has no active attempt");
69210
- if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
69211
- work = await queue.extend(lease, work, options.leaseMs);
69212
- slot.work = work;
69213
- }
69429
+ const work = message.kind === "renew" || held.expiresAt - Date.now() < options.leaseMs / 2 ? await leased(() => queue.extend(lease, held, options.leaseMs)) : held;
69430
+ slot.work = work;
69214
69431
  if (message.kind === "activity") {
69215
69432
  for (const run of message.runs)
69216
69433
  slot.runs.add(run.id);
@@ -69270,24 +69487,24 @@ class PoolCoordinator {
69270
69487
  if (message.kind === "renew")
69271
69488
  return;
69272
69489
  if (message.kind === "commit") {
69273
- const receipt = await queue.commit(lease, work, message.operation, message.mutation);
69490
+ const receipt = await leased(() => queue.commit(lease, work, message.operation, message.mutation));
69274
69491
  if (message.mutation.kind !== "session-append")
69275
69492
  this.noteUsage(await this.usageLedger.total(root));
69276
69493
  return receipt;
69277
69494
  }
69278
69495
  if (message.kind === "usage") {
69279
- this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
69496
+ this.noteUsage(await leased(() => this.usageLedger.report(lease, work, message.reports)));
69280
69497
  return;
69281
69498
  }
69282
69499
  if (message.kind === "budget") {
69283
- await queue.assertCurrent(lease, work);
69500
+ await leased(() => queue.assertCurrent(lease, work));
69284
69501
  if (this.budgetBlocked)
69285
69502
  throw new Error("Root workflow budget exhausted");
69286
- return queue.control(lease);
69503
+ return leased(() => queue.control(lease));
69287
69504
  }
69288
69505
  if (message.kind !== "complete")
69289
69506
  throw new Error("Invalid worker operation");
69290
- await queue.complete(lease, work, message.result);
69507
+ await leased(() => queue.complete(lease, work, message.result));
69291
69508
  this.endStageWatches((watch) => watch.slot === slot, "stage_finished");
69292
69509
  this.replacements = 0;
69293
69510
  slot.work = undefined;
@@ -69326,7 +69543,7 @@ class PoolCoordinator {
69326
69543
  if (this.recovering)
69327
69544
  return this.recovering;
69328
69545
  const root = this.root;
69329
- if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
69546
+ if (this.closed || root === undefined || !this.queue || this.lateRoot !== undefined || performance.now() < this.nextRecoveryAt)
69330
69547
  return Promise.resolve();
69331
69548
  const queue = this.queue;
69332
69549
  const lease = this.lease;
@@ -69335,9 +69552,12 @@ class PoolCoordinator {
69335
69552
  await new Promise((resolve) => setImmediate(resolve));
69336
69553
  await this.admission?.reconcile();
69337
69554
  const expiredBefore = Date.now();
69338
- for (const work of await queue.inspect(root)) {
69339
- if (work.state !== "leased" || work.expiresAt >= expiredBefore)
69340
- continue;
69555
+ const expired = (await queue.inspect(root)).filter((work) => work.state === "leased" && work.expiresAt < expiredBefore);
69556
+ if (expired.length > 0 && await queue.rootLease(lease, COORDINATOR_LEASE_MS) === "late") {
69557
+ this.rootRenewal(queue).catch(() => {});
69558
+ return;
69559
+ }
69560
+ for (const work of expired) {
69341
69561
  if (this.quarantined.has(work.id))
69342
69562
  continue;
69343
69563
  if (this.held.has(work.id)) {
@@ -69453,7 +69673,7 @@ function childRetryLimit(work, reason) {
69453
69673
  const exportName = work.definition.slice(work.definition.lastIndexOf("#") + 1);
69454
69674
  return `Local worker retry limit: child ${work.childRunId} (${exportName}) ${reason} ${MAX_CHILD_ATTEMPTS} times`;
69455
69675
  }
69456
- var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
69676
+ var CONTROL_ACKNOWLEDGEMENT_STALL_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, LATE_ROOT_POLL_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
69457
69677
  var init_pool_coordinator = __esm(() => {
69458
69678
  init_internal();
69459
69679
  init_pool_remote_journal();
@@ -69467,12 +69687,14 @@ var init_pool_coordinator = __esm(() => {
69467
69687
  init_pending_stage_route_capability();
69468
69688
  init_pending_stage_route_readiness();
69469
69689
  init_store();
69690
+ init_workflow_infrastructure_error();
69470
69691
  init_pool_journal();
69471
69692
  init_pooled_watch_trace();
69472
69693
  init_resource_admission();
69473
69694
  init_stage_watch_coalesce();
69474
69695
  init_startup_timeout();
69475
69696
  COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
69697
+ LATE_ROOT_POLL_MS = RECOVERY_INTERVAL_MS;
69476
69698
  STAGE_WATCH_BACKLOG_BYTES = 8 * 1024 * 1024;
69477
69699
  });
69478
69700
 
@@ -69546,7 +69768,8 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
69546
69768
  const boundedDrain = (drain, sessionId) => {
69547
69769
  const waiting = new Promise((resolve, reject) => {
69548
69770
  drain.then(resolve, reject);
69549
- watchStall(drain, sessionId, resolve);
69771
+ if (options.holdUntilDrained !== true)
69772
+ watchStall(drain, sessionId, resolve);
69550
69773
  });
69551
69774
  waiting.catch(() => {});
69552
69775
  return waiting;
@@ -69928,10 +70151,14 @@ class PoolStageHandoff {
69928
70151
  if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
69929
70152
  continue;
69930
70153
  if (run.id === work.childRunId) {
69931
- const execution = run.stages.find((stage) => stage.workerExecution !== undefined)?.workerExecution;
70154
+ const executions = new Map(run.stages.map((stage) => [stage.id, stage.workerExecution]));
70155
+ const attempt = run.stages.flatMap((stage) => stage.workerExecution === undefined ? [] : [stage.workerExecution]).reduce((latest, execution) => latest === undefined || execution.renewedAt > latest.renewedAt ? execution : latest, undefined);
69932
70156
  this.store.recordRunStart({
69933
70157
  ...run,
69934
- stages: result.stages.map((stage) => execution === undefined ? stage : { ...stage, workerExecution: execution }),
70158
+ stages: result.stages.map((stage) => {
70159
+ const execution = executions.get(stage.id) ?? attempt;
70160
+ return execution === undefined ? stage : { ...stage, workerExecution: execution };
70161
+ }),
69935
70162
  toolNodes: result.toolNodes
69936
70163
  });
69937
70164
  this.store.recordRunEnd(run.id, result.status, result.result, result.error);
@@ -69985,7 +70212,7 @@ class PoolStageHandoff {
69985
70212
  pendingStageMessages: [
69986
70213
  ...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
69987
70214
  ],
69988
- stages: withReconnectingStages(known.get(run.id), run, run.stages.map((stage) => ({ ...stage, workerExecution: execution })), work.generation)
70215
+ stages: withReconnectingStages(known.get(run.id), run, run.stages.map((stage) => ({ ...stage, workerExecution: endedUnder(known.get(run.id), stage, execution) })), work.generation)
69989
70216
  }));
69990
70217
  const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
69991
70218
  for (const run of published)
@@ -70116,7 +70343,7 @@ class PoolStageHandoff {
70116
70343
  throw new Error(STALE);
70117
70344
  if (attempt.sessions.get(key) !== sessionId)
70118
70345
  throw new Error(`Stage session ${sessionId} was replaced before it acknowledged delivery`);
70119
- }, action));
70346
+ }, action), { holdUntilDrained: true });
70120
70347
  attempt.deliveries.set(key, delivery);
70121
70348
  }
70122
70349
  return delivery;
@@ -70128,6 +70355,12 @@ function outside2(runId) {
70128
70355
  function isLive(stage) {
70129
70356
  return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
70130
70357
  }
70358
+ function endedUnder(known, stage, execution) {
70359
+ if (!isTerminalStageStatus(stage.status))
70360
+ return execution;
70361
+ const previous = known?.stages.find((candidate) => candidate.id === stage.id);
70362
+ return previous !== undefined && isTerminalStageStatus(previous.status) && previous.workerExecution?.generation === execution.generation && previous.workerExecution.worker === execution.worker ? previous.workerExecution : execution;
70363
+ }
70131
70364
  function withReconnectingStages(previous, run, stages, generation) {
70132
70365
  if (previous === undefined || run.endedAt !== undefined || isTerminalRunStatus(run.status))
70133
70366
  return stages;
@@ -70216,6 +70449,116 @@ class StageToolExecutionBuffer {
70216
70449
  }
70217
70450
  }
70218
70451
 
70452
+ // dist/builtin/workflows/src/workers/pooled-stage-task-source.ts
70453
+ function failure2(error) {
70454
+ return { ok: false, error: { message: typeof error === "string" ? error : error.message } };
70455
+ }
70456
+
70457
+ class PooledStageTaskSource {
70458
+ run;
70459
+ state = EMPTY;
70460
+ records = [];
70461
+ backgroundRecords = [];
70462
+ settlements = new Map;
70463
+ activity = new Map;
70464
+ statuses = new Map;
70465
+ stdin = new Set;
70466
+ listeners = new Set;
70467
+ constructor(run) {
70468
+ this.run = run;
70469
+ }
70470
+ apply(state) {
70471
+ this.state = state;
70472
+ this.records = [...state.tasks];
70473
+ const background = new Set(state.background);
70474
+ this.backgroundRecords = this.records.filter((task) => background.has(task.ref.taskId));
70475
+ this.settlements = new Map(state.settlement.map(([id, sequence]) => [id, BigInt(sequence)]));
70476
+ this.activity = new Map(state.activity.map((entry) => [entry.id, entry]));
70477
+ this.statuses = new Map(state.statuses.map((entry) => [entry.id, new Map(entry.entries)]));
70478
+ this.stdin = new Set(state.stdin);
70479
+ for (const listener of [...this.listeners])
70480
+ listener();
70481
+ }
70482
+ get empty() {
70483
+ return this.state.tasks.length === 0;
70484
+ }
70485
+ get tasks() {
70486
+ return this.records;
70487
+ }
70488
+ get backgroundTasks() {
70489
+ return this.backgroundRecords;
70490
+ }
70491
+ get settlementOrder() {
70492
+ return this.settlements;
70493
+ }
70494
+ recentActivity(id) {
70495
+ return [...this.activity.get(id)?.entries ?? []];
70496
+ }
70497
+ activityOmitted(id) {
70498
+ return this.activity.get(id)?.omitted === true;
70499
+ }
70500
+ subscribe(listener) {
70501
+ this.listeners.add(listener);
70502
+ return () => {
70503
+ this.listeners.delete(listener);
70504
+ };
70505
+ }
70506
+ async foregroundTask(id) {
70507
+ const reply = await this.request(id, { kind: "foreground" });
70508
+ return reply.ok ? reply.value.kind === "foreground" ? reply.value.result : unexpected() : reply;
70509
+ }
70510
+ async cancelTask(id) {
70511
+ const reply = await this.request(id, { kind: "cancel" });
70512
+ if (!reply.ok)
70513
+ return reply;
70514
+ return reply.value.kind === "cancel" ? reply.value.result ?? { ok: true, value: undefined } : unexpected();
70515
+ }
70516
+ stdinAvailable(id) {
70517
+ return this.stdin.has(id);
70518
+ }
70519
+ async writeTaskInput(id, text) {
70520
+ const reply = await this.request(id, { kind: "input", text });
70521
+ return reply.ok ? reply.value.kind === "input" ? reply.value.result : unexpected() : reply;
70522
+ }
70523
+ async readTaskOutput(id, range) {
70524
+ const reply = await this.request(id, { kind: "output", start: range.start, maximumBytes: range.maximumBytes });
70525
+ return reply.ok ? reply.value.kind === "output" ? reply.value.result : unexpected() : reply;
70526
+ }
70527
+ async taskDetailText(id) {
70528
+ const reply = await this.request(id, { kind: "detail" });
70529
+ return reply.ok && reply.value.kind === "detail" ? reply.value.detail : {};
70530
+ }
70531
+ async readTaskTranscript(id, cursor) {
70532
+ const reply = await this.request(id, { kind: "transcript", ...cursor === undefined ? {} : { cursor } });
70533
+ return reply.ok ? reply.value.kind === "transcript" ? reply.value.result : unexpected() : reply;
70534
+ }
70535
+ followTaskTranscript(id) {
70536
+ return this.records.some((task) => task.ref.taskId === id && task.kind === "agent") ? { historyOnly: true } : undefined;
70537
+ }
70538
+ extensionStatuses(id) {
70539
+ return this.statuses.get(id) ?? NO_STATUSES;
70540
+ }
70541
+ watchExtensionStatuses(_id, listener) {
70542
+ return this.subscribe(listener);
70543
+ }
70544
+ async request(id, action) {
70545
+ try {
70546
+ const reply = await this.run(id, action);
70547
+ return reply === undefined ? failure2("Task unavailable") : { ok: true, value: reply };
70548
+ } catch (error) {
70549
+ return failure2(error instanceof Error ? error : String(error));
70550
+ }
70551
+ }
70552
+ }
70553
+ function unexpected() {
70554
+ return failure2("Unexpected reply from the local worker");
70555
+ }
70556
+ var EMPTY, NO_STATUSES;
70557
+ var init_pooled_stage_task_source = __esm(() => {
70558
+ EMPTY = { tasks: [], background: [], settlement: [], activity: [], stdin: [], statuses: [] };
70559
+ NO_STATUSES = new Map;
70560
+ });
70561
+
70219
70562
  // dist/builtin/workflows/src/workers/pooled-stage-handle.ts
70220
70563
  import { randomUUID as randomUUID18 } from "node:crypto";
70221
70564
 
@@ -70246,6 +70589,11 @@ class PooledStageHandle {
70246
70589
  undelivered;
70247
70590
  disposed = false;
70248
70591
  session;
70592
+ commands;
70593
+ sessionStarted = true;
70594
+ started = Promise.withResolvers();
70595
+ bashRuns = new Map;
70596
+ taskView = new PooledStageTaskSource(async (taskId, action) => (await this.sessionCommand({ kind: "task", taskId, action })).task);
70249
70597
  pumping;
70250
70598
  pumpAgain = false;
70251
70599
  watchGone = Promise.withResolvers();
@@ -70308,6 +70656,43 @@ class PooledStageHandle {
70308
70656
  streamingMessage() {
70309
70657
  return this.streaming;
70310
70658
  }
70659
+ commandCatalog() {
70660
+ return this.commands;
70661
+ }
70662
+ taskSource() {
70663
+ return this.taskView;
70664
+ }
70665
+ get isSessionStarting() {
70666
+ return !this.sessionStarted && this.ended === undefined;
70667
+ }
70668
+ get isBashRunning() {
70669
+ return this.bashRuns.size > 0;
70670
+ }
70671
+ async executeBash(command, onChunk, options) {
70672
+ const id = randomUUID18();
70673
+ this.bashRuns.set(id, onChunk);
70674
+ try {
70675
+ const result = await this.sessionCommand({
70676
+ kind: "bash",
70677
+ id,
70678
+ command,
70679
+ ...options?.excludeFromContext === undefined ? {} : { excludeFromContext: options.excludeFromContext }
70680
+ });
70681
+ return result.bash ?? { output: "", exitCode: undefined, cancelled: true, truncated: false };
70682
+ } finally {
70683
+ this.bashRuns.delete(id);
70684
+ }
70685
+ }
70686
+ abortBash() {
70687
+ for (const id of this.bashRuns.keys())
70688
+ this.sessionCommand({ kind: "abort-bash", id }).catch(() => {});
70689
+ }
70690
+ async compact() {
70691
+ await this.sessionCommand({ kind: "compact" });
70692
+ }
70693
+ abortCompaction() {
70694
+ this.sessionCommand({ kind: "abort-compaction" }).catch(() => {});
70695
+ }
70311
70696
  pendingToolExecutionEvents() {
70312
70697
  return this.toolExecutions.replayEvents();
70313
70698
  }
@@ -70455,6 +70840,17 @@ class PooledStageHandle {
70455
70840
  if (this.disposed)
70456
70841
  throw new Error(`Stage chat "${this.stageName}" is closed.`);
70457
70842
  }
70843
+ async sessionCommand(command) {
70844
+ this.throwIfClosed();
70845
+ if (this.ended !== undefined)
70846
+ throw new PooledStageDeliveryError("StageNotLive", this.stageName);
70847
+ if (this.transport.session === undefined)
70848
+ throw new Error(`Stage "${this.stageName}" runs on a local worker that cannot run this command.`);
70849
+ const result = await this.transport.session(this.watchId, command);
70850
+ if (!result.ok)
70851
+ throw new PooledStageDeliveryError(result.refusal, this.stageName);
70852
+ return result;
70853
+ }
70458
70854
  undeliveredCount() {
70459
70855
  if (this.undelivered !== undefined)
70460
70856
  return this.undelivered;
@@ -70520,6 +70916,16 @@ class PooledStageHandle {
70520
70916
  entry.waiter = undefined;
70521
70917
  }
70522
70918
  }
70919
+ markSessionStarted(started) {
70920
+ if (started) {
70921
+ this.sessionStarted = true;
70922
+ this.started.resolve();
70923
+ return;
70924
+ }
70925
+ if (this.sessionStarted)
70926
+ this.started = Promise.withResolvers();
70927
+ this.sessionStarted = false;
70928
+ }
70523
70929
  forget(op) {
70524
70930
  const index = this.outbound.findIndex((entry) => entry.op === op);
70525
70931
  if (index >= 0)
@@ -70576,10 +70982,17 @@ class PooledStageHandle {
70576
70982
  const gone = this.watchGone.promise.then(() => ({ ok: false, refusal: "StaleAttempt" }));
70577
70983
  const sending = this.sendEntry(entry, watch);
70578
70984
  let bound;
70985
+ let stopped = false;
70579
70986
  const unconfirmed = new Promise((resolve) => {
70580
- bound = setTimeout(() => resolve("unconfirmed"), this.admissionWaitMs);
70987
+ this.started.promise.then(() => {
70988
+ if (!stopped)
70989
+ bound = setTimeout(() => resolve("unconfirmed"), this.admissionWaitMs);
70990
+ });
70991
+ });
70992
+ const outcome = await Promise.race([sending, gone, unconfirmed]).finally(() => {
70993
+ stopped = true;
70994
+ clearTimeout(bound);
70581
70995
  });
70582
- const outcome = await Promise.race([sending, gone, unconfirmed]).finally(() => clearTimeout(bound));
70583
70996
  if (outcome === "unconfirmed") {
70584
70997
  tracePooledWatch("root.outbox.unconfirmed", { op: entry.op, watch, generation });
70585
70998
  entry.unconfirmedOn = generation;
@@ -70669,6 +71082,10 @@ class PooledStageHandle {
70669
71082
  }
70670
71083
  adopt(snapshot) {
70671
71084
  this.session = snapshot.sessionId;
71085
+ this.commands = snapshot.commands;
71086
+ if (!this.taskView.empty)
71087
+ this.taskView.apply(NO_TASKS);
71088
+ this.markSessionStarted(snapshot.sessionStarted !== false);
70672
71089
  this.settled = [...snapshot.messages];
70673
71090
  this.streaming = snapshot.streamingMessage;
70674
71091
  this.currentStatus = snapshot.status;
@@ -70689,6 +71106,25 @@ class PooledStageHandle {
70689
71106
  this.forget(event.op);
70690
71107
  return false;
70691
71108
  }
71109
+ if (event.type === STAGE_SESSION_STARTED_EVENT) {
71110
+ this.commands = event.commands;
71111
+ this.markSessionStarted(true);
71112
+ const queued = this.queuedUserMessages();
71113
+ const redraw = { type: "queue_update", steering: [...queued.steering], followUp: [...queued.followUp] };
71114
+ for (const listener of [...this.listeners])
71115
+ listener(redraw);
71116
+ return false;
71117
+ }
71118
+ if (event.type === STAGE_TASKS_EVENT) {
71119
+ this.taskView.apply(event.tasks);
71120
+ return false;
71121
+ }
71122
+ if (event.type === STAGE_MESSAGES_EVENT) {
71123
+ this.settled = [...event.messages];
71124
+ return false;
71125
+ }
71126
+ if (event.type === "bash_execution_update" && event.id !== undefined)
71127
+ this.bashRuns.get(event.id)?.(event.delta);
70692
71128
  this.applyEvent(event);
70693
71129
  const turnUnderway = TURN_EVENTS.has(event.type) || (event.type === "message_start" || event.type === "message_update") && event.message.role === "assistant";
70694
71130
  if (turnUnderway && this.awaitingTurn) {
@@ -70771,10 +71207,12 @@ async function reconnectPooledStageHandle(handle, transport, wanted = () => true
70771
71207
  await rebinding;
70772
71208
  return !handle.isReconnecting;
70773
71209
  }
70774
- var DELIVERY_RECEIPT_EVENT = "atomic_pooled_delivery_receipt", POOLED_ADMISSION_WAIT_MS = 15000, RECONNECTABLE, TURN_EVENTS, PooledStageDeliveryError;
71210
+ var DELIVERY_RECEIPT_EVENT = "atomic_pooled_delivery_receipt", STAGE_SESSION_STARTED_EVENT = "atomic_pooled_stage_session_started", STAGE_MESSAGES_EVENT = "atomic_pooled_stage_messages", STAGE_TASKS_EVENT = "atomic_pooled_stage_tasks", NO_TASKS, POOLED_ADMISSION_WAIT_MS = 15000, RECONNECTABLE, TURN_EVENTS, PooledStageDeliveryError;
70775
71211
  var init_pooled_stage_handle = __esm(() => {
70776
71212
  init_stage_queued_user_messages();
71213
+ init_pooled_stage_task_source();
70777
71214
  init_pooled_watch_trace();
71215
+ NO_TASKS = { tasks: [], background: [], settlement: [], activity: [], stdin: [], statuses: [] };
70778
71216
  RECONNECTABLE = new Set(["attempt_fenced", "worker_exited", "resync"]);
70779
71217
  TURN_EVENTS = new Set(["agent_start", "turn_start", "tool_execution_start", "tool_execution_update"]);
70780
71218
  PooledStageDeliveryError = class PooledStageDeliveryError extends Error {
@@ -70834,6 +71272,9 @@ class RegisteredPooledStageHandle {
70834
71272
  pendingToolExecutionEvents() {
70835
71273
  return this.current()?.pendingToolExecutionEvents?.() ?? [];
70836
71274
  }
71275
+ commandCatalog() {
71276
+ return this.current()?.commandCatalog?.();
71277
+ }
70837
71278
  update(stage) {
70838
71279
  this.stage = stage;
70839
71280
  }
@@ -71132,6 +71573,7 @@ class LocalWorkerPool {
71132
71573
  rootSignature;
71133
71574
  completing = 0;
71134
71575
  rootFlush;
71576
+ connecting;
71135
71577
  starting;
71136
71578
  failure;
71137
71579
  closed = false;
@@ -71161,6 +71603,8 @@ class LocalWorkerPool {
71161
71603
  throw new Error("Invalid local worker startup timeout");
71162
71604
  if (options.leaseMs !== undefined && (!Number.isSafeInteger(options.leaseMs) || options.leaseMs < MINIMUM_WORK_LEASE_MS || options.leaseMs > MAX_TIMER_DELAY_MS))
71163
71605
  throw new Error(`Invalid local worker lease: must be an integer from ${MINIMUM_WORK_LEASE_MS} to ${MAX_TIMER_DELAY_MS} ms`);
71606
+ if (options.maxLoadPerCpu !== undefined && (!Number.isFinite(options.maxLoadPerCpu) || options.maxLoadPerCpu < 0))
71607
+ throw new Error("Invalid local worker load threshold: maxLoadPerCpu must be a number of at least 0");
71164
71608
  this.leaseMs = options.leaseMs ?? DEFAULT_WORK_LEASE_MS;
71165
71609
  this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
71166
71610
  if (options.admission && options.rootId === undefined)
@@ -71188,23 +71632,35 @@ class LocalWorkerPool {
71188
71632
  if (!state || state.holderId !== this.backend.holderId)
71189
71633
  throw new Error("Local worker pool requires the root owner");
71190
71634
  this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
71191
- const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
71192
- this.usageTotal = await this.command({
71193
- kind: "prepare",
71194
- root,
71195
- lease: this.lease,
71196
- options: {
71197
- journalPath: this.options.journalPath,
71198
- ...this.options.durable === undefined ? {} : { durable: this.options.durable },
71199
- workers: this.options.workers,
71200
- leaseMs: this.leaseMs,
71201
- startupTimeoutMs: this.options.startupTimeoutMs ?? defaultStartupTimeoutMs(this.options.workers),
71202
- ...this.options.admission === undefined ? {} : { admission: this.options.admission },
71203
- ...workerEntry === undefined ? {} : { workerEntry }
71204
- },
71205
- ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
71206
- });
71207
71635
  this.queue ??= await openSqlPoolQueue(poolJournal, this.options.durable === undefined ? undefined : new PoolRemoteJournal(this.backend.journal, sqlPoolStorage(poolJournal)));
71636
+ if (await this.queue.initialized(root))
71637
+ await this.connectPool();
71638
+ }
71639
+ connectPool() {
71640
+ this.connecting ??= (async () => {
71641
+ const root = this.root;
71642
+ const lease = this.lease;
71643
+ if (root === undefined || lease === undefined)
71644
+ throw new Error("Local worker pool requires root preparation");
71645
+ const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
71646
+ this.usageTotal = await this.command({
71647
+ kind: "prepare",
71648
+ root,
71649
+ lease,
71650
+ options: {
71651
+ journalPath: this.options.journalPath,
71652
+ ...this.options.durable === undefined ? {} : { durable: this.options.durable },
71653
+ workers: this.options.workers,
71654
+ leaseMs: this.leaseMs,
71655
+ startupTimeoutMs: this.options.startupTimeoutMs ?? defaultStartupTimeoutMs(this.options.workers),
71656
+ ...this.options.admission === undefined ? {} : { admission: this.options.admission },
71657
+ ...this.options.maxLoadPerCpu === undefined ? {} : { stageStartMaxLoadPerCpu: this.options.maxLoadPerCpu },
71658
+ ...workerEntry === undefined ? {} : { workerEntry }
71659
+ },
71660
+ ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
71661
+ });
71662
+ })();
71663
+ return this.connecting;
71208
71664
  }
71209
71665
  async poolJournal() {
71210
71666
  const journal = this.backend.journal;
@@ -71253,6 +71709,7 @@ class LocalWorkerPool {
71253
71709
  async start(root, promptNodes) {
71254
71710
  if (this.closed)
71255
71711
  throw new Error("Local worker pool is closed");
71712
+ await this.connectPool();
71256
71713
  if (!this.queue || !this.lease || this.lease.runId !== root)
71257
71714
  throw new Error("Local worker pool requires root preparation");
71258
71715
  this.sources = await Promise.all(this.options.definitions.map(async (binding) => {
@@ -71535,6 +71992,11 @@ class LocalWorkerPool {
71535
71992
  ...message === undefined ? {} : { message }
71536
71993
  });
71537
71994
  }
71995
+ async sessionCommandOnPooledStage(watch, command) {
71996
+ if (this.closed || this.coordinatorLost !== undefined)
71997
+ return { ok: false, refusal: "StaleAttempt" };
71998
+ return await this.command({ kind: "stage-session", watch, command });
71999
+ }
71538
72000
  stageTransport(onEnded) {
71539
72001
  return {
71540
72002
  watch: (ref, listener) => this.watchPooledStage(ref, (message) => {
@@ -71545,7 +72007,8 @@ class LocalWorkerPool {
71545
72007
  unwatch: (watch) => this.unwatchPooledStage(watch),
71546
72008
  deliver: (delivery) => this.deliverToPooledStage(delivery),
71547
72009
  pause: (watch) => this.pausePooledStage(watch),
71548
- resume: (watch, message) => this.resumePooledStage(watch, message)
72010
+ resume: (watch, message) => this.resumePooledStage(watch, message),
72011
+ session: (watch, command) => this.sessionCommandOnPooledStage(watch, command)
71549
72012
  };
71550
72013
  }
71551
72014
  async acquirePooledStage(runId, stageId) {
@@ -72208,6 +72671,9 @@ var init_local_pool = __esm(() => {
72208
72671
  completeInTurns = createTurnSlicer(2, COMPLETION_TURN_BUDGET_MS);
72209
72672
  });
72210
72673
 
72674
+ // dist/builtin/workflows/src/workers/public-options.ts
72675
+ var DEFAULT_POOLED_STAGE_START_MAX_LOAD_PER_CPU = 3;
72676
+
72211
72677
  // dist/builtin/workflows/src/workers/public-pool.ts
72212
72678
  import { resolve as resolve16 } from "node:path";
72213
72679
  import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
@@ -72268,6 +72734,7 @@ async function activatePublicPool(backend, options) {
72268
72734
  ...options.localWorkers,
72269
72735
  startupTimeoutMs: undefined,
72270
72736
  leaseMs: undefined,
72737
+ maxLoadPerCpu: undefined,
72271
72738
  definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve16(cwd, source.path) }))
72272
72739
  };
72273
72740
  const admission = options.admission && {
@@ -72311,6 +72778,7 @@ async function activatePublicPool(backend, options) {
72311
72778
  admission: resources.admission ?? {},
72312
72779
  startupTimeoutMs: options.localWorkers?.startupTimeoutMs,
72313
72780
  leaseMs: options.localWorkers?.leaseMs,
72781
+ maxLoadPerCpu: options.localWorkers?.maxLoadPerCpu ?? DEFAULT_POOLED_STAGE_START_MAX_LOAD_PER_CPU,
72314
72782
  ...options.stageControlRegistry === undefined || options[hostWorkerBindings] ? {} : { stageControlRegistry: options.stageControlRegistry }
72315
72783
  });
72316
72784
  try {
@@ -77554,7 +78022,7 @@ class StageSessionController {
77554
78022
  return this.modelRoute ? this.checkedSession(await ready) : ready;
77555
78023
  }
77556
78024
  async ensureSessionFromId(sessionId, consumer = "prompt") {
77557
- const joinsLiveGeneration = !this.sessionShutdownPromise && (this.sessionPromise !== undefined || this.session !== undefined);
78025
+ const joinsLiveGeneration = !this.sessionShutdownPromise && (this.sessionPromise !== undefined || this.session !== undefined || this.activeCreation !== undefined || this.ownedCreationPromise !== undefined);
77558
78026
  if (!this.sessionShutdownPromise && !joinsLiveGeneration)
77559
78027
  this.reattachSessionId = sessionId;
77560
78028
  const session = await this.ensureSession(consumer);
@@ -84037,6 +84505,44 @@ function effectiveWidth(width) {
84037
84505
  // dist/builtin/workflows/src/tui/run-detail.ts
84038
84506
  init_pending_stage_status();
84039
84507
 
84508
+ // dist/builtin/workflows/src/shared/stage-display-status.ts
84509
+ var STARTUP_STEP_TEXT = {
84510
+ "model-resolution": "resolving model",
84511
+ "route-authority": "waiting for model route",
84512
+ "resource-preparation": "loading resources",
84513
+ "reload-queued": "waiting to reload resources",
84514
+ "reload-active": "reloading resources",
84515
+ "sdk-creation": "creating session",
84516
+ "extension-binding": "binding extensions",
84517
+ "session-attachment": "attaching session",
84518
+ "delivery-readiness": "preparing message delivery",
84519
+ ready: "session ready",
84520
+ "first-dispatch": "session ready"
84521
+ };
84522
+ function isPooledStageStarting(stage) {
84523
+ if (stage === undefined || stage.workerExecution === undefined || stage.status !== "running")
84524
+ return false;
84525
+ if (stage.nodeKind === "tool" || stage.workflowChild !== undefined || stage.workflowChildRun !== undefined)
84526
+ return false;
84527
+ const startup = stage.startup;
84528
+ return startup !== undefined && startup.state === "active" && startup.phase !== "ready";
84529
+ }
84530
+ function stageDisplayStatus(stage) {
84531
+ return isPooledStageStarting(stage) ? "queued" : stage.status;
84532
+ }
84533
+ function pooledStageStartingStep(stage) {
84534
+ const startup = stage?.startup;
84535
+ if (startup === undefined || !isPooledStageStarting(stage))
84536
+ return;
84537
+ return STARTUP_STEP_TEXT[startup.phase];
84538
+ }
84539
+ function pooledStageStartingText(stage, now = Date.now()) {
84540
+ const step = pooledStageStartingStep(stage);
84541
+ if (step === undefined || stage?.startup === undefined)
84542
+ return;
84543
+ const seconds = Math.max(0, Math.floor((now - stage.startup.startedAt) / 1000));
84544
+ return `${step} ${seconds}s`;
84545
+ }
84040
84546
  // dist/builtin/workflows/src/tui/status-helpers.ts
84041
84547
  function statusColor(status, theme) {
84042
84548
  switch (status) {
@@ -84066,6 +84572,8 @@ function statusIcon(status) {
84066
84572
  switch (status) {
84067
84573
  case "pending":
84068
84574
  return "○";
84575
+ case "queued":
84576
+ return "○";
84069
84577
  case "awaiting_input":
84070
84578
  return "?";
84071
84579
  case "blocked":
@@ -84257,12 +84765,13 @@ function artifactRowsFor(detail) {
84257
84765
  return rows;
84258
84766
  }
84259
84767
  function stageLinePlain(stage, now, width) {
84260
- const icon = statusIcon(stage.status);
84768
+ const status = stageDisplayStatus(stage);
84769
+ const icon = statusIcon(status);
84261
84770
  const dur = stageDurationString(stage, now);
84262
- const activity = stageActivityString(stage, now);
84771
+ const activity = pooledStageStartingStep(stage) ?? stageActivityString(stage, now);
84263
84772
  const name = truncateToWidth(`${icon} ${stage.name}`, STAGE_NAME_COL + 2, "…");
84264
84773
  const activityText = activity ? truncateToWidth(activity, 16, "…") : undefined;
84265
- const parts = [pad(name, STAGE_NAME_COL + 2), pad(stage.status, 10)];
84774
+ const parts = [pad(name, STAGE_NAME_COL + 2), pad(status, 10)];
84266
84775
  if (activityText)
84267
84776
  parts.push(pad(activityText, 16));
84268
84777
  if (dur)
@@ -84270,17 +84779,18 @@ function stageLinePlain(stage, now, width) {
84270
84779
  return truncateToWidth(parts.join(""), width, "…");
84271
84780
  }
84272
84781
  function stageLineThemed(stage, now, theme, width) {
84273
- const icon = statusIcon(stage.status);
84274
- const iconFg = hexToAnsi(statusColor(stage.status, theme));
84782
+ const status = stageDisplayStatus(stage);
84783
+ const icon = statusIcon(status);
84784
+ const iconFg = hexToAnsi(statusColor(status, theme));
84275
84785
  const text = hexToAnsi(theme.text);
84276
84786
  const muted = hexToAnsi(theme.textMuted);
84277
84787
  const dim = hexToAnsi(theme.dim);
84278
- const stateFg = hexToAnsi(statusColor(stage.status, theme));
84279
- const activity = stageActivityString(stage, now);
84788
+ const stateFg = hexToAnsi(statusColor(status, theme));
84789
+ const activity = pooledStageStartingStep(stage) ?? stageActivityString(stage, now);
84280
84790
  const dur = stageDurationString(stage, now);
84281
84791
  const nameText = truncateToWidth(stage.name, STAGE_NAME_COL, "…");
84282
84792
  const namePad = pad(nameText, STAGE_NAME_COL);
84283
- const statePad = pad(stage.status, 10);
84793
+ const statePad = pad(status, 10);
84284
84794
  const activityText = activity ? truncateToWidth(activity, 16, "…") : undefined;
84285
84795
  const activitySeg = activityText ? `${muted}${pad(activityText, 16)}${RESET}` : " ".repeat(16);
84286
84796
  const durSeg = dur ? `${dim}${dur}${RESET}` : "";
@@ -85471,7 +85981,7 @@ function renderHeader(run, opts) {
85471
85981
  skipped: 0
85472
85982
  };
85473
85983
  for (const s of run.stages)
85474
- counts[s.status]++;
85984
+ counts[stageDisplayStatus(s) === "queued" ? "pending" : s.status]++;
85475
85985
  const badges = [];
85476
85986
  if (counts.completed > 0)
85477
85987
  badges.push({ text: `✓ ${counts.completed}`, fg: theme.success });
@@ -87698,7 +88208,8 @@ function renderNodeCard(stage, opts) {
87698
88208
  const focused = opts.focused ?? false;
87699
88209
  const phase = opts.pulsePhase ?? 0;
87700
88210
  const theme = opts.theme;
87701
- const borderHex = pickBorder(stage.status, focused, phase, theme);
88211
+ const status = stageDisplayStatus(stage);
88212
+ const borderHex = pickBorder(status, focused, phase, theme);
87702
88213
  const bc = hexToAnsi(borderHex);
87703
88214
  const bg = DEFAULT_BG;
87704
88215
  const innerWidth = Math.max(2, width - 2);
@@ -87709,12 +88220,12 @@ function renderNodeCard(stage, opts) {
87709
88220
  const topMiddle = `${bc}${"─".repeat(titleStart)}` + `${titleSlot}${bc}` + `${"─".repeat(Math.max(0, innerWidth - titleEnd))}`;
87710
88221
  const top = `${bg}${bc}╭${topMiddle}╮${RESET}`;
87711
88222
  const bottom = `${bg}${bc}╰${"─".repeat(innerWidth)}╯${RESET}`;
87712
- const bodyText = stage.nodeKind === "tool" ? "durable tool" : stage.status === "blocked" ? blockedBadgeText(stage, opts.stages, innerWidth) : durationText(stage);
87713
- const bodyHex = durationColor(stage.status, theme);
87714
- const statusText = `${statusIcon(stage.status)} ${stage.toolStatus ?? statusLabel(stage.status)}`;
88223
+ const bodyText = stage.nodeKind === "tool" ? "durable tool" : stage.status === "blocked" ? blockedBadgeText(stage, opts.stages, innerWidth) : pooledStageStartingText(stage) ?? durationText(stage);
88224
+ const bodyHex = durationColor(status, theme);
88225
+ const statusText = `${statusIcon(status)} ${stage.toolStatus ?? statusLabel(status)}`;
87715
88226
  const queuedCount = queuedBadgeCount(opts.queuedMessageCount);
87716
88227
  const statusLine = `${bg}${bc}│${RESET}` + centreColored(statusText, innerWidth, bodyHex, bg, {
87717
- bold: stage.status === "running" || stage.status === "awaiting_input"
88228
+ bold: status === "running" || status === "awaiting_input"
87718
88229
  }) + `${bg}${bc}│${RESET}`;
87719
88230
  const durLine = `${bg}${bc}│${RESET}` + centreColored(bodyText, innerWidth, bodyHex, bg, {
87720
88231
  bold: stage.status === "blocked"
@@ -88796,8 +89307,9 @@ function renderSwitcher(stages, state, opts) {
88796
89307
  const stage = visible[i];
88797
89308
  const idx = start + i;
88798
89309
  const isSelected = idx === state.selectedIndex;
88799
- const icon = statusIcon(stage.status);
88800
- const label = statusText(stage.status);
89310
+ const displayStatus = stageDisplayStatus(stage);
89311
+ const icon = statusIcon(displayStatus);
89312
+ const label = statusText(displayStatus);
88801
89313
  const rowTextBudget = Math.max(8, innerWidth - 14);
88802
89314
  const name = truncateToWidth(stage.name, rowTextBudget, "…");
88803
89315
  if (isSelected) {
@@ -88810,7 +89322,7 @@ function renderSwitcher(stages, state, opts) {
88810
89322
  const styled = `${accent}${accentFg}${BOLD}${padded}${RESET}`;
88811
89323
  lines.push(`${border}│${RESET}${styled}${border}│${RESET}`);
88812
89324
  } else {
88813
- const iconColor = hexToAnsi(statusColor(stage.status, theme));
89325
+ const iconColor = hexToAnsi(statusColor(displayStatus, theme));
88814
89326
  const status = `${dim}${label}${RESET}${panelBg}`;
88815
89327
  const visibleRow = ` ${icon} ${name}`;
88816
89328
  const gap = Math.max(1, innerWidth - visibleWidth(visibleRow) - visibleWidth(label) - 1);
@@ -90850,12 +91362,13 @@ function editorRuleColor(ctx, disabled, agentSession, state) {
90850
91362
  }
90851
91363
 
90852
91364
  // dist/builtin/workflows/src/tui/stage-chat-view-state.ts
90853
- init_stage_ui_broker();
90854
91365
  import {
90855
91366
  ChatSessionHost,
90856
91367
  classifyChatCommand
90857
91368
  } from "@bastani/atomic";
90858
91369
  import { Editor } from "@earendil-works/pi-tui";
91370
+ init_stage_ui_broker();
91371
+ init_store_internal();
90859
91372
 
90860
91373
  // dist/builtin/workflows/src/tui/prompt-card-input.ts
90861
91374
  function handlePromptCardInput(data, state, keybindings) {
@@ -91059,6 +91572,15 @@ function stageSkillAutocomplete(ctx) {
91059
91572
  const handle = liveHandle(ctx);
91060
91573
  if (!handle || isBlocked(ctx))
91061
91574
  throw new Error("This stage chat is not editable.");
91575
+ if (handle.agentSession === undefined && handle.commandCatalog !== undefined) {
91576
+ assertComposerAdmission(ctx, handle);
91577
+ const catalog = handle.commandCatalog();
91578
+ if (catalog !== undefined)
91579
+ return catalog;
91580
+ if (handle.isSessionStarting === true)
91581
+ return { loading: `Loading skills: stage "${handle.stageName}" is still starting on its local worker.` };
91582
+ return;
91583
+ }
91062
91584
  if (handle.agentSession === undefined) {
91063
91585
  let attachment = pendingSkillAttachments.get(handle);
91064
91586
  if (!attachment) {
@@ -91885,6 +92407,10 @@ function createChatHost(ctx, opts) {
91885
92407
  throw new Error("no live handle on this stage");
91886
92408
  await handle.ensureAttached();
91887
92409
  const agentSession = handle.agentSession;
92410
+ if (!agentSession && handle.executeBash !== undefined)
92411
+ return handle.executeBash(request.command, request.onChunk, {
92412
+ excludeFromContext: request.excludeFromContext
92413
+ });
91888
92414
  if (!agentSession)
91889
92415
  throw new Error("no live agent session on this stage");
91890
92416
  return agentSession.executeBash(request.command, request.onChunk, {
@@ -91892,16 +92418,35 @@ function createChatHost(ctx, opts) {
91892
92418
  });
91893
92419
  },
91894
92420
  abortBash: async () => {
91895
- liveHandle(ctx)?.agentSession?.abortBash();
92421
+ const handle = liveHandle(ctx);
92422
+ if (handle?.agentSession)
92423
+ handle.agentSession.abortBash();
92424
+ else
92425
+ handle?.abortBash?.();
91896
92426
  },
91897
92427
  abortCompaction: async () => {
91898
- liveHandle(ctx)?.agentSession?.abortCompaction();
92428
+ const handle = liveHandle(ctx);
92429
+ if (handle?.agentSession)
92430
+ handle.agentSession.abortCompaction();
92431
+ else
92432
+ handle?.abortCompaction?.();
91899
92433
  },
91900
92434
  handleSlashCommand: async (text) => handleSlashCommand(ctx, text)
91901
92435
  },
91902
- isBashRunning: () => liveHandle(ctx)?.agentSession?.isBashRunning === true,
92436
+ isBashRunning: () => {
92437
+ const handle = liveHandle(ctx);
92438
+ return handle?.agentSession?.isBashRunning === true || handle?.isBashRunning === true;
92439
+ },
91903
92440
  requestRender: opts.requestRender,
91904
92441
  getAgentSession: () => liveHandle(ctx)?.agentSession,
92442
+ getSessionMessages: () => {
92443
+ const handle = liveHandle(ctx);
92444
+ return handle !== undefined && handle.agentSession === undefined ? handle.messages : undefined;
92445
+ },
92446
+ getTaskSource: () => {
92447
+ const handle = liveHandle(ctx);
92448
+ return handle?.agentSession === undefined ? handle?.taskSource?.() : undefined;
92449
+ },
91905
92450
  isStreaming: () => isLiveHandleStreaming(ctx),
91906
92451
  isPaused: () => isPaused(ctx),
91907
92452
  isDisabled: () => isBlocked(ctx) || !liveHandle(ctx),
@@ -92106,6 +92651,28 @@ function viewLineCount(ctx) {
92106
92651
  function liveHandle(ctx) {
92107
92652
  return ctx.handle?.isDisposed === true ? undefined : ctx.handle;
92108
92653
  }
92654
+ function pooledStageWaitingNotice(ctx, stage = currentStage(ctx)) {
92655
+ const handle = liveHandle(ctx);
92656
+ if (handle?.agentSession !== undefined)
92657
+ return;
92658
+ const starting = pooledStageStartingText(stage);
92659
+ if (handle === undefined) {
92660
+ if (stage?.workerExecution === undefined || isTerminalStageStatus(stage.status))
92661
+ return;
92662
+ return {
92663
+ label: starting === undefined ? "CONNECTING TO LOCAL WORKER" : "STARTING ON LOCAL WORKER",
92664
+ detail: starting ?? "opening a live view",
92665
+ message: "This stage runs on a local worker. Its chat opens here by itself as soon as its session is ready."
92666
+ };
92667
+ }
92668
+ if (!(handle.isSessionStarting ?? isPooledStageStarting(stage)))
92669
+ return;
92670
+ return {
92671
+ label: "STARTING ON LOCAL WORKER",
92672
+ detail: starting ?? "starting its session",
92673
+ message: "This stage's session is still starting on its local worker. Its transcript streams here once it starts, and messages you send now are delivered then."
92674
+ };
92675
+ }
92109
92676
  function isLiveHandleStreaming(ctx) {
92110
92677
  const handle = liveHandle(ctx);
92111
92678
  if (!handle)
@@ -92149,11 +92716,15 @@ async function handleSlashCommand(ctx, text) {
92149
92716
  if (!handle)
92150
92717
  return false;
92151
92718
  await handle.ensureAttached();
92152
- if (!handle.agentSession)
92719
+ const compact = handle.agentSession !== undefined ? () => handle.agentSession.compact() : handle.compact === undefined ? undefined : () => handle.compact();
92720
+ if (compact === undefined)
92153
92721
  return false;
92154
92722
  try {
92155
- await handle.agentSession.compact();
92156
- } catch {}
92723
+ await compact();
92724
+ } catch (error) {
92725
+ if (handle.agentSession === undefined)
92726
+ ctx.chatHost.showWarning(error.message);
92727
+ }
92157
92728
  return true;
92158
92729
  }
92159
92730
  case "/quit":
@@ -92325,6 +92896,22 @@ function renderPausedBody(ctx, width, budget, indicatorLines = [], renderIndicat
92325
92896
  lines.length = budget;
92326
92897
  return lines;
92327
92898
  }
92899
+ function renderStartingBody(ctx, width, budget, notice, renderTranscript = (rows) => ctx.chatHost.renderBody(width, rows)) {
92900
+ const t = ctx.theme;
92901
+ const callout = [];
92902
+ callout.push(blankLine(width));
92903
+ callout.push(...bannerLines(ctx, width, "info", "○", notice.label, notice.detail));
92904
+ callout.push(...new Text3(paint2(notice.message, t.textMuted), 2, 0).render(width));
92905
+ const calloutRows = Math.min(callout.length, Math.max(0, budget));
92906
+ const transcriptRows = Math.max(0, budget - calloutRows);
92907
+ const lines = transcriptRows > 0 ? renderTranscript(transcriptRows) : [];
92908
+ lines.push(...callout.slice(0, calloutRows));
92909
+ while (lines.length < budget)
92910
+ lines.push(blankLine(width));
92911
+ if (lines.length > budget)
92912
+ lines.length = budget;
92913
+ return lines;
92914
+ }
92328
92915
  function renderBlockedBody(ctx, width, budget, stage) {
92329
92916
  const t = ctx.theme;
92330
92917
  const upstream = stage?.blockedByStageId ?? "upstream stage";
@@ -92794,6 +93381,7 @@ class StageChatView {
92794
93381
  keyLabel: () => keybindings?.getKeys?.(TUI_ACTION.altScreenBottom)?.join("/") ?? keyText2("tui.altScreen.bottom")
92795
93382
  });
92796
93383
  const renderTranscript = (rows) => this.chatHost.renderBody(w, rows);
93384
+ const waitingNotice = pooledStageWaitingNotice(ctx, stage);
92797
93385
  if (bodyBudget <= 0) {
92798
93386
  bodyLines = [];
92799
93387
  } else if (promptActive) {
@@ -92803,6 +93391,8 @@ class StageChatView {
92803
93391
  } else if (!readOnlyArchive && isPaused(ctx, stage)) {
92804
93392
  reservedIndicatorLines = bodyBudget > 1 ? indicator.render(w) : [];
92805
93393
  bodyLines = renderPausedBody(ctx, w, bodyBudget, reservedIndicatorLines, () => indicator.render(w), renderTranscript);
93394
+ } else if (waitingNotice !== undefined) {
93395
+ bodyLines = renderStartingBody(ctx, w, bodyBudget, waitingNotice, renderTranscript);
92806
93396
  } else if (readOnlyArchive) {
92807
93397
  reservedIndicatorLines = bodyBudget > 1 ? indicator.render(w) : [];
92808
93398
  bodyLines = renderReadOnlyArchiveBody(ctx, w, bodyBudget, stage, reservedIndicatorLines, () => indicator.render(w), renderTranscript);
@@ -92932,6 +93522,7 @@ function attachedResolution(postMortem) {
92932
93522
  // dist/builtin/workflows/src/tui/workflow-attach-pane.ts
92933
93523
  var ENTER_TRANSITION_QUARANTINE_MS = 200;
92934
93524
  var TERMINAL_STAGE_STATUSES = new Set(["completed", "failed", "skipped"]);
93525
+ var POOLED_VIEW_RETRY_MS = 1000;
92935
93526
 
92936
93527
  class WorkflowAttachPane {
92937
93528
  store;
@@ -92961,6 +93552,9 @@ class WorkflowAttachPane {
92961
93552
  chatView = null;
92962
93553
  attachedPromptPosition = null;
92963
93554
  pooledAttachment;
93555
+ awaitingPooledView = false;
93556
+ pooledViewRetryAt = 0;
93557
+ pooledViewRetrying = false;
92964
93558
  releaseChatHandle = null;
92965
93559
  unsubscribeStore = null;
92966
93560
  attachedRunId = null;
@@ -93054,7 +93648,7 @@ class WorkflowAttachPane {
93054
93648
  const run = snap.runs.find((r) => r.id === runId);
93055
93649
  return run?.name ?? "workflow";
93056
93650
  }
93057
- async _attachToStage(runId, stageId, options = { suppressInitialPromptSubmit: true }) {
93651
+ async _attachToStage(runId, stageId, options = { suppressInitialPromptSubmit: true }, pooledLease) {
93058
93652
  this._flushStoreUpdate();
93059
93653
  const generation = ++this.attachmentGeneration;
93060
93654
  this.graphEnterQuarantineUntil = 0;
@@ -93066,7 +93660,8 @@ class WorkflowAttachPane {
93066
93660
  this.attachedRunId = runId;
93067
93661
  this.lastAttachedStageId = stageId;
93068
93662
  const stage = this._stageSnapshot(runId, stageId);
93069
- const resolution = resolveAttachedStageHandle(this.registry, this.resolvePostMortemHandle, runId, stageId, stage?.workerExecution !== undefined && !TERMINAL_STAGE_STATUSES.has(stage.status) ? () => acquirePooledStage(this.store, runId, stageId) : undefined);
93663
+ const pooledLive = stage?.workerExecution !== undefined && !TERMINAL_STAGE_STATUSES.has(stage.status);
93664
+ const resolution = resolveAttachedStageHandle(this.registry, this.resolvePostMortemHandle, runId, stageId, pooledLive || pooledLease !== undefined ? () => pooledLease === undefined ? acquirePooledStage(this.store, runId, stageId) : Promise.resolve(pooledLease) : undefined);
93070
93665
  const { handle, postMortemUnavailableReason, release, onReconnected } = resolution instanceof Promise ? await resolution : resolution;
93071
93666
  if (generation !== this.attachmentGeneration || !this.visible) {
93072
93667
  release?.();
@@ -93086,8 +93681,39 @@ class WorkflowAttachPane {
93086
93681
  stopWatchingReconnects?.();
93087
93682
  release();
93088
93683
  };
93684
+ this.awaitingPooledView = pooledLive && release === undefined;
93685
+ this.pooledViewRetryAt = this.now();
93089
93686
  this._mountChatView(runId, stageId, handle, postMortemUnavailableReason);
93090
93687
  }
93688
+ _retryPooledStageView(stage) {
93689
+ const runId = this.attachedRunId;
93690
+ const stageId = this.lastAttachedStageId;
93691
+ if (!this.awaitingPooledView || this.pooledViewRetrying || this.chatView === null || !runId || !stageId)
93692
+ return;
93693
+ if (stage?.workerExecution === undefined || TERMINAL_STAGE_STATUSES.has(stage.status)) {
93694
+ this.awaitingPooledView = false;
93695
+ return;
93696
+ }
93697
+ const now = this.now();
93698
+ if (now - this.pooledViewRetryAt < POOLED_VIEW_RETRY_MS)
93699
+ return;
93700
+ this.pooledViewRetryAt = now;
93701
+ this.pooledViewRetrying = true;
93702
+ const generation = this.attachmentGeneration;
93703
+ acquirePooledStage(this.store, runId, stageId).then((lease) => {
93704
+ if (lease === undefined)
93705
+ return;
93706
+ const chat = this.chatView;
93707
+ if (generation !== this.attachmentGeneration || chat === null || !this.visible || this.attachedRunId !== runId || this.lastAttachedStageId !== stageId) {
93708
+ lease.release();
93709
+ return;
93710
+ }
93711
+ this.composerDrafts.capture(runId, stageId, chat._inputBuffer);
93712
+ this._attachToStage(runId, stageId, { suppressInitialPromptSubmit: true }, lease);
93713
+ }).catch(() => {}).finally(() => {
93714
+ this.pooledViewRetrying = false;
93715
+ });
93716
+ }
93091
93717
  _mountChatView(runId, stageId, handle, postMortemUnavailableReason) {
93092
93718
  let chatView;
93093
93719
  chatView = new StageChatView({
@@ -93122,6 +93748,7 @@ class WorkflowAttachPane {
93122
93748
  }
93123
93749
  _disposeChatView() {
93124
93750
  this.pooledAttachment = undefined;
93751
+ this.awaitingPooledView = false;
93125
93752
  this.chatView?.dispose();
93126
93753
  const release = this.releaseChatHandle;
93127
93754
  this.releaseChatHandle = null;
@@ -93224,6 +93851,7 @@ class WorkflowAttachPane {
93224
93851
  if (this.chatView !== null && stage !== undefined && stage.attached !== true)
93225
93852
  this.store.recordStageAttached(this.attachedRunId, this.lastAttachedStageId, true);
93226
93853
  this._followSupersedingPrompt(attachedRun, stage);
93854
+ this._retryPooledStageView(stage);
93227
93855
  }
93228
93856
  }
93229
93857
  _followSupersedingPrompt(run, stage) {
@@ -96108,6 +96736,7 @@ async function resumeRun(runId, opts) {
96108
96736
  await handle.resume();
96109
96737
  opts?.onResumeAcknowledged?.();
96110
96738
  }
96739
+ const stages = await settleResumeAcknowledgements(activeStore, expandedControlRunIds(activeStore, runId).flatMap((controlRunId) => registry.run(controlRunId).pausedStages().map((handle) => ({ controlRunId, handle }))), opts?.message);
96111
96740
  activeStore.recordRunResumed(runId, undefined, {
96112
96741
  source: "run_control",
96113
96742
  ...opts?.actor === undefined ? {} : { actor: opts.actor }
@@ -96116,9 +96745,9 @@ async function resumeRun(runId, opts) {
96116
96745
  ok: true,
96117
96746
  runId,
96118
96747
  snapshot: structuredClone(activeStore.runs().find((candidate) => candidate.id === runId) ?? run),
96119
- resumed: [],
96748
+ resumed: structuredClone([...stages.resumed]),
96120
96749
  mode: "paused",
96121
- message: `Resumed workflow runtime on ${runId}.`
96750
+ message: stages.failures.length === 0 ? `Resumed workflow runtime on ${runId}.` : `Resumed workflow runtime on ${runId}; failed to resume workflow stages: ${stages.failures.join("; ")}. Failed paused stages remain resumable.`
96122
96751
  };
96123
96752
  }
96124
96753
  const resumed = [];
@@ -96242,7 +96871,11 @@ async function resumeRun(runId, opts) {
96242
96871
  }
96243
96872
  function isRunFullyPaused(activeStore, runId, controlRunIds) {
96244
96873
  const runs = activeStore.runs();
96245
- const hasLiveStage = controlRunIds.some((controlRunId) => runs.find((candidate) => candidate.id === controlRunId)?.stages.some((stage) => stage.status === "running" || stage.status === "pending") === true);
96874
+ const settledChild = (stage) => {
96875
+ const child = runs.find((candidate) => candidate.id === authoritativeWorkflowChildRunId(stage));
96876
+ return child !== undefined && controlRunIds.includes(child.id) && (child.status === "paused" || child.endedAt !== undefined);
96877
+ };
96878
+ const hasLiveStage = controlRunIds.some((controlRunId) => runs.find((candidate) => candidate.id === controlRunId)?.stages.some((stage) => (stage.status === "running" || stage.status === "pending") && !settledChild(stage)) === true);
96246
96879
  return !hasLiveStage && runs.find((candidate) => candidate.id === runId)?.status === "paused";
96247
96880
  }
96248
96881
  async function pauseRunWithAction(runId, opts, action = "pause") {
@@ -96278,6 +96911,40 @@ async function pauseRunWithAction(runId, opts, action = "pause") {
96278
96911
  }
96279
96912
  const controlRunIds = expandedControlRunIds(activeStore, runId);
96280
96913
  const handles = controlRunIds.flatMap((controlRunId) => registry.run(controlRunId).stages().filter((handle) => handle.status === "running" || handle.status === "pending").map((handle) => ({ controlRunId, handle })));
96914
+ const pauseHandles = async () => {
96915
+ const paused = [];
96916
+ const pausedRunIds = new Set;
96917
+ for (const { controlRunId, handle } of handles) {
96918
+ await handle.pause();
96919
+ activeStore.recordStagePaused(controlRunId, handle.stageId);
96920
+ pausedRunIds.add(controlRunId);
96921
+ const controlRun = activeStore.runs().find((candidate) => candidate.id === controlRunId);
96922
+ const stage = controlRun?.stages.find((candidate) => candidate.id === handle.stageId);
96923
+ if (stage !== undefined)
96924
+ paused.push(structuredClone(stage));
96925
+ }
96926
+ for (const pausedRunId of pausedRunIds) {
96927
+ if (pausedRunId === runId)
96928
+ continue;
96929
+ activeStore.recordRunPaused(pausedRunId);
96930
+ }
96931
+ activeStore.recordRunPaused(runId);
96932
+ return paused;
96933
+ };
96934
+ const poolControl = (opts?.toolControlRegistry ?? toolControlRegistry).runControl(runId);
96935
+ if (poolControl !== undefined && liveWorkerStages(activeStore, controlRunIds).length > 0) {
96936
+ await poolControl.pause();
96937
+ const paused = await pauseHandles();
96938
+ const pausing = liveWorkerStages(activeStore, controlRunIds).filter((stage) => stage.status === "running").length;
96939
+ return {
96940
+ ok: true,
96941
+ runId,
96942
+ paused,
96943
+ ...pausing === 0 ? {} : {
96944
+ message: `Run ${runId} paused and can be resumed. ${pausing} stage(s) on local workers have not confirmed the pause yet; their workers pause them on their next check.`
96945
+ }
96946
+ };
96947
+ }
96281
96948
  if (handles.length === 0) {
96282
96949
  const toolControls = opts?.toolControlRegistry ?? toolControlRegistry;
96283
96950
  const runtimeControls = ownedRuntimeControls(activeStore, toolControls, runId);
@@ -96286,11 +96953,13 @@ async function pauseRunWithAction(runId, opts, action = "pause") {
96286
96953
  if (runtimeControls.some(({ controlRunId }) => controlRunId === runId) && [...activeRunIds].every((id) => toolControls.active(id).length === 0) && stageHandles.every((handle) => handle.status === "awaiting_input") && (stageHandles.length === 0 || !isRunFullyPaused(activeStore, runId, controlRunIds))) {
96287
96954
  await Promise.all(runtimeControls.map(({ handle }) => handle.pause()));
96288
96955
  activeStore.recordRunPaused(runId);
96956
+ const durable = toolControls.runControl(runId)?.durableBackend?.getWorkflow(runId);
96957
+ const resumableElsewhere = durable !== undefined && (durable.completedCheckpoints > 0 || durable.pendingPrompts > 0);
96289
96958
  return {
96290
96959
  ok: true,
96291
96960
  runId,
96292
96961
  paused: [],
96293
- message: `Run ${runId} paused. Resume with /workflow resume on this live process; untracked initialization or workflow code may still finish, but further workflow steps and completion wait for resume. Cross-process resume requires durable checkpoint or pending prompt progress.`
96962
+ message: resumableElsewhere ? `Run ${runId} paused. Resume it with /workflow resume ${runId}, here or, once this process has exited, from a new Atomic process; untracked initialization or workflow code may still finish, but further workflow steps and completion wait for resume.` : `Run ${runId} paused. Resume with /workflow resume on this live process; untracked initialization or workflow code may still finish, but further workflow steps and completion wait for resume. Cross-process resume requires durable checkpoint or pending prompt progress.`
96294
96963
  };
96295
96964
  }
96296
96965
  const controlRuns = activeStore.runs().filter((candidate) => controlRunIds.includes(candidate.id));
@@ -96307,24 +96976,11 @@ async function pauseRunWithAction(runId, opts, action = "pause") {
96307
96976
  reason: isRunFullyPaused(activeStore, runId, controlRunIds) ? "already_paused" : "no_active_stages"
96308
96977
  };
96309
96978
  }
96310
- const paused = [];
96311
- const pausedRunIds = new Set;
96312
- for (const { controlRunId, handle } of handles) {
96313
- await handle.pause();
96314
- activeStore.recordStagePaused(controlRunId, handle.stageId);
96315
- pausedRunIds.add(controlRunId);
96316
- const controlRun = activeStore.runs().find((candidate) => candidate.id === controlRunId);
96317
- const stage = controlRun?.stages.find((candidate) => candidate.id === handle.stageId);
96318
- if (stage !== undefined)
96319
- paused.push(structuredClone(stage));
96320
- }
96321
- for (const pausedRunId of pausedRunIds) {
96322
- if (pausedRunId === runId)
96323
- continue;
96324
- activeStore.recordRunPaused(pausedRunId);
96325
- }
96326
- activeStore.recordRunPaused(runId);
96327
- return { ok: true, runId, paused };
96979
+ return { ok: true, runId, paused: await pauseHandles() };
96980
+ }
96981
+ function liveWorkerStages(activeStore, runIds) {
96982
+ const ids = new Set(runIds);
96983
+ return activeStore.runs().filter((run) => ids.has(run.id) && run.endedAt === undefined).flatMap((run) => run.stages.filter((stage) => stage.workerExecution !== undefined && (stage.status === "running" || stage.status === "pending")));
96328
96984
  }
96329
96985
  async function pauseRun(runId, opts) {
96330
96986
  if (opts?.stageId === undefined) {
@@ -98071,17 +98727,17 @@ async function openCompletedDurableWorkflow(workflowId, deps, catalog) {
98071
98727
  const selectedStore = await stageSessionStore({ sessionStore: deps.sessionStore ?? deps.adapters?.sessionStore, cwd: deps.cwd }, deps.defaultSessionDir);
98072
98728
  const resolved = await resolveCompletedWorkflow(workflowId, deps.durableBackend, catalog, selectedStore);
98073
98729
  if (resolved.kind === "not_found") {
98074
- return failure2("not_found", `No completed durable workflow found for id: ${workflowId}`);
98730
+ return failure3("not_found", `No completed durable workflow found for id: ${workflowId}`);
98075
98731
  }
98076
98732
  if (resolved.kind === "malformed") {
98077
- return failure2("malformed", resolved.message);
98733
+ return failure3("malformed", resolved.message);
98078
98734
  }
98079
98735
  if (resolved.kind === "stale") {
98080
- return failure2("stale", `Completed workflow ${resolved.entry.workflowId} is stale or missing durable checkpoint/session data and cannot be opened.`);
98736
+ return failure3("stale", `Completed workflow ${resolved.entry.workflowId} is stale or missing durable checkpoint/session data and cannot be opened.`);
98081
98737
  }
98082
98738
  const existing = deps.store.runs().find((run) => run.id === resolved.snapshot.id);
98083
98739
  if (existing !== undefined && existing.status !== "completed") {
98084
- return failure2("active", `Workflow ${resolved.snapshot.id} is already active in this session; attach with /workflow connect ${resolved.snapshot.id} instead.`);
98740
+ return failure3("active", `Workflow ${resolved.snapshot.id} is already active in this session; attach with /workflow connect ${resolved.snapshot.id} instead.`);
98085
98741
  }
98086
98742
  const snapshots = await completedWorkflowRunSnapshots(deps.durableBackend, resolved.entry, selectedStore);
98087
98743
  const snapshot = snapshots.find((run) => run.id === resolved.snapshot.id) ?? resolved.snapshot;
@@ -98102,7 +98758,7 @@ async function openCompletedDurableWorkflow(workflowId, deps, catalog) {
98102
98758
  message: `Opened completed durable workflow "${snapshot.name}" (${snapshot.id}) for read-only inspection${hasReopenableStage ? " and follow-up chat" : ""}.`
98103
98759
  };
98104
98760
  }
98105
- function failure2(reason, message) {
98761
+ function failure3(reason, message) {
98106
98762
  return { ok: false, reason, message };
98107
98763
  }
98108
98764
  function registerCompletedChatHandles(snapshot, deps, sessionStore) {
@@ -103669,7 +104325,7 @@ function errorOptions(runId, cause, failedRuns) {
103669
104325
  ...failedRuns === undefined ? {} : { failedRuns }
103670
104326
  };
103671
104327
  }
103672
- function failure3(code, message, runId, cause, failedRuns) {
104328
+ function failure4(code, message, runId, cause, failedRuns) {
103673
104329
  const options = errorOptions(runId, cause, failedRuns);
103674
104330
  switch (code) {
103675
104331
  case "run_not_found":
@@ -103692,7 +104348,7 @@ function failure3(code, message, runId, cause, failedRuns) {
103692
104348
  }
103693
104349
  function translateThrown(error, runId) {
103694
104350
  const code = classifyControlError(error);
103695
- return code === "owned_elsewhere" || code === "storage_unavailable" ? failure3(code, error.message, runId, error) : error;
104351
+ return code === "owned_elsewhere" || code === "storage_unavailable" ? failure4(code, error.message, runId, error) : error;
103696
104352
  }
103697
104353
  function controlStatus(status, runId) {
103698
104354
  switch (status) {
@@ -103704,15 +104360,15 @@ function controlStatus(status, runId) {
103704
104360
  case "cancelled":
103705
104361
  return status;
103706
104362
  default:
103707
- throw failure3("control_failed", `Unexpected workflow control status: ${status}`, runId);
104363
+ throw failure4("control_failed", `Unexpected workflow control status: ${status}`, runId);
103708
104364
  }
103709
104365
  }
103710
104366
  function controlOutcome(result) {
103711
104367
  if (result.action !== "pause" && result.action !== "quit" && result.action !== "resume") {
103712
- throw failure3("control_failed", `Unexpected workflow result: ${result.action}`, undefined);
104368
+ throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, undefined);
103713
104369
  }
103714
104370
  if ("code" in result && result.code !== undefined) {
103715
- throw failure3(result.code, result.message, result.runId, undefined, "failedRuns" in result ? result.failedRuns : undefined);
104371
+ throw failure4(result.code, result.message, result.runId, undefined, "failedRuns" in result ? result.failedRuns : undefined);
103716
104372
  }
103717
104373
  return { ...result, status: controlStatus(result.status, result.runId) };
103718
104374
  }
@@ -103722,7 +104378,7 @@ function rejectUnknownRunPrefix(store, runId) {
103722
104378
  return;
103723
104379
  const resolution = resolveRunIdTarget(target, store.runs().map((run) => run.id));
103724
104380
  if (resolution.kind === "not_found") {
103725
- throw failure3("run_not_found", `Run not found: ${target} (no run in this session has that prefix; pass the full run id for a run recorded elsewhere).`, runId);
104381
+ throw failure4("run_not_found", `Run not found: ${target} (no run in this session has that prefix; pass the full run id for a run recorded elsewhere).`, runId);
103726
104382
  }
103727
104383
  }
103728
104384
  function createSessionRunControl(host) {
@@ -103744,7 +104400,7 @@ function createSessionRunControl(host) {
103744
104400
  ...filter?.status === undefined ? {} : { statusFilter: filter.status }
103745
104401
  });
103746
104402
  if (result.action !== "status")
103747
- throw failure3("control_failed", `Unexpected workflow result: ${result.action}`, undefined);
104403
+ throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, undefined);
103748
104404
  const catalog = filter?.includeDurable ? await host.catalog?.() : undefined;
103749
104405
  const cwd = host.context()?.cwd;
103750
104406
  const canonicalPaths = new Map;
@@ -103798,9 +104454,9 @@ function createSessionRunControl(host) {
103798
104454
  async getRun(runId) {
103799
104455
  const result = await request({ action: "status", runId });
103800
104456
  if (result.action !== "statusDetail")
103801
- throw failure3("control_failed", `Unexpected workflow result: ${result.action}`, runId);
104457
+ throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, runId);
103802
104458
  if ("error" in result)
103803
- throw failure3("run_not_found", result.error, runId);
104459
+ throw failure4("run_not_found", result.error, runId);
103804
104460
  return result.detail;
103805
104461
  },
103806
104462
  async getStages(runId, filter) {
@@ -103810,9 +104466,9 @@ function createSessionRunControl(host) {
103810
104466
  ...filter?.status === undefined ? {} : { statusFilter: filter.status }
103811
104467
  });
103812
104468
  if (result.action !== "stages")
103813
- throw failure3("control_failed", `Unexpected workflow result: ${result.action}`, runId);
104469
+ throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, runId);
103814
104470
  if (result.error !== undefined)
103815
- throw failure3("run_not_found", result.error, runId);
104471
+ throw failure4("run_not_found", result.error, runId);
103816
104472
  return result.stages;
103817
104473
  },
103818
104474
  async pause(runId, options) {
@@ -103835,7 +104491,7 @@ function createSessionRunControl(host) {
103835
104491
  } catch (error) {
103836
104492
  const cause = error instanceof Error ? error : new Error(String(error));
103837
104493
  const code = error instanceof JournalError ? "storage_unavailable" : classifyControlError(error);
103838
- throw failure3(code, `Run ${runId} did not settle cleanly: ${describeErrorWithCauses(error)}`, runId, cause);
104494
+ throw failure4(code, `Run ${runId} did not settle cleanly: ${describeErrorWithCauses(error)}`, runId, cause);
103839
104495
  }
103840
104496
  },
103841
104497
  async resume(runId, options) {
@@ -104421,7 +105077,8 @@ function renderStagesToolContent(result) {
104421
105077
  }
104422
105078
  lines.push("stages:");
104423
105079
  result.stages.forEach((stage, index) => {
104424
- lines.push(`[${index + 1}] ${stage.name} (${stage.id}) ${stage.status}`);
105080
+ const starting = pooledStageStartingText(stage);
105081
+ lines.push(`[${index + 1}] ${stage.name} (${stage.id}) ${stageDisplayStatus(stage)}${starting === undefined ? "" : ` (starting on its local worker: ${starting})`}`);
104425
105082
  if (stage.startup)
104426
105083
  lines.push(formatStageStartup(stage.startup));
104427
105084
  if (stage.workerExecution)