@bastani/atomic 0.9.43 → 0.9.44-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/dist/builtin/intercom/CHANGELOG.md +22 -0
  3. package/dist/builtin/intercom/broker/broker.ts +31 -7
  4. package/dist/builtin/intercom/broker/client-message-validation.ts +6 -0
  5. package/dist/builtin/intercom/broker/client.ts +69 -0
  6. package/dist/builtin/intercom/broker/send-signature.ts +4 -1
  7. package/dist/builtin/intercom/index.bundle.mjs +309 -109
  8. package/dist/builtin/intercom/live-route-refusal.ts +20 -0
  9. package/dist/builtin/intercom/package.json +1 -1
  10. package/dist/builtin/intercom/skills/intercom/SKILL.md +11 -6
  11. package/dist/builtin/intercom/types.ts +28 -1
  12. package/dist/builtin/mcp/CHANGELOG.md +7 -0
  13. package/dist/builtin/mcp/package.json +1 -1
  14. package/dist/builtin/subagents/CHANGELOG.md +9 -0
  15. package/dist/builtin/subagents/README.md +8 -8
  16. package/dist/builtin/subagents/agents/worker.md +1 -1
  17. package/dist/builtin/subagents/package.json +1 -1
  18. package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
  19. package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
  20. package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
  21. package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
  22. package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
  23. package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
  24. package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
  25. package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
  26. package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
  27. package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
  28. package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
  29. package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
  30. package/dist/builtin/web-access/package.json +1 -1
  31. package/dist/builtin/workflows/CHANGELOG.md +20 -0
  32. package/dist/builtin/workflows/README.md +1 -1
  33. package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
  34. package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
  35. package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
  36. package/dist/builtin/workflows/builtin/goal.js +2 -2
  37. package/dist/builtin/workflows/builtin/index.js +3 -3
  38. package/dist/builtin/workflows/builtin/ralph.js +2 -2
  39. package/dist/builtin/workflows/package.json +1 -1
  40. package/dist/builtin/workflows/src/extension/index.bundle.mjs +1844 -603
  41. package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1880 -605
  42. package/dist/builtin/workflows/src/index.js +1896 -655
  43. package/dist/builtin/workflows/src/local-worker-entry.mjs +1880 -605
  44. package/dist/cli/args.js +1 -1
  45. package/dist/cli/args.js.map +1 -1
  46. package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
  47. package/dist/core/agent-session-custom-message-commit.js +59 -0
  48. package/dist/core/agent-session-custom-message-commit.js.map +1 -1
  49. package/dist/core/agent-session-methods.d.ts +1 -0
  50. package/dist/core/agent-session-methods.d.ts.map +1 -1
  51. package/dist/core/agent-session-methods.js.map +1 -1
  52. package/dist/core/agent-session-prompt.d.ts +4 -0
  53. package/dist/core/agent-session-prompt.d.ts.map +1 -1
  54. package/dist/core/agent-session-prompt.js +10 -2
  55. package/dist/core/agent-session-prompt.js.map +1 -1
  56. package/dist/core/agent-session.d.ts +2 -0
  57. package/dist/core/agent-session.d.ts.map +1 -1
  58. package/dist/core/agent-session.js +9 -0
  59. package/dist/core/agent-session.js.map +1 -1
  60. package/dist/core/builtin-packages.d.ts.map +1 -1
  61. package/dist/core/builtin-packages.js +13 -13
  62. package/dist/core/builtin-packages.js.map +1 -1
  63. package/dist/core/exec.d.ts.map +1 -1
  64. package/dist/core/exec.js +11 -2
  65. package/dist/core/exec.js.map +1 -1
  66. package/dist/core/package-manager-resolver.d.ts.map +1 -1
  67. package/dist/core/package-manager-resolver.js +25 -14
  68. package/dist/core/package-manager-resolver.js.map +1 -1
  69. package/dist/core/package-manager-types.d.ts +8 -0
  70. package/dist/core/package-manager-types.d.ts.map +1 -1
  71. package/dist/core/package-manager-types.js.map +1 -1
  72. package/dist/core/package-manager.d.ts +1 -1
  73. package/dist/core/package-manager.d.ts.map +1 -1
  74. package/dist/core/package-manager.js.map +1 -1
  75. package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
  76. package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
  77. package/dist/core/resource-loader-extension-warnings.js +15 -0
  78. package/dist/core/resource-loader-extension-warnings.js.map +1 -1
  79. package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
  80. package/dist/core/resource-loader-package-resources.js +1 -0
  81. package/dist/core/resource-loader-package-resources.js.map +1 -1
  82. package/dist/core/resource-loader-reload.d.ts.map +1 -1
  83. package/dist/core/resource-loader-reload.js +5 -3
  84. package/dist/core/resource-loader-reload.js.map +1 -1
  85. package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
  86. package/dist/extensions/mcp/client/oauth/flow.js +4 -1
  87. package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
  88. package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
  89. package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
  90. package/dist/extensions/mcp/client/oauth/types.js +8 -4
  91. package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
  92. package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
  93. package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
  94. package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
  95. package/dist/index.d.ts +1 -1
  96. package/dist/index.d.ts.map +1 -1
  97. package/dist/index.js.map +1 -1
  98. package/dist/main.d.ts.map +1 -1
  99. package/dist/main.js +20 -10
  100. package/dist/main.js.map +1 -1
  101. package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
  102. package/dist/modes/interactive/components/model-selector.js +4 -2
  103. package/dist/modes/interactive/components/model-selector.js.map +1 -1
  104. package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
  105. package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
  106. package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
  107. package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
  108. package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
  109. package/dist/modes/interactive/components/tool-execution.js +11 -1
  110. package/dist/modes/interactive/components/tool-execution.js.map +1 -1
  111. package/dist/modes/interactive/model-search.d.ts +10 -0
  112. package/dist/modes/interactive/model-search.d.ts.map +1 -1
  113. package/dist/modes/interactive/model-search.js +21 -0
  114. package/dist/modes/interactive/model-search.js.map +1 -1
  115. package/dist/package-manager-cli.d.ts.map +1 -1
  116. package/dist/package-manager-cli.js +5 -0
  117. package/dist/package-manager-cli.js.map +1 -1
  118. package/dist/utils/tools-manager.d.ts.map +1 -1
  119. package/dist/utils/tools-manager.js +30 -35
  120. package/dist/utils/tools-manager.js.map +1 -1
  121. package/docs/computer-use.md +5 -5
  122. package/docs/custom-provider/streaming.md +9 -14
  123. package/docs/environment-variables.md +1 -0
  124. package/docs/intercom/operations.md +1 -1
  125. package/docs/intercom.md +16 -2
  126. package/docs/packages.md +2 -0
  127. package/docs/settings.md +2 -2
  128. package/docs/skills.md +2 -2
  129. package/docs/subagents.md +47 -10
  130. package/docs/workflows/api-reference.md +1 -1
  131. package/docs/workflows/authoring.md +1 -1
  132. package/docs/workflows/operations.md +8 -6
  133. package/docs/workflows/reliable-design.md +23 -5
  134. package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
  135. package/examples/extensions/todo.ts +7 -0
  136. package/npm-shrinkwrap.json +32 -32
  137. package/package.json +3 -3
@@ -1661,6 +1661,13 @@ function removeProcessExitCleanup(cleanup) {
1661
1661
  if (processExitCleanups.delete(cleanup) && processExitCleanups.size === 0)
1662
1662
  process.off("exit", runProcessExitCleanups);
1663
1663
  }
1664
+ function snapshotNeedsHistory(tail) {
1665
+ const event = tail.snapshot.event;
1666
+ if (event.type !== RUN_EVENT_TYPES.messageQueued)
1667
+ return false;
1668
+ const payload = event.payload;
1669
+ return hasPendingStageDeliveryDelta(payload.pendingStageMessages);
1670
+ }
1664
1671
  function* invoke(action) {
1665
1672
  try {
1666
1673
  const result = action();
@@ -1776,8 +1783,11 @@ class JournalDurableBackend {
1776
1783
  this.projection.applyWorkflowSnapshot(event.payload.handle);
1777
1784
  this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
1778
1785
  } else {
1779
- const handle = event.payload;
1780
1786
  const previous = this.projection.getWorkflow(entry.runId);
1787
+ const handle = event.type === RUN_EVENT_TYPES.messageQueued && hasPendingStageDeliveryDelta(event.payload.pendingStageMessages) ? {
1788
+ ...event.payload,
1789
+ pendingStageMessages: decodePendingStageDeliveries(previous?.pendingStageMessages ?? [], event.payload.pendingStageMessages ?? [])
1790
+ } : event.payload;
1781
1791
  this.projection.applyWorkflowSnapshot(handle);
1782
1792
  if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
1783
1793
  this.prompts.delete(entry.runId);
@@ -2073,11 +2083,23 @@ class JournalDurableBackend {
2073
2083
  for (const message of messages)
2074
2084
  if (message.runId !== logicalRunId)
2075
2085
  throw new JournalMessageScopeError(logicalRunId, message.runId);
2076
- yield* this.snapshot({
2086
+ const next = {
2077
2087
  ...handle,
2078
2088
  pendingStageMessages: replacePendingStageMessagesForRun(handle.pendingStageMessages ?? [], logicalRunId, messages),
2079
2089
  updatedAt: Math.max(Date.now(), handle.updatedAt + 1)
2080
- }, RUN_EVENT_TYPES.messageQueued);
2090
+ };
2091
+ const encoded = encodePendingStageDeliveries(handle.pendingStageMessages ?? [], next.pendingStageMessages ?? []);
2092
+ const base = this.sequences.get(runId);
2093
+ if (encoded !== undefined && base !== undefined) {
2094
+ try {
2095
+ yield* this.append(runId, { type: RUN_EVENT_TYPES.messageQueued, payload: { ...next, pendingStageMessages: encoded } }, { expectSeq: base });
2096
+ return true;
2097
+ } catch (error) {
2098
+ if (!(error instanceof JournalCasConflictError))
2099
+ throw error;
2100
+ }
2101
+ }
2102
+ yield* this.snapshot(next, RUN_EVENT_TYPES.messageQueued);
2081
2103
  return true;
2082
2104
  }
2083
2105
  async archivePendingStageMessage(runId, message) {
@@ -2171,7 +2193,7 @@ class JournalDurableBackend {
2171
2193
  if (summarized !== undefined && summarized.seq === version.seq && summarized.at === version.at)
2172
2194
  return;
2173
2195
  const tail = yield* invoke(() => journal.snapshotTail(runId));
2174
- if (tail !== undefined)
2196
+ if (tail !== undefined && !snapshotNeedsHistory(tail))
2175
2197
  return this.summarize(runId, tail);
2176
2198
  if (summarized === undefined && version.seq === 0)
2177
2199
  return;
@@ -2397,6 +2419,69 @@ class JournalDurableBackend {
2397
2419
  }
2398
2420
  }
2399
2421
  }
2422
+ function identity(message) {
2423
+ return JSON.stringify([message.runId, message.id]);
2424
+ }
2425
+ function uniqueByIdentity(messages) {
2426
+ const result = new Map;
2427
+ const repeated = new Set;
2428
+ for (const message of messages) {
2429
+ const key = identity(message);
2430
+ if (result.has(key))
2431
+ repeated.add(key);
2432
+ result.set(key, message);
2433
+ }
2434
+ for (const key of repeated)
2435
+ result.delete(key);
2436
+ return result;
2437
+ }
2438
+ function isDeltaEncoded(message) {
2439
+ return message.deliveryDelta !== undefined;
2440
+ }
2441
+ function encodePendingStageDeliveries(previous, next) {
2442
+ const bases = uniqueByIdentity(previous);
2443
+ const nextIdentities = uniqueByIdentity(next);
2444
+ let encodedAny = false;
2445
+ const encoded = next.map((message) => {
2446
+ const base = bases.get(identity(message))?.deliveries;
2447
+ const deliveries = message.deliveries;
2448
+ if (base === undefined || deliveries === undefined || !nextIdentities.has(identity(message)))
2449
+ return message;
2450
+ const drop = [];
2451
+ let kept = 0;
2452
+ base.forEach((delivery, index) => {
2453
+ if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
2454
+ kept++;
2455
+ else
2456
+ drop.push(index);
2457
+ });
2458
+ const add = deliveries.slice(kept);
2459
+ if (drop.length + add.length >= deliveries.length)
2460
+ return message;
2461
+ encodedAny = true;
2462
+ const { deliveries: _full, ...rest } = message;
2463
+ return { ...rest, deliveryDelta: drop.length === 0 ? { add } : { drop, add } };
2464
+ });
2465
+ return encodedAny ? encoded : undefined;
2466
+ }
2467
+ function decodePendingStageDeliveries(previous, messages) {
2468
+ if (!messages.some(isDeltaEncoded))
2469
+ return messages;
2470
+ const bases = uniqueByIdentity(previous);
2471
+ return messages.map((message) => {
2472
+ if (!isDeltaEncoded(message))
2473
+ return message;
2474
+ const base = bases.get(identity(message))?.deliveries;
2475
+ if (base === undefined)
2476
+ throw new Error(`atomic-workflows: journal delivery change for ${identity(message)} has no base entry`);
2477
+ const { deliveryDelta, ...rest } = message;
2478
+ const dropped = new Set(deliveryDelta.drop ?? []);
2479
+ return { ...rest, deliveries: [...base.filter((_, index) => !dropped.has(index)), ...deliveryDelta.add] };
2480
+ });
2481
+ }
2482
+ function hasPendingStageDeliveryDelta(messages) {
2483
+ return messages?.some(isDeltaEncoded) === true;
2484
+ }
2400
2485
  var PROCESS_HOLDER_ID, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError;
2401
2486
  var init_journal_backend = __esm(() => {
2402
2487
  init_workflow_instance_ownership_error();
@@ -8030,7 +8115,13 @@ function createStoreContext(state = createStoreState()) {
8030
8115
  }
8031
8116
  return cachedGraphSnapshot;
8032
8117
  }
8118
+ let batchDepth = 0;
8119
+ let batchedNotification = false;
8033
8120
  function notify() {
8121
+ if (batchDepth > 0) {
8122
+ batchedNotification = true;
8123
+ return;
8124
+ }
8034
8125
  observation.capture();
8035
8126
  for (const fn of state.invalidationListeners) {
8036
8127
  try {
@@ -8050,6 +8141,18 @@ function createStoreContext(state = createStoreState()) {
8050
8141
  state.version++;
8051
8142
  notify();
8052
8143
  }
8144
+ function batch(update) {
8145
+ batchDepth++;
8146
+ try {
8147
+ return update();
8148
+ } finally {
8149
+ batchDepth--;
8150
+ if (batchDepth === 0 && batchedNotification) {
8151
+ batchedNotification = false;
8152
+ notify();
8153
+ }
8154
+ }
8155
+ }
8053
8156
  function findRun(runId) {
8054
8157
  return state.runs.find((r) => r.id === runId);
8055
8158
  }
@@ -8103,6 +8206,7 @@ function createStoreContext(state = createStoreState()) {
8103
8206
  snapshot,
8104
8207
  graphSnapshot,
8105
8208
  notify,
8209
+ batch,
8106
8210
  bumpAndNotify,
8107
8211
  findRun,
8108
8212
  findStage,
@@ -8918,6 +9022,9 @@ function createRunStoreMethods(context) {
8918
9022
  subscribeInvalidation(fn) {
8919
9023
  state.invalidationListeners.add(fn);
8920
9024
  return () => state.invalidationListeners.delete(fn);
9025
+ },
9026
+ batch(update) {
9027
+ return context.batch(update);
8921
9028
  }
8922
9029
  };
8923
9030
  }
@@ -13403,7 +13510,7 @@ class ConcurrencyLimiter {
13403
13510
  }
13404
13511
  }
13405
13512
  function createRunLimiter(defaultConcurrency) {
13406
- return new ConcurrencyLimiter(defaultConcurrency ?? 3);
13513
+ return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
13407
13514
  }
13408
13515
 
13409
13516
  // dist/builtin/workflows/src/shared/abort.ts
@@ -57173,6 +57280,44 @@ var init_executor = __esm(() => {
57173
57280
  init_executor_run();
57174
57281
  });
57175
57282
 
57283
+ // dist/builtin/workflows/src/shared/external-run-routes.ts
57284
+ function claimExternalRunRoute(store, runId, owner) {
57285
+ let owners = registry.owners.get(store);
57286
+ if (owners === undefined) {
57287
+ owners = new Map;
57288
+ registry.owners.set(store, owners);
57289
+ }
57290
+ owners.set(runId, owner);
57291
+ return () => {
57292
+ const current = registry.owners.get(store);
57293
+ if (current?.get(runId) !== owner)
57294
+ return;
57295
+ current.delete(runId);
57296
+ if (current.size === 0)
57297
+ registry.owners.delete(store);
57298
+ };
57299
+ }
57300
+ function externalRunRouteOwner(store, runId) {
57301
+ return registry.owners.get(store)?.get(runId);
57302
+ }
57303
+ function registerPendingStageMessageRouter(store, router) {
57304
+ registry.routers.set(store, router);
57305
+ return () => {
57306
+ if (registry.routers.get(store) === router)
57307
+ registry.routers.delete(store);
57308
+ };
57309
+ }
57310
+ function pendingStageMessageRouter(store) {
57311
+ return registry.routers.get(store);
57312
+ }
57313
+ var registryKey, processState, registry;
57314
+ var init_external_run_routes = __esm(() => {
57315
+ registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
57316
+ processState = globalThis;
57317
+ registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
57318
+ processState[registryKey] = registry;
57319
+ });
57320
+
57176
57321
  // dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
57177
57322
  import { randomUUID as randomUUID10 } from "node:crypto";
57178
57323
  function capabilitiesFor(store) {
@@ -57197,12 +57342,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
57197
57342
  capabilitiesFor(store).set(runId, capability);
57198
57343
  return capability;
57199
57344
  }
57200
- var routeCapabilitiesKey, processState, routeCapabilities;
57345
+ var routeCapabilitiesKey, processState2, routeCapabilities;
57201
57346
  var init_pending_stage_route_capability = __esm(() => {
57202
57347
  routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
57203
- processState = globalThis;
57204
- routeCapabilities = processState[routeCapabilitiesKey] ?? new WeakMap;
57205
- processState[routeCapabilitiesKey] = routeCapabilities;
57348
+ processState2 = globalThis;
57349
+ routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
57350
+ processState2[routeCapabilitiesKey] = routeCapabilities;
57206
57351
  });
57207
57352
 
57208
57353
  // dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
@@ -57234,12 +57379,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
57234
57379
  function workflowPendingStageRouteReady(store, runId) {
57235
57380
  return owners.get(store)?.ready(runId);
57236
57381
  }
57237
- var routeOwnersKey, processState2, owners;
57382
+ var routeOwnersKey, processState3, owners;
57238
57383
  var init_pending_stage_route_readiness = __esm(() => {
57239
57384
  routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
57240
- processState2 = globalThis;
57241
- owners = processState2[routeOwnersKey] ?? new WeakMap;
57242
- processState2[routeOwnersKey] = owners;
57385
+ processState3 = globalThis;
57386
+ owners = processState3[routeOwnersKey] ?? new WeakMap;
57387
+ processState3[routeOwnersKey] = owners;
57243
57388
  });
57244
57389
 
57245
57390
  // dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
@@ -57540,6 +57685,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57540
57685
  notificationId,
57541
57686
  reason
57542
57687
  };
57688
+ const external = externalRunRouteOwner(activeStore, entry.runId);
57689
+ if (external !== undefined)
57690
+ return external.notifyUndeliverable(payload);
57543
57691
  pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
57544
57692
  return payload.handled && await payload.completion === true;
57545
57693
  };
@@ -57551,19 +57699,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57551
57699
  return;
57552
57700
  const ownedRunIds = new Set;
57553
57701
  const runs = activeStore.runs();
57702
+ const runById = new Map(runs.map((run) => [run.id, run]));
57703
+ const childrenByBoundary = new Map;
57704
+ for (const run of runs) {
57705
+ if (run.parentRunId === undefined || run.parentStageId === undefined)
57706
+ continue;
57707
+ const key = JSON.stringify([run.parentRunId, run.parentStageId]);
57708
+ const children = childrenByBoundary.get(key) ?? [];
57709
+ children.push(run);
57710
+ childrenByBoundary.set(key, children);
57711
+ }
57554
57712
  const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
57555
57713
  for (const run of runs) {
57556
- if (!ownsRun(run.id))
57714
+ if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
57557
57715
  continue;
57558
- const rootRunId = durableRootRunIdForRun(runs, run.id);
57716
+ const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
57559
57717
  if (rootRunId === undefined)
57560
57718
  continue;
57561
57719
  ownedRunIds.add(run.id);
57562
- const parentRun = runs.find((candidate) => candidate.id === run.parentRunId);
57720
+ const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
57563
57721
  const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
57564
57722
  const parent = parentRun === undefined || boundary === undefined ? undefined : {
57565
57723
  runId: parentRun.id,
57566
- stageKeys: [boundary.id, boundary.name].filter((key) => resolveChildRun(runs, parentRun, key)?.id === run.id)
57724
+ stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
57567
57725
  };
57568
57726
  const announcement = {
57569
57727
  runId: run.id,
@@ -57704,16 +57862,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57704
57862
  payload.handled = true;
57705
57863
  payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
57706
57864
  };
57707
- const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
57708
- const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
57865
+ const routeStickyLiveDelivered = (payload) => {
57709
57866
  if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
57710
57867
  return;
57711
57868
  payload.handled = true;
57712
57869
  payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
57870
+ };
57871
+ const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
57872
+ const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
57873
+ const disposeRouter = registerPendingStageMessageRouter(activeStore, {
57874
+ route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
57875
+ sticky: routeStickyLiveDelivered
57713
57876
  });
57714
57877
  const dispose = () => {
57715
57878
  disposed = true;
57716
57879
  disposeReadiness();
57880
+ disposeRouter();
57717
57881
  routeCompletions.clear();
57718
57882
  unsubscribeStore();
57719
57883
  if (typeof subscription === "function")
@@ -57839,6 +58003,12 @@ function isMaterializedNonAgentTarget(runs, target) {
57839
58003
  return false;
57840
58004
  return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
57841
58005
  }
58006
+ function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
58007
+ const boundariesById = parent.stages.filter((stage) => stage.id === segment);
58008
+ const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
58009
+ const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
58010
+ return children.length === 1 ? children[0] : undefined;
58011
+ }
57842
58012
  function resolveChildRun(runs, parent, segment) {
57843
58013
  const boundariesById = parent.stages.filter((stage) => stage.id === segment);
57844
58014
  const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
@@ -58178,6 +58348,7 @@ var init_pending_stage_intercom = __esm(() => {
58178
58348
  init_execution_context_boundary();
58179
58349
  init_run_owner_backend();
58180
58350
  init_store_backend();
58351
+ init_external_run_routes();
58181
58352
  init_intercom_group();
58182
58353
  init_pending_stage_route_capability();
58183
58354
  init_pending_stage_route_readiness();
@@ -58317,7 +58488,8 @@ function buildSendSignature(to, options) {
58317
58488
  attachments: canonicalizeAttachmentsForSendSignature(options.attachments) ?? [],
58318
58489
  replyTo: options.replyTo ?? null,
58319
58490
  expectsReply: options.expectsReply ?? false,
58320
- replyError: options.replyError ?? null
58491
+ replyError: options.replyError ?? null,
58492
+ ...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery }
58321
58493
  });
58322
58494
  }
58323
58495
 
@@ -58503,6 +58675,8 @@ function isMessage(value2) {
58503
58675
  return false;
58504
58676
  if (message.expectsReply !== undefined && typeof message.expectsReply !== "boolean")
58505
58677
  return false;
58678
+ if (message.delivery !== undefined && message.delivery !== "steer" && message.delivery !== "followUp" && message.delivery !== "interrupt")
58679
+ return false;
58506
58680
  if (message.replyError !== undefined && typeof message.replyError !== "string")
58507
58681
  return false;
58508
58682
  if (message.source !== undefined && !isMessageSource(message.source))
@@ -58636,7 +58810,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
58636
58810
  function isWorkflowStageRosterEntries(value2) {
58637
58811
  return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
58638
58812
  }
58639
- var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
58813
+ var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
58640
58814
  var init_client = __esm(() => {
58641
58815
  init_paths();
58642
58816
  init_pending_send_registry();
@@ -58645,6 +58819,21 @@ var init_client = __esm(() => {
58645
58819
  init_live_route_refusal();
58646
58820
  init_retry_policy();
58647
58821
  BROKER_SOCKET = getBrokerSocketPath();
58822
+ IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
58823
+ connectionKept;
58824
+ code;
58825
+ constructor(reason, refusal = { connectionKept: false }) {
58826
+ super(reason);
58827
+ this.name = "IntercomPendingStageRouteRefusedError";
58828
+ this.connectionKept = refusal.connectionKept;
58829
+ this.code = refusal.code;
58830
+ }
58831
+ };
58832
+ PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
58833
+ "not_authorized",
58834
+ "invalid_stage_roster",
58835
+ "invalid_possible_stage_roster"
58836
+ ]);
58648
58837
  IntercomClient = class IntercomClient extends EventEmitter {
58649
58838
  returnAddress;
58650
58839
  preferredSessionId;
@@ -58652,6 +58841,8 @@ var init_client = __esm(() => {
58652
58841
  connectTimeoutMs;
58653
58842
  brokerAcknowledgesReceipt = false;
58654
58843
  brokerAcknowledgesPendingStageRoutes = false;
58844
+ multiplexesPendingStageRoutes;
58845
+ brokerRefusesPendingStageRoutesInPlace = false;
58655
58846
  brokerAnswersStatus = false;
58656
58847
  socket = null;
58657
58848
  _sessionId = null;
@@ -58717,6 +58908,7 @@ var init_client = __esm(() => {
58717
58908
  this.preferredSessionId = options.preferredSessionId;
58718
58909
  this.acknowledgesReceipt = options.acknowledgesReceipt === true;
58719
58910
  this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
58911
+ this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
58720
58912
  }
58721
58913
  failPending(error) {
58722
58914
  this.pendingSends.rejectAll(error);
@@ -58829,6 +59021,7 @@ var init_client = __esm(() => {
58829
59021
  this._supervisorSessionId = null;
58830
59022
  this.brokerAcknowledgesReceipt = false;
58831
59023
  this.brokerAcknowledgesPendingStageRoutes = false;
59024
+ this.brokerRefusesPendingStageRoutesInPlace = false;
58832
59025
  this.brokerAnswersStatus = false;
58833
59026
  this._messageSource = undefined;
58834
59027
  this.disconnectError = null;
@@ -58907,7 +59100,7 @@ var init_client = __esm(() => {
58907
59100
  }
58908
59101
  switch (brokerMessage.type) {
58909
59102
  case "registered": {
58910
- if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
59103
+ if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
58911
59104
  throw new Error("Invalid registered message");
58912
59105
  }
58913
59106
  if (this._sessionId !== null) {
@@ -58917,6 +59110,7 @@ var init_client = __esm(() => {
58917
59110
  this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
58918
59111
  this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
58919
59112
  this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
59113
+ this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
58920
59114
  this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
58921
59115
  this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
58922
59116
  break;
@@ -58929,6 +59123,12 @@ var init_client = __esm(() => {
58929
59123
  this.emit("_registration_failed", refusal);
58930
59124
  break;
58931
59125
  }
59126
+ const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
59127
+ const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
59128
+ if (refusedRoute !== undefined) {
59129
+ this.pendingStageRouteRegistrations.delete(refusedRouteId);
59130
+ refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
59131
+ }
58932
59132
  this.disconnectError ??= refusal;
58933
59133
  this.failPending(this.disconnectError);
58934
59134
  this.socket?.destroy();
@@ -59054,6 +59254,21 @@ var init_client = __esm(() => {
59054
59254
  this.emit("pending_stage_route_revoked", { runId });
59055
59255
  break;
59056
59256
  }
59257
+ case "pending_stage_route_refused": {
59258
+ const { requestId, runId, code, reason } = brokerMessage;
59259
+ if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
59260
+ throw new Error("Invalid pending-stage route refusal");
59261
+ }
59262
+ const pending = this.pendingStageRouteRegistrations.get(requestId);
59263
+ if (!pending)
59264
+ return;
59265
+ this.pendingStageRouteRegistrations.delete(requestId);
59266
+ pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
59267
+ connectionKept: true,
59268
+ ...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
59269
+ }));
59270
+ break;
59271
+ }
59057
59272
  case "pending_stage_route_registered": {
59058
59273
  const { requestId } = brokerMessage;
59059
59274
  if (typeof requestId !== "string")
@@ -59557,7 +59772,8 @@ var init_client = __esm(() => {
59557
59772
  stages,
59558
59773
  possibleStages,
59559
59774
  ...parent === undefined ? {} : { parent },
59560
- ...dormant === true ? { dormant: true } : {}
59775
+ ...dormant === true ? { dormant: true } : {},
59776
+ ...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
59561
59777
  });
59562
59778
  } catch (error) {
59563
59779
  clearTimeout(timeout);
@@ -59670,6 +59886,7 @@ var init_client = __esm(() => {
59670
59886
  timestamp: Date.now(),
59671
59887
  replyTo: options.replyTo,
59672
59888
  expectsReply: options.expectsReply,
59889
+ ...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery },
59673
59890
  replyError: options.replyError,
59674
59891
  source: this._messageSource,
59675
59892
  content: { text: options.text, attachments: options.attachments }
@@ -60291,120 +60508,165 @@ var init_reconnect_backoff = __esm(() => {
60291
60508
  BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
60292
60509
  });
60293
60510
 
60294
- // dist/builtin/intercom/route-connect-gate.ts
60295
- function createConcurrencyGate(limit) {
60296
- let active = 0;
60297
- const waiting = [];
60298
- const release = () => {
60299
- const next = waiting.shift();
60300
- if (next === undefined)
60301
- active -= 1;
60302
- else
60303
- next();
60304
- };
60305
- return {
60306
- async run(operation) {
60307
- if (active < limit)
60308
- active += 1;
60309
- else
60310
- await new Promise((resolve) => waiting.push(resolve));
60311
- try {
60312
- return await operation();
60313
- } finally {
60314
- release();
60315
- }
60316
- }
60317
- };
60318
- }
60319
- var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
60320
-
60321
60511
  // dist/builtin/intercom/workflow-route-owner.ts
60322
60512
  import { randomUUID as randomUUID13 } from "node:crypto";
60323
60513
  import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
60514
+ function closedError() {
60515
+ return new Error("Workflow Intercom route owner closed");
60516
+ }
60324
60517
  function createWorkflowRouteOwner(cwd) {
60325
60518
  const events = createEventBus();
60326
60519
  const config = loadConfig();
60327
60520
  const id = randomUUID13();
60328
60521
  const routes = new Map;
60329
- const gate = createConcurrencyGate(ROUTE_CLIENT_CONNECT_CONCURRENCY);
60522
+ const connections = new Map;
60523
+ const isCurrent = (state) => routes.get(state.route.runId) === state;
60524
+ const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
60525
+ function connectionOf(state) {
60526
+ const key = connectionKey(state);
60527
+ let connection = connections.get(key);
60528
+ if (connection === undefined) {
60529
+ connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
60530
+ connections.set(key, connection);
60531
+ }
60532
+ connection.routes.set(state.route.runId, state);
60533
+ return connection;
60534
+ }
60535
+ function detach(state) {
60536
+ const key = connectionKey(state);
60537
+ const connection = connections.get(key);
60538
+ connection?.routes.delete(state.route.runId);
60539
+ if (!state.isolated || connection === undefined)
60540
+ return;
60541
+ connections.delete(key);
60542
+ connection.client?.disconnect().catch(() => {});
60543
+ }
60544
+ function settle(state, registered) {
60545
+ state.registered = registered;
60546
+ if (!registered && isCurrent(state))
60547
+ return;
60548
+ for (const waiter of state.waiters)
60549
+ waiter(registered);
60550
+ state.waiters.clear();
60551
+ }
60330
60552
  function scheduleRetry(state) {
60331
- if (state.closed || state.retry !== undefined)
60553
+ if (!isCurrent(state) || state.retry !== undefined)
60332
60554
  return;
60333
60555
  state.retry = setTimeout(() => {
60334
60556
  state.retry = undefined;
60335
- publish(state, state.route).catch(() => scheduleRetry(state));
60557
+ publish(state, state.route).catch(() => {});
60336
60558
  }, jitteredReconnectDelayMs(state.attempt++));
60337
60559
  state.retry.unref();
60338
60560
  }
60339
- async function connect(state) {
60340
- if (state.client?.isConnected())
60341
- return state.client;
60342
- const [extensionDir] = getBuiltinPackagePaths({
60343
- workflows: false,
60344
- subagents: false,
60345
- mcp: false,
60346
- "web-access": false
60347
- });
60348
- if (extensionDir === undefined)
60349
- throw new Error("Intercom builtin package is unavailable");
60350
- await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
60351
- if (state.closed)
60352
- throw new Error("Workflow Intercom route owner closed");
60353
- const client = new IntercomClient(`${id}:pending-stage-route:${state.route.runId}`);
60354
- state.client = client;
60355
- client.on("error", () => {});
60356
- client.on("disconnected", () => {
60357
- if (state.client !== client)
60358
- return;
60359
- state.client = undefined;
60360
- scheduleRetry(state);
60361
- });
60362
- client.on("pending_stage_message", (request) => {
60363
- if (state.closed || state.client !== client)
60364
- return;
60365
- const event = { ...request, handled: false };
60366
- events.emit("atomic:workflow-pending-stage-message", event);
60367
- const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
60368
- completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
60369
- });
60370
- client.on("sticky_live_delivered", (notice) => {
60371
- if (state.closed || state.client !== client)
60372
- return;
60373
- events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
60374
- });
60375
- await client.connect({
60376
- cwd,
60377
- model: "workflow",
60378
- pid: process.pid,
60379
- startedAt: Date.now(),
60380
- lastActivity: Date.now(),
60381
- group: state.route.group,
60382
- groups: [state.route.group],
60383
- recipientPurpose: "control"
60561
+ function drop(connection, client) {
60562
+ if (connection.client === client) {
60563
+ connection.client = undefined;
60564
+ connection.acknowledged = false;
60565
+ for (const state of connection.routes.values()) {
60566
+ settle(state, false);
60567
+ scheduleRetry(state);
60568
+ }
60569
+ }
60570
+ client.disconnect().catch(() => {});
60571
+ }
60572
+ function connect(connection) {
60573
+ if (connection.client?.isConnected())
60574
+ return Promise.resolve(connection.client);
60575
+ connection.connecting ??= (async () => {
60576
+ const [extensionDir] = getBuiltinPackagePaths({
60577
+ workflows: false,
60578
+ subagents: false,
60579
+ mcp: false,
60580
+ "web-access": false
60581
+ });
60582
+ if (extensionDir === undefined)
60583
+ throw new Error("Intercom builtin package is unavailable");
60584
+ await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
60585
+ if (connections.get(connection.key) !== connection)
60586
+ throw closedError();
60587
+ const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
60588
+ multiplexesPendingStageRoutes: true
60589
+ });
60590
+ connection.client = client;
60591
+ connection.acknowledged = false;
60592
+ client.on("error", () => {});
60593
+ client.on("disconnected", () => drop(connection, client));
60594
+ client.on("pending_stage_message", (request) => {
60595
+ if (connection.client !== client)
60596
+ return;
60597
+ const event = { ...request, handled: false };
60598
+ events.emit("atomic:workflow-pending-stage-message", event);
60599
+ const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
60600
+ completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
60601
+ });
60602
+ client.on("sticky_live_delivered", (notice) => {
60603
+ if (connection.client !== client)
60604
+ return;
60605
+ events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
60606
+ });
60607
+ try {
60608
+ await client.connect({
60609
+ cwd,
60610
+ model: "workflow",
60611
+ pid: process.pid,
60612
+ startedAt: Date.now(),
60613
+ lastActivity: Date.now(),
60614
+ group: connection.group,
60615
+ groups: [connection.group],
60616
+ recipientPurpose: "control"
60617
+ });
60618
+ if (connections.get(connection.key) !== connection)
60619
+ throw closedError();
60620
+ } catch (error) {
60621
+ drop(connection, client);
60622
+ throw error;
60623
+ }
60624
+ for (const state of connection.routes.values()) {
60625
+ if (state.registered || state.retry === undefined)
60626
+ continue;
60627
+ clearTimeout(state.retry);
60628
+ state.retry = undefined;
60629
+ publish(state, state.route).catch(() => {});
60630
+ }
60631
+ return client;
60632
+ })().finally(() => {
60633
+ connection.connecting = undefined;
60384
60634
  });
60385
- return client;
60635
+ return connection.connecting;
60386
60636
  }
60387
60637
  function publish(state, route) {
60388
- const completion = state.tail.catch(() => {}).then(() => gate.run(async () => {
60389
- if (state.closed)
60390
- throw new Error("Workflow Intercom route owner closed");
60391
- const establishedClient = state.client?.isConnected() ? state.client : undefined;
60638
+ const completion = state.tail.catch(() => {}).then(async () => {
60639
+ if (!isCurrent(state))
60640
+ throw closedError();
60641
+ const connection = connectionOf(state);
60642
+ const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
60643
+ let client;
60392
60644
  try {
60393
- const client = await connect(state);
60645
+ client = await connect(connection);
60394
60646
  await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
60647
+ if (connection.client !== client || !isCurrent(state))
60648
+ return;
60649
+ connection.acknowledged = true;
60395
60650
  state.attempt = 0;
60396
60651
  clearTimeout(state.retry);
60397
60652
  state.retry = undefined;
60653
+ settle(state, true);
60398
60654
  } catch (error) {
60399
- const client = state.client;
60400
- if (client !== establishedClient || !client?.isConnected()) {
60401
- state.client = undefined;
60402
- await client?.disconnect();
60655
+ if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
60656
+ scheduleRetry(state);
60657
+ throw error;
60403
60658
  }
60659
+ if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
60660
+ detach(state);
60661
+ state.isolated = true;
60662
+ connectionOf(state);
60663
+ }
60664
+ if (client !== undefined && (client !== established || !client.isConnected()))
60665
+ drop(connection, client);
60404
60666
  scheduleRetry(state);
60405
60667
  throw error;
60406
60668
  }
60407
- }));
60669
+ });
60408
60670
  state.tail = completion;
60409
60671
  completion.catch(() => {});
60410
60672
  return completion;
@@ -60413,10 +60675,15 @@ function createWorkflowRouteOwner(cwd) {
60413
60675
  const route = payload;
60414
60676
  let state = routes.get(route.runId);
60415
60677
  if (state === undefined) {
60416
- state = { route, tail: Promise.resolve(), attempt: 0, closed: false };
60678
+ state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
60417
60679
  routes.set(route.runId, state);
60680
+ } else if (state.route.group !== route.group) {
60681
+ detach(state);
60682
+ state.isolated = false;
60683
+ settle(state, false);
60418
60684
  }
60419
60685
  state.route = route;
60686
+ connectionOf(state);
60420
60687
  route.completion = publish(state, route);
60421
60688
  });
60422
60689
  events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
@@ -60427,7 +60694,7 @@ function createWorkflowRouteOwner(cwd) {
60427
60694
  event.handled = true;
60428
60695
  event.completion = (async () => {
60429
60696
  await publish(state, state.route);
60430
- const client = state.client;
60697
+ const client = connections.get(connectionKey(state))?.client;
60431
60698
  if (client === undefined || !client.isConnected())
60432
60699
  return false;
60433
60700
  const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
@@ -60437,18 +60704,33 @@ function createWorkflowRouteOwner(cwd) {
60437
60704
  });
60438
60705
  return {
60439
60706
  events,
60707
+ registered(runId) {
60708
+ const state = routes.get(runId);
60709
+ if (state === undefined)
60710
+ return Promise.resolve(false);
60711
+ if (state.registered)
60712
+ return Promise.resolve(true);
60713
+ return new Promise((resolve) => state.waiters.add(resolve));
60714
+ },
60440
60715
  async closeGroup(group) {
60441
60716
  const closing = [...routes.values()].filter((state) => state.route.group === group);
60717
+ const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
60718
+ for (const connection of closingConnections) {
60719
+ connections.delete(connection.key);
60720
+ connection.routes.clear();
60721
+ }
60442
60722
  for (const state of closing) {
60443
- state.closed = true;
60444
- clearTimeout(state.retry);
60445
60723
  routes.delete(state.route.runId);
60724
+ clearTimeout(state.retry);
60725
+ state.retry = undefined;
60726
+ settle(state, false);
60446
60727
  }
60447
- await Promise.all(closing.map(async (state) => {
60448
- await state.client?.disconnect();
60449
- await state.tail.catch(() => {});
60450
- await state.client?.disconnect();
60728
+ await Promise.all(closingConnections.map(async (connection) => {
60729
+ await connection.client?.disconnect();
60730
+ await connection.connecting?.catch(() => {});
60451
60731
  }));
60732
+ await Promise.all(closing.map((state) => state.tail.catch(() => {})));
60733
+ await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
60452
60734
  }
60453
60735
  };
60454
60736
  }
@@ -60631,7 +60913,7 @@ var init_workflow_infrastructure_error = __esm(() => {
60631
60913
  [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
60632
60914
  [/Local worker retry limit/, "worker_retry_limit"],
60633
60915
  [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
60634
- [/Worker exited before delivery acknowledgement/, "worker_lost"],
60916
+ [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
60635
60917
  [/Stale work lease/, "stale_work_lease"],
60636
60918
  [/Intercom runtime no longer active/, "intercom_unavailable"],
60637
60919
  [/Storage unavailable/, "storage_unavailable"]
@@ -60889,7 +61171,7 @@ var init_definitions = __esm(() => {
60889
61171
  // dist/builtin/workflows/src/durable/journal/pool-mutations.ts
60890
61172
  import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
60891
61173
  async function validatePoolMutation(tx, journal, root, work, mutation) {
60892
- if (!["checkpoint", "session-create", "session-append", "session-replace"].includes(mutation.kind))
61174
+ if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
60893
61175
  throw new Error("Invalid pool mutation");
60894
61176
  const checkpoint = mutation.checkpoint;
60895
61177
  const prefix = `${work.scope}:`;
@@ -60957,10 +61239,18 @@ async function sessionStore(tx, prefix) {
60957
61239
  return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
60958
61240
  }
60959
61241
  async function applyPoolSession(tx, prefix, workId, mutation) {
61242
+ const owners = `${prefix}session_owners`;
61243
+ if (mutation.kind === "session-append") {
61244
+ if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
61245
+ throw new Error("Pool sessions require database identity");
61246
+ const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
61247
+ if (owner?.work !== workId)
61248
+ throw new Error("Pool session scope mismatch");
61249
+ return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
61250
+ }
60960
61251
  const { sessionId, replayKey } = mutation.checkpoint;
60961
61252
  if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
60962
61253
  throw new Error("Pool sessions require database identity");
60963
- const owners = `${prefix}session_owners`;
60964
61254
  if (mutation.kind === "session-create") {
60965
61255
  if (mutation.header.id !== sessionId)
60966
61256
  throw new Error("Pool session identity mismatch");
@@ -60970,10 +61260,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
60970
61260
  if (owner?.work !== workId || owner.replay_key !== replayKey)
60971
61261
  throw new Error("Pool session scope mismatch");
60972
61262
  }
61263
+ return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
61264
+ }
61265
+ async function withSessionStore(tx, prefix, write) {
60973
61266
  const store = await sessionStore(tx, prefix);
60974
61267
  try {
60975
- const result = mutation.kind === "session-create" ? await store.create(mutation.header, mutation.entries) : mutation.kind === "session-append" ? await store.append(sessionId, mutation.entries, { expectSeq: mutation.expectSeq }) : await store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq });
60976
- return result.lastSeq;
61268
+ return (await write(store)).lastSeq;
60977
61269
  } finally {
60978
61270
  await store.close();
60979
61271
  }
@@ -61021,8 +61313,11 @@ var init_pool_stage_usage = __esm(() => {
61021
61313
  });
61022
61314
 
61023
61315
  // dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
61024
- import { randomUUID as randomUUID15 } from "node:crypto";
61316
+ import { createHash as createHash4, randomUUID as randomUUID15 } from "node:crypto";
61025
61317
  import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
61318
+ function requestIdentity(request) {
61319
+ return createHash4("sha256").update(request).digest("hex");
61320
+ }
61026
61321
  function fromRow(row) {
61027
61322
  return {
61028
61323
  ...JSON.parse(row.descriptor),
@@ -61078,10 +61373,11 @@ class SqlPoolQueue {
61078
61373
  const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
61079
61374
  if (versions.length > 1)
61080
61375
  throw new Error("Corrupt pool schema metadata");
61081
- if (versions.length === 0)
61082
- await tx.run(`INSERT INTO ${pool.meta} VALUES (1)`);
61083
- else
61084
- assertStorageSchemaVersion2("journal", versions[0].version, 1);
61376
+ const version = versions[0]?.version;
61377
+ if (version === undefined)
61378
+ await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
61379
+ else if (version !== 1)
61380
+ assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
61085
61381
  await tx.exec(`
61086
61382
  CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
61087
61383
  CREATE TABLE IF NOT EXISTS ${pool.work}(
@@ -61112,9 +61408,23 @@ class SqlPoolQueue {
61112
61408
  state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
61113
61409
  )`);
61114
61410
  await tx.exec(poolUsageSchema(journal.runs));
61411
+ if (version === 1)
61412
+ await pool.migrateFromVersion1(tx);
61115
61413
  });
61116
61414
  return pool;
61117
61415
  }
61416
+ async migrateFromVersion1(tx) {
61417
+ await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
61418
+ for (let after = 0;; ) {
61419
+ const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
61420
+ if (rows.length === 0)
61421
+ break;
61422
+ for (const row of rows)
61423
+ await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
61424
+ after = Number(rows.at(-1).rowid);
61425
+ }
61426
+ await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
61427
+ }
61118
61428
  async authority(tx, lease, timeout) {
61119
61429
  const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
61120
61430
  if (row?.timeout === null && timeout === undefined)
@@ -61255,31 +61565,18 @@ class SqlPoolQueue {
61255
61565
  throw new Error(STALE_WORK_LEASE);
61256
61566
  return row;
61257
61567
  }
61258
- async renew(lease, token, operationId, durationMs) {
61568
+ async renew(lease, token, durationMs) {
61259
61569
  positive(durationMs);
61260
61570
  lease = snapshotLease(lease);
61261
61571
  token = { ...token };
61262
- validateText("key", operationId);
61263
- const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
61264
61572
  return this.transaction(async (tx) => {
61265
61573
  const { now } = await this.authority(tx, lease);
61266
61574
  const row = await this.current(tx, lease, token);
61267
61575
  if (row.epoch !== lease.epoch || row.expires_at < now)
61268
61576
  throw new Error(STALE_WORK_LEASE);
61269
- const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
61270
- if (previous) {
61271
- if (previous.request !== request)
61272
- throw new Error("Pool operation identity conflict");
61273
- return JSON.parse(previous.receipt);
61274
- }
61275
- const receipt = {
61276
- ...fromRow(row),
61277
- expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
61278
- renewedAt: now
61279
- };
61280
- await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
61281
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
61282
- return receipt;
61577
+ const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
61578
+ await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
61579
+ return { ...fromRow(row), expiresAt, renewedAt: now };
61283
61580
  });
61284
61581
  }
61285
61582
  async commit(lease, token, operationId, mutation) {
@@ -61288,6 +61585,7 @@ class SqlPoolQueue {
61288
61585
  validateText("key", operationId);
61289
61586
  assertWorkflowSerializableObject(mutation, "Pool mutation");
61290
61587
  const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
61588
+ const identity = requestIdentity(request);
61291
61589
  mutation = JSON.parse(request).mutation;
61292
61590
  return this.transaction(async (tx) => {
61293
61591
  const { now, status } = await this.authority(tx, lease);
@@ -61296,7 +61594,7 @@ class SqlPoolQueue {
61296
61594
  throw new Error(STALE_WORK_LEASE);
61297
61595
  const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
61298
61596
  if (previous) {
61299
- if (previous.request !== request)
61597
+ if (previous.request !== identity)
61300
61598
  throw new Error("Pool operation identity conflict");
61301
61599
  return JSON.parse(previous.receipt);
61302
61600
  }
@@ -61304,34 +61602,41 @@ class SqlPoolQueue {
61304
61602
  throw new Error(STALE_WORK_LEASE);
61305
61603
  if (!await this.admitsAttemptWrites(tx, lease, status))
61306
61604
  throw new Error("Pool root is not running");
61307
- const checkpoint = mutation.checkpoint;
61308
- await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
61309
- if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
61310
- throw new Error("Pool checkpoint timestamp is in the future");
61311
- assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
61312
- const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
61313
- const payload = serializeJson(checkpoint);
61314
- const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
61315
- if (existing && existing.payload !== payload)
61316
- throw new Error("Pool checkpoint identity conflict");
61317
- if (existing && mutation.kind !== "checkpoint")
61318
- throw new Error("Pool session checkpoint identity conflict");
61319
- const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
61320
- const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
61321
- const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
61322
- seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
61323
- at: now,
61324
- ...lastSeq === undefined ? {} : { lastSeq }
61325
- };
61326
- if (!existing)
61327
- await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
61328
- const billed = poolCheckpointUsage(checkpoint);
61329
- if (billed && !existing)
61330
- await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
61331
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
61605
+ const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
61606
+ await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
61332
61607
  return receipt;
61333
61608
  });
61334
61609
  }
61610
+ async appendSession(tx, row, mutation, now) {
61611
+ return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
61612
+ }
61613
+ async recordCheckpoint(tx, lease, token, row, mutation, now) {
61614
+ const checkpoint = mutation.checkpoint;
61615
+ await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
61616
+ if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
61617
+ throw new Error("Pool checkpoint timestamp is in the future");
61618
+ assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
61619
+ const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
61620
+ const payload = serializeJson(checkpoint);
61621
+ const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
61622
+ if (existing && existing.payload !== payload)
61623
+ throw new Error("Pool checkpoint identity conflict");
61624
+ if (existing && mutation.kind !== "checkpoint")
61625
+ throw new Error("Pool session checkpoint identity conflict");
61626
+ const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
61627
+ const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
61628
+ const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
61629
+ seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
61630
+ at: now,
61631
+ ...lastSeq === undefined ? {} : { lastSeq }
61632
+ };
61633
+ if (!existing)
61634
+ await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
61635
+ const billed = poolCheckpointUsage(checkpoint);
61636
+ if (billed && !existing)
61637
+ await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
61638
+ return receipt;
61639
+ }
61335
61640
  async complete(lease, token, result) {
61336
61641
  lease = snapshotLease(lease);
61337
61642
  token = { ...token };
@@ -61418,7 +61723,7 @@ function openSqlPoolQueue(journal) {
61418
61723
  return Promise.reject(new Error("Worker pool requires a SQL run journal"));
61419
61724
  return SqlPoolQueue.open(storage);
61420
61725
  }
61421
- var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
61726
+ var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
61422
61727
  var init_sql_pool_queue = __esm(() => {
61423
61728
  init_serializable();
61424
61729
  init_boundary_topology();
@@ -61430,6 +61735,15 @@ var init_sql_pool_queue = __esm(() => {
61430
61735
  init_values();
61431
61736
  });
61432
61737
 
61738
+ // dist/builtin/workflows/src/workers/coordinator-protocol.ts
61739
+ var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
61740
+
61741
+ // dist/builtin/workflows/src/workers/host-options.ts
61742
+ var hostWorkerBindings;
61743
+ var init_host_options = __esm(() => {
61744
+ hostWorkerBindings = Symbol("host-worker-bindings");
61745
+ });
61746
+
61433
61747
  // dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
61434
61748
  function validate(report) {
61435
61749
  validateText("key", report.key);
@@ -61500,10 +61814,870 @@ var init_sql_pool_usage = __esm(() => {
61500
61814
  init_values();
61501
61815
  });
61502
61816
 
61503
- // dist/builtin/workflows/src/workers/host-options.ts
61504
- var hostWorkerBindings;
61505
- var init_host_options = __esm(() => {
61506
- hostWorkerBindings = Symbol("host-worker-bindings");
61817
+ // dist/builtin/workflows/src/workers/pool-coordinator.ts
61818
+ import { fork } from "node:child_process";
61819
+ import { randomUUID as randomUUID16 } from "node:crypto";
61820
+ import { createWriteStream } from "node:fs";
61821
+ import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
61822
+ import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
61823
+ import { setTimeout as delay2 } from "node:timers/promises";
61824
+ function workerLogDirectory(journalPath, root) {
61825
+ return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
61826
+ }
61827
+ function token(work) {
61828
+ return {
61829
+ id: work.id,
61830
+ generation: work.generation,
61831
+ worker: work.worker,
61832
+ epoch: work.epoch,
61833
+ renewedAt: work.renewedAt,
61834
+ expiresAt: work.expiresAt
61835
+ };
61836
+ }
61837
+ function outside(runId) {
61838
+ return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
61839
+ }
61840
+ function isLiveStage(stage) {
61841
+ return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
61842
+ }
61843
+ function projectRootRun(run) {
61844
+ return {
61845
+ id: run.id,
61846
+ name: run.name,
61847
+ status: run.status,
61848
+ startedAt: run.startedAt,
61849
+ stages: run.stages.map((stage) => ({
61850
+ id: stage.id,
61851
+ name: stage.name,
61852
+ status: stage.status,
61853
+ parentIds: [],
61854
+ toolEvents: [],
61855
+ ...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
61856
+ ...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
61857
+ ...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
61858
+ ...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
61859
+ }))
61860
+ };
61861
+ }
61862
+
61863
+ class PooledRoutes {
61864
+ root;
61865
+ owner;
61866
+ mirror = createStore();
61867
+ owners = new Map;
61868
+ attempts = new Map;
61869
+ activated = false;
61870
+ bridge;
61871
+ constructor(root, owner) {
61872
+ this.root = root;
61873
+ this.owner = owner;
61874
+ }
61875
+ static async open(root, cwd, rootRun, forward) {
61876
+ await Promise.resolve().then(() => init_workflow_route_owner());
61877
+ const owner = createWorkflowRouteOwner(cwd);
61878
+ const routes = new PooledRoutes(root, owner);
61879
+ if (rootRun !== undefined)
61880
+ routes.updateRoot(rootRun);
61881
+ routes.bridge = registerPendingStageIntercomBridge({
61882
+ events: {
61883
+ emit: (event, payload) => owner.events.emit(event, payload),
61884
+ on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
61885
+ }
61886
+ }, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
61887
+ for (const [event, kind] of [
61888
+ [PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
61889
+ [STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
61890
+ ])
61891
+ owner.events.on(event, (payload) => {
61892
+ const claimed = payload;
61893
+ if (claimed.handled !== false)
61894
+ return;
61895
+ claimed.handled = true;
61896
+ const { handled: _handled, completion: _completion, ...forwarded } = claimed;
61897
+ claimed.completion = forward({ kind, payload: forwarded });
61898
+ });
61899
+ return routes;
61900
+ }
61901
+ updateRoot(rootRun) {
61902
+ this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
61903
+ }
61904
+ accept(entries) {
61905
+ this.mirror.batch(() => {
61906
+ for (const { work, runs } of entries) {
61907
+ let attempt = this.attempts.get(work.id);
61908
+ if (attempt !== undefined && work.generation < attempt.generation)
61909
+ continue;
61910
+ if (attempt === undefined || attempt.generation !== work.generation) {
61911
+ if (attempt !== undefined)
61912
+ this.release(attempt);
61913
+ attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
61914
+ this.attempts.set(work.id, attempt);
61915
+ }
61916
+ if (attempt.fenced)
61917
+ continue;
61918
+ for (const run of runs) {
61919
+ attempt.runs.add(run.id);
61920
+ this.owners.set(run.id, work.id);
61921
+ const { pendingStageMessages: _messages, ...published } = run;
61922
+ this.mirror.recordRunStart({ ...published, rootRunId: this.root });
61923
+ if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
61924
+ this.activated = true;
61925
+ }
61926
+ }
61927
+ });
61928
+ }
61929
+ fence(work) {
61930
+ const attempt = this.attempts.get(work.id);
61931
+ if (attempt === undefined || attempt.generation > work.generation)
61932
+ return;
61933
+ if (attempt.generation < work.generation) {
61934
+ this.release(attempt);
61935
+ this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
61936
+ return;
61937
+ }
61938
+ this.release(attempt);
61939
+ attempt.fenced = true;
61940
+ }
61941
+ async route(work, runId) {
61942
+ this.requireOwned(work, runId);
61943
+ const readiness = workflowPendingStageRouteReady(this.mirror, runId);
61944
+ await readiness?.completion.catch(() => {});
61945
+ readiness?.assertCurrent();
61946
+ const waiting = new AbortController;
61947
+ await Promise.race([
61948
+ this.owner.registered(runId),
61949
+ delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
61950
+ ]);
61951
+ waiting.abort();
61952
+ this.requireOwned(work, runId);
61953
+ return workflowPendingStageRouteCapability(this.mirror, runId);
61954
+ }
61955
+ async undeliverable(payload) {
61956
+ const event = { ...payload, handled: false };
61957
+ this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
61958
+ return event.handled && await event.completion === true;
61959
+ }
61960
+ async close() {
61961
+ const bridge = this.bridge;
61962
+ if (bridge === undefined)
61963
+ return;
61964
+ this.bridge = undefined;
61965
+ try {
61966
+ await bridge.flush();
61967
+ } finally {
61968
+ bridge.refresh();
61969
+ bridge();
61970
+ await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
61971
+ }
61972
+ }
61973
+ requireOwned(work, runId) {
61974
+ const attempt = this.attempts.get(work.id);
61975
+ if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
61976
+ throw new Error(STALE_WORK_LEASE);
61977
+ if (!attempt.runs.has(runId))
61978
+ throw outside(runId);
61979
+ }
61980
+ release(attempt) {
61981
+ this.mirror.batch(() => {
61982
+ for (const id of attempt.runs) {
61983
+ rotateWorkflowPendingStageRouteCapability(this.mirror, id);
61984
+ const run = this.mirror.runs().find((candidate) => candidate.id === id);
61985
+ if (run?.stages.some(isLiveStage) !== true)
61986
+ continue;
61987
+ this.mirror.recordRunStart({
61988
+ ...run,
61989
+ stages: run.stages.map((stage) => {
61990
+ if (!isLiveStage(stage))
61991
+ return stage;
61992
+ const { sessionId: _session, sessionFile: _file, ...rest } = stage;
61993
+ return { ...rest, attachable: false, attached: false };
61994
+ })
61995
+ });
61996
+ }
61997
+ });
61998
+ }
61999
+ }
62000
+
62001
+ class PoolCoordinator {
62002
+ port;
62003
+ children = new Set;
62004
+ incarnations = new Map;
62005
+ readyWorkers = new Set;
62006
+ logs = new Map;
62007
+ cleanups = new Set;
62008
+ controlAcks = new Map;
62009
+ lastHeard = new Map;
62010
+ pendingActivity = new Map;
62011
+ batches = new Map;
62012
+ publicationFailures = new Map;
62013
+ requests = new Map;
62014
+ deliveryRequests = new Map;
62015
+ inflight = new Set;
62016
+ lastTick = performance.now();
62017
+ stalledMs = 0;
62018
+ ticker;
62019
+ recoveryTimer;
62020
+ flushTimer;
62021
+ lastFlushAt = 0;
62022
+ batchSequence = 0;
62023
+ requestSequence = 0;
62024
+ usageChanged = false;
62025
+ budgetBlocked = false;
62026
+ desiredControl;
62027
+ root;
62028
+ lease;
62029
+ options;
62030
+ journal;
62031
+ queue;
62032
+ usageLedger;
62033
+ usageTotal;
62034
+ admission;
62035
+ routes;
62036
+ publishing = false;
62037
+ sources = [];
62038
+ logDirectory;
62039
+ starting;
62040
+ recovering;
62041
+ nextRecoveryAt = 0;
62042
+ cleanupFailure;
62043
+ closed = false;
62044
+ closing;
62045
+ replacements = 0;
62046
+ constructor(port) {
62047
+ this.port = port;
62048
+ port.onMessage((message) => this.receive(message));
62049
+ this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
62050
+ }
62051
+ async shutdown() {
62052
+ await this.closing?.catch(() => {});
62053
+ while (this.inflight.size > 0)
62054
+ await Promise.allSettled([...this.inflight]);
62055
+ await this.journal?.close().catch(() => {});
62056
+ }
62057
+ receive(message) {
62058
+ if (message.kind === "request-reply") {
62059
+ const pending = this.requests.get(message.id);
62060
+ this.requests.delete(message.id);
62061
+ if (message.error !== undefined)
62062
+ pending?.reject(new Error(message.error));
62063
+ else
62064
+ pending?.resolve(message.value);
62065
+ return;
62066
+ }
62067
+ if ("id" in message) {
62068
+ const command = message;
62069
+ const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
62070
+ this.inflight.add(handled);
62071
+ handled.finally(() => this.inflight.delete(handled));
62072
+ return;
62073
+ }
62074
+ this.notice(message);
62075
+ }
62076
+ notice(notice) {
62077
+ if (notice.kind === "budget")
62078
+ this.budgetBlocked = notice.blocked;
62079
+ else if (notice.kind === "root")
62080
+ this.routes?.updateRoot(notice.rootRun);
62081
+ else {
62082
+ const batch = this.batches.get(notice.batch);
62083
+ this.batches.delete(notice.batch);
62084
+ const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
62085
+ for (const entry of rejected.values())
62086
+ this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
62087
+ this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
62088
+ batch?.settle();
62089
+ }
62090
+ }
62091
+ async execute(command) {
62092
+ switch (command.kind) {
62093
+ case "prepare":
62094
+ return this.prepare(command.root, command.lease, command.options, command.control);
62095
+ case "control":
62096
+ return this.control(command.state);
62097
+ case "start":
62098
+ this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
62099
+ return this.starting;
62100
+ case "enqueue":
62101
+ return this.enqueue(command);
62102
+ case "transport":
62103
+ return this.transport(command.request, command);
62104
+ case "undeliverable":
62105
+ return await this.routes?.undeliverable(command.payload) ?? false;
62106
+ case "close":
62107
+ this.closing ??= this.close();
62108
+ return this.closing;
62109
+ }
62110
+ }
62111
+ async prepare(root, lease, options, control) {
62112
+ if (this.closed)
62113
+ throw new Error("Local worker pool is closed");
62114
+ if (this.root !== undefined && this.root !== root)
62115
+ throw new Error("Local worker pool belongs to another root");
62116
+ this.root = root;
62117
+ this.options = options;
62118
+ this.desiredControl ??= control;
62119
+ this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
62120
+ this.usageLedger ??= await openSqlPoolUsage(this.journal);
62121
+ this.usageTotal = await this.usageLedger.total(root);
62122
+ this.lease = lease;
62123
+ const queue = await openSqlPoolQueue(this.journal);
62124
+ await queue.initialize(lease, COORDINATOR_LEASE_MS);
62125
+ await queue.releasePreviousIncarnations(lease);
62126
+ for (;; ) {
62127
+ const intended = this.desiredControl ?? "running";
62128
+ await queue.setControl(lease, intended);
62129
+ if (intended === (this.desiredControl ?? "running"))
62130
+ break;
62131
+ }
62132
+ this.queue = queue;
62133
+ return this.usageTotal;
62134
+ }
62135
+ async control(state) {
62136
+ this.desiredControl = state;
62137
+ if (!this.queue || !this.lease)
62138
+ return;
62139
+ const control = await this.queue.setControl(this.lease, state);
62140
+ const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
62141
+ while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
62142
+ if (this.desiredControl !== state)
62143
+ return;
62144
+ if (Date.now() >= deadline) {
62145
+ if (state === "quit")
62146
+ break;
62147
+ throw new Error("Worker control acknowledgement unresolved");
62148
+ }
62149
+ await delay2(25);
62150
+ }
62151
+ this.flush();
62152
+ }
62153
+ async start(sources, cwd, publish, rootRun) {
62154
+ if (this.closed)
62155
+ throw new Error("Local worker pool is closed");
62156
+ const root = this.root;
62157
+ if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
62158
+ throw new Error("Local worker pool requires root preparation");
62159
+ this.sources = sources;
62160
+ this.publishing = publish;
62161
+ if (publish) {
62162
+ this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
62163
+ if (this.closed) {
62164
+ await this.routes.close();
62165
+ throw new Error("Local worker pool is closed");
62166
+ }
62167
+ }
62168
+ if (this.options.admission)
62169
+ this.admission = await SqlResourceAdmission.open(this.journal, {
62170
+ ...this.options.admission,
62171
+ root,
62172
+ holder: "coordinator"
62173
+ });
62174
+ this.logDirectory = workerLogDirectory(this.options.journalPath, root);
62175
+ await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
62176
+ return;
62177
+ });
62178
+ this.recoveryTimer = setInterval(() => {
62179
+ this.recoverExpired().catch((error) => this.fail(error));
62180
+ }, RECOVERY_INTERVAL_MS).unref();
62181
+ const workers = Array.from({ length: this.options.workers }, () => this.spawn());
62182
+ try {
62183
+ await Promise.any(workers);
62184
+ } catch (error) {
62185
+ throw error.errors[0];
62186
+ }
62187
+ }
62188
+ async enqueue(command) {
62189
+ if (!this.queue || !this.lease || this.root === undefined)
62190
+ throw new Error("Local worker pool requires root preparation");
62191
+ if (command.rootRun !== undefined)
62192
+ this.routes?.updateRoot(command.rootRun);
62193
+ const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
62194
+ const result = await this.queue.result(this.root, work.id);
62195
+ return { work, ...result === undefined ? {} : { result } };
62196
+ }
62197
+ async transport(request, payload) {
62198
+ const target = this.deliveryRequests.get(request);
62199
+ if (target === undefined || !this.incarnations.has(target.slot.worker))
62200
+ throw new Error("Worker exited before delivery acknowledgement");
62201
+ const { slot, workerRequest } = target;
62202
+ await new Promise((resolve, reject) => {
62203
+ const id = randomUUID16();
62204
+ slot.deliveries.set(id, { resolve, reject });
62205
+ slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
62206
+ if (error) {
62207
+ slot.deliveries.delete(id);
62208
+ reject(error);
62209
+ }
62210
+ });
62211
+ });
62212
+ }
62213
+ request(body) {
62214
+ const id = ++this.requestSequence;
62215
+ return new Promise((resolve, reject) => {
62216
+ this.requests.set(id, { resolve, reject });
62217
+ this.port.send({ kind: "request", id, request: body });
62218
+ });
62219
+ }
62220
+ fail(error) {
62221
+ this.port.send({ kind: "failure", error: error.message });
62222
+ }
62223
+ publish(work, runs) {
62224
+ const failure = this.publicationFailures.get(work.id);
62225
+ if (failure?.generation === work.generation)
62226
+ throw new Error(failure.error);
62227
+ if (!this.publishing)
62228
+ return;
62229
+ this.pendingActivity.set(work.id, { work: token(work), runs });
62230
+ this.scheduleFlush();
62231
+ }
62232
+ scheduleFlush() {
62233
+ if (this.flushTimer !== undefined || this.closed)
62234
+ return;
62235
+ const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
62236
+ this.flushTimer = setTimeout(() => {
62237
+ this.flushTimer = undefined;
62238
+ if (this.batches.size > 0) {
62239
+ this.lastFlushAt = performance.now();
62240
+ this.scheduleFlush();
62241
+ return;
62242
+ }
62243
+ this.flush();
62244
+ }, wait);
62245
+ }
62246
+ flush() {
62247
+ clearTimeout(this.flushTimer);
62248
+ this.flushTimer = undefined;
62249
+ if (this.pendingActivity.size === 0 && !this.usageChanged) {
62250
+ const last = [...this.batches.values()].at(-1);
62251
+ return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
62252
+ }
62253
+ this.lastFlushAt = performance.now();
62254
+ const batch = ++this.batchSequence;
62255
+ const entries = [...this.pendingActivity.values()];
62256
+ this.pendingActivity.clear();
62257
+ const usage = this.usageChanged ? this.usageTotal : undefined;
62258
+ this.usageChanged = false;
62259
+ const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
62260
+ this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
62261
+ return settled;
62262
+ }
62263
+ chainSettle(batch, resolve) {
62264
+ const settle = batch.settle;
62265
+ batch.settle = () => {
62266
+ settle();
62267
+ resolve();
62268
+ };
62269
+ }
62270
+ noteUsage(usage) {
62271
+ this.usageTotal = usage;
62272
+ this.usageChanged = true;
62273
+ this.scheduleFlush();
62274
+ }
62275
+ async sendResult(workId) {
62276
+ const result = await this.queue.result(this.root, workId);
62277
+ if (result === undefined)
62278
+ return false;
62279
+ this.flush();
62280
+ this.port.send({
62281
+ kind: "result",
62282
+ workId,
62283
+ result,
62284
+ ...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
62285
+ });
62286
+ return true;
62287
+ }
62288
+ fence(work) {
62289
+ this.pendingActivity.delete(work.id);
62290
+ this.routes?.fence(work);
62291
+ this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
62292
+ }
62293
+ spawn() {
62294
+ if (this.closed)
62295
+ return Promise.reject(new Error("Local worker pool is closed"));
62296
+ const root = this.root;
62297
+ const options = this.options;
62298
+ const child = fork(options.workerEntry ?? process.argv[1], [
62299
+ JSON.stringify({
62300
+ journalPath: options.journalPath,
62301
+ root,
62302
+ definitions: this.sources,
62303
+ admission: options.admission
62304
+ })
62305
+ ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62306
+ const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62307
+ this.children.add(child);
62308
+ if (child.pid !== undefined)
62309
+ this.port.send({ kind: "spawned", pid: child.pid });
62310
+ const worker = randomUUID16();
62311
+ this.incarnations.set(worker, child);
62312
+ const slot = { worker, child, deliveries: new Map };
62313
+ let ready = false;
62314
+ let serial = Promise.resolve();
62315
+ const logPath = join28(this.logDirectory, `${worker}.log`);
62316
+ this.logs.set(child, logPath);
62317
+ let logFailure;
62318
+ const log = createWriteStream(logPath, { flags: "a" });
62319
+ log.on("error", (error) => {
62320
+ logFailure = error.message;
62321
+ });
62322
+ let stderr = "";
62323
+ child.stderr?.on("data", (chunk) => {
62324
+ stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62325
+ });
62326
+ child.stderr?.on("error", (error) => {
62327
+ logFailure ??= error.message;
62328
+ log.end();
62329
+ });
62330
+ child.stderr?.pipe(log);
62331
+ const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62332
+ const startupTimeoutMs = options.startupTimeoutMs;
62333
+ let timedOut = false;
62334
+ let terminated = false;
62335
+ const terminate = () => {
62336
+ if (terminated)
62337
+ return;
62338
+ terminated = true;
62339
+ child.kill("SIGKILL");
62340
+ };
62341
+ const replied = (error) => {
62342
+ if (error)
62343
+ terminate();
62344
+ };
62345
+ return new Promise((resolve, reject) => {
62346
+ const startup = setTimeout(() => {
62347
+ timedOut = true;
62348
+ terminate();
62349
+ }, startupTimeoutMs);
62350
+ child.on("error", (error) => {
62351
+ clearTimeout(startup);
62352
+ reject(error);
62353
+ terminate();
62354
+ });
62355
+ const closed = new Promise((resolve) => child.once("close", () => resolve()));
62356
+ child.once("exit", () => {
62357
+ Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62358
+ this.children.delete(child);
62359
+ this.logs.delete(child);
62360
+ if (child.pid !== undefined)
62361
+ this.port.send({ kind: "exited", pid: child.pid });
62362
+ clearTimeout(startup);
62363
+ this.incarnations.delete(worker);
62364
+ this.readyWorkers.delete(worker);
62365
+ this.controlAcks.delete(worker);
62366
+ this.lastHeard.delete(worker);
62367
+ if (slot.work)
62368
+ this.fence(slot.work);
62369
+ for (const delivery of slot.deliveries.values())
62370
+ delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62371
+ slot.deliveries.clear();
62372
+ const cleanup = disconnected.then(() => serial).then(async () => {
62373
+ try {
62374
+ if (await this.admission?.releaseStoppedSessions(worker))
62375
+ throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62376
+ } catch (error) {
62377
+ this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62378
+ throw this.cleanupFailure;
62379
+ }
62380
+ }).then(async () => {
62381
+ if (this.closed || !ready || this.desiredControl === "quit")
62382
+ return;
62383
+ let work = slot.work;
62384
+ if (work && await this.sendResult(work.id)) {
62385
+ work = undefined;
62386
+ slot.work = undefined;
62387
+ this.replacements = 0;
62388
+ }
62389
+ if (this.closed)
62390
+ return;
62391
+ if (work)
62392
+ await this.queue.releaseExited(this.lease, work);
62393
+ const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62394
+ if (exhausted)
62395
+ this.fail(diagnosed("Local worker retry limit"));
62396
+ else
62397
+ await this.spawn();
62398
+ }).catch((error) => {
62399
+ this.fail(error);
62400
+ });
62401
+ this.cleanups.add(cleanup);
62402
+ cleanup.finally(() => this.cleanups.delete(cleanup));
62403
+ });
62404
+ child.on("message", (message) => {
62405
+ this.observeResponsiveness();
62406
+ this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
62407
+ if (message.kind === "delivery-ack") {
62408
+ const pending = slot.deliveries.get(message.deliveryId);
62409
+ slot.deliveries.delete(message.deliveryId);
62410
+ if (message.error)
62411
+ pending?.reject(new Error(message.error));
62412
+ else
62413
+ pending?.resolve();
62414
+ return;
62415
+ }
62416
+ let routeReady;
62417
+ const handled = serial.then(async () => {
62418
+ if (message.kind === "ready") {
62419
+ if (terminated)
62420
+ return;
62421
+ ready = true;
62422
+ this.readyWorkers.add(worker);
62423
+ clearTimeout(startup);
62424
+ resolve();
62425
+ return;
62426
+ }
62427
+ await this.starting;
62428
+ const queue = this.queue;
62429
+ const lease = this.lease;
62430
+ if (message.kind === "control")
62431
+ return queue.control(lease);
62432
+ if (message.kind === "controlled") {
62433
+ const control = await queue.control(lease);
62434
+ if (message.revision > control.revision)
62435
+ throw new Error("Unknown control revision");
62436
+ this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62437
+ return;
62438
+ }
62439
+ if (message.kind === "claim") {
62440
+ if (slot.work)
62441
+ throw new Error("Worker already holds an attempt");
62442
+ if (this.budgetBlocked)
62443
+ return;
62444
+ await this.recoverExpired();
62445
+ slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
62446
+ definition: `${source.path}#${source.exportName}`,
62447
+ digest: source.digest
62448
+ })), options.leaseMs);
62449
+ return slot.work;
62450
+ }
62451
+ let work = slot.work;
62452
+ if (!work)
62453
+ throw new Error("Worker has no active attempt");
62454
+ if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
62455
+ work = await queue.extend(lease, work, options.leaseMs);
62456
+ slot.work = work;
62457
+ }
62458
+ if (message.kind === "activity")
62459
+ return this.publish(work, message.runs);
62460
+ if (message.kind === "route") {
62461
+ if (this.routes === undefined)
62462
+ return;
62463
+ await this.flush();
62464
+ this.assertPublished(work);
62465
+ routeReady = this.routes.route(work, message.runId);
62466
+ return;
62467
+ }
62468
+ if (message.kind === "delivery-ready") {
62469
+ if (!this.publishing)
62470
+ return;
62471
+ await this.flush();
62472
+ this.assertPublished(work);
62473
+ return this.request({
62474
+ kind: "ready",
62475
+ work: token(work),
62476
+ runId: message.runId,
62477
+ stageId: message.stageId,
62478
+ stageName: message.stageName,
62479
+ ...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
62480
+ });
62481
+ }
62482
+ if (message.kind === "delivery") {
62483
+ if (!message.recipient)
62484
+ throw new Error("Worker delivery requires a session identity");
62485
+ if (!this.publishing)
62486
+ return;
62487
+ await this.flush();
62488
+ this.assertPublished(work);
62489
+ const id = ++this.requestSequence;
62490
+ this.deliveryRequests.set(id, { slot, workerRequest: message.id });
62491
+ try {
62492
+ return await new Promise((resolve, reject) => {
62493
+ this.requests.set(id, { resolve, reject });
62494
+ this.port.send({
62495
+ kind: "request",
62496
+ id,
62497
+ request: {
62498
+ kind: "deliver",
62499
+ work: token(work),
62500
+ runId: message.runId,
62501
+ stageId: message.stageId,
62502
+ stageName: message.stageName,
62503
+ recipient: message.recipient
62504
+ }
62505
+ });
62506
+ });
62507
+ } finally {
62508
+ this.deliveryRequests.delete(id);
62509
+ }
62510
+ }
62511
+ if (message.kind === "renew")
62512
+ return;
62513
+ if (message.kind === "commit") {
62514
+ const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62515
+ this.noteUsage(await this.usageLedger.total(root));
62516
+ return receipt;
62517
+ }
62518
+ if (message.kind === "usage") {
62519
+ this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
62520
+ return;
62521
+ }
62522
+ if (message.kind === "budget") {
62523
+ await queue.assertCurrent(lease, work);
62524
+ if (this.budgetBlocked)
62525
+ throw new Error("Root workflow budget exhausted");
62526
+ return queue.control(lease);
62527
+ }
62528
+ if (message.kind !== "complete")
62529
+ throw new Error("Invalid worker operation");
62530
+ await queue.complete(lease, work, message.result);
62531
+ this.replacements = 0;
62532
+ slot.work = undefined;
62533
+ await this.sendResult(work.id);
62534
+ });
62535
+ const response = handled.then((value2) => routeReady ?? value2);
62536
+ if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62537
+ serial = handled.then(() => {}, () => {});
62538
+ response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62539
+ });
62540
+ });
62541
+ }
62542
+ assertPublished(work) {
62543
+ const failure = this.publicationFailures.get(work.id);
62544
+ if (failure?.generation === work.generation)
62545
+ throw new Error(failure.error);
62546
+ }
62547
+ recoverExpired() {
62548
+ if (this.recovering)
62549
+ return this.recovering;
62550
+ const root = this.root;
62551
+ if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
62552
+ return Promise.resolve();
62553
+ const queue = this.queue;
62554
+ const lease = this.lease;
62555
+ const leaseMs = this.options.leaseMs;
62556
+ this.recovering = (async () => {
62557
+ await new Promise((resolve) => setImmediate(resolve));
62558
+ await this.admission?.reconcile();
62559
+ const expiredBefore = Date.now();
62560
+ for (const work of await queue.inspect(root)) {
62561
+ if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62562
+ continue;
62563
+ const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62564
+ if (child && work.worker !== null && this.isLive(work.worker, child)) {
62565
+ await queue.extend(lease, work, leaseMs).catch((error) => {
62566
+ if (error.message !== STALE_WORK_LEASE)
62567
+ throw error;
62568
+ });
62569
+ continue;
62570
+ }
62571
+ if (child && !await this.stop(child).then(() => true, () => false))
62572
+ continue;
62573
+ await queue.expire(lease, work);
62574
+ }
62575
+ })().finally(() => {
62576
+ this.recovering = undefined;
62577
+ this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62578
+ });
62579
+ return this.recovering;
62580
+ }
62581
+ observeResponsiveness() {
62582
+ const now = performance.now();
62583
+ const gap = now - this.lastTick;
62584
+ if (gap > STALL_THRESHOLD_MS)
62585
+ this.stalledMs += gap - TICK_INTERVAL_MS;
62586
+ this.lastTick = now;
62587
+ }
62588
+ isLive(worker, child) {
62589
+ this.observeResponsiveness();
62590
+ const heard = this.lastHeard.get(worker);
62591
+ return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
62592
+ }
62593
+ stop(child) {
62594
+ if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62595
+ return Promise.resolve();
62596
+ return new Promise((resolve, reject) => {
62597
+ const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62598
+ child.once("exit", () => {
62599
+ clearTimeout(timeout);
62600
+ resolve();
62601
+ });
62602
+ child.kill("SIGKILL");
62603
+ });
62604
+ }
62605
+ async close() {
62606
+ this.closed = true;
62607
+ clearInterval(this.ticker);
62608
+ clearInterval(this.recoveryTimer);
62609
+ let failure;
62610
+ let stopping = [];
62611
+ try {
62612
+ await this.recovering?.catch(() => {});
62613
+ stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62614
+ await Promise.all(stopping.map(({ child }) => this.stop(child)));
62615
+ await Promise.all([...this.cleanups]);
62616
+ if (this.cleanupFailure)
62617
+ throw this.cleanupFailure;
62618
+ } catch (error) {
62619
+ failure = error;
62620
+ }
62621
+ if (failure === undefined) {
62622
+ const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62623
+ await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62624
+ if (this.logDirectory !== undefined)
62625
+ await rmdir(this.logDirectory).catch(() => {});
62626
+ }
62627
+ this.flush();
62628
+ try {
62629
+ await this.routes?.close();
62630
+ } catch (error) {
62631
+ failure ??= error;
62632
+ }
62633
+ for (const batch of this.batches.values())
62634
+ batch.settle();
62635
+ this.batches.clear();
62636
+ for (const pending of this.requests.values())
62637
+ pending.reject(new Error("Local worker pool is closed"));
62638
+ this.requests.clear();
62639
+ if (failure !== undefined)
62640
+ throw failure;
62641
+ }
62642
+ }
62643
+ function processPort() {
62644
+ return {
62645
+ send(message) {
62646
+ if (process.connected)
62647
+ process.send?.(message, () => {});
62648
+ },
62649
+ onMessage(listener) {
62650
+ process.on("message", (message) => listener(message));
62651
+ },
62652
+ onClose(listener) {
62653
+ process.once("disconnect", () => listener(new Error("Root host disconnected")));
62654
+ },
62655
+ async close() {
62656
+ if (process.connected)
62657
+ process.disconnect();
62658
+ }
62659
+ };
62660
+ }
62661
+ function runPoolCoordinator() {
62662
+ if (typeof process.send !== "function")
62663
+ throw new Error("Local worker pool coordinator requires an IPC channel");
62664
+ const port = processPort();
62665
+ const coordinator = new PoolCoordinator(port);
62666
+ port.onClose(() => process.exit(0));
62667
+ return coordinator;
62668
+ }
62669
+ var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
62670
+ var init_pool_coordinator = __esm(() => {
62671
+ init_resolution();
62672
+ init_sql_pool_queue();
62673
+ init_sql_pool_usage();
62674
+ init_pending_stage_intercom();
62675
+ init_intercom_group();
62676
+ init_pending_stage_route_capability();
62677
+ init_pending_stage_route_readiness();
62678
+ init_store();
62679
+ init_resource_admission();
62680
+ COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
61507
62681
  });
61508
62682
 
61509
62683
  // dist/builtin/workflows/src/workers/protocol.ts
@@ -61753,21 +62927,26 @@ class PoolStageHandoff {
61753
62927
  store;
61754
62928
  queue;
61755
62929
  lease;
62930
+ claimRun;
61756
62931
  attempts = new Map;
61757
62932
  owners = new Map;
61758
62933
  publications = new Map;
61759
62934
  drains = new Map;
61760
- constructor(backend, store, queue, lease) {
62935
+ constructor(backend, store, queue, lease, claimRun) {
61761
62936
  this.backend = backend;
61762
62937
  this.store = store;
61763
62938
  this.queue = queue;
61764
62939
  this.lease = lease;
62940
+ this.claimRun = claimRun;
61765
62941
  }
61766
62942
  publish(work, runs) {
61767
62943
  const previous = this.publications.get(work.id) ?? Promise.resolve();
61768
62944
  const publication = previous.catch(() => {
61769
62945
  return;
61770
- }).then(() => this.apply(work, runs));
62946
+ }).then(async () => {
62947
+ const commit = await this.prepare(work, runs);
62948
+ this.store.batch(commit);
62949
+ });
61771
62950
  const settled = publication.catch(() => {
61772
62951
  return;
61773
62952
  });
@@ -61778,6 +62957,21 @@ class PoolStageHandoff {
61778
62957
  });
61779
62958
  return publication;
61780
62959
  }
62960
+ async publishBatch(publications) {
62961
+ const prepared = [];
62962
+ for (const { work, runs } of publications)
62963
+ prepared.push(await this.prepare(work, runs).catch((error) => error));
62964
+ return this.store.batch(() => prepared.map((commit) => {
62965
+ if (commit instanceof Error)
62966
+ return commit;
62967
+ try {
62968
+ commit();
62969
+ return;
62970
+ } catch (error) {
62971
+ return error instanceof Error ? error : new Error(String(error));
62972
+ }
62973
+ }));
62974
+ }
61781
62975
  async route(work, runId) {
61782
62976
  this.requireOwned(await this.begin(work), runId);
61783
62977
  const readiness = workflowPendingStageRouteReady(this.store, runId);
@@ -61828,7 +63022,7 @@ class PoolStageHandoff {
61828
63022
  const result = await this.queue.result(root, work.id);
61829
63023
  if (result === undefined)
61830
63024
  return;
61831
- await this.backend.hydrateWorkflow(root);
63025
+ await this.backend.catchUpWorkflow(root);
61832
63026
  const terminals = new Map;
61833
63027
  for (const checkpoint of this.backend.listCheckpoints(root)) {
61834
63028
  const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
@@ -61838,6 +63032,9 @@ class PoolStageHandoff {
61838
63032
  if (child?.runId === boundary.child.runId)
61839
63033
  terminals.set(child.runId, { status: child.status, outputs: child.outputs });
61840
63034
  }
63035
+ this.store.batch(() => this.settle(work, result, terminals));
63036
+ }
63037
+ settle(work, result, terminals) {
61841
63038
  for (const run of this.store.runs()) {
61842
63039
  if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
61843
63040
  continue;
@@ -61861,22 +63058,31 @@ class PoolStageHandoff {
61861
63058
  if (current !== undefined && current.generation > work.generation)
61862
63059
  return;
61863
63060
  const attempt = current?.generation === work.generation ? current : this.start(work, current);
61864
- this.release(attempt);
63061
+ this.store.batch(() => this.release(attempt));
61865
63062
  attempt.fenced = true;
61866
63063
  }
61867
- async apply(work, runs) {
63064
+ async prepare(work, runs) {
61868
63065
  const subtree = this.subtree(work, runs);
61869
63066
  const root = this.lease.runId;
61870
- const known = new Map(this.store.runs().map((run) => [run.id, run]));
61871
- for (const run of subtree) {
61872
- const existing = known.get(run.id);
61873
- const owner = this.owners.get(run.id);
61874
- if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
61875
- throw outside(run.id);
61876
- }
61877
- if (subtree.some((run) => known.get(run.id) === undefined))
61878
- await this.backend.hydrateWorkflow(root);
63067
+ const owned = (known) => {
63068
+ for (const run of subtree) {
63069
+ const existing = known.get(run.id);
63070
+ const owner = this.owners.get(run.id);
63071
+ if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
63072
+ throw outside2(run.id);
63073
+ }
63074
+ };
63075
+ const before = new Map(this.store.runs().map((run) => [run.id, run]));
63076
+ owned(before);
63077
+ if (subtree.some((run) => before.get(run.id) === undefined))
63078
+ await this.backend.catchUpWorkflow(root);
61879
63079
  await this.queue.assertCurrent(this.lease, work);
63080
+ return () => this.commit(work, subtree, owned);
63081
+ }
63082
+ commit(work, subtree, owned) {
63083
+ const root = this.lease.runId;
63084
+ const known = new Map(this.store.runs().map((run) => [run.id, run]));
63085
+ owned(known);
61880
63086
  const attempt = this.enter(work);
61881
63087
  const execution = {
61882
63088
  worker: work.worker ?? "",
@@ -61888,11 +63094,13 @@ class PoolStageHandoff {
61888
63094
  ...run,
61889
63095
  rootRunId: root,
61890
63096
  pendingStageMessages: [
61891
- ...this.store.runs().find((current) => current.id === run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
63097
+ ...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
61892
63098
  ],
61893
63099
  stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
61894
63100
  }));
61895
63101
  const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
63102
+ for (const run of published)
63103
+ this.claimRun?.(run.id);
61896
63104
  for (const run of published) {
61897
63105
  attempt.runs.add(run.id);
61898
63106
  this.owners.set(run.id, work.id);
@@ -61917,7 +63125,7 @@ class PoolStageHandoff {
61917
63125
  const byId = new Map;
61918
63126
  for (const run of runs) {
61919
63127
  if (run.id === root || byId.has(run.id))
61920
- throw outside(run.id);
63128
+ throw outside2(run.id);
61921
63129
  byId.set(run.id, run);
61922
63130
  }
61923
63131
  if (runs.length === 0)
@@ -61930,25 +63138,25 @@ class PoolStageHandoff {
61930
63138
  const top = byId.get(work.childRunId);
61931
63139
  if (top !== undefined) {
61932
63140
  if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
61933
- throw outside(top.id);
63141
+ throw outside2(top.id);
61934
63142
  const pending = [top];
61935
63143
  for (let run = pending.shift();run !== undefined; run = pending.shift()) {
61936
63144
  if (run.rootRunId !== undefined && run.rootRunId !== root)
61937
- throw outside(run.id);
63145
+ throw outside2(run.id);
61938
63146
  ordered.push(run);
61939
63147
  for (const child of runs) {
61940
63148
  if (child.parentRunId !== run.id)
61941
63149
  continue;
61942
63150
  const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
61943
63151
  if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
61944
- throw outside(child.id);
63152
+ throw outside2(child.id);
61945
63153
  pending.push(child);
61946
63154
  }
61947
63155
  }
61948
63156
  }
61949
63157
  const stray = runs.find((run) => !ordered.includes(run));
61950
63158
  if (stray !== undefined)
61951
- throw outside(stray.id);
63159
+ throw outside2(stray.id);
61952
63160
  return ordered;
61953
63161
  }
61954
63162
  async begin(work) {
@@ -62003,7 +63211,7 @@ class PoolStageHandoff {
62003
63211
  }
62004
63212
  requireOwned(attempt, runId) {
62005
63213
  if (!attempt.runs.has(runId))
62006
- throw outside(runId);
63214
+ throw outside2(runId);
62007
63215
  }
62008
63216
  delivery(attempt, work, runId, stageId, stageName) {
62009
63217
  const key = JSON.stringify([runId, stageId]);
@@ -62021,7 +63229,7 @@ class PoolStageHandoff {
62021
63229
  return delivery;
62022
63230
  }
62023
63231
  }
62024
- function outside(runId) {
63232
+ function outside2(runId) {
62025
63233
  return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
62026
63234
  }
62027
63235
  function isLive(stage) {
@@ -62046,33 +63254,122 @@ var init_stage_handoff = __esm(() => {
62046
63254
  });
62047
63255
 
62048
63256
  // dist/builtin/workflows/src/workers/local-pool.ts
62049
- import { fork } from "node:child_process";
62050
- import { randomUUID as randomUUID16 } from "node:crypto";
62051
- import { createWriteStream } from "node:fs";
62052
- import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
62053
- import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
62054
- import { setTimeout as delay2 } from "node:timers/promises";
63257
+ import { fork as fork2 } from "node:child_process";
63258
+ import { resolve as resolve12 } from "node:path";
63259
+ import { setTimeout as delay3 } from "node:timers/promises";
62055
63260
  import { fileURLToPath as fileURLToPath5 } from "node:url";
62056
63261
  import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
62057
- function workerLogDirectory(journalPath, root) {
62058
- return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
63262
+ function defaultWorkerEntry() {
63263
+ return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
63264
+ }
63265
+ function launchCoordinatorProcess() {
63266
+ const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
63267
+ stdio: ["ignore", "ignore", "pipe", "ipc"],
63268
+ execArgv: []
63269
+ });
63270
+ let stderr = "";
63271
+ child.stderr?.on("data", (chunk) => {
63272
+ stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
63273
+ });
63274
+ const listeners = [];
63275
+ let lostReason;
63276
+ const lose = (reason) => {
63277
+ if (lostReason !== undefined)
63278
+ return;
63279
+ lostReason = reason;
63280
+ for (const listener of listeners)
63281
+ listener(reason);
63282
+ };
63283
+ const exited = new Promise((resolve) => child.once("exit", () => resolve()));
63284
+ child.on("error", lose);
63285
+ child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
63286
+ return {
63287
+ pid: child.pid,
63288
+ port: {
63289
+ send(message) {
63290
+ if (lostReason !== undefined || !child.connected)
63291
+ return;
63292
+ child.send(message, (error) => {
63293
+ if (error)
63294
+ lose(error);
63295
+ });
63296
+ },
63297
+ onMessage(listener) {
63298
+ child.on("message", (message) => listener(message));
63299
+ },
63300
+ onClose(listener) {
63301
+ if (lostReason !== undefined)
63302
+ listener(lostReason);
63303
+ else
63304
+ listeners.push(listener);
63305
+ },
63306
+ async close() {
63307
+ if (child.connected)
63308
+ child.disconnect();
63309
+ if (child.exitCode !== null || child.signalCode !== null)
63310
+ return;
63311
+ const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
63312
+ if (timedOut) {
63313
+ child.kill("SIGKILL");
63314
+ await exited;
63315
+ }
63316
+ }
63317
+ }
63318
+ };
63319
+ }
63320
+ function launchCoordinatorInProcess() {
63321
+ const mainListeners = [];
63322
+ const coordinatorListeners = [];
63323
+ const copy = (message) => JSON.parse(JSON.stringify(message));
63324
+ const coordinator = new PoolCoordinator({
63325
+ send(message) {
63326
+ const copied = copy(message);
63327
+ setImmediate(() => {
63328
+ for (const listener of mainListeners)
63329
+ listener(copied);
63330
+ });
63331
+ },
63332
+ onMessage(listener) {
63333
+ coordinatorListeners.push(listener);
63334
+ },
63335
+ onClose() {},
63336
+ async close() {}
63337
+ });
63338
+ return {
63339
+ pid: process.pid,
63340
+ port: {
63341
+ send(message) {
63342
+ const copied = copy(message);
63343
+ setImmediate(() => {
63344
+ for (const listener of coordinatorListeners)
63345
+ listener(copied);
63346
+ });
63347
+ },
63348
+ onMessage(listener) {
63349
+ mainListeners.push(listener);
63350
+ },
63351
+ onClose() {},
63352
+ async close() {
63353
+ coordinator.shutdown();
63354
+ }
63355
+ }
63356
+ };
62059
63357
  }
62060
63358
 
62061
63359
  class LocalWorkerPool {
62062
63360
  backend;
62063
63361
  options;
62064
- children = new Set;
62065
- incarnations = new Map;
62066
- readyWorkers = new Set;
62067
- logs = new Map;
62068
- logDirectory;
62069
- cleanups = new Set;
62070
- recovering;
62071
- nextRecoveryAt = 0;
62072
- controlAcks = new Map;
62073
- lastHeard = new Map;
62074
- lastTick = performance.now();
62075
- rootStalledMs = 0;
63362
+ pids = new Set;
63363
+ coordinator;
63364
+ coordinatorLost;
63365
+ commands = new Map;
63366
+ commandSequence = 0;
63367
+ inbound = Promise.resolve();
63368
+ works = new Map;
63369
+ attempts = new Map;
63370
+ results = new Map;
63371
+ changed = Promise.withResolvers();
63372
+ waiting = 0;
62076
63373
  ticker;
62077
63374
  leaseMs;
62078
63375
  desiredControl;
@@ -62080,18 +63377,17 @@ class LocalWorkerPool {
62080
63377
  store;
62081
63378
  handoff;
62082
63379
  releaseRoutes;
63380
+ routeClaims = new Map;
62083
63381
  sources = [];
62084
63382
  queue;
62085
63383
  lease;
62086
- admission;
62087
- usageLedger;
62088
63384
  usageTotal;
62089
63385
  rootBudget;
63386
+ budgetBlocked = false;
63387
+ rootSignature;
62090
63388
  starting;
62091
- cleanupFailure;
62092
63389
  failure;
62093
63390
  closed = false;
62094
- replacements = 0;
62095
63391
  releaseBinding;
62096
63392
  constructor(backend, options) {
62097
63393
  this.backend = backend;
@@ -62106,10 +63402,12 @@ class LocalWorkerPool {
62106
63402
  this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
62107
63403
  if (options.admission && options.rootId === undefined)
62108
63404
  workflowResourceAdmission.set(backend, options.admission);
62109
- this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
62110
63405
  }
62111
63406
  get workerPids() {
62112
- return [...this.children].flatMap((child) => child.pid === undefined ? [] : [child.pid]);
63407
+ return [...this.pids];
63408
+ }
63409
+ get coordinatorPid() {
63410
+ return this.coordinator?.pid;
62113
63411
  }
62114
63412
  usage() {
62115
63413
  return this.usageTotal;
@@ -62117,47 +63415,39 @@ class LocalWorkerPool {
62117
63415
  async prepare(root, store) {
62118
63416
  if (this.closed)
62119
63417
  throw new Error("Local worker pool is closed");
62120
- if (!(this.backend.journal instanceof SqlRunJournal) || resolve11(this.options.journalPath) !== resolve11(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
63418
+ if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
62121
63419
  throw new Error("Local workers require the same default-prefix SQLite journal");
62122
63420
  if (this.root !== undefined && this.root !== root)
62123
63421
  throw new Error("Local worker pool belongs to another root");
62124
63422
  this.root = root;
62125
63423
  if (store)
62126
63424
  this.store = store;
62127
- this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
62128
- this.usageTotal = await this.usageLedger.total(root);
62129
63425
  const state = await this.backend.journal.leaseState(root);
62130
63426
  if (!state || state.holderId !== this.backend.holderId)
62131
63427
  throw new Error("Local worker pool requires the root owner");
62132
63428
  this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
62133
- const queue = await openSqlPoolQueue(this.backend.journal);
62134
- await queue.initialize(this.lease, COORDINATOR_LEASE_MS);
62135
- await queue.releasePreviousIncarnations(this.lease);
62136
- for (;; ) {
62137
- const intended = this.desiredControl ?? "running";
62138
- await queue.setControl(this.lease, intended);
62139
- if (intended === (this.desiredControl ?? "running"))
62140
- break;
62141
- }
62142
- this.queue = queue;
63429
+ const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
63430
+ this.usageTotal = await this.command({
63431
+ kind: "prepare",
63432
+ root,
63433
+ lease: this.lease,
63434
+ options: {
63435
+ journalPath: this.options.journalPath,
63436
+ workers: this.options.workers,
63437
+ leaseMs: this.leaseMs,
63438
+ startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
63439
+ ...this.options.admission === undefined ? {} : { admission: this.options.admission },
63440
+ ...workerEntry === undefined ? {} : { workerEntry }
63441
+ },
63442
+ ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
63443
+ });
63444
+ this.queue ??= await openSqlPoolQueue(this.backend.journal);
62143
63445
  }
62144
63446
  async control(state) {
62145
63447
  this.desiredControl = state;
62146
63448
  try {
62147
- if (!this.queue || !this.lease)
62148
- return;
62149
- const control = await this.queue.setControl(this.lease, state);
62150
- const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
62151
- while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
62152
- if (this.desiredControl !== state)
62153
- return;
62154
- if (Date.now() >= deadline) {
62155
- if (state === "quit")
62156
- break;
62157
- throw new Error("Worker control acknowledgement unresolved");
62158
- }
62159
- await delay2(25);
62160
- }
63449
+ if (this.coordinator !== undefined)
63450
+ await this.command({ kind: "control", state });
62161
63451
  } finally {
62162
63452
  if (state === "quit")
62163
63453
  await this.close();
@@ -62203,290 +63493,61 @@ class LocalWorkerPool {
62203
63493
  throw new Error(`Worker definition is not importable: ${binding.exportName}`);
62204
63494
  return source;
62205
63495
  }));
62206
- if (this.store) {
62207
- this.releaseRoutes = await acquireStandaloneIntercom(this.store, root, String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd()));
63496
+ const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
63497
+ const store = this.store;
63498
+ if (store) {
63499
+ this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
62208
63500
  if (this.closed) {
62209
63501
  await this.releaseRoutes();
62210
63502
  this.releaseRoutes = undefined;
62211
63503
  throw new Error("Local worker pool is closed");
62212
63504
  }
62213
- this.handoff = new PoolStageHandoff(this.backend, this.store, this.queue, this.lease);
62214
- }
62215
- if (this.options.admission)
62216
- this.admission = await SqlResourceAdmission.open(this.backend.journal, {
62217
- ...this.options.admission,
62218
- root,
62219
- holder: "coordinator"
63505
+ const routeOwner = {
63506
+ notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
63507
+ };
63508
+ this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
63509
+ this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
62220
63510
  });
62221
- this.logDirectory = workerLogDirectory(this.options.journalPath, root);
62222
- await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
62223
- return;
62224
- });
62225
- const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
62226
- try {
62227
- await Promise.any(workers);
62228
- } catch (error) {
62229
- throw error.errors[0];
62230
63511
  }
62231
- }
62232
- spawn(root) {
62233
- if (this.closed)
62234
- return Promise.reject(new Error("Local worker pool is closed"));
62235
- const child = fork(isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url)), [
62236
- JSON.stringify({
62237
- journalPath: this.options.journalPath,
62238
- root,
62239
- definitions: this.sources,
62240
- admission: this.options.admission
62241
- })
62242
- ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62243
- const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62244
- this.children.add(child);
62245
- const worker = randomUUID16();
62246
- this.incarnations.set(worker, child);
62247
- let ready = false;
62248
- let serial = Promise.resolve();
62249
- let work;
62250
- const deliveries = new Map;
62251
- const logPath = join28(this.logDirectory, `${worker}.log`);
62252
- this.logs.set(child, logPath);
62253
- let logFailure;
62254
- const log = createWriteStream(logPath, { flags: "a" });
62255
- log.on("error", (error) => {
62256
- logFailure = error.message;
62257
- });
62258
- let stderr = "";
62259
- child.stderr?.on("data", (chunk) => {
62260
- stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62261
- });
62262
- child.stderr?.on("error", (error) => {
62263
- logFailure ??= error.message;
62264
- log.end();
62265
- });
62266
- child.stderr?.pipe(log);
62267
- const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62268
- const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
62269
- let timedOut = false;
62270
- let terminated = false;
62271
- const terminate = () => {
62272
- if (terminated)
62273
- return;
62274
- terminated = true;
62275
- child.kill("SIGKILL");
62276
- };
62277
- const replied = (error) => {
62278
- if (error)
62279
- terminate();
62280
- };
62281
- return new Promise((resolve, reject) => {
62282
- const startup = setTimeout(() => {
62283
- timedOut = true;
62284
- terminate();
62285
- }, startupTimeoutMs);
62286
- child.on("error", (error) => {
62287
- clearTimeout(startup);
62288
- reject(error);
62289
- terminate();
62290
- });
62291
- const closed = new Promise((resolve) => child.once("close", () => resolve()));
62292
- child.once("exit", () => {
62293
- Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62294
- this.children.delete(child);
62295
- this.logs.delete(child);
62296
- clearTimeout(startup);
62297
- this.incarnations.delete(worker);
62298
- this.readyWorkers.delete(worker);
62299
- this.controlAcks.delete(worker);
62300
- this.lastHeard.delete(worker);
62301
- if (work)
62302
- this.handoff?.fence(work);
62303
- for (const delivery of deliveries.values())
62304
- delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62305
- deliveries.clear();
62306
- const cleanup = disconnected.then(() => serial).then(async () => {
62307
- try {
62308
- if (await this.admission?.releaseStoppedSessions(worker))
62309
- throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62310
- } catch (error) {
62311
- this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62312
- throw this.cleanupFailure;
62313
- }
62314
- }).then(async () => {
62315
- if (this.closed || !ready || this.desiredControl === "quit")
62316
- return;
62317
- if (work && await this.queue.result(root, work.id)) {
62318
- work = undefined;
62319
- this.replacements = 0;
62320
- }
62321
- if (this.closed)
62322
- return;
62323
- if (work)
62324
- await this.queue.releaseExited(this.lease, work);
62325
- const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62326
- if (exhausted)
62327
- this.failure = diagnosed("Local worker retry limit");
62328
- else
62329
- await this.spawn(root);
62330
- }).catch((error) => {
62331
- this.failure = error;
62332
- });
62333
- this.cleanups.add(cleanup);
62334
- cleanup.finally(() => this.cleanups.delete(cleanup));
62335
- });
62336
- child.on("message", (message) => {
62337
- this.observeResponsiveness();
62338
- this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
62339
- if (message.kind === "delivery-ack") {
62340
- const pending = deliveries.get(message.deliveryId);
62341
- deliveries.delete(message.deliveryId);
62342
- if (message.error)
62343
- pending?.reject(new Error(message.error));
62344
- else
62345
- pending?.resolve();
62346
- return;
62347
- }
62348
- const response = serial.then(async () => {
62349
- if (message.kind === "ready") {
62350
- if (terminated)
62351
- return;
62352
- ready = true;
62353
- this.readyWorkers.add(worker);
62354
- clearTimeout(startup);
62355
- resolve();
62356
- return;
62357
- }
62358
- await this.starting;
62359
- const queue = this.queue;
62360
- const lease = this.lease;
62361
- if (message.kind === "control")
62362
- return queue.control(lease);
62363
- if (message.kind === "controlled") {
62364
- const control = await queue.control(lease);
62365
- if (message.revision > control.revision)
62366
- throw new Error("Unknown control revision");
62367
- this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62368
- return;
62369
- }
62370
- if (message.kind === "claim") {
62371
- if (work)
62372
- throw new Error("Worker already holds an attempt");
62373
- const decision = this.rootBudget?.checkpoint();
62374
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62375
- return;
62376
- await this.recoverExpired(root);
62377
- work = await queue.claim(lease, worker, this.sources.map((source) => ({
62378
- definition: `${source.path}#${source.exportName}`,
62379
- digest: source.digest
62380
- })), this.leaseMs);
62381
- return work;
62382
- }
62383
- if (!work)
62384
- throw new Error("Worker has no active attempt");
62385
- if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
62386
- work = await queue.extend(lease, work, this.leaseMs);
62387
- if (message.kind === "activity")
62388
- return this.handoff?.publish(work, message.runs);
62389
- if (message.kind === "route")
62390
- return this.handoff?.route(work, message.runId);
62391
- if (message.kind === "delivery-ready")
62392
- return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
62393
- if (message.kind === "delivery") {
62394
- if (!message.recipient)
62395
- throw new Error("Worker delivery requires a session identity");
62396
- return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
62397
- const id = randomUUID16();
62398
- deliveries.set(id, { resolve, reject });
62399
- child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
62400
- if (error) {
62401
- deliveries.delete(id);
62402
- reject(error);
62403
- }
62404
- });
62405
- }));
62406
- }
62407
- if (message.kind === "renew")
62408
- return;
62409
- if (message.kind === "commit") {
62410
- const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62411
- this.usageTotal = await this.usageLedger.total(root);
62412
- return receipt;
62413
- }
62414
- if (message.kind === "usage") {
62415
- this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
62416
- return;
62417
- }
62418
- if (message.kind === "budget") {
62419
- await queue.assertCurrent(lease, work);
62420
- const decision = this.rootBudget?.checkpoint();
62421
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62422
- throw new Error("Root workflow budget exhausted");
62423
- return queue.control(lease);
62424
- }
62425
- if (message.kind !== "complete")
62426
- throw new Error("Invalid worker operation");
62427
- await queue.complete(lease, work, message.result);
62428
- this.replacements = 0;
62429
- work = undefined;
62430
- });
62431
- if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62432
- serial = response.then(() => {}, () => {});
62433
- response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62434
- });
63512
+ await this.command({
63513
+ kind: "start",
63514
+ sources: this.sources,
63515
+ cwd,
63516
+ publish: store !== undefined,
63517
+ ...this.rootRun()
62435
63518
  });
62436
63519
  }
62437
- recoverExpired(root) {
62438
- if (this.recovering)
62439
- return this.recovering;
62440
- if (this.closed || performance.now() < this.nextRecoveryAt)
62441
- return Promise.resolve();
62442
- this.recovering = (async () => {
62443
- await new Promise((resolve) => setImmediate(resolve));
62444
- await this.admission?.reconcile();
62445
- const expiredBefore = Date.now();
62446
- for (const work of await this.queue.inspect(root)) {
62447
- if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62448
- continue;
62449
- const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62450
- if (child && work.worker !== null && this.isLive(work.worker, child)) {
62451
- await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
62452
- if (error.message !== STALE_WORK_LEASE)
62453
- throw error;
62454
- });
62455
- continue;
62456
- }
62457
- if (child && !await this.stop(child).then(() => true, () => false))
62458
- continue;
62459
- await this.queue.expire(this.lease, work);
62460
- }
62461
- })().finally(() => {
62462
- this.recovering = undefined;
62463
- this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62464
- });
62465
- return this.recovering;
63520
+ queueView() {
63521
+ return {
63522
+ assertCurrent: async (_lease, work) => {
63523
+ const current = this.attempts.get(work.id);
63524
+ if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
63525
+ throw new Error(STALE_WORK_LEASE);
63526
+ },
63527
+ withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
63528
+ result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
63529
+ };
62466
63530
  }
62467
- observeResponsiveness() {
62468
- const now = performance.now();
62469
- const gap = now - this.lastTick;
62470
- if (gap > ROOT_STALL_THRESHOLD_MS)
62471
- this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
62472
- this.lastTick = now;
63531
+ rootProjection() {
63532
+ const run = this.store?.runs().find((candidate) => candidate.id === this.root);
63533
+ return run === undefined ? undefined : projectRootRun(run);
62473
63534
  }
62474
- isLive(worker, child) {
62475
- this.observeResponsiveness();
62476
- const heard = this.lastHeard.get(worker);
62477
- return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.rootStalledMs - heard.rootStalled) < this.leaseMs;
63535
+ rootRun() {
63536
+ const rootRun = this.rootProjection();
63537
+ if (rootRun === undefined)
63538
+ return {};
63539
+ this.rootSignature = JSON.stringify(rootRun);
63540
+ return { rootRun };
62478
63541
  }
62479
- stop(child) {
62480
- if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62481
- return Promise.resolve();
62482
- return new Promise((resolve, reject) => {
62483
- const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62484
- child.once("exit", () => {
62485
- clearTimeout(timeout);
62486
- resolve();
62487
- });
62488
- child.kill("SIGKILL");
62489
- });
63542
+ refreshRootRun() {
63543
+ const rootRun = this.rootProjection();
63544
+ if (rootRun === undefined)
63545
+ return;
63546
+ const signature = JSON.stringify(rootRun);
63547
+ if (signature === this.rootSignature)
63548
+ return;
63549
+ this.rootSignature = signature;
63550
+ this.notice({ kind: "root", rootRun });
62490
63551
  }
62491
63552
  async run(definition, inputs, options) {
62492
63553
  try {
@@ -62519,7 +63580,7 @@ class LocalWorkerPool {
62519
63580
  const selectedSource = this.sources[index];
62520
63581
  if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
62521
63582
  throw new Error("Worker definition compatibility error");
62522
- await this.backend.hydrateWorkflow(root);
63583
+ await this.backend.catchUpWorkflow(root);
62523
63584
  const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
62524
63585
  if (boundary?.kind !== "stage")
62525
63586
  throw new Error("Missing durable child boundary");
@@ -62530,39 +63591,232 @@ class LocalWorkerPool {
62530
63591
  config: options.config,
62531
63592
  executionMode: options.executionMode
62532
63593
  }));
62533
- const work = await this.queue.enqueue(this.lease, {
62534
- boundaryId: options.parentRun.stageId,
62535
- childRunId: options.runId,
62536
- scope: options.durableScope.scopePrefix,
62537
- definition: `${selectedSource.path}#${selectedSource.exportName}`,
62538
- digest: selectedSource.digest,
62539
- inputs,
62540
- execution
62541
- }, boundary);
62542
- for (;; ) {
62543
- options.signal?.throwIfAborted();
62544
- if (this.closed) {
63594
+ this.evaluateBudget();
63595
+ const enqueued = await this.command({
63596
+ kind: "enqueue",
63597
+ descriptor: {
63598
+ boundaryId: options.parentRun.stageId,
63599
+ childRunId: options.runId,
63600
+ scope: options.durableScope.scopePrefix,
63601
+ definition: `${selectedSource.path}#${selectedSource.exportName}`,
63602
+ digest: selectedSource.digest,
63603
+ inputs,
63604
+ execution
63605
+ },
63606
+ boundary,
63607
+ ...this.rootRun()
63608
+ });
63609
+ const work = enqueued.work;
63610
+ this.works.set(work.id, work);
63611
+ if (enqueued.result !== undefined && !this.results.has(work.id))
63612
+ this.results.set(work.id, enqueued.result);
63613
+ this.waiting++;
63614
+ this.ticker ??= setInterval(() => {
63615
+ this.evaluateBudget();
63616
+ this.signalChange();
63617
+ }, POOL_SUMMARY_INTERVAL_MS).unref();
63618
+ try {
63619
+ for (;; ) {
63620
+ options.signal?.throwIfAborted();
63621
+ if (this.closed) {
63622
+ if (this.failure)
63623
+ throw this.failure;
63624
+ if (this.desiredControl === "quit")
63625
+ throw new WorkflowGracefulQuitError(root, "worker pool teardown");
63626
+ throw new Error("Local worker pool is closed");
63627
+ }
63628
+ const result = this.results.get(work.id);
63629
+ if (result) {
63630
+ await this.inbound;
63631
+ await this.backend.catchUpWorkflow(root);
63632
+ await this.handoff?.complete(work);
63633
+ return result;
63634
+ }
62545
63635
  if (this.failure)
62546
63636
  throw this.failure;
62547
- if (this.desiredControl === "quit")
62548
- throw new WorkflowGracefulQuitError(root, "worker pool teardown");
62549
- throw new Error("Local worker pool is closed");
63637
+ if (this.budgetBlocked)
63638
+ await this.rootBudget.stopAtBoundaryAsync(definition.name);
63639
+ await this.nextChange(options.signal);
62550
63640
  }
62551
- const result = await this.queue.result(root, work.id);
62552
- if (result) {
62553
- await this.backend.hydrateWorkflow(root);
62554
- await this.handoff?.complete(work);
62555
- return result;
63641
+ } finally {
63642
+ if (--this.waiting === 0) {
63643
+ clearInterval(this.ticker);
63644
+ this.ticker = undefined;
62556
63645
  }
62557
- if (this.failure)
62558
- throw this.failure;
62559
- const decision = this.rootBudget?.checkpoint();
62560
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62561
- await this.rootBudget.stopAtBoundaryAsync(definition.name);
62562
- await this.recoverExpired(root);
62563
- await delay2(25, undefined, { signal: options.signal });
62564
63646
  }
62565
63647
  }
63648
+ evaluateBudget() {
63649
+ const decision = this.rootBudget?.checkpoint();
63650
+ const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
63651
+ if (blocked === this.budgetBlocked)
63652
+ return;
63653
+ this.budgetBlocked = blocked;
63654
+ this.notice({ kind: "budget", blocked });
63655
+ }
63656
+ signalChange() {
63657
+ const changed = this.changed;
63658
+ this.changed = Promise.withResolvers();
63659
+ changed.resolve();
63660
+ }
63661
+ nextChange(signal) {
63662
+ const changed = this.changed.promise;
63663
+ if (signal === undefined)
63664
+ return changed;
63665
+ return new Promise((resolve, reject) => {
63666
+ const abort = () => reject(signal.reason);
63667
+ signal.addEventListener("abort", abort, { once: true });
63668
+ changed.then(() => {
63669
+ signal.removeEventListener("abort", abort);
63670
+ resolve();
63671
+ });
63672
+ });
63673
+ }
63674
+ connect() {
63675
+ if (this.coordinator !== undefined)
63676
+ return this.coordinator;
63677
+ const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
63678
+ this.coordinator = coordinator;
63679
+ coordinator.port.onMessage((event) => {
63680
+ this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
63681
+ });
63682
+ coordinator.port.onClose((reason) => {
63683
+ this.coordinatorLost = reason;
63684
+ this.inbound = this.inbound.then(() => this.lost(reason));
63685
+ });
63686
+ return coordinator;
63687
+ }
63688
+ lost(reason) {
63689
+ this.pids.clear();
63690
+ if (!this.closed)
63691
+ this.failure ??= reason;
63692
+ for (const pending of this.commands.values())
63693
+ pending.reject(reason);
63694
+ this.commands.clear();
63695
+ this.signalChange();
63696
+ }
63697
+ command(body) {
63698
+ const coordinator = this.connect();
63699
+ if (this.coordinatorLost !== undefined)
63700
+ return Promise.reject(this.coordinatorLost);
63701
+ const id = ++this.commandSequence;
63702
+ return new Promise((resolve, reject) => {
63703
+ this.commands.set(id, { resolve, reject });
63704
+ coordinator.port.send({ ...body, id });
63705
+ });
63706
+ }
63707
+ notice(notice) {
63708
+ this.coordinator?.port.send(notice);
63709
+ }
63710
+ async receive(event) {
63711
+ switch (event.kind) {
63712
+ case "reply": {
63713
+ const pending = this.commands.get(event.id);
63714
+ this.commands.delete(event.id);
63715
+ if (event.error !== undefined)
63716
+ pending?.reject(new Error(event.error));
63717
+ else
63718
+ pending?.resolve(event.value);
63719
+ return;
63720
+ }
63721
+ case "activity":
63722
+ return this.applyActivity(event.batch, event.entries, event.usage);
63723
+ case "spawned":
63724
+ this.pids.add(event.pid);
63725
+ return;
63726
+ case "exited":
63727
+ this.pids.delete(event.pid);
63728
+ return;
63729
+ case "fence": {
63730
+ const current = this.attempts.get(event.work.id);
63731
+ if (current?.generation === event.work.generation)
63732
+ current.fenced = true;
63733
+ this.handoff?.fence(event.work);
63734
+ return;
63735
+ }
63736
+ case "result":
63737
+ if (event.usage !== undefined)
63738
+ this.usageTotal = event.usage;
63739
+ this.results.set(event.workId, event.result);
63740
+ this.signalChange();
63741
+ return;
63742
+ case "failure":
63743
+ this.failure ??= new Error(event.error);
63744
+ this.signalChange();
63745
+ return;
63746
+ case "request":
63747
+ this.serve(event.id, event.request);
63748
+ return;
63749
+ }
63750
+ }
63751
+ async applyActivity(batch, entries, usage) {
63752
+ if (usage !== undefined)
63753
+ this.usageTotal = usage;
63754
+ const rejected = [];
63755
+ const handoff = this.handoff;
63756
+ if (handoff !== undefined) {
63757
+ const publications = entries.flatMap((entry) => {
63758
+ const work = this.track(entry.work);
63759
+ if (work !== undefined)
63760
+ return [{ work, runs: entry.runs }];
63761
+ rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
63762
+ return [];
63763
+ });
63764
+ const outcomes = await handoff.publishBatch(publications);
63765
+ outcomes.forEach((error, index) => {
63766
+ const { work } = publications[index];
63767
+ if (error !== undefined)
63768
+ rejected.push({ workId: work.id, generation: work.generation, error: error.message });
63769
+ });
63770
+ }
63771
+ this.notice({ kind: "published", batch, rejected });
63772
+ this.refreshRootRun();
63773
+ }
63774
+ track(attempt) {
63775
+ const work = this.works.get(attempt.id);
63776
+ if (work === undefined)
63777
+ return;
63778
+ const current = this.attempts.get(attempt.id);
63779
+ if (current === undefined || attempt.generation > current.generation)
63780
+ this.attempts.set(attempt.id, { ...attempt, fenced: false });
63781
+ return { ...work, ...attempt, state: "leased" };
63782
+ }
63783
+ async serve(id, request) {
63784
+ try {
63785
+ const value2 = await this.answer(id, request);
63786
+ this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
63787
+ } catch (error) {
63788
+ this.coordinator?.port.send({
63789
+ kind: "request-reply",
63790
+ id,
63791
+ error: error instanceof Error ? error.message : String(error)
63792
+ });
63793
+ }
63794
+ }
63795
+ async answer(id, request) {
63796
+ if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
63797
+ const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
63798
+ const event = {
63799
+ ...request.payload,
63800
+ handled: false
63801
+ };
63802
+ if (request.kind === "pending-stage-message")
63803
+ router?.route(event);
63804
+ else
63805
+ router?.sticky(event);
63806
+ if (event.handled && event.completion !== undefined)
63807
+ return await event.completion;
63808
+ return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
63809
+ }
63810
+ const handoff = this.handoff;
63811
+ const work = this.track(request.work);
63812
+ if (handoff === undefined || work === undefined)
63813
+ throw new Error(STALE_WORK_LEASE);
63814
+ if (request.kind === "ready")
63815
+ return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
63816
+ return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
63817
+ await this.command({ kind: "transport", request: id, from, message });
63818
+ });
63819
+ }
62566
63820
  closing;
62567
63821
  close() {
62568
63822
  this.closing ??= this.dispose();
@@ -62571,27 +63825,28 @@ class LocalWorkerPool {
62571
63825
  async dispose() {
62572
63826
  this.closed = true;
62573
63827
  clearInterval(this.ticker);
63828
+ this.ticker = undefined;
62574
63829
  this.releaseBinding();
62575
63830
  if (this.options.rootId === undefined)
62576
63831
  workflowResourceAdmission.delete(this.backend);
63832
+ this.signalChange();
62577
63833
  let failure;
62578
- let stopping = [];
62579
- try {
62580
- await this.recovering?.catch(() => {});
62581
- stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62582
- await Promise.all(stopping.map(({ child }) => this.stop(child)));
62583
- await Promise.all([...this.cleanups]);
62584
- if (this.cleanupFailure)
62585
- throw this.cleanupFailure;
62586
- } catch (error) {
62587
- failure = error;
62588
- }
62589
- if (failure === undefined) {
62590
- const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62591
- await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62592
- if (this.logDirectory !== undefined)
62593
- await rmdir(this.logDirectory).catch(() => {});
63834
+ const coordinator = this.coordinator;
63835
+ if (coordinator !== undefined) {
63836
+ if (this.coordinatorLost === undefined) {
63837
+ try {
63838
+ await this.command({ kind: "close" });
63839
+ } catch (error) {
63840
+ failure = error;
63841
+ }
63842
+ }
63843
+ await this.inbound;
63844
+ await coordinator.port.close();
63845
+ this.pids.clear();
62594
63846
  }
63847
+ for (const release of this.routeClaims.values())
63848
+ release();
63849
+ this.routeClaims.clear();
62595
63850
  try {
62596
63851
  await this.releaseRoutes?.();
62597
63852
  } catch (error) {
@@ -62601,27 +63856,26 @@ class LocalWorkerPool {
62601
63856
  throw failure;
62602
63857
  }
62603
63858
  }
62604
- var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, ROOT_TICK_INTERVAL_MS = 100, ROOT_STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000;
63859
+ var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
62605
63860
  var init_local_pool = __esm(() => {
62606
63861
  init_internal();
62607
63862
  init_sql_pool_queue();
62608
- init_sql_pool_usage();
62609
63863
  init_sql_run_journal();
62610
63864
  init_workflow_tool_abort();
62611
63865
  init_sdk_intercom();
63866
+ init_external_run_routes();
62612
63867
  init_workflow_infrastructure_error();
62613
63868
  init_admission_binding();
62614
63869
  init_definitions();
62615
63870
  init_dispatch();
62616
63871
  init_host_options();
62617
- init_resource_admission();
63872
+ init_pool_coordinator();
62618
63873
  init_stage_handoff();
62619
63874
  MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
62620
- COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
62621
63875
  });
62622
63876
 
62623
63877
  // dist/builtin/workflows/src/workers/public-pool.ts
62624
- import { resolve as resolve12 } from "node:path";
63878
+ import { resolve as resolve13 } from "node:path";
62625
63879
  import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
62626
63880
  async function restoreWorkerOptions(backend, options) {
62627
63881
  const source = options.continuation?.source.id ?? options.runId;
@@ -62660,11 +63914,11 @@ async function activatePublicPool(backend, options) {
62660
63914
  ...options.localWorkers,
62661
63915
  startupTimeoutMs: undefined,
62662
63916
  leaseMs: undefined,
62663
- definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve12(cwd, source.path) }))
63917
+ definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
62664
63918
  };
62665
63919
  const admission = options.admission && {
62666
63920
  ...options.admission,
62667
- ...options.admission.userDatabase ? { userDatabase: resolve12(cwd, options.admission.userDatabase) } : {}
63921
+ ...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
62668
63922
  };
62669
63923
  const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
62670
63924
  const resources = JSON.parse(JSON.stringify({
@@ -64010,7 +65264,7 @@ function hasUsableContentBlock(block) {
64010
65264
 
64011
65265
  // dist/builtin/workflows/src/shared/stage-session-store.ts
64012
65266
  import { randomUUID as randomUUID18 } from "node:crypto";
64013
- import { dirname as dirname13, resolve as resolve13 } from "node:path";
65267
+ import { dirname as dirname13, resolve as resolve14 } from "node:path";
64014
65268
  import {
64015
65269
  JsonlSessionStore,
64016
65270
  resolveSessionStore,
@@ -64046,11 +65300,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
64046
65300
  async function migrateLegacyStageSession(store, sessionId, sessionFile) {
64047
65301
  if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
64048
65302
  return;
64049
- const location = resolve13(sessionFile);
65303
+ const location = resolve14(sessionFile);
64050
65304
  const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
64051
65305
  try {
64052
65306
  for await (const file of legacy.scanFiles()) {
64053
- if (resolve13(file.location) !== location || file.header.id !== sessionId)
65307
+ if (resolve14(file.location) !== location || file.header.id !== sessionId)
64054
65308
  continue;
64055
65309
  const snapshot = await file.read();
64056
65310
  if (await store.readHeader(sessionId) === undefined)
@@ -64212,22 +65466,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
64212
65466
  }
64213
65467
  function retainReachableRunGroups(grouped, rootRunId) {
64214
65468
  const reachable = new Set([rootRunId]);
64215
- const pending = [rootRunId];
65469
+ const settled = new Set;
65470
+ const pending = [{ runId: rootRunId, live: true }];
64216
65471
  while (pending.length > 0) {
64217
- const drafts = grouped.get(pending.pop());
65472
+ const { runId, live } = pending.pop();
65473
+ const drafts = grouped.get(runId);
64218
65474
  if (drafts === undefined)
64219
65475
  continue;
64220
65476
  for (const draft of drafts) {
64221
- const childRunId = childRunIdFromDraft(draft);
64222
- if (childRunId === undefined || reachable.has(childRunId) || !grouped.has(childRunId))
65477
+ const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
65478
+ if (childRunId === undefined || !grouped.has(childRunId))
64223
65479
  continue;
64224
- reachable.add(childRunId);
64225
- pending.push(childRunId);
65480
+ if (live && childRunIdFromDraft(draft) !== undefined) {
65481
+ if (reachable.has(childRunId))
65482
+ continue;
65483
+ reachable.add(childRunId);
65484
+ pending.push({ runId: childRunId, live: true });
65485
+ } else if (!settled.has(childRunId)) {
65486
+ settled.add(childRunId);
65487
+ pending.push({ runId: childRunId, live: false });
65488
+ }
64226
65489
  }
64227
65490
  }
64228
- for (const runId of grouped.keys())
64229
- if (!reachable.has(runId))
64230
- grouped.delete(runId);
65491
+ let complete = true;
65492
+ for (const runId of grouped.keys()) {
65493
+ if (reachable.has(runId))
65494
+ continue;
65495
+ if (!settled.has(runId))
65496
+ complete = false;
65497
+ grouped.delete(runId);
65498
+ }
65499
+ return complete;
64231
65500
  }
64232
65501
  function childRunIdFromDraft(draft) {
64233
65502
  const boundary = draft.topology?.boundary;
@@ -64649,9 +65918,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
64649
65918
  }
64650
65919
  for (const group of grouped.values())
64651
65920
  group.sort(compareDraftSourceOrder);
64652
- const groupedCount = grouped.size;
64653
- retainReachableRunGroups(grouped, rootRunId);
64654
- if (policy.failOnUnreachable && grouped.size !== groupedCount)
65921
+ if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
64655
65922
  return [];
64656
65923
  if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
64657
65924
  return [];
@@ -70309,9 +71576,9 @@ var init_stage_runner_controller = __esm(() => {
70309
71576
  });
70310
71577
 
70311
71578
  // dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
70312
- import { createHash as createHash4 } from "node:crypto";
71579
+ import { createHash as createHash5 } from "node:crypto";
70313
71580
  import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
70314
- import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve14 } from "node:path";
71581
+ import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
70315
71582
  function normalizeMaxOutput2(maxOutput) {
70316
71583
  return {
70317
71584
  bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
@@ -70373,7 +71640,7 @@ function formatByteSize(bytes) {
70373
71640
  return `${value2.toFixed(1)} ${units[unitIndex]}`;
70374
71641
  }
70375
71642
  function transcriptPath(runId, outputPath) {
70376
- const digest = createHash4("sha256").update(outputPath).digest("hex").slice(0, 16);
71643
+ const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
70377
71644
  return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
70378
71645
  }
70379
71646
  function appendContent(lines, content) {
@@ -70419,7 +71686,7 @@ function renderTranscript(messages, fallbackOutput) {
70419
71686
  `);
70420
71687
  }
70421
71688
  function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
70422
- const absolutePath = resolve14(outputPath);
71689
+ const absolutePath = resolve15(outputPath);
70423
71690
  const bytes = Buffer.byteLength(fullOutput, "utf8");
70424
71691
  const lines = countLines(fullOutput);
70425
71692
  const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
@@ -70434,8 +71701,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
70434
71701
  return;
70435
71702
  if (isAbsolute11(output))
70436
71703
  return output;
70437
- const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve14(runtimeCwd, requestedCwd);
70438
- return resolve14(baseCwd, output);
71704
+ const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
71705
+ return resolve15(baseCwd, output);
70439
71706
  }
70440
71707
  function degenerateWarning(fullOutput) {
70441
71708
  return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
@@ -71444,7 +72711,7 @@ async function run(def, inputs, opts = {}) {
71444
72711
  if (!isWorkflowDefinition(def))
71445
72712
  throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
71446
72713
  const depth = opts.depth ?? 0;
71447
- const maxDepth = opts.config?.maxDepth ?? 4;
72714
+ const maxDepth = opts.config?.maxDepth ?? 3;
71448
72715
  if (depth >= maxDepth) {
71449
72716
  return {
71450
72717
  runId: opts.runId ?? crypto.randomUUID(),
@@ -72533,7 +73800,7 @@ init_admission_binding();
72533
73800
  init_definitions();
72534
73801
  init_resource_admission();
72535
73802
  import { randomUUID as randomUUID20 } from "node:crypto";
72536
- import { setTimeout as delay3 } from "node:timers/promises";
73803
+ import { setTimeout as delay4 } from "node:timers/promises";
72537
73804
  import { withResourceAdmission as withResourceAdmission2 } from "@bastani/atomic";
72538
73805
  import { StorageProvider as StorageProvider3 } from "@bastani/atomic/storage";
72539
73806
 
@@ -72542,8 +73809,8 @@ init_stage_topology();
72542
73809
  init_run_durable_topology();
72543
73810
  import { randomUUID as randomUUID19 } from "node:crypto";
72544
73811
  function workerSessionStore(input) {
72545
- const checkpoint = async (id, header) => {
72546
- const owner = (header ?? await input.reader.readHeader(id))?.workflow;
73812
+ const checkpoint = (id, header) => {
73813
+ const owner = header.workflow;
72547
73814
  const run = input.store.runs().find((run) => run.id === owner?.runId);
72548
73815
  const stage = run?.stages.find((stage) => stage.id === owner?.stageId);
72549
73816
  if (!run || !stage?.replayKey)
@@ -72571,11 +73838,11 @@ function workerSessionStore(input) {
72571
73838
  };
72572
73839
  return {
72573
73840
  descriptor: input.reader.descriptor,
72574
- create: async (header, entries) => write({ kind: "session-create", checkpoint: await checkpoint(header.id, header), header, entries }),
72575
- append: async (id, entries, options) => write({ kind: "session-append", checkpoint: await checkpoint(id), entries, expectSeq: options.expectSeq }),
73841
+ create: async (header, entries) => write({ kind: "session-create", checkpoint: checkpoint(header.id, header), header, entries }),
73842
+ append: async (id, entries, options) => write({ kind: "session-append", sessionId: id, entries, expectSeq: options.expectSeq }),
72576
73843
  replace: async (id, header, entries, options) => write({
72577
73844
  kind: "session-replace",
72578
- checkpoint: await checkpoint(id, header),
73845
+ checkpoint: checkpoint(id, header),
72579
73846
  header,
72580
73847
  entries,
72581
73848
  expectSeq: options.expectSeq
@@ -72603,6 +73870,14 @@ function collectWorkerUsage(runs) {
72603
73870
 
72604
73871
  // dist/builtin/workflows/src/workers/local-worker.ts
72605
73872
  async function runLocalWorker(start) {
73873
+ if ("role" in start && start.role === POOL_COORDINATOR_ROLE) {
73874
+ await Promise.resolve().then(() => init_pool_coordinator());
73875
+ runPoolCoordinator();
73876
+ return;
73877
+ }
73878
+ await runWorker(start);
73879
+ }
73880
+ async function runWorker(start) {
72606
73881
  let sequence = 0;
72607
73882
  const pending = new Map;
72608
73883
  process.on("message", (reply) => {
@@ -72726,13 +74001,13 @@ async function runLocalWorker(start) {
72726
74001
  for (;; ) {
72727
74002
  await observeControl();
72728
74003
  if (controlState !== "running") {
72729
- await delay3(25);
74004
+ await delay4(25);
72730
74005
  continue;
72731
74006
  }
72732
74007
  await reader.catchUpWorkflow(start.root);
72733
74008
  const work = await request({ kind: "claim" });
72734
74009
  if (!work) {
72735
- await delay3(25 + Math.random() * 25);
74010
+ await delay4(25 + Math.random() * 25);
72736
74011
  continue;
72737
74012
  }
72738
74013
  const index = start.definitions.findIndex((source) => `${source.path}#${source.exportName}` === work.definition && source.digest === work.digest);
@@ -72809,7 +74084,7 @@ async function runLocalWorker(start) {
72809
74084
  const control = await request({ kind: "budget" });
72810
74085
  if (control.state === "running")
72811
74086
  return;
72812
- await delay3(25, undefined, { signal });
74087
+ await delay4(25, undefined, { signal });
72813
74088
  }
72814
74089
  };
72815
74090
  const resourceAdmission = {
@@ -72848,7 +74123,7 @@ async function runLocalWorker(start) {
72848
74123
  await observeControl();
72849
74124
  while ((await request({ kind: "control" })).state !== "running") {
72850
74125
  controller.signal.throwIfAborted();
72851
- await delay3(25);
74126
+ await delay4(25);
72852
74127
  }
72853
74128
  const result = await withResourceAdmission2(resourceAdmission, () => run(definitions[index], work.inputs, {
72854
74129
  ...work.execution,