@bastani/atomic 0.9.43 → 0.9.44-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/dist/builtin/intercom/CHANGELOG.md +12 -0
  3. package/dist/builtin/intercom/broker/broker.ts +31 -7
  4. package/dist/builtin/intercom/broker/client.ts +65 -0
  5. package/dist/builtin/intercom/index.bundle.mjs +227 -95
  6. package/dist/builtin/intercom/live-route-refusal.ts +20 -0
  7. package/dist/builtin/intercom/package.json +1 -1
  8. package/dist/builtin/intercom/skills/intercom/SKILL.md +10 -6
  9. package/dist/builtin/intercom/types.ts +20 -1
  10. package/dist/builtin/mcp/CHANGELOG.md +7 -0
  11. package/dist/builtin/mcp/package.json +1 -1
  12. package/dist/builtin/subagents/CHANGELOG.md +9 -0
  13. package/dist/builtin/subagents/README.md +8 -8
  14. package/dist/builtin/subagents/agents/worker.md +1 -1
  15. package/dist/builtin/subagents/package.json +1 -1
  16. package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
  17. package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
  18. package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
  19. package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
  20. package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
  21. package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
  22. package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
  23. package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
  24. package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
  25. package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
  26. package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
  27. package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
  28. package/dist/builtin/web-access/package.json +1 -1
  29. package/dist/builtin/workflows/CHANGELOG.md +14 -0
  30. package/dist/builtin/workflows/README.md +1 -1
  31. package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
  32. package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
  33. package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
  34. package/dist/builtin/workflows/builtin/goal.js +2 -2
  35. package/dist/builtin/workflows/builtin/index.js +3 -3
  36. package/dist/builtin/workflows/builtin/ralph.js +2 -2
  37. package/dist/builtin/workflows/package.json +1 -1
  38. package/dist/builtin/workflows/src/extension/index.bundle.mjs +1750 -598
  39. package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1786 -600
  40. package/dist/builtin/workflows/src/index.js +1802 -650
  41. package/dist/builtin/workflows/src/local-worker-entry.mjs +1786 -600
  42. package/dist/cli/args.js +1 -1
  43. package/dist/cli/args.js.map +1 -1
  44. package/dist/core/exec.d.ts.map +1 -1
  45. package/dist/core/exec.js +11 -2
  46. package/dist/core/exec.js.map +1 -1
  47. package/dist/core/package-manager-resolver.d.ts.map +1 -1
  48. package/dist/core/package-manager-resolver.js +25 -14
  49. package/dist/core/package-manager-resolver.js.map +1 -1
  50. package/dist/core/package-manager-types.d.ts +8 -0
  51. package/dist/core/package-manager-types.d.ts.map +1 -1
  52. package/dist/core/package-manager-types.js.map +1 -1
  53. package/dist/core/package-manager.d.ts +1 -1
  54. package/dist/core/package-manager.d.ts.map +1 -1
  55. package/dist/core/package-manager.js.map +1 -1
  56. package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
  57. package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
  58. package/dist/core/resource-loader-extension-warnings.js +15 -0
  59. package/dist/core/resource-loader-extension-warnings.js.map +1 -1
  60. package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
  61. package/dist/core/resource-loader-package-resources.js +1 -0
  62. package/dist/core/resource-loader-package-resources.js.map +1 -1
  63. package/dist/core/resource-loader-reload.d.ts.map +1 -1
  64. package/dist/core/resource-loader-reload.js +5 -3
  65. package/dist/core/resource-loader-reload.js.map +1 -1
  66. package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
  67. package/dist/extensions/mcp/client/oauth/flow.js +4 -1
  68. package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
  69. package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
  70. package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
  71. package/dist/extensions/mcp/client/oauth/types.js +8 -4
  72. package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
  73. package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
  74. package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
  75. package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
  76. package/dist/index.d.ts +1 -1
  77. package/dist/index.d.ts.map +1 -1
  78. package/dist/index.js.map +1 -1
  79. package/dist/main.d.ts.map +1 -1
  80. package/dist/main.js +20 -10
  81. package/dist/main.js.map +1 -1
  82. package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
  83. package/dist/modes/interactive/components/model-selector.js +4 -2
  84. package/dist/modes/interactive/components/model-selector.js.map +1 -1
  85. package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
  86. package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
  87. package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
  88. package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
  89. package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
  90. package/dist/modes/interactive/components/tool-execution.js +11 -1
  91. package/dist/modes/interactive/components/tool-execution.js.map +1 -1
  92. package/dist/modes/interactive/model-search.d.ts +10 -0
  93. package/dist/modes/interactive/model-search.d.ts.map +1 -1
  94. package/dist/modes/interactive/model-search.js +21 -0
  95. package/dist/modes/interactive/model-search.js.map +1 -1
  96. package/dist/package-manager-cli.d.ts.map +1 -1
  97. package/dist/package-manager-cli.js +5 -0
  98. package/dist/package-manager-cli.js.map +1 -1
  99. package/dist/utils/tools-manager.d.ts.map +1 -1
  100. package/dist/utils/tools-manager.js +30 -35
  101. package/dist/utils/tools-manager.js.map +1 -1
  102. package/docs/computer-use.md +5 -5
  103. package/docs/custom-provider/streaming.md +9 -14
  104. package/docs/environment-variables.md +1 -0
  105. package/docs/intercom.md +5 -1
  106. package/docs/packages.md +2 -0
  107. package/docs/settings.md +2 -2
  108. package/docs/skills.md +2 -2
  109. package/docs/subagents.md +38 -9
  110. package/docs/workflows/api-reference.md +1 -1
  111. package/docs/workflows/authoring.md +1 -1
  112. package/docs/workflows/operations.md +8 -6
  113. package/docs/workflows/reliable-design.md +23 -5
  114. package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
  115. package/examples/extensions/todo.ts +7 -0
  116. package/npm-shrinkwrap.json +32 -32
  117. package/package.json +3 -3
@@ -8030,7 +8030,13 @@ function createStoreContext(state = createStoreState()) {
8030
8030
  }
8031
8031
  return cachedGraphSnapshot;
8032
8032
  }
8033
+ let batchDepth = 0;
8034
+ let batchedNotification = false;
8033
8035
  function notify() {
8036
+ if (batchDepth > 0) {
8037
+ batchedNotification = true;
8038
+ return;
8039
+ }
8034
8040
  observation.capture();
8035
8041
  for (const fn of state.invalidationListeners) {
8036
8042
  try {
@@ -8050,6 +8056,18 @@ function createStoreContext(state = createStoreState()) {
8050
8056
  state.version++;
8051
8057
  notify();
8052
8058
  }
8059
+ function batch(update) {
8060
+ batchDepth++;
8061
+ try {
8062
+ return update();
8063
+ } finally {
8064
+ batchDepth--;
8065
+ if (batchDepth === 0 && batchedNotification) {
8066
+ batchedNotification = false;
8067
+ notify();
8068
+ }
8069
+ }
8070
+ }
8053
8071
  function findRun(runId) {
8054
8072
  return state.runs.find((r) => r.id === runId);
8055
8073
  }
@@ -8103,6 +8121,7 @@ function createStoreContext(state = createStoreState()) {
8103
8121
  snapshot,
8104
8122
  graphSnapshot,
8105
8123
  notify,
8124
+ batch,
8106
8125
  bumpAndNotify,
8107
8126
  findRun,
8108
8127
  findStage,
@@ -8918,6 +8937,9 @@ function createRunStoreMethods(context) {
8918
8937
  subscribeInvalidation(fn) {
8919
8938
  state.invalidationListeners.add(fn);
8920
8939
  return () => state.invalidationListeners.delete(fn);
8940
+ },
8941
+ batch(update) {
8942
+ return context.batch(update);
8921
8943
  }
8922
8944
  };
8923
8945
  }
@@ -13403,7 +13425,7 @@ class ConcurrencyLimiter {
13403
13425
  }
13404
13426
  }
13405
13427
  function createRunLimiter(defaultConcurrency) {
13406
- return new ConcurrencyLimiter(defaultConcurrency ?? 3);
13428
+ return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
13407
13429
  }
13408
13430
 
13409
13431
  // dist/builtin/workflows/src/shared/abort.ts
@@ -57173,6 +57195,44 @@ var init_executor = __esm(() => {
57173
57195
  init_executor_run();
57174
57196
  });
57175
57197
 
57198
+ // dist/builtin/workflows/src/shared/external-run-routes.ts
57199
+ function claimExternalRunRoute(store, runId, owner) {
57200
+ let owners = registry.owners.get(store);
57201
+ if (owners === undefined) {
57202
+ owners = new Map;
57203
+ registry.owners.set(store, owners);
57204
+ }
57205
+ owners.set(runId, owner);
57206
+ return () => {
57207
+ const current = registry.owners.get(store);
57208
+ if (current?.get(runId) !== owner)
57209
+ return;
57210
+ current.delete(runId);
57211
+ if (current.size === 0)
57212
+ registry.owners.delete(store);
57213
+ };
57214
+ }
57215
+ function externalRunRouteOwner(store, runId) {
57216
+ return registry.owners.get(store)?.get(runId);
57217
+ }
57218
+ function registerPendingStageMessageRouter(store, router) {
57219
+ registry.routers.set(store, router);
57220
+ return () => {
57221
+ if (registry.routers.get(store) === router)
57222
+ registry.routers.delete(store);
57223
+ };
57224
+ }
57225
+ function pendingStageMessageRouter(store) {
57226
+ return registry.routers.get(store);
57227
+ }
57228
+ var registryKey, processState, registry;
57229
+ var init_external_run_routes = __esm(() => {
57230
+ registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
57231
+ processState = globalThis;
57232
+ registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
57233
+ processState[registryKey] = registry;
57234
+ });
57235
+
57176
57236
  // dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
57177
57237
  import { randomUUID as randomUUID10 } from "node:crypto";
57178
57238
  function capabilitiesFor(store) {
@@ -57197,12 +57257,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
57197
57257
  capabilitiesFor(store).set(runId, capability);
57198
57258
  return capability;
57199
57259
  }
57200
- var routeCapabilitiesKey, processState, routeCapabilities;
57260
+ var routeCapabilitiesKey, processState2, routeCapabilities;
57201
57261
  var init_pending_stage_route_capability = __esm(() => {
57202
57262
  routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
57203
- processState = globalThis;
57204
- routeCapabilities = processState[routeCapabilitiesKey] ?? new WeakMap;
57205
- processState[routeCapabilitiesKey] = routeCapabilities;
57263
+ processState2 = globalThis;
57264
+ routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
57265
+ processState2[routeCapabilitiesKey] = routeCapabilities;
57206
57266
  });
57207
57267
 
57208
57268
  // dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
@@ -57234,12 +57294,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
57234
57294
  function workflowPendingStageRouteReady(store, runId) {
57235
57295
  return owners.get(store)?.ready(runId);
57236
57296
  }
57237
- var routeOwnersKey, processState2, owners;
57297
+ var routeOwnersKey, processState3, owners;
57238
57298
  var init_pending_stage_route_readiness = __esm(() => {
57239
57299
  routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
57240
- processState2 = globalThis;
57241
- owners = processState2[routeOwnersKey] ?? new WeakMap;
57242
- processState2[routeOwnersKey] = owners;
57300
+ processState3 = globalThis;
57301
+ owners = processState3[routeOwnersKey] ?? new WeakMap;
57302
+ processState3[routeOwnersKey] = owners;
57243
57303
  });
57244
57304
 
57245
57305
  // dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
@@ -57540,6 +57600,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57540
57600
  notificationId,
57541
57601
  reason
57542
57602
  };
57603
+ const external = externalRunRouteOwner(activeStore, entry.runId);
57604
+ if (external !== undefined)
57605
+ return external.notifyUndeliverable(payload);
57543
57606
  pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
57544
57607
  return payload.handled && await payload.completion === true;
57545
57608
  };
@@ -57551,19 +57614,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57551
57614
  return;
57552
57615
  const ownedRunIds = new Set;
57553
57616
  const runs = activeStore.runs();
57617
+ const runById = new Map(runs.map((run) => [run.id, run]));
57618
+ const childrenByBoundary = new Map;
57619
+ for (const run of runs) {
57620
+ if (run.parentRunId === undefined || run.parentStageId === undefined)
57621
+ continue;
57622
+ const key = JSON.stringify([run.parentRunId, run.parentStageId]);
57623
+ const children = childrenByBoundary.get(key) ?? [];
57624
+ children.push(run);
57625
+ childrenByBoundary.set(key, children);
57626
+ }
57554
57627
  const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
57555
57628
  for (const run of runs) {
57556
- if (!ownsRun(run.id))
57629
+ if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
57557
57630
  continue;
57558
- const rootRunId = durableRootRunIdForRun(runs, run.id);
57631
+ const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
57559
57632
  if (rootRunId === undefined)
57560
57633
  continue;
57561
57634
  ownedRunIds.add(run.id);
57562
- const parentRun = runs.find((candidate) => candidate.id === run.parentRunId);
57635
+ const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
57563
57636
  const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
57564
57637
  const parent = parentRun === undefined || boundary === undefined ? undefined : {
57565
57638
  runId: parentRun.id,
57566
- stageKeys: [boundary.id, boundary.name].filter((key) => resolveChildRun(runs, parentRun, key)?.id === run.id)
57639
+ stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
57567
57640
  };
57568
57641
  const announcement = {
57569
57642
  runId: run.id,
@@ -57704,16 +57777,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57704
57777
  payload.handled = true;
57705
57778
  payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
57706
57779
  };
57707
- const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
57708
- const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
57780
+ const routeStickyLiveDelivered = (payload) => {
57709
57781
  if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
57710
57782
  return;
57711
57783
  payload.handled = true;
57712
57784
  payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
57785
+ };
57786
+ const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
57787
+ const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
57788
+ const disposeRouter = registerPendingStageMessageRouter(activeStore, {
57789
+ route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
57790
+ sticky: routeStickyLiveDelivered
57713
57791
  });
57714
57792
  const dispose = () => {
57715
57793
  disposed = true;
57716
57794
  disposeReadiness();
57795
+ disposeRouter();
57717
57796
  routeCompletions.clear();
57718
57797
  unsubscribeStore();
57719
57798
  if (typeof subscription === "function")
@@ -57839,6 +57918,12 @@ function isMaterializedNonAgentTarget(runs, target) {
57839
57918
  return false;
57840
57919
  return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
57841
57920
  }
57921
+ function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
57922
+ const boundariesById = parent.stages.filter((stage) => stage.id === segment);
57923
+ const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
57924
+ const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
57925
+ return children.length === 1 ? children[0] : undefined;
57926
+ }
57842
57927
  function resolveChildRun(runs, parent, segment) {
57843
57928
  const boundariesById = parent.stages.filter((stage) => stage.id === segment);
57844
57929
  const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
@@ -58178,6 +58263,7 @@ var init_pending_stage_intercom = __esm(() => {
58178
58263
  init_execution_context_boundary();
58179
58264
  init_run_owner_backend();
58180
58265
  init_store_backend();
58266
+ init_external_run_routes();
58181
58267
  init_intercom_group();
58182
58268
  init_pending_stage_route_capability();
58183
58269
  init_pending_stage_route_readiness();
@@ -58636,7 +58722,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
58636
58722
  function isWorkflowStageRosterEntries(value2) {
58637
58723
  return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
58638
58724
  }
58639
- var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
58725
+ var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
58640
58726
  var init_client = __esm(() => {
58641
58727
  init_paths();
58642
58728
  init_pending_send_registry();
@@ -58645,6 +58731,21 @@ var init_client = __esm(() => {
58645
58731
  init_live_route_refusal();
58646
58732
  init_retry_policy();
58647
58733
  BROKER_SOCKET = getBrokerSocketPath();
58734
+ IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
58735
+ connectionKept;
58736
+ code;
58737
+ constructor(reason, refusal = { connectionKept: false }) {
58738
+ super(reason);
58739
+ this.name = "IntercomPendingStageRouteRefusedError";
58740
+ this.connectionKept = refusal.connectionKept;
58741
+ this.code = refusal.code;
58742
+ }
58743
+ };
58744
+ PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
58745
+ "not_authorized",
58746
+ "invalid_stage_roster",
58747
+ "invalid_possible_stage_roster"
58748
+ ]);
58648
58749
  IntercomClient = class IntercomClient extends EventEmitter {
58649
58750
  returnAddress;
58650
58751
  preferredSessionId;
@@ -58652,6 +58753,8 @@ var init_client = __esm(() => {
58652
58753
  connectTimeoutMs;
58653
58754
  brokerAcknowledgesReceipt = false;
58654
58755
  brokerAcknowledgesPendingStageRoutes = false;
58756
+ multiplexesPendingStageRoutes;
58757
+ brokerRefusesPendingStageRoutesInPlace = false;
58655
58758
  brokerAnswersStatus = false;
58656
58759
  socket = null;
58657
58760
  _sessionId = null;
@@ -58717,6 +58820,7 @@ var init_client = __esm(() => {
58717
58820
  this.preferredSessionId = options.preferredSessionId;
58718
58821
  this.acknowledgesReceipt = options.acknowledgesReceipt === true;
58719
58822
  this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
58823
+ this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
58720
58824
  }
58721
58825
  failPending(error) {
58722
58826
  this.pendingSends.rejectAll(error);
@@ -58829,6 +58933,7 @@ var init_client = __esm(() => {
58829
58933
  this._supervisorSessionId = null;
58830
58934
  this.brokerAcknowledgesReceipt = false;
58831
58935
  this.brokerAcknowledgesPendingStageRoutes = false;
58936
+ this.brokerRefusesPendingStageRoutesInPlace = false;
58832
58937
  this.brokerAnswersStatus = false;
58833
58938
  this._messageSource = undefined;
58834
58939
  this.disconnectError = null;
@@ -58907,7 +59012,7 @@ var init_client = __esm(() => {
58907
59012
  }
58908
59013
  switch (brokerMessage.type) {
58909
59014
  case "registered": {
58910
- if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
59015
+ if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
58911
59016
  throw new Error("Invalid registered message");
58912
59017
  }
58913
59018
  if (this._sessionId !== null) {
@@ -58917,6 +59022,7 @@ var init_client = __esm(() => {
58917
59022
  this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
58918
59023
  this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
58919
59024
  this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
59025
+ this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
58920
59026
  this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
58921
59027
  this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
58922
59028
  break;
@@ -58929,6 +59035,12 @@ var init_client = __esm(() => {
58929
59035
  this.emit("_registration_failed", refusal);
58930
59036
  break;
58931
59037
  }
59038
+ const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
59039
+ const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
59040
+ if (refusedRoute !== undefined) {
59041
+ this.pendingStageRouteRegistrations.delete(refusedRouteId);
59042
+ refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
59043
+ }
58932
59044
  this.disconnectError ??= refusal;
58933
59045
  this.failPending(this.disconnectError);
58934
59046
  this.socket?.destroy();
@@ -59054,6 +59166,21 @@ var init_client = __esm(() => {
59054
59166
  this.emit("pending_stage_route_revoked", { runId });
59055
59167
  break;
59056
59168
  }
59169
+ case "pending_stage_route_refused": {
59170
+ const { requestId, runId, code, reason } = brokerMessage;
59171
+ if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
59172
+ throw new Error("Invalid pending-stage route refusal");
59173
+ }
59174
+ const pending = this.pendingStageRouteRegistrations.get(requestId);
59175
+ if (!pending)
59176
+ return;
59177
+ this.pendingStageRouteRegistrations.delete(requestId);
59178
+ pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
59179
+ connectionKept: true,
59180
+ ...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
59181
+ }));
59182
+ break;
59183
+ }
59057
59184
  case "pending_stage_route_registered": {
59058
59185
  const { requestId } = brokerMessage;
59059
59186
  if (typeof requestId !== "string")
@@ -59557,7 +59684,8 @@ var init_client = __esm(() => {
59557
59684
  stages,
59558
59685
  possibleStages,
59559
59686
  ...parent === undefined ? {} : { parent },
59560
- ...dormant === true ? { dormant: true } : {}
59687
+ ...dormant === true ? { dormant: true } : {},
59688
+ ...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
59561
59689
  });
59562
59690
  } catch (error) {
59563
59691
  clearTimeout(timeout);
@@ -60291,120 +60419,165 @@ var init_reconnect_backoff = __esm(() => {
60291
60419
  BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
60292
60420
  });
60293
60421
 
60294
- // dist/builtin/intercom/route-connect-gate.ts
60295
- function createConcurrencyGate(limit) {
60296
- let active = 0;
60297
- const waiting = [];
60298
- const release = () => {
60299
- const next = waiting.shift();
60300
- if (next === undefined)
60301
- active -= 1;
60302
- else
60303
- next();
60304
- };
60305
- return {
60306
- async run(operation) {
60307
- if (active < limit)
60308
- active += 1;
60309
- else
60310
- await new Promise((resolve) => waiting.push(resolve));
60311
- try {
60312
- return await operation();
60313
- } finally {
60314
- release();
60315
- }
60316
- }
60317
- };
60318
- }
60319
- var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
60320
-
60321
60422
  // dist/builtin/intercom/workflow-route-owner.ts
60322
60423
  import { randomUUID as randomUUID13 } from "node:crypto";
60323
60424
  import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
60425
+ function closedError() {
60426
+ return new Error("Workflow Intercom route owner closed");
60427
+ }
60324
60428
  function createWorkflowRouteOwner(cwd) {
60325
60429
  const events = createEventBus();
60326
60430
  const config = loadConfig();
60327
60431
  const id = randomUUID13();
60328
60432
  const routes = new Map;
60329
- const gate = createConcurrencyGate(ROUTE_CLIENT_CONNECT_CONCURRENCY);
60433
+ const connections = new Map;
60434
+ const isCurrent = (state) => routes.get(state.route.runId) === state;
60435
+ const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
60436
+ function connectionOf(state) {
60437
+ const key = connectionKey(state);
60438
+ let connection = connections.get(key);
60439
+ if (connection === undefined) {
60440
+ connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
60441
+ connections.set(key, connection);
60442
+ }
60443
+ connection.routes.set(state.route.runId, state);
60444
+ return connection;
60445
+ }
60446
+ function detach(state) {
60447
+ const key = connectionKey(state);
60448
+ const connection = connections.get(key);
60449
+ connection?.routes.delete(state.route.runId);
60450
+ if (!state.isolated || connection === undefined)
60451
+ return;
60452
+ connections.delete(key);
60453
+ connection.client?.disconnect().catch(() => {});
60454
+ }
60455
+ function settle(state, registered) {
60456
+ state.registered = registered;
60457
+ if (!registered && isCurrent(state))
60458
+ return;
60459
+ for (const waiter of state.waiters)
60460
+ waiter(registered);
60461
+ state.waiters.clear();
60462
+ }
60330
60463
  function scheduleRetry(state) {
60331
- if (state.closed || state.retry !== undefined)
60464
+ if (!isCurrent(state) || state.retry !== undefined)
60332
60465
  return;
60333
60466
  state.retry = setTimeout(() => {
60334
60467
  state.retry = undefined;
60335
- publish(state, state.route).catch(() => scheduleRetry(state));
60468
+ publish(state, state.route).catch(() => {});
60336
60469
  }, jitteredReconnectDelayMs(state.attempt++));
60337
60470
  state.retry.unref();
60338
60471
  }
60339
- async function connect(state) {
60340
- if (state.client?.isConnected())
60341
- return state.client;
60342
- const [extensionDir] = getBuiltinPackagePaths({
60343
- workflows: false,
60344
- subagents: false,
60345
- mcp: false,
60346
- "web-access": false
60347
- });
60348
- if (extensionDir === undefined)
60349
- throw new Error("Intercom builtin package is unavailable");
60350
- await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
60351
- if (state.closed)
60352
- throw new Error("Workflow Intercom route owner closed");
60353
- const client = new IntercomClient(`${id}:pending-stage-route:${state.route.runId}`);
60354
- state.client = client;
60355
- client.on("error", () => {});
60356
- client.on("disconnected", () => {
60357
- if (state.client !== client)
60358
- return;
60359
- state.client = undefined;
60360
- scheduleRetry(state);
60361
- });
60362
- client.on("pending_stage_message", (request) => {
60363
- if (state.closed || state.client !== client)
60364
- return;
60365
- const event = { ...request, handled: false };
60366
- events.emit("atomic:workflow-pending-stage-message", event);
60367
- const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
60368
- completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
60369
- });
60370
- client.on("sticky_live_delivered", (notice) => {
60371
- if (state.closed || state.client !== client)
60372
- return;
60373
- events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
60374
- });
60375
- await client.connect({
60376
- cwd,
60377
- model: "workflow",
60378
- pid: process.pid,
60379
- startedAt: Date.now(),
60380
- lastActivity: Date.now(),
60381
- group: state.route.group,
60382
- groups: [state.route.group],
60383
- recipientPurpose: "control"
60472
+ function drop(connection, client) {
60473
+ if (connection.client === client) {
60474
+ connection.client = undefined;
60475
+ connection.acknowledged = false;
60476
+ for (const state of connection.routes.values()) {
60477
+ settle(state, false);
60478
+ scheduleRetry(state);
60479
+ }
60480
+ }
60481
+ client.disconnect().catch(() => {});
60482
+ }
60483
+ function connect(connection) {
60484
+ if (connection.client?.isConnected())
60485
+ return Promise.resolve(connection.client);
60486
+ connection.connecting ??= (async () => {
60487
+ const [extensionDir] = getBuiltinPackagePaths({
60488
+ workflows: false,
60489
+ subagents: false,
60490
+ mcp: false,
60491
+ "web-access": false
60492
+ });
60493
+ if (extensionDir === undefined)
60494
+ throw new Error("Intercom builtin package is unavailable");
60495
+ await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
60496
+ if (connections.get(connection.key) !== connection)
60497
+ throw closedError();
60498
+ const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
60499
+ multiplexesPendingStageRoutes: true
60500
+ });
60501
+ connection.client = client;
60502
+ connection.acknowledged = false;
60503
+ client.on("error", () => {});
60504
+ client.on("disconnected", () => drop(connection, client));
60505
+ client.on("pending_stage_message", (request) => {
60506
+ if (connection.client !== client)
60507
+ return;
60508
+ const event = { ...request, handled: false };
60509
+ events.emit("atomic:workflow-pending-stage-message", event);
60510
+ const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
60511
+ completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
60512
+ });
60513
+ client.on("sticky_live_delivered", (notice) => {
60514
+ if (connection.client !== client)
60515
+ return;
60516
+ events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
60517
+ });
60518
+ try {
60519
+ await client.connect({
60520
+ cwd,
60521
+ model: "workflow",
60522
+ pid: process.pid,
60523
+ startedAt: Date.now(),
60524
+ lastActivity: Date.now(),
60525
+ group: connection.group,
60526
+ groups: [connection.group],
60527
+ recipientPurpose: "control"
60528
+ });
60529
+ if (connections.get(connection.key) !== connection)
60530
+ throw closedError();
60531
+ } catch (error) {
60532
+ drop(connection, client);
60533
+ throw error;
60534
+ }
60535
+ for (const state of connection.routes.values()) {
60536
+ if (state.registered || state.retry === undefined)
60537
+ continue;
60538
+ clearTimeout(state.retry);
60539
+ state.retry = undefined;
60540
+ publish(state, state.route).catch(() => {});
60541
+ }
60542
+ return client;
60543
+ })().finally(() => {
60544
+ connection.connecting = undefined;
60384
60545
  });
60385
- return client;
60546
+ return connection.connecting;
60386
60547
  }
60387
60548
  function publish(state, route) {
60388
- const completion = state.tail.catch(() => {}).then(() => gate.run(async () => {
60389
- if (state.closed)
60390
- throw new Error("Workflow Intercom route owner closed");
60391
- const establishedClient = state.client?.isConnected() ? state.client : undefined;
60549
+ const completion = state.tail.catch(() => {}).then(async () => {
60550
+ if (!isCurrent(state))
60551
+ throw closedError();
60552
+ const connection = connectionOf(state);
60553
+ const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
60554
+ let client;
60392
60555
  try {
60393
- const client = await connect(state);
60556
+ client = await connect(connection);
60394
60557
  await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
60558
+ if (connection.client !== client || !isCurrent(state))
60559
+ return;
60560
+ connection.acknowledged = true;
60395
60561
  state.attempt = 0;
60396
60562
  clearTimeout(state.retry);
60397
60563
  state.retry = undefined;
60564
+ settle(state, true);
60398
60565
  } catch (error) {
60399
- const client = state.client;
60400
- if (client !== establishedClient || !client?.isConnected()) {
60401
- state.client = undefined;
60402
- await client?.disconnect();
60566
+ if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
60567
+ scheduleRetry(state);
60568
+ throw error;
60569
+ }
60570
+ if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
60571
+ detach(state);
60572
+ state.isolated = true;
60573
+ connectionOf(state);
60403
60574
  }
60575
+ if (client !== undefined && (client !== established || !client.isConnected()))
60576
+ drop(connection, client);
60404
60577
  scheduleRetry(state);
60405
60578
  throw error;
60406
60579
  }
60407
- }));
60580
+ });
60408
60581
  state.tail = completion;
60409
60582
  completion.catch(() => {});
60410
60583
  return completion;
@@ -60413,10 +60586,15 @@ function createWorkflowRouteOwner(cwd) {
60413
60586
  const route = payload;
60414
60587
  let state = routes.get(route.runId);
60415
60588
  if (state === undefined) {
60416
- state = { route, tail: Promise.resolve(), attempt: 0, closed: false };
60589
+ state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
60417
60590
  routes.set(route.runId, state);
60591
+ } else if (state.route.group !== route.group) {
60592
+ detach(state);
60593
+ state.isolated = false;
60594
+ settle(state, false);
60418
60595
  }
60419
60596
  state.route = route;
60597
+ connectionOf(state);
60420
60598
  route.completion = publish(state, route);
60421
60599
  });
60422
60600
  events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
@@ -60427,7 +60605,7 @@ function createWorkflowRouteOwner(cwd) {
60427
60605
  event.handled = true;
60428
60606
  event.completion = (async () => {
60429
60607
  await publish(state, state.route);
60430
- const client = state.client;
60608
+ const client = connections.get(connectionKey(state))?.client;
60431
60609
  if (client === undefined || !client.isConnected())
60432
60610
  return false;
60433
60611
  const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
@@ -60437,18 +60615,33 @@ function createWorkflowRouteOwner(cwd) {
60437
60615
  });
60438
60616
  return {
60439
60617
  events,
60618
+ registered(runId) {
60619
+ const state = routes.get(runId);
60620
+ if (state === undefined)
60621
+ return Promise.resolve(false);
60622
+ if (state.registered)
60623
+ return Promise.resolve(true);
60624
+ return new Promise((resolve) => state.waiters.add(resolve));
60625
+ },
60440
60626
  async closeGroup(group) {
60441
60627
  const closing = [...routes.values()].filter((state) => state.route.group === group);
60628
+ const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
60629
+ for (const connection of closingConnections) {
60630
+ connections.delete(connection.key);
60631
+ connection.routes.clear();
60632
+ }
60442
60633
  for (const state of closing) {
60443
- state.closed = true;
60444
- clearTimeout(state.retry);
60445
60634
  routes.delete(state.route.runId);
60635
+ clearTimeout(state.retry);
60636
+ state.retry = undefined;
60637
+ settle(state, false);
60446
60638
  }
60447
- await Promise.all(closing.map(async (state) => {
60448
- await state.client?.disconnect();
60449
- await state.tail.catch(() => {});
60450
- await state.client?.disconnect();
60639
+ await Promise.all(closingConnections.map(async (connection) => {
60640
+ await connection.client?.disconnect();
60641
+ await connection.connecting?.catch(() => {});
60451
60642
  }));
60643
+ await Promise.all(closing.map((state) => state.tail.catch(() => {})));
60644
+ await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
60452
60645
  }
60453
60646
  };
60454
60647
  }
@@ -60631,7 +60824,7 @@ var init_workflow_infrastructure_error = __esm(() => {
60631
60824
  [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
60632
60825
  [/Local worker retry limit/, "worker_retry_limit"],
60633
60826
  [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
60634
- [/Worker exited before delivery acknowledgement/, "worker_lost"],
60827
+ [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
60635
60828
  [/Stale work lease/, "stale_work_lease"],
60636
60829
  [/Intercom runtime no longer active/, "intercom_unavailable"],
60637
60830
  [/Storage unavailable/, "storage_unavailable"]
@@ -60889,7 +61082,7 @@ var init_definitions = __esm(() => {
60889
61082
  // dist/builtin/workflows/src/durable/journal/pool-mutations.ts
60890
61083
  import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
60891
61084
  async function validatePoolMutation(tx, journal, root, work, mutation) {
60892
- if (!["checkpoint", "session-create", "session-append", "session-replace"].includes(mutation.kind))
61085
+ if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
60893
61086
  throw new Error("Invalid pool mutation");
60894
61087
  const checkpoint = mutation.checkpoint;
60895
61088
  const prefix = `${work.scope}:`;
@@ -60957,10 +61150,18 @@ async function sessionStore(tx, prefix) {
60957
61150
  return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
60958
61151
  }
60959
61152
  async function applyPoolSession(tx, prefix, workId, mutation) {
61153
+ const owners = `${prefix}session_owners`;
61154
+ if (mutation.kind === "session-append") {
61155
+ if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
61156
+ throw new Error("Pool sessions require database identity");
61157
+ const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
61158
+ if (owner?.work !== workId)
61159
+ throw new Error("Pool session scope mismatch");
61160
+ return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
61161
+ }
60960
61162
  const { sessionId, replayKey } = mutation.checkpoint;
60961
61163
  if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
60962
61164
  throw new Error("Pool sessions require database identity");
60963
- const owners = `${prefix}session_owners`;
60964
61165
  if (mutation.kind === "session-create") {
60965
61166
  if (mutation.header.id !== sessionId)
60966
61167
  throw new Error("Pool session identity mismatch");
@@ -60970,10 +61171,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
60970
61171
  if (owner?.work !== workId || owner.replay_key !== replayKey)
60971
61172
  throw new Error("Pool session scope mismatch");
60972
61173
  }
61174
+ return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
61175
+ }
61176
+ async function withSessionStore(tx, prefix, write) {
60973
61177
  const store = await sessionStore(tx, prefix);
60974
61178
  try {
60975
- const result = mutation.kind === "session-create" ? await store.create(mutation.header, mutation.entries) : mutation.kind === "session-append" ? await store.append(sessionId, mutation.entries, { expectSeq: mutation.expectSeq }) : await store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq });
60976
- return result.lastSeq;
61179
+ return (await write(store)).lastSeq;
60977
61180
  } finally {
60978
61181
  await store.close();
60979
61182
  }
@@ -61021,8 +61224,11 @@ var init_pool_stage_usage = __esm(() => {
61021
61224
  });
61022
61225
 
61023
61226
  // dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
61024
- import { randomUUID as randomUUID15 } from "node:crypto";
61227
+ import { createHash as createHash4, randomUUID as randomUUID15 } from "node:crypto";
61025
61228
  import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
61229
+ function requestIdentity(request) {
61230
+ return createHash4("sha256").update(request).digest("hex");
61231
+ }
61026
61232
  function fromRow(row) {
61027
61233
  return {
61028
61234
  ...JSON.parse(row.descriptor),
@@ -61078,10 +61284,11 @@ class SqlPoolQueue {
61078
61284
  const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
61079
61285
  if (versions.length > 1)
61080
61286
  throw new Error("Corrupt pool schema metadata");
61081
- if (versions.length === 0)
61082
- await tx.run(`INSERT INTO ${pool.meta} VALUES (1)`);
61083
- else
61084
- assertStorageSchemaVersion2("journal", versions[0].version, 1);
61287
+ const version = versions[0]?.version;
61288
+ if (version === undefined)
61289
+ await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
61290
+ else if (version !== 1)
61291
+ assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
61085
61292
  await tx.exec(`
61086
61293
  CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
61087
61294
  CREATE TABLE IF NOT EXISTS ${pool.work}(
@@ -61112,9 +61319,23 @@ class SqlPoolQueue {
61112
61319
  state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
61113
61320
  )`);
61114
61321
  await tx.exec(poolUsageSchema(journal.runs));
61322
+ if (version === 1)
61323
+ await pool.migrateFromVersion1(tx);
61115
61324
  });
61116
61325
  return pool;
61117
61326
  }
61327
+ async migrateFromVersion1(tx) {
61328
+ await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
61329
+ for (let after = 0;; ) {
61330
+ const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
61331
+ if (rows.length === 0)
61332
+ break;
61333
+ for (const row of rows)
61334
+ await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
61335
+ after = Number(rows.at(-1).rowid);
61336
+ }
61337
+ await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
61338
+ }
61118
61339
  async authority(tx, lease, timeout) {
61119
61340
  const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
61120
61341
  if (row?.timeout === null && timeout === undefined)
@@ -61255,31 +61476,18 @@ class SqlPoolQueue {
61255
61476
  throw new Error(STALE_WORK_LEASE);
61256
61477
  return row;
61257
61478
  }
61258
- async renew(lease, token, operationId, durationMs) {
61479
+ async renew(lease, token, durationMs) {
61259
61480
  positive(durationMs);
61260
61481
  lease = snapshotLease(lease);
61261
61482
  token = { ...token };
61262
- validateText("key", operationId);
61263
- const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
61264
61483
  return this.transaction(async (tx) => {
61265
61484
  const { now } = await this.authority(tx, lease);
61266
61485
  const row = await this.current(tx, lease, token);
61267
61486
  if (row.epoch !== lease.epoch || row.expires_at < now)
61268
61487
  throw new Error(STALE_WORK_LEASE);
61269
- const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
61270
- if (previous) {
61271
- if (previous.request !== request)
61272
- throw new Error("Pool operation identity conflict");
61273
- return JSON.parse(previous.receipt);
61274
- }
61275
- const receipt = {
61276
- ...fromRow(row),
61277
- expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
61278
- renewedAt: now
61279
- };
61280
- await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
61281
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
61282
- return receipt;
61488
+ const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
61489
+ await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
61490
+ return { ...fromRow(row), expiresAt, renewedAt: now };
61283
61491
  });
61284
61492
  }
61285
61493
  async commit(lease, token, operationId, mutation) {
@@ -61288,6 +61496,7 @@ class SqlPoolQueue {
61288
61496
  validateText("key", operationId);
61289
61497
  assertWorkflowSerializableObject(mutation, "Pool mutation");
61290
61498
  const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
61499
+ const identity = requestIdentity(request);
61291
61500
  mutation = JSON.parse(request).mutation;
61292
61501
  return this.transaction(async (tx) => {
61293
61502
  const { now, status } = await this.authority(tx, lease);
@@ -61296,7 +61505,7 @@ class SqlPoolQueue {
61296
61505
  throw new Error(STALE_WORK_LEASE);
61297
61506
  const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
61298
61507
  if (previous) {
61299
- if (previous.request !== request)
61508
+ if (previous.request !== identity)
61300
61509
  throw new Error("Pool operation identity conflict");
61301
61510
  return JSON.parse(previous.receipt);
61302
61511
  }
@@ -61304,34 +61513,41 @@ class SqlPoolQueue {
61304
61513
  throw new Error(STALE_WORK_LEASE);
61305
61514
  if (!await this.admitsAttemptWrites(tx, lease, status))
61306
61515
  throw new Error("Pool root is not running");
61307
- const checkpoint = mutation.checkpoint;
61308
- await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
61309
- if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
61310
- throw new Error("Pool checkpoint timestamp is in the future");
61311
- assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
61312
- const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
61313
- const payload = serializeJson(checkpoint);
61314
- const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
61315
- if (existing && existing.payload !== payload)
61316
- throw new Error("Pool checkpoint identity conflict");
61317
- if (existing && mutation.kind !== "checkpoint")
61318
- throw new Error("Pool session checkpoint identity conflict");
61319
- const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
61320
- const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
61321
- const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
61322
- seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
61323
- at: now,
61324
- ...lastSeq === undefined ? {} : { lastSeq }
61325
- };
61326
- if (!existing)
61327
- await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
61328
- const billed = poolCheckpointUsage(checkpoint);
61329
- if (billed && !existing)
61330
- await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
61331
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
61516
+ const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
61517
+ await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
61332
61518
  return receipt;
61333
61519
  });
61334
61520
  }
61521
+ async appendSession(tx, row, mutation, now) {
61522
+ return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
61523
+ }
61524
+ async recordCheckpoint(tx, lease, token, row, mutation, now) {
61525
+ const checkpoint = mutation.checkpoint;
61526
+ await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
61527
+ if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
61528
+ throw new Error("Pool checkpoint timestamp is in the future");
61529
+ assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
61530
+ const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
61531
+ const payload = serializeJson(checkpoint);
61532
+ const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
61533
+ if (existing && existing.payload !== payload)
61534
+ throw new Error("Pool checkpoint identity conflict");
61535
+ if (existing && mutation.kind !== "checkpoint")
61536
+ throw new Error("Pool session checkpoint identity conflict");
61537
+ const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
61538
+ const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
61539
+ const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
61540
+ seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
61541
+ at: now,
61542
+ ...lastSeq === undefined ? {} : { lastSeq }
61543
+ };
61544
+ if (!existing)
61545
+ await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
61546
+ const billed = poolCheckpointUsage(checkpoint);
61547
+ if (billed && !existing)
61548
+ await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
61549
+ return receipt;
61550
+ }
61335
61551
  async complete(lease, token, result) {
61336
61552
  lease = snapshotLease(lease);
61337
61553
  token = { ...token };
@@ -61418,7 +61634,7 @@ function openSqlPoolQueue(journal) {
61418
61634
  return Promise.reject(new Error("Worker pool requires a SQL run journal"));
61419
61635
  return SqlPoolQueue.open(storage);
61420
61636
  }
61421
- var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
61637
+ var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
61422
61638
  var init_sql_pool_queue = __esm(() => {
61423
61639
  init_serializable();
61424
61640
  init_boundary_topology();
@@ -61430,6 +61646,15 @@ var init_sql_pool_queue = __esm(() => {
61430
61646
  init_values();
61431
61647
  });
61432
61648
 
61649
+ // dist/builtin/workflows/src/workers/coordinator-protocol.ts
61650
+ var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
61651
+
61652
+ // dist/builtin/workflows/src/workers/host-options.ts
61653
+ var hostWorkerBindings;
61654
+ var init_host_options = __esm(() => {
61655
+ hostWorkerBindings = Symbol("host-worker-bindings");
61656
+ });
61657
+
61433
61658
  // dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
61434
61659
  function validate(report) {
61435
61660
  validateText("key", report.key);
@@ -61500,10 +61725,870 @@ var init_sql_pool_usage = __esm(() => {
61500
61725
  init_values();
61501
61726
  });
61502
61727
 
61503
- // dist/builtin/workflows/src/workers/host-options.ts
61504
- var hostWorkerBindings;
61505
- var init_host_options = __esm(() => {
61506
- hostWorkerBindings = Symbol("host-worker-bindings");
61728
+ // dist/builtin/workflows/src/workers/pool-coordinator.ts
61729
+ import { fork } from "node:child_process";
61730
+ import { randomUUID as randomUUID16 } from "node:crypto";
61731
+ import { createWriteStream } from "node:fs";
61732
+ import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
61733
+ import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
61734
+ import { setTimeout as delay2 } from "node:timers/promises";
61735
+ function workerLogDirectory(journalPath, root) {
61736
+ return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
61737
+ }
61738
+ function token(work) {
61739
+ return {
61740
+ id: work.id,
61741
+ generation: work.generation,
61742
+ worker: work.worker,
61743
+ epoch: work.epoch,
61744
+ renewedAt: work.renewedAt,
61745
+ expiresAt: work.expiresAt
61746
+ };
61747
+ }
61748
+ function outside(runId) {
61749
+ return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
61750
+ }
61751
+ function isLiveStage(stage) {
61752
+ return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
61753
+ }
61754
+ function projectRootRun(run) {
61755
+ return {
61756
+ id: run.id,
61757
+ name: run.name,
61758
+ status: run.status,
61759
+ startedAt: run.startedAt,
61760
+ stages: run.stages.map((stage) => ({
61761
+ id: stage.id,
61762
+ name: stage.name,
61763
+ status: stage.status,
61764
+ parentIds: [],
61765
+ toolEvents: [],
61766
+ ...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
61767
+ ...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
61768
+ ...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
61769
+ ...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
61770
+ }))
61771
+ };
61772
+ }
61773
+
61774
+ class PooledRoutes {
61775
+ root;
61776
+ owner;
61777
+ mirror = createStore();
61778
+ owners = new Map;
61779
+ attempts = new Map;
61780
+ activated = false;
61781
+ bridge;
61782
+ constructor(root, owner) {
61783
+ this.root = root;
61784
+ this.owner = owner;
61785
+ }
61786
+ static async open(root, cwd, rootRun, forward) {
61787
+ await Promise.resolve().then(() => init_workflow_route_owner());
61788
+ const owner = createWorkflowRouteOwner(cwd);
61789
+ const routes = new PooledRoutes(root, owner);
61790
+ if (rootRun !== undefined)
61791
+ routes.updateRoot(rootRun);
61792
+ routes.bridge = registerPendingStageIntercomBridge({
61793
+ events: {
61794
+ emit: (event, payload) => owner.events.emit(event, payload),
61795
+ on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
61796
+ }
61797
+ }, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
61798
+ for (const [event, kind] of [
61799
+ [PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
61800
+ [STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
61801
+ ])
61802
+ owner.events.on(event, (payload) => {
61803
+ const claimed = payload;
61804
+ if (claimed.handled !== false)
61805
+ return;
61806
+ claimed.handled = true;
61807
+ const { handled: _handled, completion: _completion, ...forwarded } = claimed;
61808
+ claimed.completion = forward({ kind, payload: forwarded });
61809
+ });
61810
+ return routes;
61811
+ }
61812
+ updateRoot(rootRun) {
61813
+ this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
61814
+ }
61815
+ accept(entries) {
61816
+ this.mirror.batch(() => {
61817
+ for (const { work, runs } of entries) {
61818
+ let attempt = this.attempts.get(work.id);
61819
+ if (attempt !== undefined && work.generation < attempt.generation)
61820
+ continue;
61821
+ if (attempt === undefined || attempt.generation !== work.generation) {
61822
+ if (attempt !== undefined)
61823
+ this.release(attempt);
61824
+ attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
61825
+ this.attempts.set(work.id, attempt);
61826
+ }
61827
+ if (attempt.fenced)
61828
+ continue;
61829
+ for (const run of runs) {
61830
+ attempt.runs.add(run.id);
61831
+ this.owners.set(run.id, work.id);
61832
+ const { pendingStageMessages: _messages, ...published } = run;
61833
+ this.mirror.recordRunStart({ ...published, rootRunId: this.root });
61834
+ if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
61835
+ this.activated = true;
61836
+ }
61837
+ }
61838
+ });
61839
+ }
61840
+ fence(work) {
61841
+ const attempt = this.attempts.get(work.id);
61842
+ if (attempt === undefined || attempt.generation > work.generation)
61843
+ return;
61844
+ if (attempt.generation < work.generation) {
61845
+ this.release(attempt);
61846
+ this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
61847
+ return;
61848
+ }
61849
+ this.release(attempt);
61850
+ attempt.fenced = true;
61851
+ }
61852
+ async route(work, runId) {
61853
+ this.requireOwned(work, runId);
61854
+ const readiness = workflowPendingStageRouteReady(this.mirror, runId);
61855
+ await readiness?.completion.catch(() => {});
61856
+ readiness?.assertCurrent();
61857
+ const waiting = new AbortController;
61858
+ await Promise.race([
61859
+ this.owner.registered(runId),
61860
+ delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
61861
+ ]);
61862
+ waiting.abort();
61863
+ this.requireOwned(work, runId);
61864
+ return workflowPendingStageRouteCapability(this.mirror, runId);
61865
+ }
61866
+ async undeliverable(payload) {
61867
+ const event = { ...payload, handled: false };
61868
+ this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
61869
+ return event.handled && await event.completion === true;
61870
+ }
61871
+ async close() {
61872
+ const bridge = this.bridge;
61873
+ if (bridge === undefined)
61874
+ return;
61875
+ this.bridge = undefined;
61876
+ try {
61877
+ await bridge.flush();
61878
+ } finally {
61879
+ bridge.refresh();
61880
+ bridge();
61881
+ await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
61882
+ }
61883
+ }
61884
+ requireOwned(work, runId) {
61885
+ const attempt = this.attempts.get(work.id);
61886
+ if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
61887
+ throw new Error(STALE_WORK_LEASE);
61888
+ if (!attempt.runs.has(runId))
61889
+ throw outside(runId);
61890
+ }
61891
+ release(attempt) {
61892
+ this.mirror.batch(() => {
61893
+ for (const id of attempt.runs) {
61894
+ rotateWorkflowPendingStageRouteCapability(this.mirror, id);
61895
+ const run = this.mirror.runs().find((candidate) => candidate.id === id);
61896
+ if (run?.stages.some(isLiveStage) !== true)
61897
+ continue;
61898
+ this.mirror.recordRunStart({
61899
+ ...run,
61900
+ stages: run.stages.map((stage) => {
61901
+ if (!isLiveStage(stage))
61902
+ return stage;
61903
+ const { sessionId: _session, sessionFile: _file, ...rest } = stage;
61904
+ return { ...rest, attachable: false, attached: false };
61905
+ })
61906
+ });
61907
+ }
61908
+ });
61909
+ }
61910
+ }
61911
+
61912
+ class PoolCoordinator {
61913
+ port;
61914
+ children = new Set;
61915
+ incarnations = new Map;
61916
+ readyWorkers = new Set;
61917
+ logs = new Map;
61918
+ cleanups = new Set;
61919
+ controlAcks = new Map;
61920
+ lastHeard = new Map;
61921
+ pendingActivity = new Map;
61922
+ batches = new Map;
61923
+ publicationFailures = new Map;
61924
+ requests = new Map;
61925
+ deliveryRequests = new Map;
61926
+ inflight = new Set;
61927
+ lastTick = performance.now();
61928
+ stalledMs = 0;
61929
+ ticker;
61930
+ recoveryTimer;
61931
+ flushTimer;
61932
+ lastFlushAt = 0;
61933
+ batchSequence = 0;
61934
+ requestSequence = 0;
61935
+ usageChanged = false;
61936
+ budgetBlocked = false;
61937
+ desiredControl;
61938
+ root;
61939
+ lease;
61940
+ options;
61941
+ journal;
61942
+ queue;
61943
+ usageLedger;
61944
+ usageTotal;
61945
+ admission;
61946
+ routes;
61947
+ publishing = false;
61948
+ sources = [];
61949
+ logDirectory;
61950
+ starting;
61951
+ recovering;
61952
+ nextRecoveryAt = 0;
61953
+ cleanupFailure;
61954
+ closed = false;
61955
+ closing;
61956
+ replacements = 0;
61957
+ constructor(port) {
61958
+ this.port = port;
61959
+ port.onMessage((message) => this.receive(message));
61960
+ this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
61961
+ }
61962
+ async shutdown() {
61963
+ await this.closing?.catch(() => {});
61964
+ while (this.inflight.size > 0)
61965
+ await Promise.allSettled([...this.inflight]);
61966
+ await this.journal?.close().catch(() => {});
61967
+ }
61968
+ receive(message) {
61969
+ if (message.kind === "request-reply") {
61970
+ const pending = this.requests.get(message.id);
61971
+ this.requests.delete(message.id);
61972
+ if (message.error !== undefined)
61973
+ pending?.reject(new Error(message.error));
61974
+ else
61975
+ pending?.resolve(message.value);
61976
+ return;
61977
+ }
61978
+ if ("id" in message) {
61979
+ const command = message;
61980
+ const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
61981
+ this.inflight.add(handled);
61982
+ handled.finally(() => this.inflight.delete(handled));
61983
+ return;
61984
+ }
61985
+ this.notice(message);
61986
+ }
61987
+ notice(notice) {
61988
+ if (notice.kind === "budget")
61989
+ this.budgetBlocked = notice.blocked;
61990
+ else if (notice.kind === "root")
61991
+ this.routes?.updateRoot(notice.rootRun);
61992
+ else {
61993
+ const batch = this.batches.get(notice.batch);
61994
+ this.batches.delete(notice.batch);
61995
+ const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
61996
+ for (const entry of rejected.values())
61997
+ this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
61998
+ this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
61999
+ batch?.settle();
62000
+ }
62001
+ }
62002
+ async execute(command) {
62003
+ switch (command.kind) {
62004
+ case "prepare":
62005
+ return this.prepare(command.root, command.lease, command.options, command.control);
62006
+ case "control":
62007
+ return this.control(command.state);
62008
+ case "start":
62009
+ this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
62010
+ return this.starting;
62011
+ case "enqueue":
62012
+ return this.enqueue(command);
62013
+ case "transport":
62014
+ return this.transport(command.request, command);
62015
+ case "undeliverable":
62016
+ return await this.routes?.undeliverable(command.payload) ?? false;
62017
+ case "close":
62018
+ this.closing ??= this.close();
62019
+ return this.closing;
62020
+ }
62021
+ }
62022
+ async prepare(root, lease, options, control) {
62023
+ if (this.closed)
62024
+ throw new Error("Local worker pool is closed");
62025
+ if (this.root !== undefined && this.root !== root)
62026
+ throw new Error("Local worker pool belongs to another root");
62027
+ this.root = root;
62028
+ this.options = options;
62029
+ this.desiredControl ??= control;
62030
+ this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
62031
+ this.usageLedger ??= await openSqlPoolUsage(this.journal);
62032
+ this.usageTotal = await this.usageLedger.total(root);
62033
+ this.lease = lease;
62034
+ const queue = await openSqlPoolQueue(this.journal);
62035
+ await queue.initialize(lease, COORDINATOR_LEASE_MS);
62036
+ await queue.releasePreviousIncarnations(lease);
62037
+ for (;; ) {
62038
+ const intended = this.desiredControl ?? "running";
62039
+ await queue.setControl(lease, intended);
62040
+ if (intended === (this.desiredControl ?? "running"))
62041
+ break;
62042
+ }
62043
+ this.queue = queue;
62044
+ return this.usageTotal;
62045
+ }
62046
+ async control(state) {
62047
+ this.desiredControl = state;
62048
+ if (!this.queue || !this.lease)
62049
+ return;
62050
+ const control = await this.queue.setControl(this.lease, state);
62051
+ const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
62052
+ while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
62053
+ if (this.desiredControl !== state)
62054
+ return;
62055
+ if (Date.now() >= deadline) {
62056
+ if (state === "quit")
62057
+ break;
62058
+ throw new Error("Worker control acknowledgement unresolved");
62059
+ }
62060
+ await delay2(25);
62061
+ }
62062
+ this.flush();
62063
+ }
62064
+ async start(sources, cwd, publish, rootRun) {
62065
+ if (this.closed)
62066
+ throw new Error("Local worker pool is closed");
62067
+ const root = this.root;
62068
+ if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
62069
+ throw new Error("Local worker pool requires root preparation");
62070
+ this.sources = sources;
62071
+ this.publishing = publish;
62072
+ if (publish) {
62073
+ this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
62074
+ if (this.closed) {
62075
+ await this.routes.close();
62076
+ throw new Error("Local worker pool is closed");
62077
+ }
62078
+ }
62079
+ if (this.options.admission)
62080
+ this.admission = await SqlResourceAdmission.open(this.journal, {
62081
+ ...this.options.admission,
62082
+ root,
62083
+ holder: "coordinator"
62084
+ });
62085
+ this.logDirectory = workerLogDirectory(this.options.journalPath, root);
62086
+ await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
62087
+ return;
62088
+ });
62089
+ this.recoveryTimer = setInterval(() => {
62090
+ this.recoverExpired().catch((error) => this.fail(error));
62091
+ }, RECOVERY_INTERVAL_MS).unref();
62092
+ const workers = Array.from({ length: this.options.workers }, () => this.spawn());
62093
+ try {
62094
+ await Promise.any(workers);
62095
+ } catch (error) {
62096
+ throw error.errors[0];
62097
+ }
62098
+ }
62099
+ async enqueue(command) {
62100
+ if (!this.queue || !this.lease || this.root === undefined)
62101
+ throw new Error("Local worker pool requires root preparation");
62102
+ if (command.rootRun !== undefined)
62103
+ this.routes?.updateRoot(command.rootRun);
62104
+ const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
62105
+ const result = await this.queue.result(this.root, work.id);
62106
+ return { work, ...result === undefined ? {} : { result } };
62107
+ }
62108
+ async transport(request, payload) {
62109
+ const target = this.deliveryRequests.get(request);
62110
+ if (target === undefined || !this.incarnations.has(target.slot.worker))
62111
+ throw new Error("Worker exited before delivery acknowledgement");
62112
+ const { slot, workerRequest } = target;
62113
+ await new Promise((resolve, reject) => {
62114
+ const id = randomUUID16();
62115
+ slot.deliveries.set(id, { resolve, reject });
62116
+ slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
62117
+ if (error) {
62118
+ slot.deliveries.delete(id);
62119
+ reject(error);
62120
+ }
62121
+ });
62122
+ });
62123
+ }
62124
+ request(body) {
62125
+ const id = ++this.requestSequence;
62126
+ return new Promise((resolve, reject) => {
62127
+ this.requests.set(id, { resolve, reject });
62128
+ this.port.send({ kind: "request", id, request: body });
62129
+ });
62130
+ }
62131
+ fail(error) {
62132
+ this.port.send({ kind: "failure", error: error.message });
62133
+ }
62134
+ publish(work, runs) {
62135
+ const failure = this.publicationFailures.get(work.id);
62136
+ if (failure?.generation === work.generation)
62137
+ throw new Error(failure.error);
62138
+ if (!this.publishing)
62139
+ return;
62140
+ this.pendingActivity.set(work.id, { work: token(work), runs });
62141
+ this.scheduleFlush();
62142
+ }
62143
+ scheduleFlush() {
62144
+ if (this.flushTimer !== undefined || this.closed)
62145
+ return;
62146
+ const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
62147
+ this.flushTimer = setTimeout(() => {
62148
+ this.flushTimer = undefined;
62149
+ if (this.batches.size > 0) {
62150
+ this.lastFlushAt = performance.now();
62151
+ this.scheduleFlush();
62152
+ return;
62153
+ }
62154
+ this.flush();
62155
+ }, wait);
62156
+ }
62157
+ flush() {
62158
+ clearTimeout(this.flushTimer);
62159
+ this.flushTimer = undefined;
62160
+ if (this.pendingActivity.size === 0 && !this.usageChanged) {
62161
+ const last = [...this.batches.values()].at(-1);
62162
+ return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
62163
+ }
62164
+ this.lastFlushAt = performance.now();
62165
+ const batch = ++this.batchSequence;
62166
+ const entries = [...this.pendingActivity.values()];
62167
+ this.pendingActivity.clear();
62168
+ const usage = this.usageChanged ? this.usageTotal : undefined;
62169
+ this.usageChanged = false;
62170
+ const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
62171
+ this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
62172
+ return settled;
62173
+ }
62174
+ chainSettle(batch, resolve) {
62175
+ const settle = batch.settle;
62176
+ batch.settle = () => {
62177
+ settle();
62178
+ resolve();
62179
+ };
62180
+ }
62181
+ noteUsage(usage) {
62182
+ this.usageTotal = usage;
62183
+ this.usageChanged = true;
62184
+ this.scheduleFlush();
62185
+ }
62186
+ async sendResult(workId) {
62187
+ const result = await this.queue.result(this.root, workId);
62188
+ if (result === undefined)
62189
+ return false;
62190
+ this.flush();
62191
+ this.port.send({
62192
+ kind: "result",
62193
+ workId,
62194
+ result,
62195
+ ...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
62196
+ });
62197
+ return true;
62198
+ }
62199
+ fence(work) {
62200
+ this.pendingActivity.delete(work.id);
62201
+ this.routes?.fence(work);
62202
+ this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
62203
+ }
62204
+ spawn() {
62205
+ if (this.closed)
62206
+ return Promise.reject(new Error("Local worker pool is closed"));
62207
+ const root = this.root;
62208
+ const options = this.options;
62209
+ const child = fork(options.workerEntry ?? process.argv[1], [
62210
+ JSON.stringify({
62211
+ journalPath: options.journalPath,
62212
+ root,
62213
+ definitions: this.sources,
62214
+ admission: options.admission
62215
+ })
62216
+ ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62217
+ const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62218
+ this.children.add(child);
62219
+ if (child.pid !== undefined)
62220
+ this.port.send({ kind: "spawned", pid: child.pid });
62221
+ const worker = randomUUID16();
62222
+ this.incarnations.set(worker, child);
62223
+ const slot = { worker, child, deliveries: new Map };
62224
+ let ready = false;
62225
+ let serial = Promise.resolve();
62226
+ const logPath = join28(this.logDirectory, `${worker}.log`);
62227
+ this.logs.set(child, logPath);
62228
+ let logFailure;
62229
+ const log = createWriteStream(logPath, { flags: "a" });
62230
+ log.on("error", (error) => {
62231
+ logFailure = error.message;
62232
+ });
62233
+ let stderr = "";
62234
+ child.stderr?.on("data", (chunk) => {
62235
+ stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62236
+ });
62237
+ child.stderr?.on("error", (error) => {
62238
+ logFailure ??= error.message;
62239
+ log.end();
62240
+ });
62241
+ child.stderr?.pipe(log);
62242
+ const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62243
+ const startupTimeoutMs = options.startupTimeoutMs;
62244
+ let timedOut = false;
62245
+ let terminated = false;
62246
+ const terminate = () => {
62247
+ if (terminated)
62248
+ return;
62249
+ terminated = true;
62250
+ child.kill("SIGKILL");
62251
+ };
62252
+ const replied = (error) => {
62253
+ if (error)
62254
+ terminate();
62255
+ };
62256
+ return new Promise((resolve, reject) => {
62257
+ const startup = setTimeout(() => {
62258
+ timedOut = true;
62259
+ terminate();
62260
+ }, startupTimeoutMs);
62261
+ child.on("error", (error) => {
62262
+ clearTimeout(startup);
62263
+ reject(error);
62264
+ terminate();
62265
+ });
62266
+ const closed = new Promise((resolve) => child.once("close", () => resolve()));
62267
+ child.once("exit", () => {
62268
+ Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62269
+ this.children.delete(child);
62270
+ this.logs.delete(child);
62271
+ if (child.pid !== undefined)
62272
+ this.port.send({ kind: "exited", pid: child.pid });
62273
+ clearTimeout(startup);
62274
+ this.incarnations.delete(worker);
62275
+ this.readyWorkers.delete(worker);
62276
+ this.controlAcks.delete(worker);
62277
+ this.lastHeard.delete(worker);
62278
+ if (slot.work)
62279
+ this.fence(slot.work);
62280
+ for (const delivery of slot.deliveries.values())
62281
+ delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62282
+ slot.deliveries.clear();
62283
+ const cleanup = disconnected.then(() => serial).then(async () => {
62284
+ try {
62285
+ if (await this.admission?.releaseStoppedSessions(worker))
62286
+ throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62287
+ } catch (error) {
62288
+ this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62289
+ throw this.cleanupFailure;
62290
+ }
62291
+ }).then(async () => {
62292
+ if (this.closed || !ready || this.desiredControl === "quit")
62293
+ return;
62294
+ let work = slot.work;
62295
+ if (work && await this.sendResult(work.id)) {
62296
+ work = undefined;
62297
+ slot.work = undefined;
62298
+ this.replacements = 0;
62299
+ }
62300
+ if (this.closed)
62301
+ return;
62302
+ if (work)
62303
+ await this.queue.releaseExited(this.lease, work);
62304
+ const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62305
+ if (exhausted)
62306
+ this.fail(diagnosed("Local worker retry limit"));
62307
+ else
62308
+ await this.spawn();
62309
+ }).catch((error) => {
62310
+ this.fail(error);
62311
+ });
62312
+ this.cleanups.add(cleanup);
62313
+ cleanup.finally(() => this.cleanups.delete(cleanup));
62314
+ });
62315
+ child.on("message", (message) => {
62316
+ this.observeResponsiveness();
62317
+ this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
62318
+ if (message.kind === "delivery-ack") {
62319
+ const pending = slot.deliveries.get(message.deliveryId);
62320
+ slot.deliveries.delete(message.deliveryId);
62321
+ if (message.error)
62322
+ pending?.reject(new Error(message.error));
62323
+ else
62324
+ pending?.resolve();
62325
+ return;
62326
+ }
62327
+ let routeReady;
62328
+ const handled = serial.then(async () => {
62329
+ if (message.kind === "ready") {
62330
+ if (terminated)
62331
+ return;
62332
+ ready = true;
62333
+ this.readyWorkers.add(worker);
62334
+ clearTimeout(startup);
62335
+ resolve();
62336
+ return;
62337
+ }
62338
+ await this.starting;
62339
+ const queue = this.queue;
62340
+ const lease = this.lease;
62341
+ if (message.kind === "control")
62342
+ return queue.control(lease);
62343
+ if (message.kind === "controlled") {
62344
+ const control = await queue.control(lease);
62345
+ if (message.revision > control.revision)
62346
+ throw new Error("Unknown control revision");
62347
+ this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62348
+ return;
62349
+ }
62350
+ if (message.kind === "claim") {
62351
+ if (slot.work)
62352
+ throw new Error("Worker already holds an attempt");
62353
+ if (this.budgetBlocked)
62354
+ return;
62355
+ await this.recoverExpired();
62356
+ slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
62357
+ definition: `${source.path}#${source.exportName}`,
62358
+ digest: source.digest
62359
+ })), options.leaseMs);
62360
+ return slot.work;
62361
+ }
62362
+ let work = slot.work;
62363
+ if (!work)
62364
+ throw new Error("Worker has no active attempt");
62365
+ if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
62366
+ work = await queue.extend(lease, work, options.leaseMs);
62367
+ slot.work = work;
62368
+ }
62369
+ if (message.kind === "activity")
62370
+ return this.publish(work, message.runs);
62371
+ if (message.kind === "route") {
62372
+ if (this.routes === undefined)
62373
+ return;
62374
+ await this.flush();
62375
+ this.assertPublished(work);
62376
+ routeReady = this.routes.route(work, message.runId);
62377
+ return;
62378
+ }
62379
+ if (message.kind === "delivery-ready") {
62380
+ if (!this.publishing)
62381
+ return;
62382
+ await this.flush();
62383
+ this.assertPublished(work);
62384
+ return this.request({
62385
+ kind: "ready",
62386
+ work: token(work),
62387
+ runId: message.runId,
62388
+ stageId: message.stageId,
62389
+ stageName: message.stageName,
62390
+ ...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
62391
+ });
62392
+ }
62393
+ if (message.kind === "delivery") {
62394
+ if (!message.recipient)
62395
+ throw new Error("Worker delivery requires a session identity");
62396
+ if (!this.publishing)
62397
+ return;
62398
+ await this.flush();
62399
+ this.assertPublished(work);
62400
+ const id = ++this.requestSequence;
62401
+ this.deliveryRequests.set(id, { slot, workerRequest: message.id });
62402
+ try {
62403
+ return await new Promise((resolve, reject) => {
62404
+ this.requests.set(id, { resolve, reject });
62405
+ this.port.send({
62406
+ kind: "request",
62407
+ id,
62408
+ request: {
62409
+ kind: "deliver",
62410
+ work: token(work),
62411
+ runId: message.runId,
62412
+ stageId: message.stageId,
62413
+ stageName: message.stageName,
62414
+ recipient: message.recipient
62415
+ }
62416
+ });
62417
+ });
62418
+ } finally {
62419
+ this.deliveryRequests.delete(id);
62420
+ }
62421
+ }
62422
+ if (message.kind === "renew")
62423
+ return;
62424
+ if (message.kind === "commit") {
62425
+ const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62426
+ this.noteUsage(await this.usageLedger.total(root));
62427
+ return receipt;
62428
+ }
62429
+ if (message.kind === "usage") {
62430
+ this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
62431
+ return;
62432
+ }
62433
+ if (message.kind === "budget") {
62434
+ await queue.assertCurrent(lease, work);
62435
+ if (this.budgetBlocked)
62436
+ throw new Error("Root workflow budget exhausted");
62437
+ return queue.control(lease);
62438
+ }
62439
+ if (message.kind !== "complete")
62440
+ throw new Error("Invalid worker operation");
62441
+ await queue.complete(lease, work, message.result);
62442
+ this.replacements = 0;
62443
+ slot.work = undefined;
62444
+ await this.sendResult(work.id);
62445
+ });
62446
+ const response = handled.then((value2) => routeReady ?? value2);
62447
+ if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62448
+ serial = handled.then(() => {}, () => {});
62449
+ response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62450
+ });
62451
+ });
62452
+ }
62453
+ assertPublished(work) {
62454
+ const failure = this.publicationFailures.get(work.id);
62455
+ if (failure?.generation === work.generation)
62456
+ throw new Error(failure.error);
62457
+ }
62458
+ recoverExpired() {
62459
+ if (this.recovering)
62460
+ return this.recovering;
62461
+ const root = this.root;
62462
+ if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
62463
+ return Promise.resolve();
62464
+ const queue = this.queue;
62465
+ const lease = this.lease;
62466
+ const leaseMs = this.options.leaseMs;
62467
+ this.recovering = (async () => {
62468
+ await new Promise((resolve) => setImmediate(resolve));
62469
+ await this.admission?.reconcile();
62470
+ const expiredBefore = Date.now();
62471
+ for (const work of await queue.inspect(root)) {
62472
+ if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62473
+ continue;
62474
+ const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62475
+ if (child && work.worker !== null && this.isLive(work.worker, child)) {
62476
+ await queue.extend(lease, work, leaseMs).catch((error) => {
62477
+ if (error.message !== STALE_WORK_LEASE)
62478
+ throw error;
62479
+ });
62480
+ continue;
62481
+ }
62482
+ if (child && !await this.stop(child).then(() => true, () => false))
62483
+ continue;
62484
+ await queue.expire(lease, work);
62485
+ }
62486
+ })().finally(() => {
62487
+ this.recovering = undefined;
62488
+ this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62489
+ });
62490
+ return this.recovering;
62491
+ }
62492
+ observeResponsiveness() {
62493
+ const now = performance.now();
62494
+ const gap = now - this.lastTick;
62495
+ if (gap > STALL_THRESHOLD_MS)
62496
+ this.stalledMs += gap - TICK_INTERVAL_MS;
62497
+ this.lastTick = now;
62498
+ }
62499
+ isLive(worker, child) {
62500
+ this.observeResponsiveness();
62501
+ const heard = this.lastHeard.get(worker);
62502
+ return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
62503
+ }
62504
+ stop(child) {
62505
+ if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62506
+ return Promise.resolve();
62507
+ return new Promise((resolve, reject) => {
62508
+ const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62509
+ child.once("exit", () => {
62510
+ clearTimeout(timeout);
62511
+ resolve();
62512
+ });
62513
+ child.kill("SIGKILL");
62514
+ });
62515
+ }
62516
+ async close() {
62517
+ this.closed = true;
62518
+ clearInterval(this.ticker);
62519
+ clearInterval(this.recoveryTimer);
62520
+ let failure;
62521
+ let stopping = [];
62522
+ try {
62523
+ await this.recovering?.catch(() => {});
62524
+ stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62525
+ await Promise.all(stopping.map(({ child }) => this.stop(child)));
62526
+ await Promise.all([...this.cleanups]);
62527
+ if (this.cleanupFailure)
62528
+ throw this.cleanupFailure;
62529
+ } catch (error) {
62530
+ failure = error;
62531
+ }
62532
+ if (failure === undefined) {
62533
+ const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62534
+ await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62535
+ if (this.logDirectory !== undefined)
62536
+ await rmdir(this.logDirectory).catch(() => {});
62537
+ }
62538
+ this.flush();
62539
+ try {
62540
+ await this.routes?.close();
62541
+ } catch (error) {
62542
+ failure ??= error;
62543
+ }
62544
+ for (const batch of this.batches.values())
62545
+ batch.settle();
62546
+ this.batches.clear();
62547
+ for (const pending of this.requests.values())
62548
+ pending.reject(new Error("Local worker pool is closed"));
62549
+ this.requests.clear();
62550
+ if (failure !== undefined)
62551
+ throw failure;
62552
+ }
62553
+ }
62554
+ function processPort() {
62555
+ return {
62556
+ send(message) {
62557
+ if (process.connected)
62558
+ process.send?.(message, () => {});
62559
+ },
62560
+ onMessage(listener) {
62561
+ process.on("message", (message) => listener(message));
62562
+ },
62563
+ onClose(listener) {
62564
+ process.once("disconnect", () => listener(new Error("Root host disconnected")));
62565
+ },
62566
+ async close() {
62567
+ if (process.connected)
62568
+ process.disconnect();
62569
+ }
62570
+ };
62571
+ }
62572
+ function runPoolCoordinator() {
62573
+ if (typeof process.send !== "function")
62574
+ throw new Error("Local worker pool coordinator requires an IPC channel");
62575
+ const port = processPort();
62576
+ const coordinator = new PoolCoordinator(port);
62577
+ port.onClose(() => process.exit(0));
62578
+ return coordinator;
62579
+ }
62580
+ var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
62581
+ var init_pool_coordinator = __esm(() => {
62582
+ init_resolution();
62583
+ init_sql_pool_queue();
62584
+ init_sql_pool_usage();
62585
+ init_pending_stage_intercom();
62586
+ init_intercom_group();
62587
+ init_pending_stage_route_capability();
62588
+ init_pending_stage_route_readiness();
62589
+ init_store();
62590
+ init_resource_admission();
62591
+ COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
61507
62592
  });
61508
62593
 
61509
62594
  // dist/builtin/workflows/src/workers/protocol.ts
@@ -61753,21 +62838,26 @@ class PoolStageHandoff {
61753
62838
  store;
61754
62839
  queue;
61755
62840
  lease;
62841
+ claimRun;
61756
62842
  attempts = new Map;
61757
62843
  owners = new Map;
61758
62844
  publications = new Map;
61759
62845
  drains = new Map;
61760
- constructor(backend, store, queue, lease) {
62846
+ constructor(backend, store, queue, lease, claimRun) {
61761
62847
  this.backend = backend;
61762
62848
  this.store = store;
61763
62849
  this.queue = queue;
61764
62850
  this.lease = lease;
62851
+ this.claimRun = claimRun;
61765
62852
  }
61766
62853
  publish(work, runs) {
61767
62854
  const previous = this.publications.get(work.id) ?? Promise.resolve();
61768
62855
  const publication = previous.catch(() => {
61769
62856
  return;
61770
- }).then(() => this.apply(work, runs));
62857
+ }).then(async () => {
62858
+ const commit = await this.prepare(work, runs);
62859
+ this.store.batch(commit);
62860
+ });
61771
62861
  const settled = publication.catch(() => {
61772
62862
  return;
61773
62863
  });
@@ -61778,6 +62868,21 @@ class PoolStageHandoff {
61778
62868
  });
61779
62869
  return publication;
61780
62870
  }
62871
+ async publishBatch(publications) {
62872
+ const prepared = [];
62873
+ for (const { work, runs } of publications)
62874
+ prepared.push(await this.prepare(work, runs).catch((error) => error));
62875
+ return this.store.batch(() => prepared.map((commit) => {
62876
+ if (commit instanceof Error)
62877
+ return commit;
62878
+ try {
62879
+ commit();
62880
+ return;
62881
+ } catch (error) {
62882
+ return error instanceof Error ? error : new Error(String(error));
62883
+ }
62884
+ }));
62885
+ }
61781
62886
  async route(work, runId) {
61782
62887
  this.requireOwned(await this.begin(work), runId);
61783
62888
  const readiness = workflowPendingStageRouteReady(this.store, runId);
@@ -61828,7 +62933,7 @@ class PoolStageHandoff {
61828
62933
  const result = await this.queue.result(root, work.id);
61829
62934
  if (result === undefined)
61830
62935
  return;
61831
- await this.backend.hydrateWorkflow(root);
62936
+ await this.backend.catchUpWorkflow(root);
61832
62937
  const terminals = new Map;
61833
62938
  for (const checkpoint of this.backend.listCheckpoints(root)) {
61834
62939
  const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
@@ -61838,6 +62943,9 @@ class PoolStageHandoff {
61838
62943
  if (child?.runId === boundary.child.runId)
61839
62944
  terminals.set(child.runId, { status: child.status, outputs: child.outputs });
61840
62945
  }
62946
+ this.store.batch(() => this.settle(work, result, terminals));
62947
+ }
62948
+ settle(work, result, terminals) {
61841
62949
  for (const run of this.store.runs()) {
61842
62950
  if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
61843
62951
  continue;
@@ -61861,22 +62969,31 @@ class PoolStageHandoff {
61861
62969
  if (current !== undefined && current.generation > work.generation)
61862
62970
  return;
61863
62971
  const attempt = current?.generation === work.generation ? current : this.start(work, current);
61864
- this.release(attempt);
62972
+ this.store.batch(() => this.release(attempt));
61865
62973
  attempt.fenced = true;
61866
62974
  }
61867
- async apply(work, runs) {
62975
+ async prepare(work, runs) {
61868
62976
  const subtree = this.subtree(work, runs);
61869
62977
  const root = this.lease.runId;
61870
- const known = new Map(this.store.runs().map((run) => [run.id, run]));
61871
- for (const run of subtree) {
61872
- const existing = known.get(run.id);
61873
- const owner = this.owners.get(run.id);
61874
- if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
61875
- throw outside(run.id);
61876
- }
61877
- if (subtree.some((run) => known.get(run.id) === undefined))
61878
- await this.backend.hydrateWorkflow(root);
62978
+ const owned = (known) => {
62979
+ for (const run of subtree) {
62980
+ const existing = known.get(run.id);
62981
+ const owner = this.owners.get(run.id);
62982
+ if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
62983
+ throw outside2(run.id);
62984
+ }
62985
+ };
62986
+ const before = new Map(this.store.runs().map((run) => [run.id, run]));
62987
+ owned(before);
62988
+ if (subtree.some((run) => before.get(run.id) === undefined))
62989
+ await this.backend.catchUpWorkflow(root);
61879
62990
  await this.queue.assertCurrent(this.lease, work);
62991
+ return () => this.commit(work, subtree, owned);
62992
+ }
62993
+ commit(work, subtree, owned) {
62994
+ const root = this.lease.runId;
62995
+ const known = new Map(this.store.runs().map((run) => [run.id, run]));
62996
+ owned(known);
61880
62997
  const attempt = this.enter(work);
61881
62998
  const execution = {
61882
62999
  worker: work.worker ?? "",
@@ -61888,11 +63005,13 @@ class PoolStageHandoff {
61888
63005
  ...run,
61889
63006
  rootRunId: root,
61890
63007
  pendingStageMessages: [
61891
- ...this.store.runs().find((current) => current.id === run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
63008
+ ...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
61892
63009
  ],
61893
63010
  stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
61894
63011
  }));
61895
63012
  const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
63013
+ for (const run of published)
63014
+ this.claimRun?.(run.id);
61896
63015
  for (const run of published) {
61897
63016
  attempt.runs.add(run.id);
61898
63017
  this.owners.set(run.id, work.id);
@@ -61917,7 +63036,7 @@ class PoolStageHandoff {
61917
63036
  const byId = new Map;
61918
63037
  for (const run of runs) {
61919
63038
  if (run.id === root || byId.has(run.id))
61920
- throw outside(run.id);
63039
+ throw outside2(run.id);
61921
63040
  byId.set(run.id, run);
61922
63041
  }
61923
63042
  if (runs.length === 0)
@@ -61930,25 +63049,25 @@ class PoolStageHandoff {
61930
63049
  const top = byId.get(work.childRunId);
61931
63050
  if (top !== undefined) {
61932
63051
  if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
61933
- throw outside(top.id);
63052
+ throw outside2(top.id);
61934
63053
  const pending = [top];
61935
63054
  for (let run = pending.shift();run !== undefined; run = pending.shift()) {
61936
63055
  if (run.rootRunId !== undefined && run.rootRunId !== root)
61937
- throw outside(run.id);
63056
+ throw outside2(run.id);
61938
63057
  ordered.push(run);
61939
63058
  for (const child of runs) {
61940
63059
  if (child.parentRunId !== run.id)
61941
63060
  continue;
61942
63061
  const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
61943
63062
  if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
61944
- throw outside(child.id);
63063
+ throw outside2(child.id);
61945
63064
  pending.push(child);
61946
63065
  }
61947
63066
  }
61948
63067
  }
61949
63068
  const stray = runs.find((run) => !ordered.includes(run));
61950
63069
  if (stray !== undefined)
61951
- throw outside(stray.id);
63070
+ throw outside2(stray.id);
61952
63071
  return ordered;
61953
63072
  }
61954
63073
  async begin(work) {
@@ -62003,7 +63122,7 @@ class PoolStageHandoff {
62003
63122
  }
62004
63123
  requireOwned(attempt, runId) {
62005
63124
  if (!attempt.runs.has(runId))
62006
- throw outside(runId);
63125
+ throw outside2(runId);
62007
63126
  }
62008
63127
  delivery(attempt, work, runId, stageId, stageName) {
62009
63128
  const key = JSON.stringify([runId, stageId]);
@@ -62021,7 +63140,7 @@ class PoolStageHandoff {
62021
63140
  return delivery;
62022
63141
  }
62023
63142
  }
62024
- function outside(runId) {
63143
+ function outside2(runId) {
62025
63144
  return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
62026
63145
  }
62027
63146
  function isLive(stage) {
@@ -62046,33 +63165,122 @@ var init_stage_handoff = __esm(() => {
62046
63165
  });
62047
63166
 
62048
63167
  // dist/builtin/workflows/src/workers/local-pool.ts
62049
- import { fork } from "node:child_process";
62050
- import { randomUUID as randomUUID16 } from "node:crypto";
62051
- import { createWriteStream } from "node:fs";
62052
- import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
62053
- import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
62054
- import { setTimeout as delay2 } from "node:timers/promises";
63168
+ import { fork as fork2 } from "node:child_process";
63169
+ import { resolve as resolve12 } from "node:path";
63170
+ import { setTimeout as delay3 } from "node:timers/promises";
62055
63171
  import { fileURLToPath as fileURLToPath5 } from "node:url";
62056
63172
  import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
62057
- function workerLogDirectory(journalPath, root) {
62058
- return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
63173
+ function defaultWorkerEntry() {
63174
+ return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
63175
+ }
63176
+ function launchCoordinatorProcess() {
63177
+ const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
63178
+ stdio: ["ignore", "ignore", "pipe", "ipc"],
63179
+ execArgv: []
63180
+ });
63181
+ let stderr = "";
63182
+ child.stderr?.on("data", (chunk) => {
63183
+ stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
63184
+ });
63185
+ const listeners = [];
63186
+ let lostReason;
63187
+ const lose = (reason) => {
63188
+ if (lostReason !== undefined)
63189
+ return;
63190
+ lostReason = reason;
63191
+ for (const listener of listeners)
63192
+ listener(reason);
63193
+ };
63194
+ const exited = new Promise((resolve) => child.once("exit", () => resolve()));
63195
+ child.on("error", lose);
63196
+ child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
63197
+ return {
63198
+ pid: child.pid,
63199
+ port: {
63200
+ send(message) {
63201
+ if (lostReason !== undefined || !child.connected)
63202
+ return;
63203
+ child.send(message, (error) => {
63204
+ if (error)
63205
+ lose(error);
63206
+ });
63207
+ },
63208
+ onMessage(listener) {
63209
+ child.on("message", (message) => listener(message));
63210
+ },
63211
+ onClose(listener) {
63212
+ if (lostReason !== undefined)
63213
+ listener(lostReason);
63214
+ else
63215
+ listeners.push(listener);
63216
+ },
63217
+ async close() {
63218
+ if (child.connected)
63219
+ child.disconnect();
63220
+ if (child.exitCode !== null || child.signalCode !== null)
63221
+ return;
63222
+ const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
63223
+ if (timedOut) {
63224
+ child.kill("SIGKILL");
63225
+ await exited;
63226
+ }
63227
+ }
63228
+ }
63229
+ };
63230
+ }
63231
+ function launchCoordinatorInProcess() {
63232
+ const mainListeners = [];
63233
+ const coordinatorListeners = [];
63234
+ const copy = (message) => JSON.parse(JSON.stringify(message));
63235
+ const coordinator = new PoolCoordinator({
63236
+ send(message) {
63237
+ const copied = copy(message);
63238
+ setImmediate(() => {
63239
+ for (const listener of mainListeners)
63240
+ listener(copied);
63241
+ });
63242
+ },
63243
+ onMessage(listener) {
63244
+ coordinatorListeners.push(listener);
63245
+ },
63246
+ onClose() {},
63247
+ async close() {}
63248
+ });
63249
+ return {
63250
+ pid: process.pid,
63251
+ port: {
63252
+ send(message) {
63253
+ const copied = copy(message);
63254
+ setImmediate(() => {
63255
+ for (const listener of coordinatorListeners)
63256
+ listener(copied);
63257
+ });
63258
+ },
63259
+ onMessage(listener) {
63260
+ mainListeners.push(listener);
63261
+ },
63262
+ onClose() {},
63263
+ async close() {
63264
+ coordinator.shutdown();
63265
+ }
63266
+ }
63267
+ };
62059
63268
  }
62060
63269
 
62061
63270
  class LocalWorkerPool {
62062
63271
  backend;
62063
63272
  options;
62064
- children = new Set;
62065
- incarnations = new Map;
62066
- readyWorkers = new Set;
62067
- logs = new Map;
62068
- logDirectory;
62069
- cleanups = new Set;
62070
- recovering;
62071
- nextRecoveryAt = 0;
62072
- controlAcks = new Map;
62073
- lastHeard = new Map;
62074
- lastTick = performance.now();
62075
- rootStalledMs = 0;
63273
+ pids = new Set;
63274
+ coordinator;
63275
+ coordinatorLost;
63276
+ commands = new Map;
63277
+ commandSequence = 0;
63278
+ inbound = Promise.resolve();
63279
+ works = new Map;
63280
+ attempts = new Map;
63281
+ results = new Map;
63282
+ changed = Promise.withResolvers();
63283
+ waiting = 0;
62076
63284
  ticker;
62077
63285
  leaseMs;
62078
63286
  desiredControl;
@@ -62080,18 +63288,17 @@ class LocalWorkerPool {
62080
63288
  store;
62081
63289
  handoff;
62082
63290
  releaseRoutes;
63291
+ routeClaims = new Map;
62083
63292
  sources = [];
62084
63293
  queue;
62085
63294
  lease;
62086
- admission;
62087
- usageLedger;
62088
63295
  usageTotal;
62089
63296
  rootBudget;
63297
+ budgetBlocked = false;
63298
+ rootSignature;
62090
63299
  starting;
62091
- cleanupFailure;
62092
63300
  failure;
62093
63301
  closed = false;
62094
- replacements = 0;
62095
63302
  releaseBinding;
62096
63303
  constructor(backend, options) {
62097
63304
  this.backend = backend;
@@ -62106,10 +63313,12 @@ class LocalWorkerPool {
62106
63313
  this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
62107
63314
  if (options.admission && options.rootId === undefined)
62108
63315
  workflowResourceAdmission.set(backend, options.admission);
62109
- this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
62110
63316
  }
62111
63317
  get workerPids() {
62112
- return [...this.children].flatMap((child) => child.pid === undefined ? [] : [child.pid]);
63318
+ return [...this.pids];
63319
+ }
63320
+ get coordinatorPid() {
63321
+ return this.coordinator?.pid;
62113
63322
  }
62114
63323
  usage() {
62115
63324
  return this.usageTotal;
@@ -62117,47 +63326,39 @@ class LocalWorkerPool {
62117
63326
  async prepare(root, store) {
62118
63327
  if (this.closed)
62119
63328
  throw new Error("Local worker pool is closed");
62120
- if (!(this.backend.journal instanceof SqlRunJournal) || resolve11(this.options.journalPath) !== resolve11(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
63329
+ if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
62121
63330
  throw new Error("Local workers require the same default-prefix SQLite journal");
62122
63331
  if (this.root !== undefined && this.root !== root)
62123
63332
  throw new Error("Local worker pool belongs to another root");
62124
63333
  this.root = root;
62125
63334
  if (store)
62126
63335
  this.store = store;
62127
- this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
62128
- this.usageTotal = await this.usageLedger.total(root);
62129
63336
  const state = await this.backend.journal.leaseState(root);
62130
63337
  if (!state || state.holderId !== this.backend.holderId)
62131
63338
  throw new Error("Local worker pool requires the root owner");
62132
63339
  this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
62133
- const queue = await openSqlPoolQueue(this.backend.journal);
62134
- await queue.initialize(this.lease, COORDINATOR_LEASE_MS);
62135
- await queue.releasePreviousIncarnations(this.lease);
62136
- for (;; ) {
62137
- const intended = this.desiredControl ?? "running";
62138
- await queue.setControl(this.lease, intended);
62139
- if (intended === (this.desiredControl ?? "running"))
62140
- break;
62141
- }
62142
- this.queue = queue;
63340
+ const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
63341
+ this.usageTotal = await this.command({
63342
+ kind: "prepare",
63343
+ root,
63344
+ lease: this.lease,
63345
+ options: {
63346
+ journalPath: this.options.journalPath,
63347
+ workers: this.options.workers,
63348
+ leaseMs: this.leaseMs,
63349
+ startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
63350
+ ...this.options.admission === undefined ? {} : { admission: this.options.admission },
63351
+ ...workerEntry === undefined ? {} : { workerEntry }
63352
+ },
63353
+ ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
63354
+ });
63355
+ this.queue ??= await openSqlPoolQueue(this.backend.journal);
62143
63356
  }
62144
63357
  async control(state) {
62145
63358
  this.desiredControl = state;
62146
63359
  try {
62147
- if (!this.queue || !this.lease)
62148
- return;
62149
- const control = await this.queue.setControl(this.lease, state);
62150
- const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
62151
- while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
62152
- if (this.desiredControl !== state)
62153
- return;
62154
- if (Date.now() >= deadline) {
62155
- if (state === "quit")
62156
- break;
62157
- throw new Error("Worker control acknowledgement unresolved");
62158
- }
62159
- await delay2(25);
62160
- }
63360
+ if (this.coordinator !== undefined)
63361
+ await this.command({ kind: "control", state });
62161
63362
  } finally {
62162
63363
  if (state === "quit")
62163
63364
  await this.close();
@@ -62203,290 +63404,61 @@ class LocalWorkerPool {
62203
63404
  throw new Error(`Worker definition is not importable: ${binding.exportName}`);
62204
63405
  return source;
62205
63406
  }));
62206
- if (this.store) {
62207
- this.releaseRoutes = await acquireStandaloneIntercom(this.store, root, String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd()));
63407
+ const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
63408
+ const store = this.store;
63409
+ if (store) {
63410
+ this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
62208
63411
  if (this.closed) {
62209
63412
  await this.releaseRoutes();
62210
63413
  this.releaseRoutes = undefined;
62211
63414
  throw new Error("Local worker pool is closed");
62212
63415
  }
62213
- this.handoff = new PoolStageHandoff(this.backend, this.store, this.queue, this.lease);
62214
- }
62215
- if (this.options.admission)
62216
- this.admission = await SqlResourceAdmission.open(this.backend.journal, {
62217
- ...this.options.admission,
62218
- root,
62219
- holder: "coordinator"
63416
+ const routeOwner = {
63417
+ notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
63418
+ };
63419
+ this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
63420
+ this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
62220
63421
  });
62221
- this.logDirectory = workerLogDirectory(this.options.journalPath, root);
62222
- await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
62223
- return;
62224
- });
62225
- const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
62226
- try {
62227
- await Promise.any(workers);
62228
- } catch (error) {
62229
- throw error.errors[0];
62230
63422
  }
62231
- }
62232
- spawn(root) {
62233
- if (this.closed)
62234
- return Promise.reject(new Error("Local worker pool is closed"));
62235
- const child = fork(isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url)), [
62236
- JSON.stringify({
62237
- journalPath: this.options.journalPath,
62238
- root,
62239
- definitions: this.sources,
62240
- admission: this.options.admission
62241
- })
62242
- ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62243
- const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62244
- this.children.add(child);
62245
- const worker = randomUUID16();
62246
- this.incarnations.set(worker, child);
62247
- let ready = false;
62248
- let serial = Promise.resolve();
62249
- let work;
62250
- const deliveries = new Map;
62251
- const logPath = join28(this.logDirectory, `${worker}.log`);
62252
- this.logs.set(child, logPath);
62253
- let logFailure;
62254
- const log = createWriteStream(logPath, { flags: "a" });
62255
- log.on("error", (error) => {
62256
- logFailure = error.message;
62257
- });
62258
- let stderr = "";
62259
- child.stderr?.on("data", (chunk) => {
62260
- stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62261
- });
62262
- child.stderr?.on("error", (error) => {
62263
- logFailure ??= error.message;
62264
- log.end();
62265
- });
62266
- child.stderr?.pipe(log);
62267
- const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62268
- const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
62269
- let timedOut = false;
62270
- let terminated = false;
62271
- const terminate = () => {
62272
- if (terminated)
62273
- return;
62274
- terminated = true;
62275
- child.kill("SIGKILL");
62276
- };
62277
- const replied = (error) => {
62278
- if (error)
62279
- terminate();
62280
- };
62281
- return new Promise((resolve, reject) => {
62282
- const startup = setTimeout(() => {
62283
- timedOut = true;
62284
- terminate();
62285
- }, startupTimeoutMs);
62286
- child.on("error", (error) => {
62287
- clearTimeout(startup);
62288
- reject(error);
62289
- terminate();
62290
- });
62291
- const closed = new Promise((resolve) => child.once("close", () => resolve()));
62292
- child.once("exit", () => {
62293
- Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62294
- this.children.delete(child);
62295
- this.logs.delete(child);
62296
- clearTimeout(startup);
62297
- this.incarnations.delete(worker);
62298
- this.readyWorkers.delete(worker);
62299
- this.controlAcks.delete(worker);
62300
- this.lastHeard.delete(worker);
62301
- if (work)
62302
- this.handoff?.fence(work);
62303
- for (const delivery of deliveries.values())
62304
- delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62305
- deliveries.clear();
62306
- const cleanup = disconnected.then(() => serial).then(async () => {
62307
- try {
62308
- if (await this.admission?.releaseStoppedSessions(worker))
62309
- throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62310
- } catch (error) {
62311
- this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62312
- throw this.cleanupFailure;
62313
- }
62314
- }).then(async () => {
62315
- if (this.closed || !ready || this.desiredControl === "quit")
62316
- return;
62317
- if (work && await this.queue.result(root, work.id)) {
62318
- work = undefined;
62319
- this.replacements = 0;
62320
- }
62321
- if (this.closed)
62322
- return;
62323
- if (work)
62324
- await this.queue.releaseExited(this.lease, work);
62325
- const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62326
- if (exhausted)
62327
- this.failure = diagnosed("Local worker retry limit");
62328
- else
62329
- await this.spawn(root);
62330
- }).catch((error) => {
62331
- this.failure = error;
62332
- });
62333
- this.cleanups.add(cleanup);
62334
- cleanup.finally(() => this.cleanups.delete(cleanup));
62335
- });
62336
- child.on("message", (message) => {
62337
- this.observeResponsiveness();
62338
- this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
62339
- if (message.kind === "delivery-ack") {
62340
- const pending = deliveries.get(message.deliveryId);
62341
- deliveries.delete(message.deliveryId);
62342
- if (message.error)
62343
- pending?.reject(new Error(message.error));
62344
- else
62345
- pending?.resolve();
62346
- return;
62347
- }
62348
- const response = serial.then(async () => {
62349
- if (message.kind === "ready") {
62350
- if (terminated)
62351
- return;
62352
- ready = true;
62353
- this.readyWorkers.add(worker);
62354
- clearTimeout(startup);
62355
- resolve();
62356
- return;
62357
- }
62358
- await this.starting;
62359
- const queue = this.queue;
62360
- const lease = this.lease;
62361
- if (message.kind === "control")
62362
- return queue.control(lease);
62363
- if (message.kind === "controlled") {
62364
- const control = await queue.control(lease);
62365
- if (message.revision > control.revision)
62366
- throw new Error("Unknown control revision");
62367
- this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62368
- return;
62369
- }
62370
- if (message.kind === "claim") {
62371
- if (work)
62372
- throw new Error("Worker already holds an attempt");
62373
- const decision = this.rootBudget?.checkpoint();
62374
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62375
- return;
62376
- await this.recoverExpired(root);
62377
- work = await queue.claim(lease, worker, this.sources.map((source) => ({
62378
- definition: `${source.path}#${source.exportName}`,
62379
- digest: source.digest
62380
- })), this.leaseMs);
62381
- return work;
62382
- }
62383
- if (!work)
62384
- throw new Error("Worker has no active attempt");
62385
- if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
62386
- work = await queue.extend(lease, work, this.leaseMs);
62387
- if (message.kind === "activity")
62388
- return this.handoff?.publish(work, message.runs);
62389
- if (message.kind === "route")
62390
- return this.handoff?.route(work, message.runId);
62391
- if (message.kind === "delivery-ready")
62392
- return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
62393
- if (message.kind === "delivery") {
62394
- if (!message.recipient)
62395
- throw new Error("Worker delivery requires a session identity");
62396
- return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
62397
- const id = randomUUID16();
62398
- deliveries.set(id, { resolve, reject });
62399
- child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
62400
- if (error) {
62401
- deliveries.delete(id);
62402
- reject(error);
62403
- }
62404
- });
62405
- }));
62406
- }
62407
- if (message.kind === "renew")
62408
- return;
62409
- if (message.kind === "commit") {
62410
- const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62411
- this.usageTotal = await this.usageLedger.total(root);
62412
- return receipt;
62413
- }
62414
- if (message.kind === "usage") {
62415
- this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
62416
- return;
62417
- }
62418
- if (message.kind === "budget") {
62419
- await queue.assertCurrent(lease, work);
62420
- const decision = this.rootBudget?.checkpoint();
62421
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62422
- throw new Error("Root workflow budget exhausted");
62423
- return queue.control(lease);
62424
- }
62425
- if (message.kind !== "complete")
62426
- throw new Error("Invalid worker operation");
62427
- await queue.complete(lease, work, message.result);
62428
- this.replacements = 0;
62429
- work = undefined;
62430
- });
62431
- if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62432
- serial = response.then(() => {}, () => {});
62433
- response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62434
- });
63423
+ await this.command({
63424
+ kind: "start",
63425
+ sources: this.sources,
63426
+ cwd,
63427
+ publish: store !== undefined,
63428
+ ...this.rootRun()
62435
63429
  });
62436
63430
  }
62437
- recoverExpired(root) {
62438
- if (this.recovering)
62439
- return this.recovering;
62440
- if (this.closed || performance.now() < this.nextRecoveryAt)
62441
- return Promise.resolve();
62442
- this.recovering = (async () => {
62443
- await new Promise((resolve) => setImmediate(resolve));
62444
- await this.admission?.reconcile();
62445
- const expiredBefore = Date.now();
62446
- for (const work of await this.queue.inspect(root)) {
62447
- if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62448
- continue;
62449
- const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62450
- if (child && work.worker !== null && this.isLive(work.worker, child)) {
62451
- await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
62452
- if (error.message !== STALE_WORK_LEASE)
62453
- throw error;
62454
- });
62455
- continue;
62456
- }
62457
- if (child && !await this.stop(child).then(() => true, () => false))
62458
- continue;
62459
- await this.queue.expire(this.lease, work);
62460
- }
62461
- })().finally(() => {
62462
- this.recovering = undefined;
62463
- this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62464
- });
62465
- return this.recovering;
63431
+ queueView() {
63432
+ return {
63433
+ assertCurrent: async (_lease, work) => {
63434
+ const current = this.attempts.get(work.id);
63435
+ if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
63436
+ throw new Error(STALE_WORK_LEASE);
63437
+ },
63438
+ withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
63439
+ result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
63440
+ };
62466
63441
  }
62467
- observeResponsiveness() {
62468
- const now = performance.now();
62469
- const gap = now - this.lastTick;
62470
- if (gap > ROOT_STALL_THRESHOLD_MS)
62471
- this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
62472
- this.lastTick = now;
63442
+ rootProjection() {
63443
+ const run = this.store?.runs().find((candidate) => candidate.id === this.root);
63444
+ return run === undefined ? undefined : projectRootRun(run);
62473
63445
  }
62474
- isLive(worker, child) {
62475
- this.observeResponsiveness();
62476
- const heard = this.lastHeard.get(worker);
62477
- return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.rootStalledMs - heard.rootStalled) < this.leaseMs;
63446
+ rootRun() {
63447
+ const rootRun = this.rootProjection();
63448
+ if (rootRun === undefined)
63449
+ return {};
63450
+ this.rootSignature = JSON.stringify(rootRun);
63451
+ return { rootRun };
62478
63452
  }
62479
- stop(child) {
62480
- if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62481
- return Promise.resolve();
62482
- return new Promise((resolve, reject) => {
62483
- const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62484
- child.once("exit", () => {
62485
- clearTimeout(timeout);
62486
- resolve();
62487
- });
62488
- child.kill("SIGKILL");
62489
- });
63453
+ refreshRootRun() {
63454
+ const rootRun = this.rootProjection();
63455
+ if (rootRun === undefined)
63456
+ return;
63457
+ const signature = JSON.stringify(rootRun);
63458
+ if (signature === this.rootSignature)
63459
+ return;
63460
+ this.rootSignature = signature;
63461
+ this.notice({ kind: "root", rootRun });
62490
63462
  }
62491
63463
  async run(definition, inputs, options) {
62492
63464
  try {
@@ -62519,7 +63491,7 @@ class LocalWorkerPool {
62519
63491
  const selectedSource = this.sources[index];
62520
63492
  if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
62521
63493
  throw new Error("Worker definition compatibility error");
62522
- await this.backend.hydrateWorkflow(root);
63494
+ await this.backend.catchUpWorkflow(root);
62523
63495
  const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
62524
63496
  if (boundary?.kind !== "stage")
62525
63497
  throw new Error("Missing durable child boundary");
@@ -62530,39 +63502,232 @@ class LocalWorkerPool {
62530
63502
  config: options.config,
62531
63503
  executionMode: options.executionMode
62532
63504
  }));
62533
- const work = await this.queue.enqueue(this.lease, {
62534
- boundaryId: options.parentRun.stageId,
62535
- childRunId: options.runId,
62536
- scope: options.durableScope.scopePrefix,
62537
- definition: `${selectedSource.path}#${selectedSource.exportName}`,
62538
- digest: selectedSource.digest,
62539
- inputs,
62540
- execution
62541
- }, boundary);
62542
- for (;; ) {
62543
- options.signal?.throwIfAborted();
62544
- if (this.closed) {
63505
+ this.evaluateBudget();
63506
+ const enqueued = await this.command({
63507
+ kind: "enqueue",
63508
+ descriptor: {
63509
+ boundaryId: options.parentRun.stageId,
63510
+ childRunId: options.runId,
63511
+ scope: options.durableScope.scopePrefix,
63512
+ definition: `${selectedSource.path}#${selectedSource.exportName}`,
63513
+ digest: selectedSource.digest,
63514
+ inputs,
63515
+ execution
63516
+ },
63517
+ boundary,
63518
+ ...this.rootRun()
63519
+ });
63520
+ const work = enqueued.work;
63521
+ this.works.set(work.id, work);
63522
+ if (enqueued.result !== undefined && !this.results.has(work.id))
63523
+ this.results.set(work.id, enqueued.result);
63524
+ this.waiting++;
63525
+ this.ticker ??= setInterval(() => {
63526
+ this.evaluateBudget();
63527
+ this.signalChange();
63528
+ }, POOL_SUMMARY_INTERVAL_MS).unref();
63529
+ try {
63530
+ for (;; ) {
63531
+ options.signal?.throwIfAborted();
63532
+ if (this.closed) {
63533
+ if (this.failure)
63534
+ throw this.failure;
63535
+ if (this.desiredControl === "quit")
63536
+ throw new WorkflowGracefulQuitError(root, "worker pool teardown");
63537
+ throw new Error("Local worker pool is closed");
63538
+ }
63539
+ const result = this.results.get(work.id);
63540
+ if (result) {
63541
+ await this.inbound;
63542
+ await this.backend.catchUpWorkflow(root);
63543
+ await this.handoff?.complete(work);
63544
+ return result;
63545
+ }
62545
63546
  if (this.failure)
62546
63547
  throw this.failure;
62547
- if (this.desiredControl === "quit")
62548
- throw new WorkflowGracefulQuitError(root, "worker pool teardown");
62549
- throw new Error("Local worker pool is closed");
63548
+ if (this.budgetBlocked)
63549
+ await this.rootBudget.stopAtBoundaryAsync(definition.name);
63550
+ await this.nextChange(options.signal);
62550
63551
  }
62551
- const result = await this.queue.result(root, work.id);
62552
- if (result) {
62553
- await this.backend.hydrateWorkflow(root);
62554
- await this.handoff?.complete(work);
62555
- return result;
63552
+ } finally {
63553
+ if (--this.waiting === 0) {
63554
+ clearInterval(this.ticker);
63555
+ this.ticker = undefined;
62556
63556
  }
62557
- if (this.failure)
62558
- throw this.failure;
62559
- const decision = this.rootBudget?.checkpoint();
62560
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62561
- await this.rootBudget.stopAtBoundaryAsync(definition.name);
62562
- await this.recoverExpired(root);
62563
- await delay2(25, undefined, { signal: options.signal });
62564
63557
  }
62565
63558
  }
63559
+ evaluateBudget() {
63560
+ const decision = this.rootBudget?.checkpoint();
63561
+ const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
63562
+ if (blocked === this.budgetBlocked)
63563
+ return;
63564
+ this.budgetBlocked = blocked;
63565
+ this.notice({ kind: "budget", blocked });
63566
+ }
63567
+ signalChange() {
63568
+ const changed = this.changed;
63569
+ this.changed = Promise.withResolvers();
63570
+ changed.resolve();
63571
+ }
63572
+ nextChange(signal) {
63573
+ const changed = this.changed.promise;
63574
+ if (signal === undefined)
63575
+ return changed;
63576
+ return new Promise((resolve, reject) => {
63577
+ const abort = () => reject(signal.reason);
63578
+ signal.addEventListener("abort", abort, { once: true });
63579
+ changed.then(() => {
63580
+ signal.removeEventListener("abort", abort);
63581
+ resolve();
63582
+ });
63583
+ });
63584
+ }
63585
+ connect() {
63586
+ if (this.coordinator !== undefined)
63587
+ return this.coordinator;
63588
+ const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
63589
+ this.coordinator = coordinator;
63590
+ coordinator.port.onMessage((event) => {
63591
+ this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
63592
+ });
63593
+ coordinator.port.onClose((reason) => {
63594
+ this.coordinatorLost = reason;
63595
+ this.inbound = this.inbound.then(() => this.lost(reason));
63596
+ });
63597
+ return coordinator;
63598
+ }
63599
+ lost(reason) {
63600
+ this.pids.clear();
63601
+ if (!this.closed)
63602
+ this.failure ??= reason;
63603
+ for (const pending of this.commands.values())
63604
+ pending.reject(reason);
63605
+ this.commands.clear();
63606
+ this.signalChange();
63607
+ }
63608
+ command(body) {
63609
+ const coordinator = this.connect();
63610
+ if (this.coordinatorLost !== undefined)
63611
+ return Promise.reject(this.coordinatorLost);
63612
+ const id = ++this.commandSequence;
63613
+ return new Promise((resolve, reject) => {
63614
+ this.commands.set(id, { resolve, reject });
63615
+ coordinator.port.send({ ...body, id });
63616
+ });
63617
+ }
63618
+ notice(notice) {
63619
+ this.coordinator?.port.send(notice);
63620
+ }
63621
+ async receive(event) {
63622
+ switch (event.kind) {
63623
+ case "reply": {
63624
+ const pending = this.commands.get(event.id);
63625
+ this.commands.delete(event.id);
63626
+ if (event.error !== undefined)
63627
+ pending?.reject(new Error(event.error));
63628
+ else
63629
+ pending?.resolve(event.value);
63630
+ return;
63631
+ }
63632
+ case "activity":
63633
+ return this.applyActivity(event.batch, event.entries, event.usage);
63634
+ case "spawned":
63635
+ this.pids.add(event.pid);
63636
+ return;
63637
+ case "exited":
63638
+ this.pids.delete(event.pid);
63639
+ return;
63640
+ case "fence": {
63641
+ const current = this.attempts.get(event.work.id);
63642
+ if (current?.generation === event.work.generation)
63643
+ current.fenced = true;
63644
+ this.handoff?.fence(event.work);
63645
+ return;
63646
+ }
63647
+ case "result":
63648
+ if (event.usage !== undefined)
63649
+ this.usageTotal = event.usage;
63650
+ this.results.set(event.workId, event.result);
63651
+ this.signalChange();
63652
+ return;
63653
+ case "failure":
63654
+ this.failure ??= new Error(event.error);
63655
+ this.signalChange();
63656
+ return;
63657
+ case "request":
63658
+ this.serve(event.id, event.request);
63659
+ return;
63660
+ }
63661
+ }
63662
+ async applyActivity(batch, entries, usage) {
63663
+ if (usage !== undefined)
63664
+ this.usageTotal = usage;
63665
+ const rejected = [];
63666
+ const handoff = this.handoff;
63667
+ if (handoff !== undefined) {
63668
+ const publications = entries.flatMap((entry) => {
63669
+ const work = this.track(entry.work);
63670
+ if (work !== undefined)
63671
+ return [{ work, runs: entry.runs }];
63672
+ rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
63673
+ return [];
63674
+ });
63675
+ const outcomes = await handoff.publishBatch(publications);
63676
+ outcomes.forEach((error, index) => {
63677
+ const { work } = publications[index];
63678
+ if (error !== undefined)
63679
+ rejected.push({ workId: work.id, generation: work.generation, error: error.message });
63680
+ });
63681
+ }
63682
+ this.notice({ kind: "published", batch, rejected });
63683
+ this.refreshRootRun();
63684
+ }
63685
+ track(attempt) {
63686
+ const work = this.works.get(attempt.id);
63687
+ if (work === undefined)
63688
+ return;
63689
+ const current = this.attempts.get(attempt.id);
63690
+ if (current === undefined || attempt.generation > current.generation)
63691
+ this.attempts.set(attempt.id, { ...attempt, fenced: false });
63692
+ return { ...work, ...attempt, state: "leased" };
63693
+ }
63694
+ async serve(id, request) {
63695
+ try {
63696
+ const value2 = await this.answer(id, request);
63697
+ this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
63698
+ } catch (error) {
63699
+ this.coordinator?.port.send({
63700
+ kind: "request-reply",
63701
+ id,
63702
+ error: error instanceof Error ? error.message : String(error)
63703
+ });
63704
+ }
63705
+ }
63706
+ async answer(id, request) {
63707
+ if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
63708
+ const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
63709
+ const event = {
63710
+ ...request.payload,
63711
+ handled: false
63712
+ };
63713
+ if (request.kind === "pending-stage-message")
63714
+ router?.route(event);
63715
+ else
63716
+ router?.sticky(event);
63717
+ if (event.handled && event.completion !== undefined)
63718
+ return await event.completion;
63719
+ return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
63720
+ }
63721
+ const handoff = this.handoff;
63722
+ const work = this.track(request.work);
63723
+ if (handoff === undefined || work === undefined)
63724
+ throw new Error(STALE_WORK_LEASE);
63725
+ if (request.kind === "ready")
63726
+ return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
63727
+ return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
63728
+ await this.command({ kind: "transport", request: id, from, message });
63729
+ });
63730
+ }
62566
63731
  closing;
62567
63732
  close() {
62568
63733
  this.closing ??= this.dispose();
@@ -62571,27 +63736,28 @@ class LocalWorkerPool {
62571
63736
  async dispose() {
62572
63737
  this.closed = true;
62573
63738
  clearInterval(this.ticker);
63739
+ this.ticker = undefined;
62574
63740
  this.releaseBinding();
62575
63741
  if (this.options.rootId === undefined)
62576
63742
  workflowResourceAdmission.delete(this.backend);
63743
+ this.signalChange();
62577
63744
  let failure;
62578
- let stopping = [];
62579
- try {
62580
- await this.recovering?.catch(() => {});
62581
- stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62582
- await Promise.all(stopping.map(({ child }) => this.stop(child)));
62583
- await Promise.all([...this.cleanups]);
62584
- if (this.cleanupFailure)
62585
- throw this.cleanupFailure;
62586
- } catch (error) {
62587
- failure = error;
62588
- }
62589
- if (failure === undefined) {
62590
- const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62591
- await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62592
- if (this.logDirectory !== undefined)
62593
- await rmdir(this.logDirectory).catch(() => {});
63745
+ const coordinator = this.coordinator;
63746
+ if (coordinator !== undefined) {
63747
+ if (this.coordinatorLost === undefined) {
63748
+ try {
63749
+ await this.command({ kind: "close" });
63750
+ } catch (error) {
63751
+ failure = error;
63752
+ }
63753
+ }
63754
+ await this.inbound;
63755
+ await coordinator.port.close();
63756
+ this.pids.clear();
62594
63757
  }
63758
+ for (const release of this.routeClaims.values())
63759
+ release();
63760
+ this.routeClaims.clear();
62595
63761
  try {
62596
63762
  await this.releaseRoutes?.();
62597
63763
  } catch (error) {
@@ -62601,27 +63767,26 @@ class LocalWorkerPool {
62601
63767
  throw failure;
62602
63768
  }
62603
63769
  }
62604
- var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, ROOT_TICK_INTERVAL_MS = 100, ROOT_STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000;
63770
+ var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
62605
63771
  var init_local_pool = __esm(() => {
62606
63772
  init_internal();
62607
63773
  init_sql_pool_queue();
62608
- init_sql_pool_usage();
62609
63774
  init_sql_run_journal();
62610
63775
  init_workflow_tool_abort();
62611
63776
  init_sdk_intercom();
63777
+ init_external_run_routes();
62612
63778
  init_workflow_infrastructure_error();
62613
63779
  init_admission_binding();
62614
63780
  init_definitions();
62615
63781
  init_dispatch();
62616
63782
  init_host_options();
62617
- init_resource_admission();
63783
+ init_pool_coordinator();
62618
63784
  init_stage_handoff();
62619
63785
  MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
62620
- COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
62621
63786
  });
62622
63787
 
62623
63788
  // dist/builtin/workflows/src/workers/public-pool.ts
62624
- import { resolve as resolve12 } from "node:path";
63789
+ import { resolve as resolve13 } from "node:path";
62625
63790
  import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
62626
63791
  async function restoreWorkerOptions(backend, options) {
62627
63792
  const source = options.continuation?.source.id ?? options.runId;
@@ -62660,11 +63825,11 @@ async function activatePublicPool(backend, options) {
62660
63825
  ...options.localWorkers,
62661
63826
  startupTimeoutMs: undefined,
62662
63827
  leaseMs: undefined,
62663
- definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve12(cwd, source.path) }))
63828
+ definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
62664
63829
  };
62665
63830
  const admission = options.admission && {
62666
63831
  ...options.admission,
62667
- ...options.admission.userDatabase ? { userDatabase: resolve12(cwd, options.admission.userDatabase) } : {}
63832
+ ...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
62668
63833
  };
62669
63834
  const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
62670
63835
  const resources = JSON.parse(JSON.stringify({
@@ -64010,7 +65175,7 @@ function hasUsableContentBlock(block) {
64010
65175
 
64011
65176
  // dist/builtin/workflows/src/shared/stage-session-store.ts
64012
65177
  import { randomUUID as randomUUID18 } from "node:crypto";
64013
- import { dirname as dirname13, resolve as resolve13 } from "node:path";
65178
+ import { dirname as dirname13, resolve as resolve14 } from "node:path";
64014
65179
  import {
64015
65180
  JsonlSessionStore,
64016
65181
  resolveSessionStore,
@@ -64046,11 +65211,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
64046
65211
  async function migrateLegacyStageSession(store, sessionId, sessionFile) {
64047
65212
  if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
64048
65213
  return;
64049
- const location = resolve13(sessionFile);
65214
+ const location = resolve14(sessionFile);
64050
65215
  const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
64051
65216
  try {
64052
65217
  for await (const file of legacy.scanFiles()) {
64053
- if (resolve13(file.location) !== location || file.header.id !== sessionId)
65218
+ if (resolve14(file.location) !== location || file.header.id !== sessionId)
64054
65219
  continue;
64055
65220
  const snapshot = await file.read();
64056
65221
  if (await store.readHeader(sessionId) === undefined)
@@ -64212,22 +65377,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
64212
65377
  }
64213
65378
  function retainReachableRunGroups(grouped, rootRunId) {
64214
65379
  const reachable = new Set([rootRunId]);
64215
- const pending = [rootRunId];
65380
+ const settled = new Set;
65381
+ const pending = [{ runId: rootRunId, live: true }];
64216
65382
  while (pending.length > 0) {
64217
- const drafts = grouped.get(pending.pop());
65383
+ const { runId, live } = pending.pop();
65384
+ const drafts = grouped.get(runId);
64218
65385
  if (drafts === undefined)
64219
65386
  continue;
64220
65387
  for (const draft of drafts) {
64221
- const childRunId = childRunIdFromDraft(draft);
64222
- if (childRunId === undefined || reachable.has(childRunId) || !grouped.has(childRunId))
65388
+ const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
65389
+ if (childRunId === undefined || !grouped.has(childRunId))
64223
65390
  continue;
64224
- reachable.add(childRunId);
64225
- pending.push(childRunId);
65391
+ if (live && childRunIdFromDraft(draft) !== undefined) {
65392
+ if (reachable.has(childRunId))
65393
+ continue;
65394
+ reachable.add(childRunId);
65395
+ pending.push({ runId: childRunId, live: true });
65396
+ } else if (!settled.has(childRunId)) {
65397
+ settled.add(childRunId);
65398
+ pending.push({ runId: childRunId, live: false });
65399
+ }
64226
65400
  }
64227
65401
  }
64228
- for (const runId of grouped.keys())
64229
- if (!reachable.has(runId))
64230
- grouped.delete(runId);
65402
+ let complete = true;
65403
+ for (const runId of grouped.keys()) {
65404
+ if (reachable.has(runId))
65405
+ continue;
65406
+ if (!settled.has(runId))
65407
+ complete = false;
65408
+ grouped.delete(runId);
65409
+ }
65410
+ return complete;
64231
65411
  }
64232
65412
  function childRunIdFromDraft(draft) {
64233
65413
  const boundary = draft.topology?.boundary;
@@ -64649,9 +65829,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
64649
65829
  }
64650
65830
  for (const group of grouped.values())
64651
65831
  group.sort(compareDraftSourceOrder);
64652
- const groupedCount = grouped.size;
64653
- retainReachableRunGroups(grouped, rootRunId);
64654
- if (policy.failOnUnreachable && grouped.size !== groupedCount)
65832
+ if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
64655
65833
  return [];
64656
65834
  if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
64657
65835
  return [];
@@ -70309,9 +71487,9 @@ var init_stage_runner_controller = __esm(() => {
70309
71487
  });
70310
71488
 
70311
71489
  // dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
70312
- import { createHash as createHash4 } from "node:crypto";
71490
+ import { createHash as createHash5 } from "node:crypto";
70313
71491
  import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
70314
- import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve14 } from "node:path";
71492
+ import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
70315
71493
  function normalizeMaxOutput2(maxOutput) {
70316
71494
  return {
70317
71495
  bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
@@ -70373,7 +71551,7 @@ function formatByteSize(bytes) {
70373
71551
  return `${value2.toFixed(1)} ${units[unitIndex]}`;
70374
71552
  }
70375
71553
  function transcriptPath(runId, outputPath) {
70376
- const digest = createHash4("sha256").update(outputPath).digest("hex").slice(0, 16);
71554
+ const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
70377
71555
  return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
70378
71556
  }
70379
71557
  function appendContent(lines, content) {
@@ -70419,7 +71597,7 @@ function renderTranscript(messages, fallbackOutput) {
70419
71597
  `);
70420
71598
  }
70421
71599
  function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
70422
- const absolutePath = resolve14(outputPath);
71600
+ const absolutePath = resolve15(outputPath);
70423
71601
  const bytes = Buffer.byteLength(fullOutput, "utf8");
70424
71602
  const lines = countLines(fullOutput);
70425
71603
  const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
@@ -70434,8 +71612,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
70434
71612
  return;
70435
71613
  if (isAbsolute11(output))
70436
71614
  return output;
70437
- const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve14(runtimeCwd, requestedCwd);
70438
- return resolve14(baseCwd, output);
71615
+ const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
71616
+ return resolve15(baseCwd, output);
70439
71617
  }
70440
71618
  function degenerateWarning(fullOutput) {
70441
71619
  return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
@@ -71444,7 +72622,7 @@ async function run(def, inputs, opts = {}) {
71444
72622
  if (!isWorkflowDefinition(def))
71445
72623
  throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
71446
72624
  const depth = opts.depth ?? 0;
71447
- const maxDepth = opts.config?.maxDepth ?? 4;
72625
+ const maxDepth = opts.config?.maxDepth ?? 3;
71448
72626
  if (depth >= maxDepth) {
71449
72627
  return {
71450
72628
  runId: opts.runId ?? crypto.randomUUID(),
@@ -72533,7 +73711,7 @@ init_admission_binding();
72533
73711
  init_definitions();
72534
73712
  init_resource_admission();
72535
73713
  import { randomUUID as randomUUID20 } from "node:crypto";
72536
- import { setTimeout as delay3 } from "node:timers/promises";
73714
+ import { setTimeout as delay4 } from "node:timers/promises";
72537
73715
  import { withResourceAdmission as withResourceAdmission2 } from "@bastani/atomic";
72538
73716
  import { StorageProvider as StorageProvider3 } from "@bastani/atomic/storage";
72539
73717
 
@@ -72542,8 +73720,8 @@ init_stage_topology();
72542
73720
  init_run_durable_topology();
72543
73721
  import { randomUUID as randomUUID19 } from "node:crypto";
72544
73722
  function workerSessionStore(input) {
72545
- const checkpoint = async (id, header) => {
72546
- const owner = (header ?? await input.reader.readHeader(id))?.workflow;
73723
+ const checkpoint = (id, header) => {
73724
+ const owner = header.workflow;
72547
73725
  const run = input.store.runs().find((run) => run.id === owner?.runId);
72548
73726
  const stage = run?.stages.find((stage) => stage.id === owner?.stageId);
72549
73727
  if (!run || !stage?.replayKey)
@@ -72571,11 +73749,11 @@ function workerSessionStore(input) {
72571
73749
  };
72572
73750
  return {
72573
73751
  descriptor: input.reader.descriptor,
72574
- create: async (header, entries) => write({ kind: "session-create", checkpoint: await checkpoint(header.id, header), header, entries }),
72575
- append: async (id, entries, options) => write({ kind: "session-append", checkpoint: await checkpoint(id), entries, expectSeq: options.expectSeq }),
73752
+ create: async (header, entries) => write({ kind: "session-create", checkpoint: checkpoint(header.id, header), header, entries }),
73753
+ append: async (id, entries, options) => write({ kind: "session-append", sessionId: id, entries, expectSeq: options.expectSeq }),
72576
73754
  replace: async (id, header, entries, options) => write({
72577
73755
  kind: "session-replace",
72578
- checkpoint: await checkpoint(id, header),
73756
+ checkpoint: checkpoint(id, header),
72579
73757
  header,
72580
73758
  entries,
72581
73759
  expectSeq: options.expectSeq
@@ -72603,6 +73781,14 @@ function collectWorkerUsage(runs) {
72603
73781
 
72604
73782
  // dist/builtin/workflows/src/workers/local-worker.ts
72605
73783
  async function runLocalWorker(start) {
73784
+ if ("role" in start && start.role === POOL_COORDINATOR_ROLE) {
73785
+ await Promise.resolve().then(() => init_pool_coordinator());
73786
+ runPoolCoordinator();
73787
+ return;
73788
+ }
73789
+ await runWorker(start);
73790
+ }
73791
+ async function runWorker(start) {
72606
73792
  let sequence = 0;
72607
73793
  const pending = new Map;
72608
73794
  process.on("message", (reply) => {
@@ -72726,13 +73912,13 @@ async function runLocalWorker(start) {
72726
73912
  for (;; ) {
72727
73913
  await observeControl();
72728
73914
  if (controlState !== "running") {
72729
- await delay3(25);
73915
+ await delay4(25);
72730
73916
  continue;
72731
73917
  }
72732
73918
  await reader.catchUpWorkflow(start.root);
72733
73919
  const work = await request({ kind: "claim" });
72734
73920
  if (!work) {
72735
- await delay3(25 + Math.random() * 25);
73921
+ await delay4(25 + Math.random() * 25);
72736
73922
  continue;
72737
73923
  }
72738
73924
  const index = start.definitions.findIndex((source) => `${source.path}#${source.exportName}` === work.definition && source.digest === work.digest);
@@ -72809,7 +73995,7 @@ async function runLocalWorker(start) {
72809
73995
  const control = await request({ kind: "budget" });
72810
73996
  if (control.state === "running")
72811
73997
  return;
72812
- await delay3(25, undefined, { signal });
73998
+ await delay4(25, undefined, { signal });
72813
73999
  }
72814
74000
  };
72815
74001
  const resourceAdmission = {
@@ -72848,7 +74034,7 @@ async function runLocalWorker(start) {
72848
74034
  await observeControl();
72849
74035
  while ((await request({ kind: "control" })).state !== "running") {
72850
74036
  controller.signal.throwIfAborted();
72851
- await delay3(25);
74037
+ await delay4(25);
72852
74038
  }
72853
74039
  const result = await withResourceAdmission2(resourceAdmission, () => run(definitions[index], work.inputs, {
72854
74040
  ...work.execution,