@bastani/atomic 0.9.43 → 0.9.44-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/dist/builtin/intercom/CHANGELOG.md +22 -0
  3. package/dist/builtin/intercom/broker/broker.ts +31 -7
  4. package/dist/builtin/intercom/broker/client-message-validation.ts +6 -0
  5. package/dist/builtin/intercom/broker/client.ts +69 -0
  6. package/dist/builtin/intercom/broker/send-signature.ts +4 -1
  7. package/dist/builtin/intercom/index.bundle.mjs +309 -109
  8. package/dist/builtin/intercom/live-route-refusal.ts +20 -0
  9. package/dist/builtin/intercom/package.json +1 -1
  10. package/dist/builtin/intercom/skills/intercom/SKILL.md +11 -6
  11. package/dist/builtin/intercom/types.ts +28 -1
  12. package/dist/builtin/mcp/CHANGELOG.md +7 -0
  13. package/dist/builtin/mcp/package.json +1 -1
  14. package/dist/builtin/subagents/CHANGELOG.md +9 -0
  15. package/dist/builtin/subagents/README.md +8 -8
  16. package/dist/builtin/subagents/agents/worker.md +1 -1
  17. package/dist/builtin/subagents/package.json +1 -1
  18. package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
  19. package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
  20. package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
  21. package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
  22. package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
  23. package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
  24. package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
  25. package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
  26. package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
  27. package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
  28. package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
  29. package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
  30. package/dist/builtin/web-access/package.json +1 -1
  31. package/dist/builtin/workflows/CHANGELOG.md +20 -0
  32. package/dist/builtin/workflows/README.md +1 -1
  33. package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
  34. package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
  35. package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
  36. package/dist/builtin/workflows/builtin/goal.js +2 -2
  37. package/dist/builtin/workflows/builtin/index.js +3 -3
  38. package/dist/builtin/workflows/builtin/ralph.js +2 -2
  39. package/dist/builtin/workflows/package.json +1 -1
  40. package/dist/builtin/workflows/src/extension/index.bundle.mjs +1844 -603
  41. package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1880 -605
  42. package/dist/builtin/workflows/src/index.js +1896 -655
  43. package/dist/builtin/workflows/src/local-worker-entry.mjs +1880 -605
  44. package/dist/cli/args.js +1 -1
  45. package/dist/cli/args.js.map +1 -1
  46. package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
  47. package/dist/core/agent-session-custom-message-commit.js +59 -0
  48. package/dist/core/agent-session-custom-message-commit.js.map +1 -1
  49. package/dist/core/agent-session-methods.d.ts +1 -0
  50. package/dist/core/agent-session-methods.d.ts.map +1 -1
  51. package/dist/core/agent-session-methods.js.map +1 -1
  52. package/dist/core/agent-session-prompt.d.ts +4 -0
  53. package/dist/core/agent-session-prompt.d.ts.map +1 -1
  54. package/dist/core/agent-session-prompt.js +10 -2
  55. package/dist/core/agent-session-prompt.js.map +1 -1
  56. package/dist/core/agent-session.d.ts +2 -0
  57. package/dist/core/agent-session.d.ts.map +1 -1
  58. package/dist/core/agent-session.js +9 -0
  59. package/dist/core/agent-session.js.map +1 -1
  60. package/dist/core/builtin-packages.d.ts.map +1 -1
  61. package/dist/core/builtin-packages.js +13 -13
  62. package/dist/core/builtin-packages.js.map +1 -1
  63. package/dist/core/exec.d.ts.map +1 -1
  64. package/dist/core/exec.js +11 -2
  65. package/dist/core/exec.js.map +1 -1
  66. package/dist/core/package-manager-resolver.d.ts.map +1 -1
  67. package/dist/core/package-manager-resolver.js +25 -14
  68. package/dist/core/package-manager-resolver.js.map +1 -1
  69. package/dist/core/package-manager-types.d.ts +8 -0
  70. package/dist/core/package-manager-types.d.ts.map +1 -1
  71. package/dist/core/package-manager-types.js.map +1 -1
  72. package/dist/core/package-manager.d.ts +1 -1
  73. package/dist/core/package-manager.d.ts.map +1 -1
  74. package/dist/core/package-manager.js.map +1 -1
  75. package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
  76. package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
  77. package/dist/core/resource-loader-extension-warnings.js +15 -0
  78. package/dist/core/resource-loader-extension-warnings.js.map +1 -1
  79. package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
  80. package/dist/core/resource-loader-package-resources.js +1 -0
  81. package/dist/core/resource-loader-package-resources.js.map +1 -1
  82. package/dist/core/resource-loader-reload.d.ts.map +1 -1
  83. package/dist/core/resource-loader-reload.js +5 -3
  84. package/dist/core/resource-loader-reload.js.map +1 -1
  85. package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
  86. package/dist/extensions/mcp/client/oauth/flow.js +4 -1
  87. package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
  88. package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
  89. package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
  90. package/dist/extensions/mcp/client/oauth/types.js +8 -4
  91. package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
  92. package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
  93. package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
  94. package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
  95. package/dist/index.d.ts +1 -1
  96. package/dist/index.d.ts.map +1 -1
  97. package/dist/index.js.map +1 -1
  98. package/dist/main.d.ts.map +1 -1
  99. package/dist/main.js +20 -10
  100. package/dist/main.js.map +1 -1
  101. package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
  102. package/dist/modes/interactive/components/model-selector.js +4 -2
  103. package/dist/modes/interactive/components/model-selector.js.map +1 -1
  104. package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
  105. package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
  106. package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
  107. package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
  108. package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
  109. package/dist/modes/interactive/components/tool-execution.js +11 -1
  110. package/dist/modes/interactive/components/tool-execution.js.map +1 -1
  111. package/dist/modes/interactive/model-search.d.ts +10 -0
  112. package/dist/modes/interactive/model-search.d.ts.map +1 -1
  113. package/dist/modes/interactive/model-search.js +21 -0
  114. package/dist/modes/interactive/model-search.js.map +1 -1
  115. package/dist/package-manager-cli.d.ts.map +1 -1
  116. package/dist/package-manager-cli.js +5 -0
  117. package/dist/package-manager-cli.js.map +1 -1
  118. package/dist/utils/tools-manager.d.ts.map +1 -1
  119. package/dist/utils/tools-manager.js +30 -35
  120. package/dist/utils/tools-manager.js.map +1 -1
  121. package/docs/computer-use.md +5 -5
  122. package/docs/custom-provider/streaming.md +9 -14
  123. package/docs/environment-variables.md +1 -0
  124. package/docs/intercom/operations.md +1 -1
  125. package/docs/intercom.md +16 -2
  126. package/docs/packages.md +2 -0
  127. package/docs/settings.md +2 -2
  128. package/docs/skills.md +2 -2
  129. package/docs/subagents.md +47 -10
  130. package/docs/workflows/api-reference.md +1 -1
  131. package/docs/workflows/authoring.md +1 -1
  132. package/docs/workflows/operations.md +8 -6
  133. package/docs/workflows/reliable-design.md +23 -5
  134. package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
  135. package/examples/extensions/todo.ts +7 -0
  136. package/npm-shrinkwrap.json +32 -32
  137. package/package.json +3 -3
@@ -3221,7 +3221,13 @@ function createStoreContext(state = createStoreState()) {
3221
3221
  }
3222
3222
  return cachedGraphSnapshot;
3223
3223
  }
3224
+ let batchDepth = 0;
3225
+ let batchedNotification = false;
3224
3226
  function notify() {
3227
+ if (batchDepth > 0) {
3228
+ batchedNotification = true;
3229
+ return;
3230
+ }
3225
3231
  observation.capture();
3226
3232
  for (const fn of state.invalidationListeners) {
3227
3233
  try {
@@ -3241,6 +3247,18 @@ function createStoreContext(state = createStoreState()) {
3241
3247
  state.version++;
3242
3248
  notify();
3243
3249
  }
3250
+ function batch(update) {
3251
+ batchDepth++;
3252
+ try {
3253
+ return update();
3254
+ } finally {
3255
+ batchDepth--;
3256
+ if (batchDepth === 0 && batchedNotification) {
3257
+ batchedNotification = false;
3258
+ notify();
3259
+ }
3260
+ }
3261
+ }
3244
3262
  function findRun(runId) {
3245
3263
  return state.runs.find((r) => r.id === runId);
3246
3264
  }
@@ -3294,6 +3312,7 @@ function createStoreContext(state = createStoreState()) {
3294
3312
  snapshot,
3295
3313
  graphSnapshot,
3296
3314
  notify,
3315
+ batch,
3297
3316
  bumpAndNotify,
3298
3317
  findRun,
3299
3318
  findStage,
@@ -4432,6 +4451,9 @@ function createRunStoreMethods(context) {
4432
4451
  subscribeInvalidation(fn) {
4433
4452
  state.invalidationListeners.add(fn);
4434
4453
  return () => state.invalidationListeners.delete(fn);
4454
+ },
4455
+ batch(update) {
4456
+ return context.batch(update);
4435
4457
  }
4436
4458
  };
4437
4459
  }
@@ -5165,7 +5187,7 @@ class ConcurrencyLimiter {
5165
5187
  }
5166
5188
  }
5167
5189
  function createRunLimiter(defaultConcurrency) {
5168
- return new ConcurrencyLimiter(defaultConcurrency ?? 3);
5190
+ return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
5169
5191
  }
5170
5192
 
5171
5193
  // dist/builtin/workflows/src/extension/atomic-stage-session.ts
@@ -9848,6 +9870,13 @@ function removeProcessExitCleanup(cleanup) {
9848
9870
  if (processExitCleanups.delete(cleanup) && processExitCleanups.size === 0)
9849
9871
  process.off("exit", runProcessExitCleanups);
9850
9872
  }
9873
+ function snapshotNeedsHistory(tail) {
9874
+ const event = tail.snapshot.event;
9875
+ if (event.type !== RUN_EVENT_TYPES.messageQueued)
9876
+ return false;
9877
+ const payload = event.payload;
9878
+ return hasPendingStageDeliveryDelta(payload.pendingStageMessages);
9879
+ }
9851
9880
  function* invoke(action) {
9852
9881
  try {
9853
9882
  const result = action();
@@ -9963,8 +9992,11 @@ class JournalDurableBackend {
9963
9992
  this.projection.applyWorkflowSnapshot(event.payload.handle);
9964
9993
  this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
9965
9994
  } else {
9966
- const handle = event.payload;
9967
9995
  const previous = this.projection.getWorkflow(entry.runId);
9996
+ const handle = event.type === RUN_EVENT_TYPES.messageQueued && hasPendingStageDeliveryDelta(event.payload.pendingStageMessages) ? {
9997
+ ...event.payload,
9998
+ pendingStageMessages: decodePendingStageDeliveries(previous?.pendingStageMessages ?? [], event.payload.pendingStageMessages ?? [])
9999
+ } : event.payload;
9968
10000
  this.projection.applyWorkflowSnapshot(handle);
9969
10001
  if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
9970
10002
  this.prompts.delete(entry.runId);
@@ -10260,11 +10292,23 @@ class JournalDurableBackend {
10260
10292
  for (const message of messages)
10261
10293
  if (message.runId !== logicalRunId)
10262
10294
  throw new JournalMessageScopeError(logicalRunId, message.runId);
10263
- yield* this.snapshot({
10295
+ const next = {
10264
10296
  ...handle,
10265
10297
  pendingStageMessages: replacePendingStageMessagesForRun(handle.pendingStageMessages ?? [], logicalRunId, messages),
10266
10298
  updatedAt: Math.max(Date.now(), handle.updatedAt + 1)
10267
- }, RUN_EVENT_TYPES.messageQueued);
10299
+ };
10300
+ const encoded = encodePendingStageDeliveries(handle.pendingStageMessages ?? [], next.pendingStageMessages ?? []);
10301
+ const base = this.sequences.get(runId);
10302
+ if (encoded !== undefined && base !== undefined) {
10303
+ try {
10304
+ yield* this.append(runId, { type: RUN_EVENT_TYPES.messageQueued, payload: { ...next, pendingStageMessages: encoded } }, { expectSeq: base });
10305
+ return true;
10306
+ } catch (error) {
10307
+ if (!(error instanceof JournalCasConflictError))
10308
+ throw error;
10309
+ }
10310
+ }
10311
+ yield* this.snapshot(next, RUN_EVENT_TYPES.messageQueued);
10268
10312
  return true;
10269
10313
  }
10270
10314
  async archivePendingStageMessage(runId, message) {
@@ -10358,7 +10402,7 @@ class JournalDurableBackend {
10358
10402
  if (summarized !== undefined && summarized.seq === version.seq && summarized.at === version.at)
10359
10403
  return;
10360
10404
  const tail = yield* invoke(() => journal.snapshotTail(runId));
10361
- if (tail !== undefined)
10405
+ if (tail !== undefined && !snapshotNeedsHistory(tail))
10362
10406
  return this.summarize(runId, tail);
10363
10407
  if (summarized === undefined && version.seq === 0)
10364
10408
  return;
@@ -10584,6 +10628,69 @@ class JournalDurableBackend {
10584
10628
  }
10585
10629
  }
10586
10630
  }
10631
+ function identity(message) {
10632
+ return JSON.stringify([message.runId, message.id]);
10633
+ }
10634
+ function uniqueByIdentity(messages) {
10635
+ const result = new Map;
10636
+ const repeated = new Set;
10637
+ for (const message of messages) {
10638
+ const key = identity(message);
10639
+ if (result.has(key))
10640
+ repeated.add(key);
10641
+ result.set(key, message);
10642
+ }
10643
+ for (const key of repeated)
10644
+ result.delete(key);
10645
+ return result;
10646
+ }
10647
+ function isDeltaEncoded(message) {
10648
+ return message.deliveryDelta !== undefined;
10649
+ }
10650
+ function encodePendingStageDeliveries(previous, next) {
10651
+ const bases = uniqueByIdentity(previous);
10652
+ const nextIdentities = uniqueByIdentity(next);
10653
+ let encodedAny = false;
10654
+ const encoded = next.map((message) => {
10655
+ const base = bases.get(identity(message))?.deliveries;
10656
+ const deliveries = message.deliveries;
10657
+ if (base === undefined || deliveries === undefined || !nextIdentities.has(identity(message)))
10658
+ return message;
10659
+ const drop = [];
10660
+ let kept = 0;
10661
+ base.forEach((delivery, index) => {
10662
+ if (kept < deliveries.length && JSON.stringify(delivery) === JSON.stringify(deliveries[kept]))
10663
+ kept++;
10664
+ else
10665
+ drop.push(index);
10666
+ });
10667
+ const add = deliveries.slice(kept);
10668
+ if (drop.length + add.length >= deliveries.length)
10669
+ return message;
10670
+ encodedAny = true;
10671
+ const { deliveries: _full, ...rest } = message;
10672
+ return { ...rest, deliveryDelta: drop.length === 0 ? { add } : { drop, add } };
10673
+ });
10674
+ return encodedAny ? encoded : undefined;
10675
+ }
10676
+ function decodePendingStageDeliveries(previous, messages) {
10677
+ if (!messages.some(isDeltaEncoded))
10678
+ return messages;
10679
+ const bases = uniqueByIdentity(previous);
10680
+ return messages.map((message) => {
10681
+ if (!isDeltaEncoded(message))
10682
+ return message;
10683
+ const base = bases.get(identity(message))?.deliveries;
10684
+ if (base === undefined)
10685
+ throw new Error(`atomic-workflows: journal delivery change for ${identity(message)} has no base entry`);
10686
+ const { deliveryDelta, ...rest } = message;
10687
+ const dropped = new Set(deliveryDelta.drop ?? []);
10688
+ return { ...rest, deliveries: [...base.filter((_, index) => !dropped.has(index)), ...deliveryDelta.add] };
10689
+ });
10690
+ }
10691
+ function hasPendingStageDeliveryDelta(messages) {
10692
+ return messages?.some(isDeltaEncoded) === true;
10693
+ }
10587
10694
  var PROCESS_HOLDER_ID, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError;
10588
10695
  var init_journal_backend = __esm(() => {
10589
10696
  init_workflow_instance_ownership_error();
@@ -57292,7 +57399,7 @@ var init_definitions = __esm(() => {
57292
57399
  // dist/builtin/workflows/src/durable/journal/pool-mutations.ts
57293
57400
  import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
57294
57401
  async function validatePoolMutation(tx, journal, root, work, mutation) {
57295
- if (!["checkpoint", "session-create", "session-append", "session-replace"].includes(mutation.kind))
57402
+ if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
57296
57403
  throw new Error("Invalid pool mutation");
57297
57404
  const checkpoint = mutation.checkpoint;
57298
57405
  const prefix = `${work.scope}:`;
@@ -57360,10 +57467,18 @@ async function sessionStore(tx, prefix) {
57360
57467
  return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
57361
57468
  }
57362
57469
  async function applyPoolSession(tx, prefix, workId, mutation) {
57470
+ const owners = `${prefix}session_owners`;
57471
+ if (mutation.kind === "session-append") {
57472
+ if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
57473
+ throw new Error("Pool sessions require database identity");
57474
+ const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
57475
+ if (owner?.work !== workId)
57476
+ throw new Error("Pool session scope mismatch");
57477
+ return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
57478
+ }
57363
57479
  const { sessionId, replayKey } = mutation.checkpoint;
57364
57480
  if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
57365
57481
  throw new Error("Pool sessions require database identity");
57366
- const owners = `${prefix}session_owners`;
57367
57482
  if (mutation.kind === "session-create") {
57368
57483
  if (mutation.header.id !== sessionId)
57369
57484
  throw new Error("Pool session identity mismatch");
@@ -57373,10 +57488,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
57373
57488
  if (owner?.work !== workId || owner.replay_key !== replayKey)
57374
57489
  throw new Error("Pool session scope mismatch");
57375
57490
  }
57491
+ return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
57492
+ }
57493
+ async function withSessionStore(tx, prefix, write) {
57376
57494
  const store = await sessionStore(tx, prefix);
57377
57495
  try {
57378
- const result = mutation.kind === "session-create" ? await store.create(mutation.header, mutation.entries) : mutation.kind === "session-append" ? await store.append(sessionId, mutation.entries, { expectSeq: mutation.expectSeq }) : await store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq });
57379
- return result.lastSeq;
57496
+ return (await write(store)).lastSeq;
57380
57497
  } finally {
57381
57498
  await store.close();
57382
57499
  }
@@ -57424,8 +57541,11 @@ var init_pool_stage_usage = __esm(() => {
57424
57541
  });
57425
57542
 
57426
57543
  // dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
57427
- import { randomUUID as randomUUID10 } from "node:crypto";
57544
+ import { createHash as createHash4, randomUUID as randomUUID10 } from "node:crypto";
57428
57545
  import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
57546
+ function requestIdentity(request) {
57547
+ return createHash4("sha256").update(request).digest("hex");
57548
+ }
57429
57549
  function fromRow(row) {
57430
57550
  return {
57431
57551
  ...JSON.parse(row.descriptor),
@@ -57481,10 +57601,11 @@ class SqlPoolQueue {
57481
57601
  const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
57482
57602
  if (versions.length > 1)
57483
57603
  throw new Error("Corrupt pool schema metadata");
57484
- if (versions.length === 0)
57485
- await tx.run(`INSERT INTO ${pool.meta} VALUES (1)`);
57486
- else
57487
- assertStorageSchemaVersion2("journal", versions[0].version, 1);
57604
+ const version = versions[0]?.version;
57605
+ if (version === undefined)
57606
+ await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
57607
+ else if (version !== 1)
57608
+ assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
57488
57609
  await tx.exec(`
57489
57610
  CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
57490
57611
  CREATE TABLE IF NOT EXISTS ${pool.work}(
@@ -57515,9 +57636,23 @@ class SqlPoolQueue {
57515
57636
  state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
57516
57637
  )`);
57517
57638
  await tx.exec(poolUsageSchema(journal.runs));
57639
+ if (version === 1)
57640
+ await pool.migrateFromVersion1(tx);
57518
57641
  });
57519
57642
  return pool;
57520
57643
  }
57644
+ async migrateFromVersion1(tx) {
57645
+ await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
57646
+ for (let after = 0;; ) {
57647
+ const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
57648
+ if (rows.length === 0)
57649
+ break;
57650
+ for (const row of rows)
57651
+ await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
57652
+ after = Number(rows.at(-1).rowid);
57653
+ }
57654
+ await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
57655
+ }
57521
57656
  async authority(tx, lease, timeout) {
57522
57657
  const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
57523
57658
  if (row?.timeout === null && timeout === undefined)
@@ -57658,31 +57793,18 @@ class SqlPoolQueue {
57658
57793
  throw new Error(STALE_WORK_LEASE);
57659
57794
  return row;
57660
57795
  }
57661
- async renew(lease, token, operationId, durationMs) {
57796
+ async renew(lease, token, durationMs) {
57662
57797
  positive(durationMs);
57663
57798
  lease = snapshotLease(lease);
57664
57799
  token = { ...token };
57665
- validateText("key", operationId);
57666
- const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
57667
57800
  return this.transaction(async (tx) => {
57668
57801
  const { now } = await this.authority(tx, lease);
57669
57802
  const row = await this.current(tx, lease, token);
57670
57803
  if (row.epoch !== lease.epoch || row.expires_at < now)
57671
57804
  throw new Error(STALE_WORK_LEASE);
57672
- const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
57673
- if (previous) {
57674
- if (previous.request !== request)
57675
- throw new Error("Pool operation identity conflict");
57676
- return JSON.parse(previous.receipt);
57677
- }
57678
- const receipt = {
57679
- ...fromRow(row),
57680
- expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
57681
- renewedAt: now
57682
- };
57683
- await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
57684
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
57685
- return receipt;
57805
+ const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
57806
+ await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
57807
+ return { ...fromRow(row), expiresAt, renewedAt: now };
57686
57808
  });
57687
57809
  }
57688
57810
  async commit(lease, token, operationId, mutation) {
@@ -57691,6 +57813,7 @@ class SqlPoolQueue {
57691
57813
  validateText("key", operationId);
57692
57814
  assertWorkflowSerializableObject(mutation, "Pool mutation");
57693
57815
  const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
57816
+ const identity = requestIdentity(request);
57694
57817
  mutation = JSON.parse(request).mutation;
57695
57818
  return this.transaction(async (tx) => {
57696
57819
  const { now, status } = await this.authority(tx, lease);
@@ -57699,7 +57822,7 @@ class SqlPoolQueue {
57699
57822
  throw new Error(STALE_WORK_LEASE);
57700
57823
  const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
57701
57824
  if (previous) {
57702
- if (previous.request !== request)
57825
+ if (previous.request !== identity)
57703
57826
  throw new Error("Pool operation identity conflict");
57704
57827
  return JSON.parse(previous.receipt);
57705
57828
  }
@@ -57707,34 +57830,41 @@ class SqlPoolQueue {
57707
57830
  throw new Error(STALE_WORK_LEASE);
57708
57831
  if (!await this.admitsAttemptWrites(tx, lease, status))
57709
57832
  throw new Error("Pool root is not running");
57710
- const checkpoint = mutation.checkpoint;
57711
- await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
57712
- if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
57713
- throw new Error("Pool checkpoint timestamp is in the future");
57714
- assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
57715
- const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
57716
- const payload = serializeJson(checkpoint);
57717
- const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
57718
- if (existing && existing.payload !== payload)
57719
- throw new Error("Pool checkpoint identity conflict");
57720
- if (existing && mutation.kind !== "checkpoint")
57721
- throw new Error("Pool session checkpoint identity conflict");
57722
- const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
57723
- const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
57724
- const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
57725
- seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
57726
- at: now,
57727
- ...lastSeq === undefined ? {} : { lastSeq }
57728
- };
57729
- if (!existing)
57730
- await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
57731
- const billed = poolCheckpointUsage(checkpoint);
57732
- if (billed && !existing)
57733
- await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
57734
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
57833
+ const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
57834
+ await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
57735
57835
  return receipt;
57736
57836
  });
57737
57837
  }
57838
+ async appendSession(tx, row, mutation, now) {
57839
+ return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
57840
+ }
57841
+ async recordCheckpoint(tx, lease, token, row, mutation, now) {
57842
+ const checkpoint = mutation.checkpoint;
57843
+ await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
57844
+ if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
57845
+ throw new Error("Pool checkpoint timestamp is in the future");
57846
+ assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
57847
+ const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
57848
+ const payload = serializeJson(checkpoint);
57849
+ const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
57850
+ if (existing && existing.payload !== payload)
57851
+ throw new Error("Pool checkpoint identity conflict");
57852
+ if (existing && mutation.kind !== "checkpoint")
57853
+ throw new Error("Pool session checkpoint identity conflict");
57854
+ const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
57855
+ const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
57856
+ const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
57857
+ seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
57858
+ at: now,
57859
+ ...lastSeq === undefined ? {} : { lastSeq }
57860
+ };
57861
+ if (!existing)
57862
+ await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
57863
+ const billed = poolCheckpointUsage(checkpoint);
57864
+ if (billed && !existing)
57865
+ await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
57866
+ return receipt;
57867
+ }
57738
57868
  async complete(lease, token, result) {
57739
57869
  lease = snapshotLease(lease);
57740
57870
  token = { ...token };
@@ -57821,7 +57951,7 @@ function openSqlPoolQueue(journal) {
57821
57951
  return Promise.reject(new Error("Worker pool requires a SQL run journal"));
57822
57952
  return SqlPoolQueue.open(storage);
57823
57953
  }
57824
- var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
57954
+ var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
57825
57955
  var init_sql_pool_queue = __esm(() => {
57826
57956
  init_serializable();
57827
57957
  init_boundary_topology();
@@ -57833,74 +57963,42 @@ var init_sql_pool_queue = __esm(() => {
57833
57963
  init_values();
57834
57964
  });
57835
57965
 
57836
- // dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
57837
- function validate(report) {
57838
- validateText("key", report.key);
57839
- if (report.key === "")
57840
- throw new Error("Pool usage key must not be empty");
57841
- for (const field of POOL_USAGE_FIELDS) {
57842
- const value2 = report.usage[field];
57843
- if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
57844
- throw new Error(`Pool usage ${field} must be a finite non-negative number`);
57966
+ // dist/builtin/workflows/src/shared/external-run-routes.ts
57967
+ function claimExternalRunRoute(store, runId, owner) {
57968
+ let owners = registry.owners.get(store);
57969
+ if (owners === undefined) {
57970
+ owners = new Map;
57971
+ registry.owners.set(store, owners);
57845
57972
  }
57973
+ owners.set(runId, owner);
57974
+ return () => {
57975
+ const current = registry.owners.get(store);
57976
+ if (current?.get(runId) !== owner)
57977
+ return;
57978
+ current.delete(runId);
57979
+ if (current.size === 0)
57980
+ registry.owners.delete(store);
57981
+ };
57846
57982
  }
57847
-
57848
- class SqlPoolUsage {
57849
- transaction;
57850
- journal;
57851
- table;
57852
- constructor(transaction, journal) {
57853
- this.transaction = transaction;
57854
- this.journal = journal;
57855
- this.table = poolUsageTable(journal.runs);
57856
- }
57857
- static async open(transaction, journal) {
57858
- const usage = new SqlPoolUsage(transaction, journal);
57859
- await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
57860
- return usage;
57861
- }
57862
- async totalIn(tx, root) {
57863
- const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
57864
- FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
57865
- return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
57866
- }
57867
- async report(lease, token, reports) {
57868
- validateText("runId", lease.runId);
57869
- validateText("holderId", lease.holderId);
57870
- for (const report of reports)
57871
- validate(report);
57872
- const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
57873
- return this.transaction(async (tx) => {
57874
- const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
57875
- LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
57876
- if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
57877
- throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
57878
- const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
57879
- if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
57880
- throw new Error("Stale work lease");
57881
- for (const { key, usage } of snapshot)
57882
- await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
57883
- return this.totalIn(tx, lease.runId);
57884
- });
57885
- }
57886
- total(root) {
57887
- return this.transaction((tx) => this.totalIn(tx, root));
57888
- }
57983
+ function externalRunRouteOwner(store, runId) {
57984
+ return registry.owners.get(store)?.get(runId);
57889
57985
  }
57890
- async function openSqlPoolUsage(journal) {
57891
- const storage = sqlPoolStorage(journal);
57892
- if (!storage)
57893
- throw new Error("Worker pool requires a SQL run journal");
57894
- await openSqlPoolQueue(journal);
57895
- return SqlPoolUsage.open(storage.transaction, storage.tables);
57986
+ function registerPendingStageMessageRouter(store, router) {
57987
+ registry.routers.set(store, router);
57988
+ return () => {
57989
+ if (registry.routers.get(store) === router)
57990
+ registry.routers.delete(store);
57991
+ };
57896
57992
  }
57897
- var STORE_NOW3 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
57898
- var init_sql_pool_usage = __esm(() => {
57899
- init_internal();
57900
- init_pool_stage_usage();
57901
- init_sql_pool_queue();
57902
- init_types();
57903
- init_values();
57993
+ function pendingStageMessageRouter(store) {
57994
+ return registry.routers.get(store);
57995
+ }
57996
+ var registryKey, processState, registry;
57997
+ var init_external_run_routes = __esm(() => {
57998
+ registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
57999
+ processState = globalThis;
58000
+ registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
58001
+ processState[registryKey] = registry;
57904
58002
  });
57905
58003
 
57906
58004
  // dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
@@ -57927,12 +58025,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
57927
58025
  capabilitiesFor(store).set(runId, capability);
57928
58026
  return capability;
57929
58027
  }
57930
- var routeCapabilitiesKey, processState, routeCapabilities;
58028
+ var routeCapabilitiesKey, processState2, routeCapabilities;
57931
58029
  var init_pending_stage_route_capability = __esm(() => {
57932
58030
  routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
57933
- processState = globalThis;
57934
- routeCapabilities = processState[routeCapabilitiesKey] ?? new WeakMap;
57935
- processState[routeCapabilitiesKey] = routeCapabilities;
58031
+ processState2 = globalThis;
58032
+ routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
58033
+ processState2[routeCapabilitiesKey] = routeCapabilities;
57936
58034
  });
57937
58035
 
57938
58036
  // dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
@@ -57964,12 +58062,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
57964
58062
  function workflowPendingStageRouteReady(store, runId) {
57965
58063
  return owners.get(store)?.ready(runId);
57966
58064
  }
57967
- var routeOwnersKey, processState2, owners;
58065
+ var routeOwnersKey, processState3, owners;
57968
58066
  var init_pending_stage_route_readiness = __esm(() => {
57969
58067
  routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
57970
- processState2 = globalThis;
57971
- owners = processState2[routeOwnersKey] ?? new WeakMap;
57972
- processState2[routeOwnersKey] = owners;
58068
+ processState3 = globalThis;
58069
+ owners = processState3[routeOwnersKey] ?? new WeakMap;
58070
+ processState3[routeOwnersKey] = owners;
57973
58071
  });
57974
58072
 
57975
58073
  // dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
@@ -58270,6 +58368,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
58270
58368
  notificationId,
58271
58369
  reason
58272
58370
  };
58371
+ const external = externalRunRouteOwner(activeStore, entry.runId);
58372
+ if (external !== undefined)
58373
+ return external.notifyUndeliverable(payload);
58273
58374
  pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
58274
58375
  return payload.handled && await payload.completion === true;
58275
58376
  };
@@ -58281,19 +58382,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
58281
58382
  return;
58282
58383
  const ownedRunIds = new Set;
58283
58384
  const runs = activeStore.runs();
58385
+ const runById = new Map(runs.map((run) => [run.id, run]));
58386
+ const childrenByBoundary = new Map;
58387
+ for (const run of runs) {
58388
+ if (run.parentRunId === undefined || run.parentStageId === undefined)
58389
+ continue;
58390
+ const key = JSON.stringify([run.parentRunId, run.parentStageId]);
58391
+ const children = childrenByBoundary.get(key) ?? [];
58392
+ children.push(run);
58393
+ childrenByBoundary.set(key, children);
58394
+ }
58284
58395
  const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
58285
58396
  for (const run of runs) {
58286
- if (!ownsRun(run.id))
58397
+ if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
58287
58398
  continue;
58288
- const rootRunId = durableRootRunIdForRun(runs, run.id);
58399
+ const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
58289
58400
  if (rootRunId === undefined)
58290
58401
  continue;
58291
58402
  ownedRunIds.add(run.id);
58292
- const parentRun = runs.find((candidate) => candidate.id === run.parentRunId);
58403
+ const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
58293
58404
  const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
58294
58405
  const parent = parentRun === undefined || boundary === undefined ? undefined : {
58295
58406
  runId: parentRun.id,
58296
- stageKeys: [boundary.id, boundary.name].filter((key) => resolveChildRun(runs, parentRun, key)?.id === run.id)
58407
+ stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
58297
58408
  };
58298
58409
  const announcement = {
58299
58410
  runId: run.id,
@@ -58434,16 +58545,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
58434
58545
  payload.handled = true;
58435
58546
  payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
58436
58547
  };
58437
- const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
58438
- const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
58548
+ const routeStickyLiveDelivered = (payload) => {
58439
58549
  if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
58440
58550
  return;
58441
58551
  payload.handled = true;
58442
58552
  payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
58553
+ };
58554
+ const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
58555
+ const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
58556
+ const disposeRouter = registerPendingStageMessageRouter(activeStore, {
58557
+ route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
58558
+ sticky: routeStickyLiveDelivered
58443
58559
  });
58444
58560
  const dispose = () => {
58445
58561
  disposed = true;
58446
58562
  disposeReadiness();
58563
+ disposeRouter();
58447
58564
  routeCompletions.clear();
58448
58565
  unsubscribeStore();
58449
58566
  if (typeof subscription === "function")
@@ -58569,6 +58686,12 @@ function isMaterializedNonAgentTarget(runs, target) {
58569
58686
  return false;
58570
58687
  return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
58571
58688
  }
58689
+ function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
58690
+ const boundariesById = parent.stages.filter((stage) => stage.id === segment);
58691
+ const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
58692
+ const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
58693
+ return children.length === 1 ? children[0] : undefined;
58694
+ }
58572
58695
  function resolveChildRun(runs, parent, segment) {
58573
58696
  const boundariesById = parent.stages.filter((stage) => stage.id === segment);
58574
58697
  const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
@@ -58908,6 +59031,7 @@ var init_pending_stage_intercom = __esm(() => {
58908
59031
  init_execution_context_boundary();
58909
59032
  init_run_owner_backend();
58910
59033
  init_store_backend();
59034
+ init_external_run_routes();
58911
59035
  init_intercom_group();
58912
59036
  init_pending_stage_route_capability();
58913
59037
  init_pending_stage_route_readiness();
@@ -59047,7 +59171,8 @@ function buildSendSignature(to, options) {
59047
59171
  attachments: canonicalizeAttachmentsForSendSignature(options.attachments) ?? [],
59048
59172
  replyTo: options.replyTo ?? null,
59049
59173
  expectsReply: options.expectsReply ?? false,
59050
- replyError: options.replyError ?? null
59174
+ replyError: options.replyError ?? null,
59175
+ ...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery }
59051
59176
  });
59052
59177
  }
59053
59178
 
@@ -59233,6 +59358,8 @@ function isMessage(value2) {
59233
59358
  return false;
59234
59359
  if (message.expectsReply !== undefined && typeof message.expectsReply !== "boolean")
59235
59360
  return false;
59361
+ if (message.delivery !== undefined && message.delivery !== "steer" && message.delivery !== "followUp" && message.delivery !== "interrupt")
59362
+ return false;
59236
59363
  if (message.replyError !== undefined && typeof message.replyError !== "string")
59237
59364
  return false;
59238
59365
  if (message.source !== undefined && !isMessageSource(message.source))
@@ -59366,7 +59493,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
59366
59493
  function isWorkflowStageRosterEntries(value2) {
59367
59494
  return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
59368
59495
  }
59369
- var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
59496
+ var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
59370
59497
  var init_client = __esm(() => {
59371
59498
  init_paths();
59372
59499
  init_pending_send_registry();
@@ -59375,6 +59502,21 @@ var init_client = __esm(() => {
59375
59502
  init_live_route_refusal();
59376
59503
  init_retry_policy();
59377
59504
  BROKER_SOCKET = getBrokerSocketPath();
59505
+ IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
59506
+ connectionKept;
59507
+ code;
59508
+ constructor(reason, refusal = { connectionKept: false }) {
59509
+ super(reason);
59510
+ this.name = "IntercomPendingStageRouteRefusedError";
59511
+ this.connectionKept = refusal.connectionKept;
59512
+ this.code = refusal.code;
59513
+ }
59514
+ };
59515
+ PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
59516
+ "not_authorized",
59517
+ "invalid_stage_roster",
59518
+ "invalid_possible_stage_roster"
59519
+ ]);
59378
59520
  IntercomClient = class IntercomClient extends EventEmitter {
59379
59521
  returnAddress;
59380
59522
  preferredSessionId;
@@ -59382,6 +59524,8 @@ var init_client = __esm(() => {
59382
59524
  connectTimeoutMs;
59383
59525
  brokerAcknowledgesReceipt = false;
59384
59526
  brokerAcknowledgesPendingStageRoutes = false;
59527
+ multiplexesPendingStageRoutes;
59528
+ brokerRefusesPendingStageRoutesInPlace = false;
59385
59529
  brokerAnswersStatus = false;
59386
59530
  socket = null;
59387
59531
  _sessionId = null;
@@ -59447,6 +59591,7 @@ var init_client = __esm(() => {
59447
59591
  this.preferredSessionId = options.preferredSessionId;
59448
59592
  this.acknowledgesReceipt = options.acknowledgesReceipt === true;
59449
59593
  this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
59594
+ this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
59450
59595
  }
59451
59596
  failPending(error) {
59452
59597
  this.pendingSends.rejectAll(error);
@@ -59559,6 +59704,7 @@ var init_client = __esm(() => {
59559
59704
  this._supervisorSessionId = null;
59560
59705
  this.brokerAcknowledgesReceipt = false;
59561
59706
  this.brokerAcknowledgesPendingStageRoutes = false;
59707
+ this.brokerRefusesPendingStageRoutesInPlace = false;
59562
59708
  this.brokerAnswersStatus = false;
59563
59709
  this._messageSource = undefined;
59564
59710
  this.disconnectError = null;
@@ -59637,7 +59783,7 @@ var init_client = __esm(() => {
59637
59783
  }
59638
59784
  switch (brokerMessage.type) {
59639
59785
  case "registered": {
59640
- if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
59786
+ if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
59641
59787
  throw new Error("Invalid registered message");
59642
59788
  }
59643
59789
  if (this._sessionId !== null) {
@@ -59647,6 +59793,7 @@ var init_client = __esm(() => {
59647
59793
  this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
59648
59794
  this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
59649
59795
  this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
59796
+ this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
59650
59797
  this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
59651
59798
  this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
59652
59799
  break;
@@ -59659,6 +59806,12 @@ var init_client = __esm(() => {
59659
59806
  this.emit("_registration_failed", refusal);
59660
59807
  break;
59661
59808
  }
59809
+ const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
59810
+ const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
59811
+ if (refusedRoute !== undefined) {
59812
+ this.pendingStageRouteRegistrations.delete(refusedRouteId);
59813
+ refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
59814
+ }
59662
59815
  this.disconnectError ??= refusal;
59663
59816
  this.failPending(this.disconnectError);
59664
59817
  this.socket?.destroy();
@@ -59784,6 +59937,21 @@ var init_client = __esm(() => {
59784
59937
  this.emit("pending_stage_route_revoked", { runId });
59785
59938
  break;
59786
59939
  }
59940
+ case "pending_stage_route_refused": {
59941
+ const { requestId, runId, code, reason } = brokerMessage;
59942
+ if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
59943
+ throw new Error("Invalid pending-stage route refusal");
59944
+ }
59945
+ const pending = this.pendingStageRouteRegistrations.get(requestId);
59946
+ if (!pending)
59947
+ return;
59948
+ this.pendingStageRouteRegistrations.delete(requestId);
59949
+ pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
59950
+ connectionKept: true,
59951
+ ...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
59952
+ }));
59953
+ break;
59954
+ }
59787
59955
  case "pending_stage_route_registered": {
59788
59956
  const { requestId } = brokerMessage;
59789
59957
  if (typeof requestId !== "string")
@@ -60287,7 +60455,8 @@ var init_client = __esm(() => {
60287
60455
  stages,
60288
60456
  possibleStages,
60289
60457
  ...parent === undefined ? {} : { parent },
60290
- ...dormant === true ? { dormant: true } : {}
60458
+ ...dormant === true ? { dormant: true } : {},
60459
+ ...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
60291
60460
  });
60292
60461
  } catch (error) {
60293
60462
  clearTimeout(timeout);
@@ -60400,6 +60569,7 @@ var init_client = __esm(() => {
60400
60569
  timestamp: Date.now(),
60401
60570
  replyTo: options.replyTo,
60402
60571
  expectsReply: options.expectsReply,
60572
+ ...options.delivery === undefined || options.delivery === "steer" ? {} : { delivery: options.delivery },
60403
60573
  replyError: options.replyError,
60404
60574
  source: this._messageSource,
60405
60575
  content: { text: options.text, attachments: options.attachments }
@@ -61021,120 +61191,165 @@ var init_reconnect_backoff = __esm(() => {
61021
61191
  BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
61022
61192
  });
61023
61193
 
61024
- // dist/builtin/intercom/route-connect-gate.ts
61025
- function createConcurrencyGate(limit) {
61026
- let active = 0;
61027
- const waiting = [];
61028
- const release = () => {
61029
- const next = waiting.shift();
61030
- if (next === undefined)
61031
- active -= 1;
61032
- else
61033
- next();
61034
- };
61035
- return {
61036
- async run(operation) {
61037
- if (active < limit)
61038
- active += 1;
61039
- else
61040
- await new Promise((resolve) => waiting.push(resolve));
61041
- try {
61042
- return await operation();
61043
- } finally {
61044
- release();
61045
- }
61046
- }
61047
- };
61048
- }
61049
- var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
61050
-
61051
61194
  // dist/builtin/intercom/workflow-route-owner.ts
61052
61195
  import { randomUUID as randomUUID14 } from "node:crypto";
61053
61196
  import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
61197
+ function closedError() {
61198
+ return new Error("Workflow Intercom route owner closed");
61199
+ }
61054
61200
  function createWorkflowRouteOwner(cwd) {
61055
61201
  const events = createEventBus();
61056
61202
  const config = loadConfig();
61057
61203
  const id = randomUUID14();
61058
61204
  const routes = new Map;
61059
- const gate = createConcurrencyGate(ROUTE_CLIENT_CONNECT_CONCURRENCY);
61205
+ const connections = new Map;
61206
+ const isCurrent = (state) => routes.get(state.route.runId) === state;
61207
+ const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
61208
+ function connectionOf(state) {
61209
+ const key = connectionKey(state);
61210
+ let connection = connections.get(key);
61211
+ if (connection === undefined) {
61212
+ connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
61213
+ connections.set(key, connection);
61214
+ }
61215
+ connection.routes.set(state.route.runId, state);
61216
+ return connection;
61217
+ }
61218
+ function detach(state) {
61219
+ const key = connectionKey(state);
61220
+ const connection = connections.get(key);
61221
+ connection?.routes.delete(state.route.runId);
61222
+ if (!state.isolated || connection === undefined)
61223
+ return;
61224
+ connections.delete(key);
61225
+ connection.client?.disconnect().catch(() => {});
61226
+ }
61227
+ function settle(state, registered) {
61228
+ state.registered = registered;
61229
+ if (!registered && isCurrent(state))
61230
+ return;
61231
+ for (const waiter of state.waiters)
61232
+ waiter(registered);
61233
+ state.waiters.clear();
61234
+ }
61060
61235
  function scheduleRetry(state) {
61061
- if (state.closed || state.retry !== undefined)
61236
+ if (!isCurrent(state) || state.retry !== undefined)
61062
61237
  return;
61063
61238
  state.retry = setTimeout(() => {
61064
61239
  state.retry = undefined;
61065
- publish(state, state.route).catch(() => scheduleRetry(state));
61240
+ publish(state, state.route).catch(() => {});
61066
61241
  }, jitteredReconnectDelayMs(state.attempt++));
61067
61242
  state.retry.unref();
61068
61243
  }
61069
- async function connect(state) {
61070
- if (state.client?.isConnected())
61071
- return state.client;
61072
- const [extensionDir] = getBuiltinPackagePaths({
61073
- workflows: false,
61074
- subagents: false,
61075
- mcp: false,
61076
- "web-access": false
61077
- });
61078
- if (extensionDir === undefined)
61079
- throw new Error("Intercom builtin package is unavailable");
61080
- await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
61081
- if (state.closed)
61082
- throw new Error("Workflow Intercom route owner closed");
61083
- const client = new IntercomClient(`${id}:pending-stage-route:${state.route.runId}`);
61084
- state.client = client;
61085
- client.on("error", () => {});
61086
- client.on("disconnected", () => {
61087
- if (state.client !== client)
61088
- return;
61089
- state.client = undefined;
61090
- scheduleRetry(state);
61091
- });
61092
- client.on("pending_stage_message", (request) => {
61093
- if (state.closed || state.client !== client)
61094
- return;
61095
- const event = { ...request, handled: false };
61096
- events.emit("atomic:workflow-pending-stage-message", event);
61097
- const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
61098
- completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
61099
- });
61100
- client.on("sticky_live_delivered", (notice) => {
61101
- if (state.closed || state.client !== client)
61102
- return;
61103
- events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
61104
- });
61105
- await client.connect({
61106
- cwd,
61107
- model: "workflow",
61108
- pid: process.pid,
61109
- startedAt: Date.now(),
61110
- lastActivity: Date.now(),
61111
- group: state.route.group,
61112
- groups: [state.route.group],
61113
- recipientPurpose: "control"
61244
+ function drop(connection, client) {
61245
+ if (connection.client === client) {
61246
+ connection.client = undefined;
61247
+ connection.acknowledged = false;
61248
+ for (const state of connection.routes.values()) {
61249
+ settle(state, false);
61250
+ scheduleRetry(state);
61251
+ }
61252
+ }
61253
+ client.disconnect().catch(() => {});
61254
+ }
61255
+ function connect(connection) {
61256
+ if (connection.client?.isConnected())
61257
+ return Promise.resolve(connection.client);
61258
+ connection.connecting ??= (async () => {
61259
+ const [extensionDir] = getBuiltinPackagePaths({
61260
+ workflows: false,
61261
+ subagents: false,
61262
+ mcp: false,
61263
+ "web-access": false
61264
+ });
61265
+ if (extensionDir === undefined)
61266
+ throw new Error("Intercom builtin package is unavailable");
61267
+ await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
61268
+ if (connections.get(connection.key) !== connection)
61269
+ throw closedError();
61270
+ const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
61271
+ multiplexesPendingStageRoutes: true
61272
+ });
61273
+ connection.client = client;
61274
+ connection.acknowledged = false;
61275
+ client.on("error", () => {});
61276
+ client.on("disconnected", () => drop(connection, client));
61277
+ client.on("pending_stage_message", (request) => {
61278
+ if (connection.client !== client)
61279
+ return;
61280
+ const event = { ...request, handled: false };
61281
+ events.emit("atomic:workflow-pending-stage-message", event);
61282
+ const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
61283
+ completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
61284
+ });
61285
+ client.on("sticky_live_delivered", (notice) => {
61286
+ if (connection.client !== client)
61287
+ return;
61288
+ events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
61289
+ });
61290
+ try {
61291
+ await client.connect({
61292
+ cwd,
61293
+ model: "workflow",
61294
+ pid: process.pid,
61295
+ startedAt: Date.now(),
61296
+ lastActivity: Date.now(),
61297
+ group: connection.group,
61298
+ groups: [connection.group],
61299
+ recipientPurpose: "control"
61300
+ });
61301
+ if (connections.get(connection.key) !== connection)
61302
+ throw closedError();
61303
+ } catch (error) {
61304
+ drop(connection, client);
61305
+ throw error;
61306
+ }
61307
+ for (const state of connection.routes.values()) {
61308
+ if (state.registered || state.retry === undefined)
61309
+ continue;
61310
+ clearTimeout(state.retry);
61311
+ state.retry = undefined;
61312
+ publish(state, state.route).catch(() => {});
61313
+ }
61314
+ return client;
61315
+ })().finally(() => {
61316
+ connection.connecting = undefined;
61114
61317
  });
61115
- return client;
61318
+ return connection.connecting;
61116
61319
  }
61117
61320
  function publish(state, route) {
61118
- const completion = state.tail.catch(() => {}).then(() => gate.run(async () => {
61119
- if (state.closed)
61120
- throw new Error("Workflow Intercom route owner closed");
61121
- const establishedClient = state.client?.isConnected() ? state.client : undefined;
61321
+ const completion = state.tail.catch(() => {}).then(async () => {
61322
+ if (!isCurrent(state))
61323
+ throw closedError();
61324
+ const connection = connectionOf(state);
61325
+ const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
61326
+ let client;
61122
61327
  try {
61123
- const client = await connect(state);
61328
+ client = await connect(connection);
61124
61329
  await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
61330
+ if (connection.client !== client || !isCurrent(state))
61331
+ return;
61332
+ connection.acknowledged = true;
61125
61333
  state.attempt = 0;
61126
61334
  clearTimeout(state.retry);
61127
61335
  state.retry = undefined;
61336
+ settle(state, true);
61128
61337
  } catch (error) {
61129
- const client = state.client;
61130
- if (client !== establishedClient || !client?.isConnected()) {
61131
- state.client = undefined;
61132
- await client?.disconnect();
61338
+ if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
61339
+ scheduleRetry(state);
61340
+ throw error;
61133
61341
  }
61342
+ if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
61343
+ detach(state);
61344
+ state.isolated = true;
61345
+ connectionOf(state);
61346
+ }
61347
+ if (client !== undefined && (client !== established || !client.isConnected()))
61348
+ drop(connection, client);
61134
61349
  scheduleRetry(state);
61135
61350
  throw error;
61136
61351
  }
61137
- }));
61352
+ });
61138
61353
  state.tail = completion;
61139
61354
  completion.catch(() => {});
61140
61355
  return completion;
@@ -61143,10 +61358,15 @@ function createWorkflowRouteOwner(cwd) {
61143
61358
  const route = payload;
61144
61359
  let state = routes.get(route.runId);
61145
61360
  if (state === undefined) {
61146
- state = { route, tail: Promise.resolve(), attempt: 0, closed: false };
61361
+ state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
61147
61362
  routes.set(route.runId, state);
61363
+ } else if (state.route.group !== route.group) {
61364
+ detach(state);
61365
+ state.isolated = false;
61366
+ settle(state, false);
61148
61367
  }
61149
61368
  state.route = route;
61369
+ connectionOf(state);
61150
61370
  route.completion = publish(state, route);
61151
61371
  });
61152
61372
  events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
@@ -61157,7 +61377,7 @@ function createWorkflowRouteOwner(cwd) {
61157
61377
  event.handled = true;
61158
61378
  event.completion = (async () => {
61159
61379
  await publish(state, state.route);
61160
- const client = state.client;
61380
+ const client = connections.get(connectionKey(state))?.client;
61161
61381
  if (client === undefined || !client.isConnected())
61162
61382
  return false;
61163
61383
  const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
@@ -61167,18 +61387,33 @@ function createWorkflowRouteOwner(cwd) {
61167
61387
  });
61168
61388
  return {
61169
61389
  events,
61390
+ registered(runId) {
61391
+ const state = routes.get(runId);
61392
+ if (state === undefined)
61393
+ return Promise.resolve(false);
61394
+ if (state.registered)
61395
+ return Promise.resolve(true);
61396
+ return new Promise((resolve) => state.waiters.add(resolve));
61397
+ },
61170
61398
  async closeGroup(group) {
61171
61399
  const closing = [...routes.values()].filter((state) => state.route.group === group);
61400
+ const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
61401
+ for (const connection of closingConnections) {
61402
+ connections.delete(connection.key);
61403
+ connection.routes.clear();
61404
+ }
61172
61405
  for (const state of closing) {
61173
- state.closed = true;
61174
- clearTimeout(state.retry);
61175
61406
  routes.delete(state.route.runId);
61407
+ clearTimeout(state.retry);
61408
+ state.retry = undefined;
61409
+ settle(state, false);
61176
61410
  }
61177
- await Promise.all(closing.map(async (state) => {
61178
- await state.client?.disconnect();
61179
- await state.tail.catch(() => {});
61180
- await state.client?.disconnect();
61411
+ await Promise.all(closingConnections.map(async (connection) => {
61412
+ await connection.client?.disconnect();
61413
+ await connection.connecting?.catch(() => {});
61181
61414
  }));
61415
+ await Promise.all(closing.map((state) => state.tail.catch(() => {})));
61416
+ await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
61182
61417
  }
61183
61418
  };
61184
61419
  }
@@ -61297,7 +61532,7 @@ var init_workflow_infrastructure_error = __esm(() => {
61297
61532
  [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
61298
61533
  [/Local worker retry limit/, "worker_retry_limit"],
61299
61534
  [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
61300
- [/Worker exited before delivery acknowledgement/, "worker_lost"],
61535
+ [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
61301
61536
  [/Stale work lease/, "stale_work_lease"],
61302
61537
  [/Intercom runtime no longer active/, "intercom_unavailable"],
61303
61538
  [/Storage unavailable/, "storage_unavailable"]
@@ -61313,12 +61548,925 @@ var init_workflow_infrastructure_error = __esm(() => {
61313
61548
  };
61314
61549
  });
61315
61550
 
61551
+ // dist/builtin/workflows/src/workers/coordinator-protocol.ts
61552
+ var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
61553
+
61316
61554
  // dist/builtin/workflows/src/workers/host-options.ts
61317
61555
  var hostWorkerBindings;
61318
61556
  var init_host_options = __esm(() => {
61319
61557
  hostWorkerBindings = Symbol("host-worker-bindings");
61320
61558
  });
61321
61559
 
61560
+ // dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
61561
+ function validate(report) {
61562
+ validateText("key", report.key);
61563
+ if (report.key === "")
61564
+ throw new Error("Pool usage key must not be empty");
61565
+ for (const field of POOL_USAGE_FIELDS) {
61566
+ const value2 = report.usage[field];
61567
+ if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
61568
+ throw new Error(`Pool usage ${field} must be a finite non-negative number`);
61569
+ }
61570
+ }
61571
+
61572
+ class SqlPoolUsage {
61573
+ transaction;
61574
+ journal;
61575
+ table;
61576
+ constructor(transaction, journal) {
61577
+ this.transaction = transaction;
61578
+ this.journal = journal;
61579
+ this.table = poolUsageTable(journal.runs);
61580
+ }
61581
+ static async open(transaction, journal) {
61582
+ const usage = new SqlPoolUsage(transaction, journal);
61583
+ await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
61584
+ return usage;
61585
+ }
61586
+ async totalIn(tx, root) {
61587
+ const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
61588
+ FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
61589
+ return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
61590
+ }
61591
+ async report(lease, token, reports) {
61592
+ validateText("runId", lease.runId);
61593
+ validateText("holderId", lease.holderId);
61594
+ for (const report of reports)
61595
+ validate(report);
61596
+ const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
61597
+ return this.transaction(async (tx) => {
61598
+ const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
61599
+ LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
61600
+ if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
61601
+ throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
61602
+ const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
61603
+ if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
61604
+ throw new Error("Stale work lease");
61605
+ for (const { key, usage } of snapshot)
61606
+ await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
61607
+ return this.totalIn(tx, lease.runId);
61608
+ });
61609
+ }
61610
+ total(root) {
61611
+ return this.transaction((tx) => this.totalIn(tx, root));
61612
+ }
61613
+ }
61614
+ async function openSqlPoolUsage(journal) {
61615
+ const storage = sqlPoolStorage(journal);
61616
+ if (!storage)
61617
+ throw new Error("Worker pool requires a SQL run journal");
61618
+ await openSqlPoolQueue(journal);
61619
+ return SqlPoolUsage.open(storage.transaction, storage.tables);
61620
+ }
61621
+ var STORE_NOW3 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
61622
+ var init_sql_pool_usage = __esm(() => {
61623
+ init_internal();
61624
+ init_pool_stage_usage();
61625
+ init_sql_pool_queue();
61626
+ init_types();
61627
+ init_values();
61628
+ });
61629
+
61630
+ // dist/builtin/workflows/src/workers/pool-coordinator.ts
61631
+ import { fork } from "node:child_process";
61632
+ import { randomUUID as randomUUID15 } from "node:crypto";
61633
+ import { createWriteStream } from "node:fs";
61634
+ import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
61635
+ import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
61636
+ import { setTimeout as delay2 } from "node:timers/promises";
61637
+ function workerLogDirectory(journalPath, root) {
61638
+ return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
61639
+ }
61640
+ function token(work) {
61641
+ return {
61642
+ id: work.id,
61643
+ generation: work.generation,
61644
+ worker: work.worker,
61645
+ epoch: work.epoch,
61646
+ renewedAt: work.renewedAt,
61647
+ expiresAt: work.expiresAt
61648
+ };
61649
+ }
61650
+ function outside(runId) {
61651
+ return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
61652
+ }
61653
+ function isLiveStage(stage) {
61654
+ return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
61655
+ }
61656
+ function projectRootRun(run) {
61657
+ return {
61658
+ id: run.id,
61659
+ name: run.name,
61660
+ status: run.status,
61661
+ startedAt: run.startedAt,
61662
+ stages: run.stages.map((stage) => ({
61663
+ id: stage.id,
61664
+ name: stage.name,
61665
+ status: stage.status,
61666
+ parentIds: [],
61667
+ toolEvents: [],
61668
+ ...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
61669
+ ...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
61670
+ ...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
61671
+ ...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
61672
+ }))
61673
+ };
61674
+ }
61675
+
61676
+ class PooledRoutes {
61677
+ root;
61678
+ owner;
61679
+ mirror = createStore();
61680
+ owners = new Map;
61681
+ attempts = new Map;
61682
+ activated = false;
61683
+ bridge;
61684
+ constructor(root, owner) {
61685
+ this.root = root;
61686
+ this.owner = owner;
61687
+ }
61688
+ static async open(root, cwd, rootRun, forward) {
61689
+ await Promise.resolve().then(() => init_workflow_route_owner());
61690
+ const owner = createWorkflowRouteOwner(cwd);
61691
+ const routes = new PooledRoutes(root, owner);
61692
+ if (rootRun !== undefined)
61693
+ routes.updateRoot(rootRun);
61694
+ routes.bridge = registerPendingStageIntercomBridge({
61695
+ events: {
61696
+ emit: (event, payload) => owner.events.emit(event, payload),
61697
+ on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
61698
+ }
61699
+ }, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
61700
+ for (const [event, kind] of [
61701
+ [PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
61702
+ [STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
61703
+ ])
61704
+ owner.events.on(event, (payload) => {
61705
+ const claimed = payload;
61706
+ if (claimed.handled !== false)
61707
+ return;
61708
+ claimed.handled = true;
61709
+ const { handled: _handled, completion: _completion, ...forwarded } = claimed;
61710
+ claimed.completion = forward({ kind, payload: forwarded });
61711
+ });
61712
+ return routes;
61713
+ }
61714
+ updateRoot(rootRun) {
61715
+ this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
61716
+ }
61717
+ accept(entries) {
61718
+ this.mirror.batch(() => {
61719
+ for (const { work, runs } of entries) {
61720
+ let attempt = this.attempts.get(work.id);
61721
+ if (attempt !== undefined && work.generation < attempt.generation)
61722
+ continue;
61723
+ if (attempt === undefined || attempt.generation !== work.generation) {
61724
+ if (attempt !== undefined)
61725
+ this.release(attempt);
61726
+ attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
61727
+ this.attempts.set(work.id, attempt);
61728
+ }
61729
+ if (attempt.fenced)
61730
+ continue;
61731
+ for (const run of runs) {
61732
+ attempt.runs.add(run.id);
61733
+ this.owners.set(run.id, work.id);
61734
+ const { pendingStageMessages: _messages, ...published } = run;
61735
+ this.mirror.recordRunStart({ ...published, rootRunId: this.root });
61736
+ if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
61737
+ this.activated = true;
61738
+ }
61739
+ }
61740
+ });
61741
+ }
61742
+ fence(work) {
61743
+ const attempt = this.attempts.get(work.id);
61744
+ if (attempt === undefined || attempt.generation > work.generation)
61745
+ return;
61746
+ if (attempt.generation < work.generation) {
61747
+ this.release(attempt);
61748
+ this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
61749
+ return;
61750
+ }
61751
+ this.release(attempt);
61752
+ attempt.fenced = true;
61753
+ }
61754
+ async route(work, runId) {
61755
+ this.requireOwned(work, runId);
61756
+ const readiness = workflowPendingStageRouteReady(this.mirror, runId);
61757
+ await readiness?.completion.catch(() => {});
61758
+ readiness?.assertCurrent();
61759
+ const waiting = new AbortController;
61760
+ await Promise.race([
61761
+ this.owner.registered(runId),
61762
+ delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
61763
+ ]);
61764
+ waiting.abort();
61765
+ this.requireOwned(work, runId);
61766
+ return workflowPendingStageRouteCapability(this.mirror, runId);
61767
+ }
61768
+ async undeliverable(payload) {
61769
+ const event = { ...payload, handled: false };
61770
+ this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
61771
+ return event.handled && await event.completion === true;
61772
+ }
61773
+ async close() {
61774
+ const bridge = this.bridge;
61775
+ if (bridge === undefined)
61776
+ return;
61777
+ this.bridge = undefined;
61778
+ try {
61779
+ await bridge.flush();
61780
+ } finally {
61781
+ bridge.refresh();
61782
+ bridge();
61783
+ await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
61784
+ }
61785
+ }
61786
+ requireOwned(work, runId) {
61787
+ const attempt = this.attempts.get(work.id);
61788
+ if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
61789
+ throw new Error(STALE_WORK_LEASE);
61790
+ if (!attempt.runs.has(runId))
61791
+ throw outside(runId);
61792
+ }
61793
+ release(attempt) {
61794
+ this.mirror.batch(() => {
61795
+ for (const id of attempt.runs) {
61796
+ rotateWorkflowPendingStageRouteCapability(this.mirror, id);
61797
+ const run = this.mirror.runs().find((candidate) => candidate.id === id);
61798
+ if (run?.stages.some(isLiveStage) !== true)
61799
+ continue;
61800
+ this.mirror.recordRunStart({
61801
+ ...run,
61802
+ stages: run.stages.map((stage) => {
61803
+ if (!isLiveStage(stage))
61804
+ return stage;
61805
+ const { sessionId: _session, sessionFile: _file, ...rest } = stage;
61806
+ return { ...rest, attachable: false, attached: false };
61807
+ })
61808
+ });
61809
+ }
61810
+ });
61811
+ }
61812
+ }
61813
+
61814
+ class PoolCoordinator {
61815
+ port;
61816
+ children = new Set;
61817
+ incarnations = new Map;
61818
+ readyWorkers = new Set;
61819
+ logs = new Map;
61820
+ cleanups = new Set;
61821
+ controlAcks = new Map;
61822
+ lastHeard = new Map;
61823
+ pendingActivity = new Map;
61824
+ batches = new Map;
61825
+ publicationFailures = new Map;
61826
+ requests = new Map;
61827
+ deliveryRequests = new Map;
61828
+ inflight = new Set;
61829
+ lastTick = performance.now();
61830
+ stalledMs = 0;
61831
+ ticker;
61832
+ recoveryTimer;
61833
+ flushTimer;
61834
+ lastFlushAt = 0;
61835
+ batchSequence = 0;
61836
+ requestSequence = 0;
61837
+ usageChanged = false;
61838
+ budgetBlocked = false;
61839
+ desiredControl;
61840
+ root;
61841
+ lease;
61842
+ options;
61843
+ journal;
61844
+ queue;
61845
+ usageLedger;
61846
+ usageTotal;
61847
+ admission;
61848
+ routes;
61849
+ publishing = false;
61850
+ sources = [];
61851
+ logDirectory;
61852
+ starting;
61853
+ recovering;
61854
+ nextRecoveryAt = 0;
61855
+ cleanupFailure;
61856
+ closed = false;
61857
+ closing;
61858
+ replacements = 0;
61859
+ constructor(port) {
61860
+ this.port = port;
61861
+ port.onMessage((message) => this.receive(message));
61862
+ this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
61863
+ }
61864
+ async shutdown() {
61865
+ await this.closing?.catch(() => {});
61866
+ while (this.inflight.size > 0)
61867
+ await Promise.allSettled([...this.inflight]);
61868
+ await this.journal?.close().catch(() => {});
61869
+ }
61870
+ receive(message) {
61871
+ if (message.kind === "request-reply") {
61872
+ const pending = this.requests.get(message.id);
61873
+ this.requests.delete(message.id);
61874
+ if (message.error !== undefined)
61875
+ pending?.reject(new Error(message.error));
61876
+ else
61877
+ pending?.resolve(message.value);
61878
+ return;
61879
+ }
61880
+ if ("id" in message) {
61881
+ const command = message;
61882
+ const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
61883
+ this.inflight.add(handled);
61884
+ handled.finally(() => this.inflight.delete(handled));
61885
+ return;
61886
+ }
61887
+ this.notice(message);
61888
+ }
61889
+ notice(notice) {
61890
+ if (notice.kind === "budget")
61891
+ this.budgetBlocked = notice.blocked;
61892
+ else if (notice.kind === "root")
61893
+ this.routes?.updateRoot(notice.rootRun);
61894
+ else {
61895
+ const batch = this.batches.get(notice.batch);
61896
+ this.batches.delete(notice.batch);
61897
+ const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
61898
+ for (const entry of rejected.values())
61899
+ this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
61900
+ this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
61901
+ batch?.settle();
61902
+ }
61903
+ }
61904
+ async execute(command) {
61905
+ switch (command.kind) {
61906
+ case "prepare":
61907
+ return this.prepare(command.root, command.lease, command.options, command.control);
61908
+ case "control":
61909
+ return this.control(command.state);
61910
+ case "start":
61911
+ this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
61912
+ return this.starting;
61913
+ case "enqueue":
61914
+ return this.enqueue(command);
61915
+ case "transport":
61916
+ return this.transport(command.request, command);
61917
+ case "undeliverable":
61918
+ return await this.routes?.undeliverable(command.payload) ?? false;
61919
+ case "close":
61920
+ this.closing ??= this.close();
61921
+ return this.closing;
61922
+ }
61923
+ }
61924
+ async prepare(root, lease, options, control) {
61925
+ if (this.closed)
61926
+ throw new Error("Local worker pool is closed");
61927
+ if (this.root !== undefined && this.root !== root)
61928
+ throw new Error("Local worker pool belongs to another root");
61929
+ this.root = root;
61930
+ this.options = options;
61931
+ this.desiredControl ??= control;
61932
+ this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
61933
+ this.usageLedger ??= await openSqlPoolUsage(this.journal);
61934
+ this.usageTotal = await this.usageLedger.total(root);
61935
+ this.lease = lease;
61936
+ const queue = await openSqlPoolQueue(this.journal);
61937
+ await queue.initialize(lease, COORDINATOR_LEASE_MS);
61938
+ await queue.releasePreviousIncarnations(lease);
61939
+ for (;; ) {
61940
+ const intended = this.desiredControl ?? "running";
61941
+ await queue.setControl(lease, intended);
61942
+ if (intended === (this.desiredControl ?? "running"))
61943
+ break;
61944
+ }
61945
+ this.queue = queue;
61946
+ return this.usageTotal;
61947
+ }
61948
+ async control(state) {
61949
+ this.desiredControl = state;
61950
+ if (!this.queue || !this.lease)
61951
+ return;
61952
+ const control = await this.queue.setControl(this.lease, state);
61953
+ const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
61954
+ while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
61955
+ if (this.desiredControl !== state)
61956
+ return;
61957
+ if (Date.now() >= deadline) {
61958
+ if (state === "quit")
61959
+ break;
61960
+ throw new Error("Worker control acknowledgement unresolved");
61961
+ }
61962
+ await delay2(25);
61963
+ }
61964
+ this.flush();
61965
+ }
61966
+ async start(sources, cwd, publish, rootRun) {
61967
+ if (this.closed)
61968
+ throw new Error("Local worker pool is closed");
61969
+ const root = this.root;
61970
+ if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
61971
+ throw new Error("Local worker pool requires root preparation");
61972
+ this.sources = sources;
61973
+ this.publishing = publish;
61974
+ if (publish) {
61975
+ this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
61976
+ if (this.closed) {
61977
+ await this.routes.close();
61978
+ throw new Error("Local worker pool is closed");
61979
+ }
61980
+ }
61981
+ if (this.options.admission)
61982
+ this.admission = await SqlResourceAdmission.open(this.journal, {
61983
+ ...this.options.admission,
61984
+ root,
61985
+ holder: "coordinator"
61986
+ });
61987
+ this.logDirectory = workerLogDirectory(this.options.journalPath, root);
61988
+ await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
61989
+ return;
61990
+ });
61991
+ this.recoveryTimer = setInterval(() => {
61992
+ this.recoverExpired().catch((error) => this.fail(error));
61993
+ }, RECOVERY_INTERVAL_MS).unref();
61994
+ const workers = Array.from({ length: this.options.workers }, () => this.spawn());
61995
+ try {
61996
+ await Promise.any(workers);
61997
+ } catch (error) {
61998
+ throw error.errors[0];
61999
+ }
62000
+ }
62001
+ async enqueue(command) {
62002
+ if (!this.queue || !this.lease || this.root === undefined)
62003
+ throw new Error("Local worker pool requires root preparation");
62004
+ if (command.rootRun !== undefined)
62005
+ this.routes?.updateRoot(command.rootRun);
62006
+ const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
62007
+ const result = await this.queue.result(this.root, work.id);
62008
+ return { work, ...result === undefined ? {} : { result } };
62009
+ }
62010
+ async transport(request, payload) {
62011
+ const target = this.deliveryRequests.get(request);
62012
+ if (target === undefined || !this.incarnations.has(target.slot.worker))
62013
+ throw new Error("Worker exited before delivery acknowledgement");
62014
+ const { slot, workerRequest } = target;
62015
+ await new Promise((resolve, reject) => {
62016
+ const id = randomUUID15();
62017
+ slot.deliveries.set(id, { resolve, reject });
62018
+ slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
62019
+ if (error) {
62020
+ slot.deliveries.delete(id);
62021
+ reject(error);
62022
+ }
62023
+ });
62024
+ });
62025
+ }
62026
+ request(body) {
62027
+ const id = ++this.requestSequence;
62028
+ return new Promise((resolve, reject) => {
62029
+ this.requests.set(id, { resolve, reject });
62030
+ this.port.send({ kind: "request", id, request: body });
62031
+ });
62032
+ }
62033
+ fail(error) {
62034
+ this.port.send({ kind: "failure", error: error.message });
62035
+ }
62036
+ publish(work, runs) {
62037
+ const failure = this.publicationFailures.get(work.id);
62038
+ if (failure?.generation === work.generation)
62039
+ throw new Error(failure.error);
62040
+ if (!this.publishing)
62041
+ return;
62042
+ this.pendingActivity.set(work.id, { work: token(work), runs });
62043
+ this.scheduleFlush();
62044
+ }
62045
+ scheduleFlush() {
62046
+ if (this.flushTimer !== undefined || this.closed)
62047
+ return;
62048
+ const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
62049
+ this.flushTimer = setTimeout(() => {
62050
+ this.flushTimer = undefined;
62051
+ if (this.batches.size > 0) {
62052
+ this.lastFlushAt = performance.now();
62053
+ this.scheduleFlush();
62054
+ return;
62055
+ }
62056
+ this.flush();
62057
+ }, wait);
62058
+ }
62059
+ flush() {
62060
+ clearTimeout(this.flushTimer);
62061
+ this.flushTimer = undefined;
62062
+ if (this.pendingActivity.size === 0 && !this.usageChanged) {
62063
+ const last = [...this.batches.values()].at(-1);
62064
+ return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
62065
+ }
62066
+ this.lastFlushAt = performance.now();
62067
+ const batch = ++this.batchSequence;
62068
+ const entries = [...this.pendingActivity.values()];
62069
+ this.pendingActivity.clear();
62070
+ const usage = this.usageChanged ? this.usageTotal : undefined;
62071
+ this.usageChanged = false;
62072
+ const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
62073
+ this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
62074
+ return settled;
62075
+ }
62076
+ chainSettle(batch, resolve) {
62077
+ const settle = batch.settle;
62078
+ batch.settle = () => {
62079
+ settle();
62080
+ resolve();
62081
+ };
62082
+ }
62083
+ noteUsage(usage) {
62084
+ this.usageTotal = usage;
62085
+ this.usageChanged = true;
62086
+ this.scheduleFlush();
62087
+ }
62088
+ async sendResult(workId) {
62089
+ const result = await this.queue.result(this.root, workId);
62090
+ if (result === undefined)
62091
+ return false;
62092
+ this.flush();
62093
+ this.port.send({
62094
+ kind: "result",
62095
+ workId,
62096
+ result,
62097
+ ...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
62098
+ });
62099
+ return true;
62100
+ }
62101
+ fence(work) {
62102
+ this.pendingActivity.delete(work.id);
62103
+ this.routes?.fence(work);
62104
+ this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
62105
+ }
62106
+ spawn() {
62107
+ if (this.closed)
62108
+ return Promise.reject(new Error("Local worker pool is closed"));
62109
+ const root = this.root;
62110
+ const options = this.options;
62111
+ const child = fork(options.workerEntry ?? process.argv[1], [
62112
+ JSON.stringify({
62113
+ journalPath: options.journalPath,
62114
+ root,
62115
+ definitions: this.sources,
62116
+ admission: options.admission
62117
+ })
62118
+ ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62119
+ const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62120
+ this.children.add(child);
62121
+ if (child.pid !== undefined)
62122
+ this.port.send({ kind: "spawned", pid: child.pid });
62123
+ const worker = randomUUID15();
62124
+ this.incarnations.set(worker, child);
62125
+ const slot = { worker, child, deliveries: new Map };
62126
+ let ready = false;
62127
+ let serial = Promise.resolve();
62128
+ const logPath = join28(this.logDirectory, `${worker}.log`);
62129
+ this.logs.set(child, logPath);
62130
+ let logFailure;
62131
+ const log = createWriteStream(logPath, { flags: "a" });
62132
+ log.on("error", (error) => {
62133
+ logFailure = error.message;
62134
+ });
62135
+ let stderr = "";
62136
+ child.stderr?.on("data", (chunk) => {
62137
+ stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62138
+ });
62139
+ child.stderr?.on("error", (error) => {
62140
+ logFailure ??= error.message;
62141
+ log.end();
62142
+ });
62143
+ child.stderr?.pipe(log);
62144
+ const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62145
+ const startupTimeoutMs = options.startupTimeoutMs;
62146
+ let timedOut = false;
62147
+ let terminated = false;
62148
+ const terminate = () => {
62149
+ if (terminated)
62150
+ return;
62151
+ terminated = true;
62152
+ child.kill("SIGKILL");
62153
+ };
62154
+ const replied = (error) => {
62155
+ if (error)
62156
+ terminate();
62157
+ };
62158
+ return new Promise((resolve, reject) => {
62159
+ const startup = setTimeout(() => {
62160
+ timedOut = true;
62161
+ terminate();
62162
+ }, startupTimeoutMs);
62163
+ child.on("error", (error) => {
62164
+ clearTimeout(startup);
62165
+ reject(error);
62166
+ terminate();
62167
+ });
62168
+ const closed = new Promise((resolve) => child.once("close", () => resolve()));
62169
+ child.once("exit", () => {
62170
+ Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62171
+ this.children.delete(child);
62172
+ this.logs.delete(child);
62173
+ if (child.pid !== undefined)
62174
+ this.port.send({ kind: "exited", pid: child.pid });
62175
+ clearTimeout(startup);
62176
+ this.incarnations.delete(worker);
62177
+ this.readyWorkers.delete(worker);
62178
+ this.controlAcks.delete(worker);
62179
+ this.lastHeard.delete(worker);
62180
+ if (slot.work)
62181
+ this.fence(slot.work);
62182
+ for (const delivery of slot.deliveries.values())
62183
+ delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62184
+ slot.deliveries.clear();
62185
+ const cleanup = disconnected.then(() => serial).then(async () => {
62186
+ try {
62187
+ if (await this.admission?.releaseStoppedSessions(worker))
62188
+ throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62189
+ } catch (error) {
62190
+ this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62191
+ throw this.cleanupFailure;
62192
+ }
62193
+ }).then(async () => {
62194
+ if (this.closed || !ready || this.desiredControl === "quit")
62195
+ return;
62196
+ let work = slot.work;
62197
+ if (work && await this.sendResult(work.id)) {
62198
+ work = undefined;
62199
+ slot.work = undefined;
62200
+ this.replacements = 0;
62201
+ }
62202
+ if (this.closed)
62203
+ return;
62204
+ if (work)
62205
+ await this.queue.releaseExited(this.lease, work);
62206
+ const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62207
+ if (exhausted)
62208
+ this.fail(diagnosed("Local worker retry limit"));
62209
+ else
62210
+ await this.spawn();
62211
+ }).catch((error) => {
62212
+ this.fail(error);
62213
+ });
62214
+ this.cleanups.add(cleanup);
62215
+ cleanup.finally(() => this.cleanups.delete(cleanup));
62216
+ });
62217
+ child.on("message", (message) => {
62218
+ this.observeResponsiveness();
62219
+ this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
62220
+ if (message.kind === "delivery-ack") {
62221
+ const pending = slot.deliveries.get(message.deliveryId);
62222
+ slot.deliveries.delete(message.deliveryId);
62223
+ if (message.error)
62224
+ pending?.reject(new Error(message.error));
62225
+ else
62226
+ pending?.resolve();
62227
+ return;
62228
+ }
62229
+ let routeReady;
62230
+ const handled = serial.then(async () => {
62231
+ if (message.kind === "ready") {
62232
+ if (terminated)
62233
+ return;
62234
+ ready = true;
62235
+ this.readyWorkers.add(worker);
62236
+ clearTimeout(startup);
62237
+ resolve();
62238
+ return;
62239
+ }
62240
+ await this.starting;
62241
+ const queue = this.queue;
62242
+ const lease = this.lease;
62243
+ if (message.kind === "control")
62244
+ return queue.control(lease);
62245
+ if (message.kind === "controlled") {
62246
+ const control = await queue.control(lease);
62247
+ if (message.revision > control.revision)
62248
+ throw new Error("Unknown control revision");
62249
+ this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62250
+ return;
62251
+ }
62252
+ if (message.kind === "claim") {
62253
+ if (slot.work)
62254
+ throw new Error("Worker already holds an attempt");
62255
+ if (this.budgetBlocked)
62256
+ return;
62257
+ await this.recoverExpired();
62258
+ slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
62259
+ definition: `${source.path}#${source.exportName}`,
62260
+ digest: source.digest
62261
+ })), options.leaseMs);
62262
+ return slot.work;
62263
+ }
62264
+ let work = slot.work;
62265
+ if (!work)
62266
+ throw new Error("Worker has no active attempt");
62267
+ if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
62268
+ work = await queue.extend(lease, work, options.leaseMs);
62269
+ slot.work = work;
62270
+ }
62271
+ if (message.kind === "activity")
62272
+ return this.publish(work, message.runs);
62273
+ if (message.kind === "route") {
62274
+ if (this.routes === undefined)
62275
+ return;
62276
+ await this.flush();
62277
+ this.assertPublished(work);
62278
+ routeReady = this.routes.route(work, message.runId);
62279
+ return;
62280
+ }
62281
+ if (message.kind === "delivery-ready") {
62282
+ if (!this.publishing)
62283
+ return;
62284
+ await this.flush();
62285
+ this.assertPublished(work);
62286
+ return this.request({
62287
+ kind: "ready",
62288
+ work: token(work),
62289
+ runId: message.runId,
62290
+ stageId: message.stageId,
62291
+ stageName: message.stageName,
62292
+ ...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
62293
+ });
62294
+ }
62295
+ if (message.kind === "delivery") {
62296
+ if (!message.recipient)
62297
+ throw new Error("Worker delivery requires a session identity");
62298
+ if (!this.publishing)
62299
+ return;
62300
+ await this.flush();
62301
+ this.assertPublished(work);
62302
+ const id = ++this.requestSequence;
62303
+ this.deliveryRequests.set(id, { slot, workerRequest: message.id });
62304
+ try {
62305
+ return await new Promise((resolve, reject) => {
62306
+ this.requests.set(id, { resolve, reject });
62307
+ this.port.send({
62308
+ kind: "request",
62309
+ id,
62310
+ request: {
62311
+ kind: "deliver",
62312
+ work: token(work),
62313
+ runId: message.runId,
62314
+ stageId: message.stageId,
62315
+ stageName: message.stageName,
62316
+ recipient: message.recipient
62317
+ }
62318
+ });
62319
+ });
62320
+ } finally {
62321
+ this.deliveryRequests.delete(id);
62322
+ }
62323
+ }
62324
+ if (message.kind === "renew")
62325
+ return;
62326
+ if (message.kind === "commit") {
62327
+ const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62328
+ this.noteUsage(await this.usageLedger.total(root));
62329
+ return receipt;
62330
+ }
62331
+ if (message.kind === "usage") {
62332
+ this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
62333
+ return;
62334
+ }
62335
+ if (message.kind === "budget") {
62336
+ await queue.assertCurrent(lease, work);
62337
+ if (this.budgetBlocked)
62338
+ throw new Error("Root workflow budget exhausted");
62339
+ return queue.control(lease);
62340
+ }
62341
+ if (message.kind !== "complete")
62342
+ throw new Error("Invalid worker operation");
62343
+ await queue.complete(lease, work, message.result);
62344
+ this.replacements = 0;
62345
+ slot.work = undefined;
62346
+ await this.sendResult(work.id);
62347
+ });
62348
+ const response = handled.then((value2) => routeReady ?? value2);
62349
+ if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62350
+ serial = handled.then(() => {}, () => {});
62351
+ response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62352
+ });
62353
+ });
62354
+ }
62355
+ assertPublished(work) {
62356
+ const failure = this.publicationFailures.get(work.id);
62357
+ if (failure?.generation === work.generation)
62358
+ throw new Error(failure.error);
62359
+ }
62360
+ recoverExpired() {
62361
+ if (this.recovering)
62362
+ return this.recovering;
62363
+ const root = this.root;
62364
+ if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
62365
+ return Promise.resolve();
62366
+ const queue = this.queue;
62367
+ const lease = this.lease;
62368
+ const leaseMs = this.options.leaseMs;
62369
+ this.recovering = (async () => {
62370
+ await new Promise((resolve) => setImmediate(resolve));
62371
+ await this.admission?.reconcile();
62372
+ const expiredBefore = Date.now();
62373
+ for (const work of await queue.inspect(root)) {
62374
+ if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62375
+ continue;
62376
+ const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62377
+ if (child && work.worker !== null && this.isLive(work.worker, child)) {
62378
+ await queue.extend(lease, work, leaseMs).catch((error) => {
62379
+ if (error.message !== STALE_WORK_LEASE)
62380
+ throw error;
62381
+ });
62382
+ continue;
62383
+ }
62384
+ if (child && !await this.stop(child).then(() => true, () => false))
62385
+ continue;
62386
+ await queue.expire(lease, work);
62387
+ }
62388
+ })().finally(() => {
62389
+ this.recovering = undefined;
62390
+ this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62391
+ });
62392
+ return this.recovering;
62393
+ }
62394
+ observeResponsiveness() {
62395
+ const now = performance.now();
62396
+ const gap = now - this.lastTick;
62397
+ if (gap > STALL_THRESHOLD_MS)
62398
+ this.stalledMs += gap - TICK_INTERVAL_MS;
62399
+ this.lastTick = now;
62400
+ }
62401
+ isLive(worker, child) {
62402
+ this.observeResponsiveness();
62403
+ const heard = this.lastHeard.get(worker);
62404
+ return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
62405
+ }
62406
+ stop(child) {
62407
+ if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62408
+ return Promise.resolve();
62409
+ return new Promise((resolve, reject) => {
62410
+ const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62411
+ child.once("exit", () => {
62412
+ clearTimeout(timeout);
62413
+ resolve();
62414
+ });
62415
+ child.kill("SIGKILL");
62416
+ });
62417
+ }
62418
+ async close() {
62419
+ this.closed = true;
62420
+ clearInterval(this.ticker);
62421
+ clearInterval(this.recoveryTimer);
62422
+ let failure;
62423
+ let stopping = [];
62424
+ try {
62425
+ await this.recovering?.catch(() => {});
62426
+ stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62427
+ await Promise.all(stopping.map(({ child }) => this.stop(child)));
62428
+ await Promise.all([...this.cleanups]);
62429
+ if (this.cleanupFailure)
62430
+ throw this.cleanupFailure;
62431
+ } catch (error) {
62432
+ failure = error;
62433
+ }
62434
+ if (failure === undefined) {
62435
+ const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62436
+ await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62437
+ if (this.logDirectory !== undefined)
62438
+ await rmdir(this.logDirectory).catch(() => {});
62439
+ }
62440
+ this.flush();
62441
+ try {
62442
+ await this.routes?.close();
62443
+ } catch (error) {
62444
+ failure ??= error;
62445
+ }
62446
+ for (const batch of this.batches.values())
62447
+ batch.settle();
62448
+ this.batches.clear();
62449
+ for (const pending of this.requests.values())
62450
+ pending.reject(new Error("Local worker pool is closed"));
62451
+ this.requests.clear();
62452
+ if (failure !== undefined)
62453
+ throw failure;
62454
+ }
62455
+ }
62456
+ var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
62457
+ var init_pool_coordinator = __esm(() => {
62458
+ init_resolution();
62459
+ init_sql_pool_queue();
62460
+ init_sql_pool_usage();
62461
+ init_pending_stage_intercom();
62462
+ init_intercom_group();
62463
+ init_pending_stage_route_capability();
62464
+ init_pending_stage_route_readiness();
62465
+ init_store();
62466
+ init_resource_admission();
62467
+ COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
62468
+ });
62469
+
61322
62470
  // dist/builtin/workflows/src/workers/protocol.ts
61323
62471
  var WORKER_HEARTBEAT_INTERVAL_MS = 1000;
61324
62472
 
@@ -61566,21 +62714,26 @@ class PoolStageHandoff {
61566
62714
  store;
61567
62715
  queue;
61568
62716
  lease;
62717
+ claimRun;
61569
62718
  attempts = new Map;
61570
62719
  owners = new Map;
61571
62720
  publications = new Map;
61572
62721
  drains = new Map;
61573
- constructor(backend, store, queue, lease) {
62722
+ constructor(backend, store, queue, lease, claimRun) {
61574
62723
  this.backend = backend;
61575
62724
  this.store = store;
61576
62725
  this.queue = queue;
61577
62726
  this.lease = lease;
62727
+ this.claimRun = claimRun;
61578
62728
  }
61579
62729
  publish(work, runs) {
61580
62730
  const previous = this.publications.get(work.id) ?? Promise.resolve();
61581
62731
  const publication = previous.catch(() => {
61582
62732
  return;
61583
- }).then(() => this.apply(work, runs));
62733
+ }).then(async () => {
62734
+ const commit = await this.prepare(work, runs);
62735
+ this.store.batch(commit);
62736
+ });
61584
62737
  const settled = publication.catch(() => {
61585
62738
  return;
61586
62739
  });
@@ -61591,6 +62744,21 @@ class PoolStageHandoff {
61591
62744
  });
61592
62745
  return publication;
61593
62746
  }
62747
+ async publishBatch(publications) {
62748
+ const prepared = [];
62749
+ for (const { work, runs } of publications)
62750
+ prepared.push(await this.prepare(work, runs).catch((error) => error));
62751
+ return this.store.batch(() => prepared.map((commit) => {
62752
+ if (commit instanceof Error)
62753
+ return commit;
62754
+ try {
62755
+ commit();
62756
+ return;
62757
+ } catch (error) {
62758
+ return error instanceof Error ? error : new Error(String(error));
62759
+ }
62760
+ }));
62761
+ }
61594
62762
  async route(work, runId) {
61595
62763
  this.requireOwned(await this.begin(work), runId);
61596
62764
  const readiness = workflowPendingStageRouteReady(this.store, runId);
@@ -61641,7 +62809,7 @@ class PoolStageHandoff {
61641
62809
  const result = await this.queue.result(root, work.id);
61642
62810
  if (result === undefined)
61643
62811
  return;
61644
- await this.backend.hydrateWorkflow(root);
62812
+ await this.backend.catchUpWorkflow(root);
61645
62813
  const terminals = new Map;
61646
62814
  for (const checkpoint of this.backend.listCheckpoints(root)) {
61647
62815
  const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
@@ -61651,6 +62819,9 @@ class PoolStageHandoff {
61651
62819
  if (child?.runId === boundary.child.runId)
61652
62820
  terminals.set(child.runId, { status: child.status, outputs: child.outputs });
61653
62821
  }
62822
+ this.store.batch(() => this.settle(work, result, terminals));
62823
+ }
62824
+ settle(work, result, terminals) {
61654
62825
  for (const run of this.store.runs()) {
61655
62826
  if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
61656
62827
  continue;
@@ -61674,22 +62845,31 @@ class PoolStageHandoff {
61674
62845
  if (current !== undefined && current.generation > work.generation)
61675
62846
  return;
61676
62847
  const attempt = current?.generation === work.generation ? current : this.start(work, current);
61677
- this.release(attempt);
62848
+ this.store.batch(() => this.release(attempt));
61678
62849
  attempt.fenced = true;
61679
62850
  }
61680
- async apply(work, runs) {
62851
+ async prepare(work, runs) {
61681
62852
  const subtree = this.subtree(work, runs);
61682
62853
  const root = this.lease.runId;
61683
- const known = new Map(this.store.runs().map((run) => [run.id, run]));
61684
- for (const run of subtree) {
61685
- const existing = known.get(run.id);
61686
- const owner = this.owners.get(run.id);
61687
- if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
61688
- throw outside(run.id);
61689
- }
61690
- if (subtree.some((run) => known.get(run.id) === undefined))
61691
- await this.backend.hydrateWorkflow(root);
62854
+ const owned = (known) => {
62855
+ for (const run of subtree) {
62856
+ const existing = known.get(run.id);
62857
+ const owner = this.owners.get(run.id);
62858
+ if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
62859
+ throw outside2(run.id);
62860
+ }
62861
+ };
62862
+ const before = new Map(this.store.runs().map((run) => [run.id, run]));
62863
+ owned(before);
62864
+ if (subtree.some((run) => before.get(run.id) === undefined))
62865
+ await this.backend.catchUpWorkflow(root);
61692
62866
  await this.queue.assertCurrent(this.lease, work);
62867
+ return () => this.commit(work, subtree, owned);
62868
+ }
62869
+ commit(work, subtree, owned) {
62870
+ const root = this.lease.runId;
62871
+ const known = new Map(this.store.runs().map((run) => [run.id, run]));
62872
+ owned(known);
61693
62873
  const attempt = this.enter(work);
61694
62874
  const execution = {
61695
62875
  worker: work.worker ?? "",
@@ -61701,11 +62881,13 @@ class PoolStageHandoff {
61701
62881
  ...run,
61702
62882
  rootRunId: root,
61703
62883
  pendingStageMessages: [
61704
- ...this.store.runs().find((current) => current.id === run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
62884
+ ...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
61705
62885
  ],
61706
62886
  stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
61707
62887
  }));
61708
62888
  const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
62889
+ for (const run of published)
62890
+ this.claimRun?.(run.id);
61709
62891
  for (const run of published) {
61710
62892
  attempt.runs.add(run.id);
61711
62893
  this.owners.set(run.id, work.id);
@@ -61730,7 +62912,7 @@ class PoolStageHandoff {
61730
62912
  const byId = new Map;
61731
62913
  for (const run of runs) {
61732
62914
  if (run.id === root || byId.has(run.id))
61733
- throw outside(run.id);
62915
+ throw outside2(run.id);
61734
62916
  byId.set(run.id, run);
61735
62917
  }
61736
62918
  if (runs.length === 0)
@@ -61743,25 +62925,25 @@ class PoolStageHandoff {
61743
62925
  const top = byId.get(work.childRunId);
61744
62926
  if (top !== undefined) {
61745
62927
  if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
61746
- throw outside(top.id);
62928
+ throw outside2(top.id);
61747
62929
  const pending = [top];
61748
62930
  for (let run = pending.shift();run !== undefined; run = pending.shift()) {
61749
62931
  if (run.rootRunId !== undefined && run.rootRunId !== root)
61750
- throw outside(run.id);
62932
+ throw outside2(run.id);
61751
62933
  ordered.push(run);
61752
62934
  for (const child of runs) {
61753
62935
  if (child.parentRunId !== run.id)
61754
62936
  continue;
61755
62937
  const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
61756
62938
  if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
61757
- throw outside(child.id);
62939
+ throw outside2(child.id);
61758
62940
  pending.push(child);
61759
62941
  }
61760
62942
  }
61761
62943
  }
61762
62944
  const stray = runs.find((run) => !ordered.includes(run));
61763
62945
  if (stray !== undefined)
61764
- throw outside(stray.id);
62946
+ throw outside2(stray.id);
61765
62947
  return ordered;
61766
62948
  }
61767
62949
  async begin(work) {
@@ -61816,7 +62998,7 @@ class PoolStageHandoff {
61816
62998
  }
61817
62999
  requireOwned(attempt, runId) {
61818
63000
  if (!attempt.runs.has(runId))
61819
- throw outside(runId);
63001
+ throw outside2(runId);
61820
63002
  }
61821
63003
  delivery(attempt, work, runId, stageId, stageName) {
61822
63004
  const key = JSON.stringify([runId, stageId]);
@@ -61834,7 +63016,7 @@ class PoolStageHandoff {
61834
63016
  return delivery;
61835
63017
  }
61836
63018
  }
61837
- function outside(runId) {
63019
+ function outside2(runId) {
61838
63020
  return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
61839
63021
  }
61840
63022
  function isLive(stage) {
@@ -61859,33 +63041,122 @@ var init_stage_handoff = __esm(() => {
61859
63041
  });
61860
63042
 
61861
63043
  // dist/builtin/workflows/src/workers/local-pool.ts
61862
- import { fork } from "node:child_process";
61863
- import { randomUUID as randomUUID15 } from "node:crypto";
61864
- import { createWriteStream } from "node:fs";
61865
- import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
61866
- import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
61867
- import { setTimeout as delay2 } from "node:timers/promises";
63044
+ import { fork as fork2 } from "node:child_process";
63045
+ import { resolve as resolve12 } from "node:path";
63046
+ import { setTimeout as delay3 } from "node:timers/promises";
61868
63047
  import { fileURLToPath as fileURLToPath5 } from "node:url";
61869
63048
  import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
61870
- function workerLogDirectory(journalPath, root) {
61871
- return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
63049
+ function defaultWorkerEntry() {
63050
+ return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
63051
+ }
63052
+ function launchCoordinatorProcess() {
63053
+ const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
63054
+ stdio: ["ignore", "ignore", "pipe", "ipc"],
63055
+ execArgv: []
63056
+ });
63057
+ let stderr = "";
63058
+ child.stderr?.on("data", (chunk) => {
63059
+ stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
63060
+ });
63061
+ const listeners = [];
63062
+ let lostReason;
63063
+ const lose = (reason) => {
63064
+ if (lostReason !== undefined)
63065
+ return;
63066
+ lostReason = reason;
63067
+ for (const listener of listeners)
63068
+ listener(reason);
63069
+ };
63070
+ const exited = new Promise((resolve) => child.once("exit", () => resolve()));
63071
+ child.on("error", lose);
63072
+ child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
63073
+ return {
63074
+ pid: child.pid,
63075
+ port: {
63076
+ send(message) {
63077
+ if (lostReason !== undefined || !child.connected)
63078
+ return;
63079
+ child.send(message, (error) => {
63080
+ if (error)
63081
+ lose(error);
63082
+ });
63083
+ },
63084
+ onMessage(listener) {
63085
+ child.on("message", (message) => listener(message));
63086
+ },
63087
+ onClose(listener) {
63088
+ if (lostReason !== undefined)
63089
+ listener(lostReason);
63090
+ else
63091
+ listeners.push(listener);
63092
+ },
63093
+ async close() {
63094
+ if (child.connected)
63095
+ child.disconnect();
63096
+ if (child.exitCode !== null || child.signalCode !== null)
63097
+ return;
63098
+ const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
63099
+ if (timedOut) {
63100
+ child.kill("SIGKILL");
63101
+ await exited;
63102
+ }
63103
+ }
63104
+ }
63105
+ };
63106
+ }
63107
+ function launchCoordinatorInProcess() {
63108
+ const mainListeners = [];
63109
+ const coordinatorListeners = [];
63110
+ const copy = (message) => JSON.parse(JSON.stringify(message));
63111
+ const coordinator = new PoolCoordinator({
63112
+ send(message) {
63113
+ const copied = copy(message);
63114
+ setImmediate(() => {
63115
+ for (const listener of mainListeners)
63116
+ listener(copied);
63117
+ });
63118
+ },
63119
+ onMessage(listener) {
63120
+ coordinatorListeners.push(listener);
63121
+ },
63122
+ onClose() {},
63123
+ async close() {}
63124
+ });
63125
+ return {
63126
+ pid: process.pid,
63127
+ port: {
63128
+ send(message) {
63129
+ const copied = copy(message);
63130
+ setImmediate(() => {
63131
+ for (const listener of coordinatorListeners)
63132
+ listener(copied);
63133
+ });
63134
+ },
63135
+ onMessage(listener) {
63136
+ mainListeners.push(listener);
63137
+ },
63138
+ onClose() {},
63139
+ async close() {
63140
+ coordinator.shutdown();
63141
+ }
63142
+ }
63143
+ };
61872
63144
  }
61873
63145
 
61874
63146
  class LocalWorkerPool {
61875
63147
  backend;
61876
63148
  options;
61877
- children = new Set;
61878
- incarnations = new Map;
61879
- readyWorkers = new Set;
61880
- logs = new Map;
61881
- logDirectory;
61882
- cleanups = new Set;
61883
- recovering;
61884
- nextRecoveryAt = 0;
61885
- controlAcks = new Map;
61886
- lastHeard = new Map;
61887
- lastTick = performance.now();
61888
- rootStalledMs = 0;
63149
+ pids = new Set;
63150
+ coordinator;
63151
+ coordinatorLost;
63152
+ commands = new Map;
63153
+ commandSequence = 0;
63154
+ inbound = Promise.resolve();
63155
+ works = new Map;
63156
+ attempts = new Map;
63157
+ results = new Map;
63158
+ changed = Promise.withResolvers();
63159
+ waiting = 0;
61889
63160
  ticker;
61890
63161
  leaseMs;
61891
63162
  desiredControl;
@@ -61893,18 +63164,17 @@ class LocalWorkerPool {
61893
63164
  store;
61894
63165
  handoff;
61895
63166
  releaseRoutes;
63167
+ routeClaims = new Map;
61896
63168
  sources = [];
61897
63169
  queue;
61898
63170
  lease;
61899
- admission;
61900
- usageLedger;
61901
63171
  usageTotal;
61902
63172
  rootBudget;
63173
+ budgetBlocked = false;
63174
+ rootSignature;
61903
63175
  starting;
61904
- cleanupFailure;
61905
63176
  failure;
61906
63177
  closed = false;
61907
- replacements = 0;
61908
63178
  releaseBinding;
61909
63179
  constructor(backend, options) {
61910
63180
  this.backend = backend;
@@ -61919,10 +63189,12 @@ class LocalWorkerPool {
61919
63189
  this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
61920
63190
  if (options.admission && options.rootId === undefined)
61921
63191
  workflowResourceAdmission.set(backend, options.admission);
61922
- this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
61923
63192
  }
61924
63193
  get workerPids() {
61925
- return [...this.children].flatMap((child) => child.pid === undefined ? [] : [child.pid]);
63194
+ return [...this.pids];
63195
+ }
63196
+ get coordinatorPid() {
63197
+ return this.coordinator?.pid;
61926
63198
  }
61927
63199
  usage() {
61928
63200
  return this.usageTotal;
@@ -61930,47 +63202,39 @@ class LocalWorkerPool {
61930
63202
  async prepare(root, store) {
61931
63203
  if (this.closed)
61932
63204
  throw new Error("Local worker pool is closed");
61933
- if (!(this.backend.journal instanceof SqlRunJournal) || resolve11(this.options.journalPath) !== resolve11(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
63205
+ if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
61934
63206
  throw new Error("Local workers require the same default-prefix SQLite journal");
61935
63207
  if (this.root !== undefined && this.root !== root)
61936
63208
  throw new Error("Local worker pool belongs to another root");
61937
63209
  this.root = root;
61938
63210
  if (store)
61939
63211
  this.store = store;
61940
- this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
61941
- this.usageTotal = await this.usageLedger.total(root);
61942
63212
  const state = await this.backend.journal.leaseState(root);
61943
63213
  if (!state || state.holderId !== this.backend.holderId)
61944
63214
  throw new Error("Local worker pool requires the root owner");
61945
63215
  this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
61946
- const queue = await openSqlPoolQueue(this.backend.journal);
61947
- await queue.initialize(this.lease, COORDINATOR_LEASE_MS);
61948
- await queue.releasePreviousIncarnations(this.lease);
61949
- for (;; ) {
61950
- const intended = this.desiredControl ?? "running";
61951
- await queue.setControl(this.lease, intended);
61952
- if (intended === (this.desiredControl ?? "running"))
61953
- break;
61954
- }
61955
- this.queue = queue;
63216
+ const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
63217
+ this.usageTotal = await this.command({
63218
+ kind: "prepare",
63219
+ root,
63220
+ lease: this.lease,
63221
+ options: {
63222
+ journalPath: this.options.journalPath,
63223
+ workers: this.options.workers,
63224
+ leaseMs: this.leaseMs,
63225
+ startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
63226
+ ...this.options.admission === undefined ? {} : { admission: this.options.admission },
63227
+ ...workerEntry === undefined ? {} : { workerEntry }
63228
+ },
63229
+ ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
63230
+ });
63231
+ this.queue ??= await openSqlPoolQueue(this.backend.journal);
61956
63232
  }
61957
63233
  async control(state) {
61958
63234
  this.desiredControl = state;
61959
63235
  try {
61960
- if (!this.queue || !this.lease)
61961
- return;
61962
- const control = await this.queue.setControl(this.lease, state);
61963
- const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
61964
- while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
61965
- if (this.desiredControl !== state)
61966
- return;
61967
- if (Date.now() >= deadline) {
61968
- if (state === "quit")
61969
- break;
61970
- throw new Error("Worker control acknowledgement unresolved");
61971
- }
61972
- await delay2(25);
61973
- }
63236
+ if (this.coordinator !== undefined)
63237
+ await this.command({ kind: "control", state });
61974
63238
  } finally {
61975
63239
  if (state === "quit")
61976
63240
  await this.close();
@@ -62016,290 +63280,61 @@ class LocalWorkerPool {
62016
63280
  throw new Error(`Worker definition is not importable: ${binding.exportName}`);
62017
63281
  return source;
62018
63282
  }));
62019
- if (this.store) {
62020
- this.releaseRoutes = await acquireStandaloneIntercom(this.store, root, String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd()));
63283
+ const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
63284
+ const store = this.store;
63285
+ if (store) {
63286
+ this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
62021
63287
  if (this.closed) {
62022
63288
  await this.releaseRoutes();
62023
63289
  this.releaseRoutes = undefined;
62024
63290
  throw new Error("Local worker pool is closed");
62025
63291
  }
62026
- this.handoff = new PoolStageHandoff(this.backend, this.store, this.queue, this.lease);
62027
- }
62028
- if (this.options.admission)
62029
- this.admission = await SqlResourceAdmission.open(this.backend.journal, {
62030
- ...this.options.admission,
62031
- root,
62032
- holder: "coordinator"
63292
+ const routeOwner = {
63293
+ notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
63294
+ };
63295
+ this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
63296
+ this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
62033
63297
  });
62034
- this.logDirectory = workerLogDirectory(this.options.journalPath, root);
62035
- await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
62036
- return;
62037
- });
62038
- const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
62039
- try {
62040
- await Promise.any(workers);
62041
- } catch (error) {
62042
- throw error.errors[0];
62043
63298
  }
62044
- }
62045
- spawn(root) {
62046
- if (this.closed)
62047
- return Promise.reject(new Error("Local worker pool is closed"));
62048
- const child = fork(isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url)), [
62049
- JSON.stringify({
62050
- journalPath: this.options.journalPath,
62051
- root,
62052
- definitions: this.sources,
62053
- admission: this.options.admission
62054
- })
62055
- ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62056
- const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62057
- this.children.add(child);
62058
- const worker = randomUUID15();
62059
- this.incarnations.set(worker, child);
62060
- let ready = false;
62061
- let serial = Promise.resolve();
62062
- let work;
62063
- const deliveries = new Map;
62064
- const logPath = join28(this.logDirectory, `${worker}.log`);
62065
- this.logs.set(child, logPath);
62066
- let logFailure;
62067
- const log = createWriteStream(logPath, { flags: "a" });
62068
- log.on("error", (error) => {
62069
- logFailure = error.message;
62070
- });
62071
- let stderr = "";
62072
- child.stderr?.on("data", (chunk) => {
62073
- stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62074
- });
62075
- child.stderr?.on("error", (error) => {
62076
- logFailure ??= error.message;
62077
- log.end();
62078
- });
62079
- child.stderr?.pipe(log);
62080
- const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62081
- const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
62082
- let timedOut = false;
62083
- let terminated = false;
62084
- const terminate = () => {
62085
- if (terminated)
62086
- return;
62087
- terminated = true;
62088
- child.kill("SIGKILL");
62089
- };
62090
- const replied = (error) => {
62091
- if (error)
62092
- terminate();
62093
- };
62094
- return new Promise((resolve, reject) => {
62095
- const startup = setTimeout(() => {
62096
- timedOut = true;
62097
- terminate();
62098
- }, startupTimeoutMs);
62099
- child.on("error", (error) => {
62100
- clearTimeout(startup);
62101
- reject(error);
62102
- terminate();
62103
- });
62104
- const closed = new Promise((resolve) => child.once("close", () => resolve()));
62105
- child.once("exit", () => {
62106
- Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62107
- this.children.delete(child);
62108
- this.logs.delete(child);
62109
- clearTimeout(startup);
62110
- this.incarnations.delete(worker);
62111
- this.readyWorkers.delete(worker);
62112
- this.controlAcks.delete(worker);
62113
- this.lastHeard.delete(worker);
62114
- if (work)
62115
- this.handoff?.fence(work);
62116
- for (const delivery of deliveries.values())
62117
- delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62118
- deliveries.clear();
62119
- const cleanup = disconnected.then(() => serial).then(async () => {
62120
- try {
62121
- if (await this.admission?.releaseStoppedSessions(worker))
62122
- throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62123
- } catch (error) {
62124
- this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62125
- throw this.cleanupFailure;
62126
- }
62127
- }).then(async () => {
62128
- if (this.closed || !ready || this.desiredControl === "quit")
62129
- return;
62130
- if (work && await this.queue.result(root, work.id)) {
62131
- work = undefined;
62132
- this.replacements = 0;
62133
- }
62134
- if (this.closed)
62135
- return;
62136
- if (work)
62137
- await this.queue.releaseExited(this.lease, work);
62138
- const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62139
- if (exhausted)
62140
- this.failure = diagnosed("Local worker retry limit");
62141
- else
62142
- await this.spawn(root);
62143
- }).catch((error) => {
62144
- this.failure = error;
62145
- });
62146
- this.cleanups.add(cleanup);
62147
- cleanup.finally(() => this.cleanups.delete(cleanup));
62148
- });
62149
- child.on("message", (message) => {
62150
- this.observeResponsiveness();
62151
- this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
62152
- if (message.kind === "delivery-ack") {
62153
- const pending = deliveries.get(message.deliveryId);
62154
- deliveries.delete(message.deliveryId);
62155
- if (message.error)
62156
- pending?.reject(new Error(message.error));
62157
- else
62158
- pending?.resolve();
62159
- return;
62160
- }
62161
- const response = serial.then(async () => {
62162
- if (message.kind === "ready") {
62163
- if (terminated)
62164
- return;
62165
- ready = true;
62166
- this.readyWorkers.add(worker);
62167
- clearTimeout(startup);
62168
- resolve();
62169
- return;
62170
- }
62171
- await this.starting;
62172
- const queue = this.queue;
62173
- const lease = this.lease;
62174
- if (message.kind === "control")
62175
- return queue.control(lease);
62176
- if (message.kind === "controlled") {
62177
- const control = await queue.control(lease);
62178
- if (message.revision > control.revision)
62179
- throw new Error("Unknown control revision");
62180
- this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62181
- return;
62182
- }
62183
- if (message.kind === "claim") {
62184
- if (work)
62185
- throw new Error("Worker already holds an attempt");
62186
- const decision = this.rootBudget?.checkpoint();
62187
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62188
- return;
62189
- await this.recoverExpired(root);
62190
- work = await queue.claim(lease, worker, this.sources.map((source) => ({
62191
- definition: `${source.path}#${source.exportName}`,
62192
- digest: source.digest
62193
- })), this.leaseMs);
62194
- return work;
62195
- }
62196
- if (!work)
62197
- throw new Error("Worker has no active attempt");
62198
- if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
62199
- work = await queue.extend(lease, work, this.leaseMs);
62200
- if (message.kind === "activity")
62201
- return this.handoff?.publish(work, message.runs);
62202
- if (message.kind === "route")
62203
- return this.handoff?.route(work, message.runId);
62204
- if (message.kind === "delivery-ready")
62205
- return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
62206
- if (message.kind === "delivery") {
62207
- if (!message.recipient)
62208
- throw new Error("Worker delivery requires a session identity");
62209
- return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
62210
- const id = randomUUID15();
62211
- deliveries.set(id, { resolve, reject });
62212
- child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
62213
- if (error) {
62214
- deliveries.delete(id);
62215
- reject(error);
62216
- }
62217
- });
62218
- }));
62219
- }
62220
- if (message.kind === "renew")
62221
- return;
62222
- if (message.kind === "commit") {
62223
- const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62224
- this.usageTotal = await this.usageLedger.total(root);
62225
- return receipt;
62226
- }
62227
- if (message.kind === "usage") {
62228
- this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
62229
- return;
62230
- }
62231
- if (message.kind === "budget") {
62232
- await queue.assertCurrent(lease, work);
62233
- const decision = this.rootBudget?.checkpoint();
62234
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62235
- throw new Error("Root workflow budget exhausted");
62236
- return queue.control(lease);
62237
- }
62238
- if (message.kind !== "complete")
62239
- throw new Error("Invalid worker operation");
62240
- await queue.complete(lease, work, message.result);
62241
- this.replacements = 0;
62242
- work = undefined;
62243
- });
62244
- if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62245
- serial = response.then(() => {}, () => {});
62246
- response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62247
- });
63299
+ await this.command({
63300
+ kind: "start",
63301
+ sources: this.sources,
63302
+ cwd,
63303
+ publish: store !== undefined,
63304
+ ...this.rootRun()
62248
63305
  });
62249
63306
  }
62250
- recoverExpired(root) {
62251
- if (this.recovering)
62252
- return this.recovering;
62253
- if (this.closed || performance.now() < this.nextRecoveryAt)
62254
- return Promise.resolve();
62255
- this.recovering = (async () => {
62256
- await new Promise((resolve) => setImmediate(resolve));
62257
- await this.admission?.reconcile();
62258
- const expiredBefore = Date.now();
62259
- for (const work of await this.queue.inspect(root)) {
62260
- if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62261
- continue;
62262
- const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62263
- if (child && work.worker !== null && this.isLive(work.worker, child)) {
62264
- await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
62265
- if (error.message !== STALE_WORK_LEASE)
62266
- throw error;
62267
- });
62268
- continue;
62269
- }
62270
- if (child && !await this.stop(child).then(() => true, () => false))
62271
- continue;
62272
- await this.queue.expire(this.lease, work);
62273
- }
62274
- })().finally(() => {
62275
- this.recovering = undefined;
62276
- this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62277
- });
62278
- return this.recovering;
63307
+ queueView() {
63308
+ return {
63309
+ assertCurrent: async (_lease, work) => {
63310
+ const current = this.attempts.get(work.id);
63311
+ if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
63312
+ throw new Error(STALE_WORK_LEASE);
63313
+ },
63314
+ withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
63315
+ result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
63316
+ };
62279
63317
  }
62280
- observeResponsiveness() {
62281
- const now = performance.now();
62282
- const gap = now - this.lastTick;
62283
- if (gap > ROOT_STALL_THRESHOLD_MS)
62284
- this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
62285
- this.lastTick = now;
63318
+ rootProjection() {
63319
+ const run = this.store?.runs().find((candidate) => candidate.id === this.root);
63320
+ return run === undefined ? undefined : projectRootRun(run);
62286
63321
  }
62287
- isLive(worker, child) {
62288
- this.observeResponsiveness();
62289
- const heard = this.lastHeard.get(worker);
62290
- return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.rootStalledMs - heard.rootStalled) < this.leaseMs;
63322
+ rootRun() {
63323
+ const rootRun = this.rootProjection();
63324
+ if (rootRun === undefined)
63325
+ return {};
63326
+ this.rootSignature = JSON.stringify(rootRun);
63327
+ return { rootRun };
62291
63328
  }
62292
- stop(child) {
62293
- if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62294
- return Promise.resolve();
62295
- return new Promise((resolve, reject) => {
62296
- const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62297
- child.once("exit", () => {
62298
- clearTimeout(timeout);
62299
- resolve();
62300
- });
62301
- child.kill("SIGKILL");
62302
- });
63329
+ refreshRootRun() {
63330
+ const rootRun = this.rootProjection();
63331
+ if (rootRun === undefined)
63332
+ return;
63333
+ const signature = JSON.stringify(rootRun);
63334
+ if (signature === this.rootSignature)
63335
+ return;
63336
+ this.rootSignature = signature;
63337
+ this.notice({ kind: "root", rootRun });
62303
63338
  }
62304
63339
  async run(definition, inputs, options) {
62305
63340
  try {
@@ -62332,7 +63367,7 @@ class LocalWorkerPool {
62332
63367
  const selectedSource = this.sources[index];
62333
63368
  if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
62334
63369
  throw new Error("Worker definition compatibility error");
62335
- await this.backend.hydrateWorkflow(root);
63370
+ await this.backend.catchUpWorkflow(root);
62336
63371
  const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
62337
63372
  if (boundary?.kind !== "stage")
62338
63373
  throw new Error("Missing durable child boundary");
@@ -62343,39 +63378,232 @@ class LocalWorkerPool {
62343
63378
  config: options.config,
62344
63379
  executionMode: options.executionMode
62345
63380
  }));
62346
- const work = await this.queue.enqueue(this.lease, {
62347
- boundaryId: options.parentRun.stageId,
62348
- childRunId: options.runId,
62349
- scope: options.durableScope.scopePrefix,
62350
- definition: `${selectedSource.path}#${selectedSource.exportName}`,
62351
- digest: selectedSource.digest,
62352
- inputs,
62353
- execution
62354
- }, boundary);
62355
- for (;; ) {
62356
- options.signal?.throwIfAborted();
62357
- if (this.closed) {
63381
+ this.evaluateBudget();
63382
+ const enqueued = await this.command({
63383
+ kind: "enqueue",
63384
+ descriptor: {
63385
+ boundaryId: options.parentRun.stageId,
63386
+ childRunId: options.runId,
63387
+ scope: options.durableScope.scopePrefix,
63388
+ definition: `${selectedSource.path}#${selectedSource.exportName}`,
63389
+ digest: selectedSource.digest,
63390
+ inputs,
63391
+ execution
63392
+ },
63393
+ boundary,
63394
+ ...this.rootRun()
63395
+ });
63396
+ const work = enqueued.work;
63397
+ this.works.set(work.id, work);
63398
+ if (enqueued.result !== undefined && !this.results.has(work.id))
63399
+ this.results.set(work.id, enqueued.result);
63400
+ this.waiting++;
63401
+ this.ticker ??= setInterval(() => {
63402
+ this.evaluateBudget();
63403
+ this.signalChange();
63404
+ }, POOL_SUMMARY_INTERVAL_MS).unref();
63405
+ try {
63406
+ for (;; ) {
63407
+ options.signal?.throwIfAborted();
63408
+ if (this.closed) {
63409
+ if (this.failure)
63410
+ throw this.failure;
63411
+ if (this.desiredControl === "quit")
63412
+ throw new WorkflowGracefulQuitError(root, "worker pool teardown");
63413
+ throw new Error("Local worker pool is closed");
63414
+ }
63415
+ const result = this.results.get(work.id);
63416
+ if (result) {
63417
+ await this.inbound;
63418
+ await this.backend.catchUpWorkflow(root);
63419
+ await this.handoff?.complete(work);
63420
+ return result;
63421
+ }
62358
63422
  if (this.failure)
62359
63423
  throw this.failure;
62360
- if (this.desiredControl === "quit")
62361
- throw new WorkflowGracefulQuitError(root, "worker pool teardown");
62362
- throw new Error("Local worker pool is closed");
63424
+ if (this.budgetBlocked)
63425
+ await this.rootBudget.stopAtBoundaryAsync(definition.name);
63426
+ await this.nextChange(options.signal);
62363
63427
  }
62364
- const result = await this.queue.result(root, work.id);
62365
- if (result) {
62366
- await this.backend.hydrateWorkflow(root);
62367
- await this.handoff?.complete(work);
62368
- return result;
63428
+ } finally {
63429
+ if (--this.waiting === 0) {
63430
+ clearInterval(this.ticker);
63431
+ this.ticker = undefined;
63432
+ }
63433
+ }
63434
+ }
63435
+ evaluateBudget() {
63436
+ const decision = this.rootBudget?.checkpoint();
63437
+ const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
63438
+ if (blocked === this.budgetBlocked)
63439
+ return;
63440
+ this.budgetBlocked = blocked;
63441
+ this.notice({ kind: "budget", blocked });
63442
+ }
63443
+ signalChange() {
63444
+ const changed = this.changed;
63445
+ this.changed = Promise.withResolvers();
63446
+ changed.resolve();
63447
+ }
63448
+ nextChange(signal) {
63449
+ const changed = this.changed.promise;
63450
+ if (signal === undefined)
63451
+ return changed;
63452
+ return new Promise((resolve, reject) => {
63453
+ const abort = () => reject(signal.reason);
63454
+ signal.addEventListener("abort", abort, { once: true });
63455
+ changed.then(() => {
63456
+ signal.removeEventListener("abort", abort);
63457
+ resolve();
63458
+ });
63459
+ });
63460
+ }
63461
+ connect() {
63462
+ if (this.coordinator !== undefined)
63463
+ return this.coordinator;
63464
+ const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
63465
+ this.coordinator = coordinator;
63466
+ coordinator.port.onMessage((event) => {
63467
+ this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
63468
+ });
63469
+ coordinator.port.onClose((reason) => {
63470
+ this.coordinatorLost = reason;
63471
+ this.inbound = this.inbound.then(() => this.lost(reason));
63472
+ });
63473
+ return coordinator;
63474
+ }
63475
+ lost(reason) {
63476
+ this.pids.clear();
63477
+ if (!this.closed)
63478
+ this.failure ??= reason;
63479
+ for (const pending of this.commands.values())
63480
+ pending.reject(reason);
63481
+ this.commands.clear();
63482
+ this.signalChange();
63483
+ }
63484
+ command(body) {
63485
+ const coordinator = this.connect();
63486
+ if (this.coordinatorLost !== undefined)
63487
+ return Promise.reject(this.coordinatorLost);
63488
+ const id = ++this.commandSequence;
63489
+ return new Promise((resolve, reject) => {
63490
+ this.commands.set(id, { resolve, reject });
63491
+ coordinator.port.send({ ...body, id });
63492
+ });
63493
+ }
63494
+ notice(notice) {
63495
+ this.coordinator?.port.send(notice);
63496
+ }
63497
+ async receive(event) {
63498
+ switch (event.kind) {
63499
+ case "reply": {
63500
+ const pending = this.commands.get(event.id);
63501
+ this.commands.delete(event.id);
63502
+ if (event.error !== undefined)
63503
+ pending?.reject(new Error(event.error));
63504
+ else
63505
+ pending?.resolve(event.value);
63506
+ return;
63507
+ }
63508
+ case "activity":
63509
+ return this.applyActivity(event.batch, event.entries, event.usage);
63510
+ case "spawned":
63511
+ this.pids.add(event.pid);
63512
+ return;
63513
+ case "exited":
63514
+ this.pids.delete(event.pid);
63515
+ return;
63516
+ case "fence": {
63517
+ const current = this.attempts.get(event.work.id);
63518
+ if (current?.generation === event.work.generation)
63519
+ current.fenced = true;
63520
+ this.handoff?.fence(event.work);
63521
+ return;
62369
63522
  }
62370
- if (this.failure)
62371
- throw this.failure;
62372
- const decision = this.rootBudget?.checkpoint();
62373
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62374
- await this.rootBudget.stopAtBoundaryAsync(definition.name);
62375
- await this.recoverExpired(root);
62376
- await delay2(25, undefined, { signal: options.signal });
63523
+ case "result":
63524
+ if (event.usage !== undefined)
63525
+ this.usageTotal = event.usage;
63526
+ this.results.set(event.workId, event.result);
63527
+ this.signalChange();
63528
+ return;
63529
+ case "failure":
63530
+ this.failure ??= new Error(event.error);
63531
+ this.signalChange();
63532
+ return;
63533
+ case "request":
63534
+ this.serve(event.id, event.request);
63535
+ return;
62377
63536
  }
62378
63537
  }
63538
+ async applyActivity(batch, entries, usage) {
63539
+ if (usage !== undefined)
63540
+ this.usageTotal = usage;
63541
+ const rejected = [];
63542
+ const handoff = this.handoff;
63543
+ if (handoff !== undefined) {
63544
+ const publications = entries.flatMap((entry) => {
63545
+ const work = this.track(entry.work);
63546
+ if (work !== undefined)
63547
+ return [{ work, runs: entry.runs }];
63548
+ rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
63549
+ return [];
63550
+ });
63551
+ const outcomes = await handoff.publishBatch(publications);
63552
+ outcomes.forEach((error, index) => {
63553
+ const { work } = publications[index];
63554
+ if (error !== undefined)
63555
+ rejected.push({ workId: work.id, generation: work.generation, error: error.message });
63556
+ });
63557
+ }
63558
+ this.notice({ kind: "published", batch, rejected });
63559
+ this.refreshRootRun();
63560
+ }
63561
+ track(attempt) {
63562
+ const work = this.works.get(attempt.id);
63563
+ if (work === undefined)
63564
+ return;
63565
+ const current = this.attempts.get(attempt.id);
63566
+ if (current === undefined || attempt.generation > current.generation)
63567
+ this.attempts.set(attempt.id, { ...attempt, fenced: false });
63568
+ return { ...work, ...attempt, state: "leased" };
63569
+ }
63570
+ async serve(id, request) {
63571
+ try {
63572
+ const value2 = await this.answer(id, request);
63573
+ this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
63574
+ } catch (error) {
63575
+ this.coordinator?.port.send({
63576
+ kind: "request-reply",
63577
+ id,
63578
+ error: error instanceof Error ? error.message : String(error)
63579
+ });
63580
+ }
63581
+ }
63582
+ async answer(id, request) {
63583
+ if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
63584
+ const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
63585
+ const event = {
63586
+ ...request.payload,
63587
+ handled: false
63588
+ };
63589
+ if (request.kind === "pending-stage-message")
63590
+ router?.route(event);
63591
+ else
63592
+ router?.sticky(event);
63593
+ if (event.handled && event.completion !== undefined)
63594
+ return await event.completion;
63595
+ return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
63596
+ }
63597
+ const handoff = this.handoff;
63598
+ const work = this.track(request.work);
63599
+ if (handoff === undefined || work === undefined)
63600
+ throw new Error(STALE_WORK_LEASE);
63601
+ if (request.kind === "ready")
63602
+ return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
63603
+ return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
63604
+ await this.command({ kind: "transport", request: id, from, message });
63605
+ });
63606
+ }
62379
63607
  closing;
62380
63608
  close() {
62381
63609
  this.closing ??= this.dispose();
@@ -62384,27 +63612,28 @@ class LocalWorkerPool {
62384
63612
  async dispose() {
62385
63613
  this.closed = true;
62386
63614
  clearInterval(this.ticker);
63615
+ this.ticker = undefined;
62387
63616
  this.releaseBinding();
62388
63617
  if (this.options.rootId === undefined)
62389
63618
  workflowResourceAdmission.delete(this.backend);
63619
+ this.signalChange();
62390
63620
  let failure;
62391
- let stopping = [];
62392
- try {
62393
- await this.recovering?.catch(() => {});
62394
- stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62395
- await Promise.all(stopping.map(({ child }) => this.stop(child)));
62396
- await Promise.all([...this.cleanups]);
62397
- if (this.cleanupFailure)
62398
- throw this.cleanupFailure;
62399
- } catch (error) {
62400
- failure = error;
62401
- }
62402
- if (failure === undefined) {
62403
- const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62404
- await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62405
- if (this.logDirectory !== undefined)
62406
- await rmdir(this.logDirectory).catch(() => {});
63621
+ const coordinator = this.coordinator;
63622
+ if (coordinator !== undefined) {
63623
+ if (this.coordinatorLost === undefined) {
63624
+ try {
63625
+ await this.command({ kind: "close" });
63626
+ } catch (error) {
63627
+ failure = error;
63628
+ }
63629
+ }
63630
+ await this.inbound;
63631
+ await coordinator.port.close();
63632
+ this.pids.clear();
62407
63633
  }
63634
+ for (const release of this.routeClaims.values())
63635
+ release();
63636
+ this.routeClaims.clear();
62408
63637
  try {
62409
63638
  await this.releaseRoutes?.();
62410
63639
  } catch (error) {
@@ -62414,27 +63643,26 @@ class LocalWorkerPool {
62414
63643
  throw failure;
62415
63644
  }
62416
63645
  }
62417
- var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, ROOT_TICK_INTERVAL_MS = 100, ROOT_STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000;
63646
+ var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
62418
63647
  var init_local_pool = __esm(() => {
62419
63648
  init_internal();
62420
63649
  init_sql_pool_queue();
62421
- init_sql_pool_usage();
62422
63650
  init_sql_run_journal();
62423
63651
  init_workflow_tool_abort();
62424
63652
  init_sdk_intercom();
63653
+ init_external_run_routes();
62425
63654
  init_workflow_infrastructure_error();
62426
63655
  init_admission_binding();
62427
63656
  init_definitions();
62428
63657
  init_dispatch();
62429
63658
  init_host_options();
62430
- init_resource_admission();
63659
+ init_pool_coordinator();
62431
63660
  init_stage_handoff();
62432
63661
  MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
62433
- COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
62434
63662
  });
62435
63663
 
62436
63664
  // dist/builtin/workflows/src/workers/public-pool.ts
62437
- import { resolve as resolve12 } from "node:path";
63665
+ import { resolve as resolve13 } from "node:path";
62438
63666
  import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
62439
63667
  async function restoreWorkerOptions(backend, options) {
62440
63668
  const source = options.continuation?.source.id ?? options.runId;
@@ -62473,11 +63701,11 @@ async function activatePublicPool(backend, options) {
62473
63701
  ...options.localWorkers,
62474
63702
  startupTimeoutMs: undefined,
62475
63703
  leaseMs: undefined,
62476
- definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve12(cwd, source.path) }))
63704
+ definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
62477
63705
  };
62478
63706
  const admission = options.admission && {
62479
63707
  ...options.admission,
62480
- ...options.admission.userDatabase ? { userDatabase: resolve12(cwd, options.admission.userDatabase) } : {}
63708
+ ...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
62481
63709
  };
62482
63710
  const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
62483
63711
  const resources = JSON.parse(JSON.stringify({
@@ -63823,7 +65051,7 @@ function hasUsableContentBlock(block) {
63823
65051
 
63824
65052
  // dist/builtin/workflows/src/shared/stage-session-store.ts
63825
65053
  import { randomUUID as randomUUID17 } from "node:crypto";
63826
- import { dirname as dirname13, resolve as resolve13 } from "node:path";
65054
+ import { dirname as dirname13, resolve as resolve14 } from "node:path";
63827
65055
  import {
63828
65056
  JsonlSessionStore,
63829
65057
  resolveSessionStore,
@@ -63859,11 +65087,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
63859
65087
  async function migrateLegacyStageSession(store, sessionId, sessionFile) {
63860
65088
  if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
63861
65089
  return;
63862
- const location = resolve13(sessionFile);
65090
+ const location = resolve14(sessionFile);
63863
65091
  const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
63864
65092
  try {
63865
65093
  for await (const file of legacy.scanFiles()) {
63866
- if (resolve13(file.location) !== location || file.header.id !== sessionId)
65094
+ if (resolve14(file.location) !== location || file.header.id !== sessionId)
63867
65095
  continue;
63868
65096
  const snapshot = await file.read();
63869
65097
  if (await store.readHeader(sessionId) === undefined)
@@ -64025,22 +65253,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
64025
65253
  }
64026
65254
  function retainReachableRunGroups(grouped, rootRunId) {
64027
65255
  const reachable = new Set([rootRunId]);
64028
- const pending = [rootRunId];
65256
+ const settled = new Set;
65257
+ const pending = [{ runId: rootRunId, live: true }];
64029
65258
  while (pending.length > 0) {
64030
- const drafts = grouped.get(pending.pop());
65259
+ const { runId, live } = pending.pop();
65260
+ const drafts = grouped.get(runId);
64031
65261
  if (drafts === undefined)
64032
65262
  continue;
64033
65263
  for (const draft of drafts) {
64034
- const childRunId = childRunIdFromDraft(draft);
64035
- if (childRunId === undefined || reachable.has(childRunId) || !grouped.has(childRunId))
65264
+ const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
65265
+ if (childRunId === undefined || !grouped.has(childRunId))
64036
65266
  continue;
64037
- reachable.add(childRunId);
64038
- pending.push(childRunId);
65267
+ if (live && childRunIdFromDraft(draft) !== undefined) {
65268
+ if (reachable.has(childRunId))
65269
+ continue;
65270
+ reachable.add(childRunId);
65271
+ pending.push({ runId: childRunId, live: true });
65272
+ } else if (!settled.has(childRunId)) {
65273
+ settled.add(childRunId);
65274
+ pending.push({ runId: childRunId, live: false });
65275
+ }
64039
65276
  }
64040
65277
  }
64041
- for (const runId of grouped.keys())
64042
- if (!reachable.has(runId))
64043
- grouped.delete(runId);
65278
+ let complete = true;
65279
+ for (const runId of grouped.keys()) {
65280
+ if (reachable.has(runId))
65281
+ continue;
65282
+ if (!settled.has(runId))
65283
+ complete = false;
65284
+ grouped.delete(runId);
65285
+ }
65286
+ return complete;
64044
65287
  }
64045
65288
  function childRunIdFromDraft(draft) {
64046
65289
  const boundary = draft.topology?.boundary;
@@ -64462,9 +65705,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
64462
65705
  }
64463
65706
  for (const group of grouped.values())
64464
65707
  group.sort(compareDraftSourceOrder);
64465
- const groupedCount = grouped.size;
64466
- retainReachableRunGroups(grouped, rootRunId);
64467
- if (policy.failOnUnreachable && grouped.size !== groupedCount)
65708
+ if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
64468
65709
  return [];
64469
65710
  if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
64470
65711
  return [];
@@ -70122,9 +71363,9 @@ var init_stage_runner_controller = __esm(() => {
70122
71363
  });
70123
71364
 
70124
71365
  // dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
70125
- import { createHash as createHash4 } from "node:crypto";
71366
+ import { createHash as createHash5 } from "node:crypto";
70126
71367
  import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
70127
- import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve14 } from "node:path";
71368
+ import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
70128
71369
  function normalizeMaxOutput2(maxOutput) {
70129
71370
  return {
70130
71371
  bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
@@ -70186,7 +71427,7 @@ function formatByteSize(bytes) {
70186
71427
  return `${value2.toFixed(1)} ${units[unitIndex]}`;
70187
71428
  }
70188
71429
  function transcriptPath(runId, outputPath) {
70189
- const digest = createHash4("sha256").update(outputPath).digest("hex").slice(0, 16);
71430
+ const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
70190
71431
  return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
70191
71432
  }
70192
71433
  function appendContent(lines, content) {
@@ -70232,7 +71473,7 @@ function renderTranscript(messages, fallbackOutput) {
70232
71473
  `);
70233
71474
  }
70234
71475
  function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
70235
- const absolutePath = resolve14(outputPath);
71476
+ const absolutePath = resolve15(outputPath);
70236
71477
  const bytes = Buffer.byteLength(fullOutput, "utf8");
70237
71478
  const lines = countLines(fullOutput);
70238
71479
  const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
@@ -70247,8 +71488,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
70247
71488
  return;
70248
71489
  if (isAbsolute11(output))
70249
71490
  return output;
70250
- const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve14(runtimeCwd, requestedCwd);
70251
- return resolve14(baseCwd, output);
71491
+ const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
71492
+ return resolve15(baseCwd, output);
70252
71493
  }
70253
71494
  function degenerateWarning(fullOutput) {
70254
71495
  return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
@@ -71257,7 +72498,7 @@ async function run(def, inputs, opts = {}) {
71257
72498
  if (!isWorkflowDefinition(def))
71258
72499
  throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
71259
72500
  const depth = opts.depth ?? 0;
71260
- const maxDepth = opts.config?.maxDepth ?? 4;
72501
+ const maxDepth = opts.config?.maxDepth ?? 3;
71261
72502
  if (depth >= maxDepth) {
71262
72503
  return {
71263
72504
  runId: opts.runId ?? crypto.randomUUID(),