@bastani/atomic 0.9.43 → 0.9.44-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/dist/builtin/intercom/CHANGELOG.md +12 -0
  3. package/dist/builtin/intercom/broker/broker.ts +31 -7
  4. package/dist/builtin/intercom/broker/client.ts +65 -0
  5. package/dist/builtin/intercom/index.bundle.mjs +227 -95
  6. package/dist/builtin/intercom/live-route-refusal.ts +20 -0
  7. package/dist/builtin/intercom/package.json +1 -1
  8. package/dist/builtin/intercom/skills/intercom/SKILL.md +10 -6
  9. package/dist/builtin/intercom/types.ts +20 -1
  10. package/dist/builtin/mcp/CHANGELOG.md +7 -0
  11. package/dist/builtin/mcp/package.json +1 -1
  12. package/dist/builtin/subagents/CHANGELOG.md +9 -0
  13. package/dist/builtin/subagents/README.md +8 -8
  14. package/dist/builtin/subagents/agents/worker.md +1 -1
  15. package/dist/builtin/subagents/package.json +1 -1
  16. package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
  17. package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +21 -2
  18. package/dist/builtin/subagents/skills/cua-driver/LINUX.md +2 -2
  19. package/dist/builtin/subagents/skills/cua-driver/MACOS.md +342 -10
  20. package/dist/builtin/subagents/skills/cua-driver/README.md +4 -3
  21. package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +3 -1
  22. package/dist/builtin/subagents/skills/cua-driver/SKILL.md +7 -4
  23. package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +6 -5
  24. package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +15 -0
  25. package/dist/builtin/subagents/skills/herdr/SKILL.md +22 -10
  26. package/dist/builtin/subagents/skills/subagent/SKILL.md +8 -5
  27. package/dist/builtin/subagents/src/extension/index.bundle.mjs +6 -6
  28. package/dist/builtin/web-access/package.json +1 -1
  29. package/dist/builtin/workflows/CHANGELOG.md +14 -0
  30. package/dist/builtin/workflows/README.md +1 -1
  31. package/dist/builtin/workflows/builtin/{chunk-frqxd4d1.js → chunk-k8d7ehjx.js} +22 -0
  32. package/dist/builtin/workflows/builtin/{chunk-3s95f7ry.js → chunk-qxpymnaf.js} +1 -1
  33. package/dist/builtin/workflows/builtin/{chunk-edhf30ef.js → chunk-zjecnjxx.js} +1 -1
  34. package/dist/builtin/workflows/builtin/goal.js +2 -2
  35. package/dist/builtin/workflows/builtin/index.js +3 -3
  36. package/dist/builtin/workflows/builtin/ralph.js +2 -2
  37. package/dist/builtin/workflows/package.json +1 -1
  38. package/dist/builtin/workflows/src/extension/index.bundle.mjs +1750 -598
  39. package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +1786 -600
  40. package/dist/builtin/workflows/src/index.js +1802 -650
  41. package/dist/builtin/workflows/src/local-worker-entry.mjs +1786 -600
  42. package/dist/cli/args.js +1 -1
  43. package/dist/cli/args.js.map +1 -1
  44. package/dist/core/exec.d.ts.map +1 -1
  45. package/dist/core/exec.js +11 -2
  46. package/dist/core/exec.js.map +1 -1
  47. package/dist/core/package-manager-resolver.d.ts.map +1 -1
  48. package/dist/core/package-manager-resolver.js +25 -14
  49. package/dist/core/package-manager-resolver.js.map +1 -1
  50. package/dist/core/package-manager-types.d.ts +8 -0
  51. package/dist/core/package-manager-types.d.ts.map +1 -1
  52. package/dist/core/package-manager-types.js.map +1 -1
  53. package/dist/core/package-manager.d.ts +1 -1
  54. package/dist/core/package-manager.d.ts.map +1 -1
  55. package/dist/core/package-manager.js.map +1 -1
  56. package/dist/core/resource-loader-extension-warnings.d.ts +4 -1
  57. package/dist/core/resource-loader-extension-warnings.d.ts.map +1 -1
  58. package/dist/core/resource-loader-extension-warnings.js +15 -0
  59. package/dist/core/resource-loader-extension-warnings.js.map +1 -1
  60. package/dist/core/resource-loader-package-resources.d.ts.map +1 -1
  61. package/dist/core/resource-loader-package-resources.js +1 -0
  62. package/dist/core/resource-loader-package-resources.js.map +1 -1
  63. package/dist/core/resource-loader-reload.d.ts.map +1 -1
  64. package/dist/core/resource-loader-reload.js +5 -3
  65. package/dist/core/resource-loader-reload.js.map +1 -1
  66. package/dist/extensions/mcp/client/oauth/flow.d.ts.map +1 -1
  67. package/dist/extensions/mcp/client/oauth/flow.js +4 -1
  68. package/dist/extensions/mcp/client/oauth/flow.js.map +1 -1
  69. package/dist/extensions/mcp/client/oauth/types.d.ts +1 -0
  70. package/dist/extensions/mcp/client/oauth/types.d.ts.map +1 -1
  71. package/dist/extensions/mcp/client/oauth/types.js +8 -4
  72. package/dist/extensions/mcp/client/oauth/types.js.map +1 -1
  73. package/dist/extensions/mcp/client/transports/streamable-http.d.ts.map +1 -1
  74. package/dist/extensions/mcp/client/transports/streamable-http.js +2 -1
  75. package/dist/extensions/mcp/client/transports/streamable-http.js.map +1 -1
  76. package/dist/index.d.ts +1 -1
  77. package/dist/index.d.ts.map +1 -1
  78. package/dist/index.js.map +1 -1
  79. package/dist/main.d.ts.map +1 -1
  80. package/dist/main.js +20 -10
  81. package/dist/main.js.map +1 -1
  82. package/dist/modes/interactive/components/model-selector.d.ts.map +1 -1
  83. package/dist/modes/interactive/components/model-selector.js +4 -2
  84. package/dist/modes/interactive/components/model-selector.js.map +1 -1
  85. package/dist/modes/interactive/components/scoped-models-selector.d.ts.map +1 -1
  86. package/dist/modes/interactive/components/scoped-models-selector.js +4 -2
  87. package/dist/modes/interactive/components/scoped-models-selector.js.map +1 -1
  88. package/dist/modes/interactive/components/tool-execution.d.ts +6 -0
  89. package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
  90. package/dist/modes/interactive/components/tool-execution.js +11 -1
  91. package/dist/modes/interactive/components/tool-execution.js.map +1 -1
  92. package/dist/modes/interactive/model-search.d.ts +10 -0
  93. package/dist/modes/interactive/model-search.d.ts.map +1 -1
  94. package/dist/modes/interactive/model-search.js +21 -0
  95. package/dist/modes/interactive/model-search.js.map +1 -1
  96. package/dist/package-manager-cli.d.ts.map +1 -1
  97. package/dist/package-manager-cli.js +5 -0
  98. package/dist/package-manager-cli.js.map +1 -1
  99. package/dist/utils/tools-manager.d.ts.map +1 -1
  100. package/dist/utils/tools-manager.js +30 -35
  101. package/dist/utils/tools-manager.js.map +1 -1
  102. package/docs/computer-use.md +5 -5
  103. package/docs/custom-provider/streaming.md +9 -14
  104. package/docs/environment-variables.md +1 -0
  105. package/docs/intercom.md +5 -1
  106. package/docs/packages.md +2 -0
  107. package/docs/settings.md +2 -2
  108. package/docs/skills.md +2 -2
  109. package/docs/subagents.md +38 -9
  110. package/docs/workflows/api-reference.md +1 -1
  111. package/docs/workflows/authoring.md +1 -1
  112. package/docs/workflows/operations.md +8 -6
  113. package/docs/workflows/reliable-design.md +23 -5
  114. package/examples/extensions/custom-provider-anthropic/index.ts +31 -31
  115. package/examples/extensions/todo.ts +7 -0
  116. package/npm-shrinkwrap.json +32 -32
  117. package/package.json +3 -3
@@ -3221,7 +3221,13 @@ function createStoreContext(state = createStoreState()) {
3221
3221
  }
3222
3222
  return cachedGraphSnapshot;
3223
3223
  }
3224
+ let batchDepth = 0;
3225
+ let batchedNotification = false;
3224
3226
  function notify() {
3227
+ if (batchDepth > 0) {
3228
+ batchedNotification = true;
3229
+ return;
3230
+ }
3225
3231
  observation.capture();
3226
3232
  for (const fn of state.invalidationListeners) {
3227
3233
  try {
@@ -3241,6 +3247,18 @@ function createStoreContext(state = createStoreState()) {
3241
3247
  state.version++;
3242
3248
  notify();
3243
3249
  }
3250
+ function batch(update) {
3251
+ batchDepth++;
3252
+ try {
3253
+ return update();
3254
+ } finally {
3255
+ batchDepth--;
3256
+ if (batchDepth === 0 && batchedNotification) {
3257
+ batchedNotification = false;
3258
+ notify();
3259
+ }
3260
+ }
3261
+ }
3244
3262
  function findRun(runId) {
3245
3263
  return state.runs.find((r) => r.id === runId);
3246
3264
  }
@@ -3294,6 +3312,7 @@ function createStoreContext(state = createStoreState()) {
3294
3312
  snapshot,
3295
3313
  graphSnapshot,
3296
3314
  notify,
3315
+ batch,
3297
3316
  bumpAndNotify,
3298
3317
  findRun,
3299
3318
  findStage,
@@ -4432,6 +4451,9 @@ function createRunStoreMethods(context) {
4432
4451
  subscribeInvalidation(fn) {
4433
4452
  state.invalidationListeners.add(fn);
4434
4453
  return () => state.invalidationListeners.delete(fn);
4454
+ },
4455
+ batch(update) {
4456
+ return context.batch(update);
4435
4457
  }
4436
4458
  };
4437
4459
  }
@@ -5165,7 +5187,7 @@ class ConcurrencyLimiter {
5165
5187
  }
5166
5188
  }
5167
5189
  function createRunLimiter(defaultConcurrency) {
5168
- return new ConcurrencyLimiter(defaultConcurrency ?? 3);
5190
+ return new ConcurrencyLimiter(defaultConcurrency ?? 1000);
5169
5191
  }
5170
5192
 
5171
5193
  // dist/builtin/workflows/src/extension/atomic-stage-session.ts
@@ -57292,7 +57314,7 @@ var init_definitions = __esm(() => {
57292
57314
  // dist/builtin/workflows/src/durable/journal/pool-mutations.ts
57293
57315
  import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
57294
57316
  async function validatePoolMutation(tx, journal, root, work, mutation) {
57295
- if (!["checkpoint", "session-create", "session-append", "session-replace"].includes(mutation.kind))
57317
+ if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
57296
57318
  throw new Error("Invalid pool mutation");
57297
57319
  const checkpoint = mutation.checkpoint;
57298
57320
  const prefix = `${work.scope}:`;
@@ -57360,10 +57382,18 @@ async function sessionStore(tx, prefix) {
57360
57382
  return new StorageProvider2().openSessionStore({ kind: "database", database, prefix }, { agentDir: "", domain: "sessions" });
57361
57383
  }
57362
57384
  async function applyPoolSession(tx, prefix, workId, mutation) {
57385
+ const owners = `${prefix}session_owners`;
57386
+ if (mutation.kind === "session-append") {
57387
+ if (typeof mutation.sessionId !== "string" || mutation.sessionId === "")
57388
+ throw new Error("Pool sessions require database identity");
57389
+ const owner = await tx.get(`SELECT work FROM ${owners} WHERE session = ?`, mutation.sessionId);
57390
+ if (owner?.work !== workId)
57391
+ throw new Error("Pool session scope mismatch");
57392
+ return withSessionStore(tx, prefix, (store) => store.append(mutation.sessionId, mutation.entries, { expectSeq: mutation.expectSeq }));
57393
+ }
57363
57394
  const { sessionId, replayKey } = mutation.checkpoint;
57364
57395
  if (!sessionId || mutation.checkpoint.sessionFile !== undefined)
57365
57396
  throw new Error("Pool sessions require database identity");
57366
- const owners = `${prefix}session_owners`;
57367
57397
  if (mutation.kind === "session-create") {
57368
57398
  if (mutation.header.id !== sessionId)
57369
57399
  throw new Error("Pool session identity mismatch");
@@ -57373,10 +57403,12 @@ async function applyPoolSession(tx, prefix, workId, mutation) {
57373
57403
  if (owner?.work !== workId || owner.replay_key !== replayKey)
57374
57404
  throw new Error("Pool session scope mismatch");
57375
57405
  }
57406
+ return withSessionStore(tx, prefix, (store) => mutation.kind === "session-create" ? store.create(mutation.header, mutation.entries) : store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq }));
57407
+ }
57408
+ async function withSessionStore(tx, prefix, write) {
57376
57409
  const store = await sessionStore(tx, prefix);
57377
57410
  try {
57378
- const result = mutation.kind === "session-create" ? await store.create(mutation.header, mutation.entries) : mutation.kind === "session-append" ? await store.append(sessionId, mutation.entries, { expectSeq: mutation.expectSeq }) : await store.replace(sessionId, mutation.header, mutation.entries, { expectSeq: mutation.expectSeq });
57379
- return result.lastSeq;
57411
+ return (await write(store)).lastSeq;
57380
57412
  } finally {
57381
57413
  await store.close();
57382
57414
  }
@@ -57424,8 +57456,11 @@ var init_pool_stage_usage = __esm(() => {
57424
57456
  });
57425
57457
 
57426
57458
  // dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
57427
- import { randomUUID as randomUUID10 } from "node:crypto";
57459
+ import { createHash as createHash4, randomUUID as randomUUID10 } from "node:crypto";
57428
57460
  import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
57461
+ function requestIdentity(request) {
57462
+ return createHash4("sha256").update(request).digest("hex");
57463
+ }
57429
57464
  function fromRow(row) {
57430
57465
  return {
57431
57466
  ...JSON.parse(row.descriptor),
@@ -57481,10 +57516,11 @@ class SqlPoolQueue {
57481
57516
  const versions = await tx.all(`SELECT version FROM ${pool.meta}`);
57482
57517
  if (versions.length > 1)
57483
57518
  throw new Error("Corrupt pool schema metadata");
57484
- if (versions.length === 0)
57485
- await tx.run(`INSERT INTO ${pool.meta} VALUES (1)`);
57486
- else
57487
- assertStorageSchemaVersion2("journal", versions[0].version, 1);
57519
+ const version = versions[0]?.version;
57520
+ if (version === undefined)
57521
+ await tx.run(`INSERT INTO ${pool.meta} VALUES (?)`, POOL_SCHEMA_VERSION);
57522
+ else if (version !== 1)
57523
+ assertStorageSchemaVersion2("journal", version, POOL_SCHEMA_VERSION);
57488
57524
  await tx.exec(`
57489
57525
  CREATE TABLE IF NOT EXISTS ${pool.roots}(root TEXT PRIMARY KEY REFERENCES ${journal.runs}(run_id) ON DELETE CASCADE, timeout INTEGER NOT NULL);
57490
57526
  CREATE TABLE IF NOT EXISTS ${pool.work}(
@@ -57515,9 +57551,23 @@ class SqlPoolQueue {
57515
57551
  state TEXT NOT NULL DEFAULT 'running', revision INTEGER NOT NULL DEFAULT 0
57516
57552
  )`);
57517
57553
  await tx.exec(poolUsageSchema(journal.runs));
57554
+ if (version === 1)
57555
+ await pool.migrateFromVersion1(tx);
57518
57556
  });
57519
57557
  return pool;
57520
57558
  }
57559
+ async migrateFromVersion1(tx) {
57560
+ await tx.run(`DELETE FROM ${this.operations} WHERE substr(request, 1, 1) = '['`);
57561
+ for (let after = 0;; ) {
57562
+ const rows = await tx.all(`SELECT rowid, request FROM ${this.operations} WHERE rowid > ? ORDER BY rowid LIMIT 256`, after);
57563
+ if (rows.length === 0)
57564
+ break;
57565
+ for (const row of rows)
57566
+ await tx.run(`UPDATE ${this.operations} SET request = ? WHERE rowid = ?`, requestIdentity(String(row.request)), row.rowid);
57567
+ after = Number(rows.at(-1).rowid);
57568
+ }
57569
+ await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
57570
+ }
57521
57571
  async authority(tx, lease, timeout) {
57522
57572
  const row = await tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
57523
57573
  if (row?.timeout === null && timeout === undefined)
@@ -57658,31 +57708,18 @@ class SqlPoolQueue {
57658
57708
  throw new Error(STALE_WORK_LEASE);
57659
57709
  return row;
57660
57710
  }
57661
- async renew(lease, token, operationId, durationMs) {
57711
+ async renew(lease, token, durationMs) {
57662
57712
  positive(durationMs);
57663
57713
  lease = snapshotLease(lease);
57664
57714
  token = { ...token };
57665
- validateText("key", operationId);
57666
- const request = JSON.stringify([token.epoch, token.generation, token.worker, durationMs]);
57667
57715
  return this.transaction(async (tx) => {
57668
57716
  const { now } = await this.authority(tx, lease);
57669
57717
  const row = await this.current(tx, lease, token);
57670
57718
  if (row.epoch !== lease.epoch || row.expires_at < now)
57671
57719
  throw new Error(STALE_WORK_LEASE);
57672
- const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
57673
- if (previous) {
57674
- if (previous.request !== request)
57675
- throw new Error("Pool operation identity conflict");
57676
- return JSON.parse(previous.receipt);
57677
- }
57678
- const receipt = {
57679
- ...fromRow(row),
57680
- expiresAt: Math.max(Number(row.expires_at), expiry(now, durationMs)),
57681
- renewedAt: now
57682
- };
57683
- await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, receipt.expiresAt, now, row.id);
57684
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, row.id, token.generation, operationId, request, JSON.stringify(receipt));
57685
- return receipt;
57720
+ const expiresAt = Math.max(Number(row.expires_at), expiry(now, durationMs));
57721
+ await tx.run(`UPDATE ${this.work} SET expires_at = ?, renewed_at = ? WHERE id = ?`, expiresAt, now, row.id);
57722
+ return { ...fromRow(row), expiresAt, renewedAt: now };
57686
57723
  });
57687
57724
  }
57688
57725
  async commit(lease, token, operationId, mutation) {
@@ -57691,6 +57728,7 @@ class SqlPoolQueue {
57691
57728
  validateText("key", operationId);
57692
57729
  assertWorkflowSerializableObject(mutation, "Pool mutation");
57693
57730
  const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
57731
+ const identity = requestIdentity(request);
57694
57732
  mutation = JSON.parse(request).mutation;
57695
57733
  return this.transaction(async (tx) => {
57696
57734
  const { now, status } = await this.authority(tx, lease);
@@ -57699,7 +57737,7 @@ class SqlPoolQueue {
57699
57737
  throw new Error(STALE_WORK_LEASE);
57700
57738
  const previous = await tx.get(`SELECT * FROM ${this.operations} WHERE work = ? AND generation = ? AND operation = ?`, token.id, token.generation, operationId);
57701
57739
  if (previous) {
57702
- if (previous.request !== request)
57740
+ if (previous.request !== identity)
57703
57741
  throw new Error("Pool operation identity conflict");
57704
57742
  return JSON.parse(previous.receipt);
57705
57743
  }
@@ -57707,34 +57745,41 @@ class SqlPoolQueue {
57707
57745
  throw new Error(STALE_WORK_LEASE);
57708
57746
  if (!await this.admitsAttemptWrites(tx, lease, status))
57709
57747
  throw new Error("Pool root is not running");
57710
- const checkpoint = mutation.checkpoint;
57711
- await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
57712
- if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
57713
- throw new Error("Pool checkpoint timestamp is in the future");
57714
- assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
57715
- const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
57716
- const payload = serializeJson(checkpoint);
57717
- const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
57718
- if (existing && existing.payload !== payload)
57719
- throw new Error("Pool checkpoint identity conflict");
57720
- if (existing && mutation.kind !== "checkpoint")
57721
- throw new Error("Pool session checkpoint identity conflict");
57722
- const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
57723
- const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
57724
- const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
57725
- seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
57726
- at: now,
57727
- ...lastSeq === undefined ? {} : { lastSeq }
57728
- };
57729
- if (!existing)
57730
- await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
57731
- const billed = poolCheckpointUsage(checkpoint);
57732
- if (billed && !existing)
57733
- await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
57734
- await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, request, JSON.stringify(receipt));
57748
+ const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
57749
+ await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
57735
57750
  return receipt;
57736
57751
  });
57737
57752
  }
57753
+ async appendSession(tx, row, mutation, now) {
57754
+ return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
57755
+ }
57756
+ async recordCheckpoint(tx, lease, token, row, mutation, now) {
57757
+ const checkpoint = mutation.checkpoint;
57758
+ await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
57759
+ if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
57760
+ throw new Error("Pool checkpoint timestamp is in the future");
57761
+ assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
57762
+ const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
57763
+ const payload = serializeJson(checkpoint);
57764
+ const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
57765
+ if (existing && existing.payload !== payload)
57766
+ throw new Error("Pool checkpoint identity conflict");
57767
+ if (existing && mutation.kind !== "checkpoint")
57768
+ throw new Error("Pool session checkpoint identity conflict");
57769
+ const lastSeq = mutation.kind === "checkpoint" ? undefined : await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation);
57770
+ const latest = await tx.get(`SELECT COALESCE(MAX(seq), 0) AS seq FROM ${this.journal.events} WHERE run_id = ?`, lease.runId);
57771
+ const receipt = existing ? { seq: Number(existing.seq), at: Number(existing.at) } : {
57772
+ seq: journalCounter(lease.runId, "seq", Number(latest.seq) + 1, "overflow"),
57773
+ at: now,
57774
+ ...lastSeq === undefined ? {} : { lastSeq }
57775
+ };
57776
+ if (!existing)
57777
+ await tx.run(`INSERT INTO ${this.journal.events} VALUES (?, ?, ?, ?, ?, ?, ?)`, lease.runId, receipt.seq, lease.epoch, key, RUN_EVENT_TYPES.checkpointRecorded, payload, now);
57778
+ const billed = poolCheckpointUsage(checkpoint);
57779
+ if (billed && !existing)
57780
+ await upsertPoolUsage(tx, this.journal.runs, row.id, token.generation, billed.key, billed.usage);
57781
+ return receipt;
57782
+ }
57738
57783
  async complete(lease, token, result) {
57739
57784
  lease = snapshotLease(lease);
57740
57785
  token = { ...token };
@@ -57821,7 +57866,7 @@ function openSqlPoolQueue(journal) {
57821
57866
  return Promise.reject(new Error("Worker pool requires a SQL run journal"));
57822
57867
  return SqlPoolQueue.open(storage);
57823
57868
  }
57824
- var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, STALE_WORK_LEASE = "Stale work lease";
57869
+ var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
57825
57870
  var init_sql_pool_queue = __esm(() => {
57826
57871
  init_serializable();
57827
57872
  init_boundary_topology();
@@ -57833,74 +57878,42 @@ var init_sql_pool_queue = __esm(() => {
57833
57878
  init_values();
57834
57879
  });
57835
57880
 
57836
- // dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
57837
- function validate(report) {
57838
- validateText("key", report.key);
57839
- if (report.key === "")
57840
- throw new Error("Pool usage key must not be empty");
57841
- for (const field of POOL_USAGE_FIELDS) {
57842
- const value2 = report.usage[field];
57843
- if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
57844
- throw new Error(`Pool usage ${field} must be a finite non-negative number`);
57881
+ // dist/builtin/workflows/src/shared/external-run-routes.ts
57882
+ function claimExternalRunRoute(store, runId, owner) {
57883
+ let owners = registry.owners.get(store);
57884
+ if (owners === undefined) {
57885
+ owners = new Map;
57886
+ registry.owners.set(store, owners);
57845
57887
  }
57888
+ owners.set(runId, owner);
57889
+ return () => {
57890
+ const current = registry.owners.get(store);
57891
+ if (current?.get(runId) !== owner)
57892
+ return;
57893
+ current.delete(runId);
57894
+ if (current.size === 0)
57895
+ registry.owners.delete(store);
57896
+ };
57846
57897
  }
57847
-
57848
- class SqlPoolUsage {
57849
- transaction;
57850
- journal;
57851
- table;
57852
- constructor(transaction, journal) {
57853
- this.transaction = transaction;
57854
- this.journal = journal;
57855
- this.table = poolUsageTable(journal.runs);
57856
- }
57857
- static async open(transaction, journal) {
57858
- const usage = new SqlPoolUsage(transaction, journal);
57859
- await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
57860
- return usage;
57861
- }
57862
- async totalIn(tx, root) {
57863
- const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
57864
- FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
57865
- return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
57866
- }
57867
- async report(lease, token, reports) {
57868
- validateText("runId", lease.runId);
57869
- validateText("holderId", lease.holderId);
57870
- for (const report of reports)
57871
- validate(report);
57872
- const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
57873
- return this.transaction(async (tx) => {
57874
- const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
57875
- LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
57876
- if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
57877
- throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
57878
- const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
57879
- if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
57880
- throw new Error("Stale work lease");
57881
- for (const { key, usage } of snapshot)
57882
- await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
57883
- return this.totalIn(tx, lease.runId);
57884
- });
57885
- }
57886
- total(root) {
57887
- return this.transaction((tx) => this.totalIn(tx, root));
57888
- }
57898
+ function externalRunRouteOwner(store, runId) {
57899
+ return registry.owners.get(store)?.get(runId);
57889
57900
  }
57890
- async function openSqlPoolUsage(journal) {
57891
- const storage = sqlPoolStorage(journal);
57892
- if (!storage)
57893
- throw new Error("Worker pool requires a SQL run journal");
57894
- await openSqlPoolQueue(journal);
57895
- return SqlPoolUsage.open(storage.transaction, storage.tables);
57901
+ function registerPendingStageMessageRouter(store, router) {
57902
+ registry.routers.set(store, router);
57903
+ return () => {
57904
+ if (registry.routers.get(store) === router)
57905
+ registry.routers.delete(store);
57906
+ };
57896
57907
  }
57897
- var STORE_NOW3 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
57898
- var init_sql_pool_usage = __esm(() => {
57899
- init_internal();
57900
- init_pool_stage_usage();
57901
- init_sql_pool_queue();
57902
- init_types();
57903
- init_values();
57908
+ function pendingStageMessageRouter(store) {
57909
+ return registry.routers.get(store);
57910
+ }
57911
+ var registryKey, processState, registry;
57912
+ var init_external_run_routes = __esm(() => {
57913
+ registryKey = Symbol.for("atomic-workflows/external-run-routes@1");
57914
+ processState = globalThis;
57915
+ registry = processState[registryKey] ?? { owners: new WeakMap, routers: new WeakMap };
57916
+ processState[registryKey] = registry;
57904
57917
  });
57905
57918
 
57906
57919
  // dist/builtin/workflows/src/shared/pending-stage-route-capability.ts
@@ -57927,12 +57940,12 @@ function rotateWorkflowPendingStageRouteCapability(store, runId) {
57927
57940
  capabilitiesFor(store).set(runId, capability);
57928
57941
  return capability;
57929
57942
  }
57930
- var routeCapabilitiesKey, processState, routeCapabilities;
57943
+ var routeCapabilitiesKey, processState2, routeCapabilities;
57931
57944
  var init_pending_stage_route_capability = __esm(() => {
57932
57945
  routeCapabilitiesKey = Symbol.for("atomic-workflows/pending-stage-route-capabilities@1");
57933
- processState = globalThis;
57934
- routeCapabilities = processState[routeCapabilitiesKey] ?? new WeakMap;
57935
- processState[routeCapabilitiesKey] = routeCapabilities;
57946
+ processState2 = globalThis;
57947
+ routeCapabilities = processState2[routeCapabilitiesKey] ?? new WeakMap;
57948
+ processState2[routeCapabilitiesKey] = routeCapabilities;
57936
57949
  });
57937
57950
 
57938
57951
  // dist/builtin/workflows/src/shared/pending-stage-route-readiness.ts
@@ -57964,12 +57977,12 @@ function hasWorkflowPendingStageRouteOwner(store) {
57964
57977
  function workflowPendingStageRouteReady(store, runId) {
57965
57978
  return owners.get(store)?.ready(runId);
57966
57979
  }
57967
- var routeOwnersKey, processState2, owners;
57980
+ var routeOwnersKey, processState3, owners;
57968
57981
  var init_pending_stage_route_readiness = __esm(() => {
57969
57982
  routeOwnersKey = Symbol.for("atomic-workflows/pending-stage-route-owners@1");
57970
- processState2 = globalThis;
57971
- owners = processState2[routeOwnersKey] ?? new WeakMap;
57972
- processState2[routeOwnersKey] = owners;
57983
+ processState3 = globalThis;
57984
+ owners = processState3[routeOwnersKey] ?? new WeakMap;
57985
+ processState3[routeOwnersKey] = owners;
57973
57986
  });
57974
57987
 
57975
57988
  // dist/builtin/workflows/src/shared/workflow-stage-path-matching.ts
@@ -58270,6 +58283,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
58270
58283
  notificationId,
58271
58284
  reason
58272
58285
  };
58286
+ const external = externalRunRouteOwner(activeStore, entry.runId);
58287
+ if (external !== undefined)
58288
+ return external.notifyUndeliverable(payload);
58273
58289
  pi.events?.emit?.(PENDING_STAGE_UNDELIVERABLE_EVENT, payload);
58274
58290
  return payload.handled && await payload.completion === true;
58275
58291
  };
@@ -58281,19 +58297,29 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
58281
58297
  return;
58282
58298
  const ownedRunIds = new Set;
58283
58299
  const runs = activeStore.runs();
58300
+ const runById = new Map(runs.map((run) => [run.id, run]));
58301
+ const childrenByBoundary = new Map;
58302
+ for (const run of runs) {
58303
+ if (run.parentRunId === undefined || run.parentStageId === undefined)
58304
+ continue;
58305
+ const key = JSON.stringify([run.parentRunId, run.parentStageId]);
58306
+ const children = childrenByBoundary.get(key) ?? [];
58307
+ children.push(run);
58308
+ childrenByBoundary.set(key, children);
58309
+ }
58284
58310
  const resolveBoundarySegments = createWorkflowBoundarySegmentsResolver(runs);
58285
58311
  for (const run of runs) {
58286
- if (!ownsRun(run.id))
58312
+ if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
58287
58313
  continue;
58288
- const rootRunId = durableRootRunIdForRun(runs, run.id);
58314
+ const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
58289
58315
  if (rootRunId === undefined)
58290
58316
  continue;
58291
58317
  ownedRunIds.add(run.id);
58292
- const parentRun = runs.find((candidate) => candidate.id === run.parentRunId);
58318
+ const parentRun = run.parentRunId === undefined ? undefined : runById.get(run.parentRunId);
58293
58319
  const boundary = parentRun?.stages.find((stage) => stage.id === run.parentStageId);
58294
58320
  const parent = parentRun === undefined || boundary === undefined ? undefined : {
58295
58321
  runId: parentRun.id,
58296
- stageKeys: [boundary.id, boundary.name].filter((key) => resolveChildRun(runs, parentRun, key)?.id === run.id)
58322
+ stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
58297
58323
  };
58298
58324
  const announcement = {
58299
58325
  runId: run.id,
@@ -58434,16 +58460,22 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
58434
58460
  payload.handled = true;
58435
58461
  payload.completion = queueAndPersist(activeStore, resolvedEvent, workflowInvocationIntercomGroup(rootRunId), stage.pendingStageDeliveryAvailable === true);
58436
58462
  };
58437
- const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
58438
- const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, (payload) => {
58463
+ const routeStickyLiveDelivered = (payload) => {
58439
58464
  if (disposed || !isStickyLiveDeliveredEvent(payload) || payload.handled)
58440
58465
  return;
58441
58466
  payload.handled = true;
58442
58467
  payload.completion = outsideWorkflowExecutions(() => recordConfirmedStickyDeliveries(activeStore, payload));
58468
+ };
58469
+ const subscription = pi.events?.on?.(PENDING_STAGE_MESSAGE_EVENT, (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)));
58470
+ const stickySubscription = pi.events?.on?.(STICKY_LIVE_DELIVERED_EVENT, routeStickyLiveDelivered);
58471
+ const disposeRouter = registerPendingStageMessageRouter(activeStore, {
58472
+ route: (payload) => outsideWorkflowExecutions(() => routePendingStageMessage(payload)),
58473
+ sticky: routeStickyLiveDelivered
58443
58474
  });
58444
58475
  const dispose = () => {
58445
58476
  disposed = true;
58446
58477
  disposeReadiness();
58478
+ disposeRouter();
58447
58479
  routeCompletions.clear();
58448
58480
  unsubscribeStore();
58449
58481
  if (typeof subscription === "function")
@@ -58569,6 +58601,12 @@ function isMaterializedNonAgentTarget(runs, target) {
58569
58601
  return false;
58570
58602
  return nonAgentNodes(run).some((node) => node.id === stageKey || node.name === stageKey);
58571
58603
  }
58604
+ function resolveIndexedChildRun(childrenByBoundary, parent, segment) {
58605
+ const boundariesById = parent.stages.filter((stage) => stage.id === segment);
58606
+ const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
58607
+ const children = boundaries.flatMap((boundary) => childrenByBoundary.get(JSON.stringify([parent.id, boundary.id])) ?? []);
58608
+ return children.length === 1 ? children[0] : undefined;
58609
+ }
58572
58610
  function resolveChildRun(runs, parent, segment) {
58573
58611
  const boundariesById = parent.stages.filter((stage) => stage.id === segment);
58574
58612
  const boundaries = boundariesById.length > 0 ? boundariesById : parent.stages.filter((stage) => stage.name === segment);
@@ -58908,6 +58946,7 @@ var init_pending_stage_intercom = __esm(() => {
58908
58946
  init_execution_context_boundary();
58909
58947
  init_run_owner_backend();
58910
58948
  init_store_backend();
58949
+ init_external_run_routes();
58911
58950
  init_intercom_group();
58912
58951
  init_pending_stage_route_capability();
58913
58952
  init_pending_stage_route_readiness();
@@ -59366,7 +59405,7 @@ function isWorkflowFutureStageRosterEntries(value2) {
59366
59405
  function isWorkflowStageRosterEntries(value2) {
59367
59406
  return Array.isArray(value2) && value2.every((entry) => typeof entry === "object" && entry !== null && entry.kind === "workflow-stage" && typeof entry.runId === "string" && typeof entry.stageId === "string" && typeof entry.stageName === "string" && typeof entry.target === "string" && (entry.lifecycle === "pending" || entry.lifecycle === "running") && typeof entry.group === "string" && (entry.sessionId === undefined || typeof entry.sessionId === "string"));
59368
59407
  }
59369
- var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomClient;
59408
+ var BROKER_SOCKET, BROKER_CONNECT_TIMEOUT_MS = 1e4, PENDING_STAGE_ROUTE_ACK_TIMEOUT_MS = 1e4, GROUP_REQUEST_TIMEOUT_MS = 5000, PRESENCE_ACK_TIMEOUT_MS = 5000, RESTORING_RECIPIENT_REQUEST_TIMEOUT_MS = 5000, HELD_SEND_GRACE_MS = 5000, IntercomPendingStageRouteRefusedError, PENDING_STAGE_ROUTE_REFUSAL_CODES, IntercomClient;
59370
59409
  var init_client = __esm(() => {
59371
59410
  init_paths();
59372
59411
  init_pending_send_registry();
@@ -59375,6 +59414,21 @@ var init_client = __esm(() => {
59375
59414
  init_live_route_refusal();
59376
59415
  init_retry_policy();
59377
59416
  BROKER_SOCKET = getBrokerSocketPath();
59417
+ IntercomPendingStageRouteRefusedError = class IntercomPendingStageRouteRefusedError extends Error {
59418
+ connectionKept;
59419
+ code;
59420
+ constructor(reason, refusal = { connectionKept: false }) {
59421
+ super(reason);
59422
+ this.name = "IntercomPendingStageRouteRefusedError";
59423
+ this.connectionKept = refusal.connectionKept;
59424
+ this.code = refusal.code;
59425
+ }
59426
+ };
59427
+ PENDING_STAGE_ROUTE_REFUSAL_CODES = new Set([
59428
+ "not_authorized",
59429
+ "invalid_stage_roster",
59430
+ "invalid_possible_stage_roster"
59431
+ ]);
59378
59432
  IntercomClient = class IntercomClient extends EventEmitter {
59379
59433
  returnAddress;
59380
59434
  preferredSessionId;
@@ -59382,6 +59436,8 @@ var init_client = __esm(() => {
59382
59436
  connectTimeoutMs;
59383
59437
  brokerAcknowledgesReceipt = false;
59384
59438
  brokerAcknowledgesPendingStageRoutes = false;
59439
+ multiplexesPendingStageRoutes;
59440
+ brokerRefusesPendingStageRoutesInPlace = false;
59385
59441
  brokerAnswersStatus = false;
59386
59442
  socket = null;
59387
59443
  _sessionId = null;
@@ -59447,6 +59503,7 @@ var init_client = __esm(() => {
59447
59503
  this.preferredSessionId = options.preferredSessionId;
59448
59504
  this.acknowledgesReceipt = options.acknowledgesReceipt === true;
59449
59505
  this.connectTimeoutMs = options.connectTimeoutMs ?? BROKER_CONNECT_TIMEOUT_MS;
59506
+ this.multiplexesPendingStageRoutes = options.multiplexesPendingStageRoutes === true;
59450
59507
  }
59451
59508
  failPending(error) {
59452
59509
  this.pendingSends.rejectAll(error);
@@ -59559,6 +59616,7 @@ var init_client = __esm(() => {
59559
59616
  this._supervisorSessionId = null;
59560
59617
  this.brokerAcknowledgesReceipt = false;
59561
59618
  this.brokerAcknowledgesPendingStageRoutes = false;
59619
+ this.brokerRefusesPendingStageRoutesInPlace = false;
59562
59620
  this.brokerAnswersStatus = false;
59563
59621
  this._messageSource = undefined;
59564
59622
  this.disconnectError = null;
@@ -59637,7 +59695,7 @@ var init_client = __esm(() => {
59637
59695
  }
59638
59696
  switch (brokerMessage.type) {
59639
59697
  case "registered": {
59640
- if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
59698
+ if (typeof brokerMessage.sessionId !== "string" || brokerMessage.supervisorSessionId !== undefined && typeof brokerMessage.supervisorSessionId !== "string" || brokerMessage.acknowledgesReceipt !== undefined && brokerMessage.acknowledgesReceipt !== true || brokerMessage.acknowledgesPendingStageRoutes !== undefined && brokerMessage.acknowledgesPendingStageRoutes !== true || brokerMessage.refusesPendingStageRoutesInPlace !== undefined && brokerMessage.refusesPendingStageRoutesInPlace !== true || brokerMessage.brokerStatus !== undefined && brokerMessage.brokerStatus !== true) {
59641
59699
  throw new Error("Invalid registered message");
59642
59700
  }
59643
59701
  if (this._sessionId !== null) {
@@ -59647,6 +59705,7 @@ var init_client = __esm(() => {
59647
59705
  this._supervisorSessionId = typeof brokerMessage.supervisorSessionId === "string" ? brokerMessage.supervisorSessionId : null;
59648
59706
  this.brokerAcknowledgesReceipt = this.acknowledgesReceipt && brokerMessage.acknowledgesReceipt === true;
59649
59707
  this.brokerAcknowledgesPendingStageRoutes = brokerMessage.acknowledgesPendingStageRoutes === true;
59708
+ this.brokerRefusesPendingStageRoutesInPlace = brokerMessage.refusesPendingStageRoutesInPlace === true;
59650
59709
  this.brokerAnswersStatus = brokerMessage.brokerStatus === true;
59651
59710
  this.emit("_registered", { type: "registered", sessionId: brokerMessage.sessionId });
59652
59711
  break;
@@ -59659,6 +59718,12 @@ var init_client = __esm(() => {
59659
59718
  this.emit("_registration_failed", refusal);
59660
59719
  break;
59661
59720
  }
59721
+ const refusedRouteId = typeof brokerMessage.requestId === "string" ? brokerMessage.requestId : "";
59722
+ const refusedRoute = this.pendingStageRouteRegistrations.get(refusedRouteId);
59723
+ if (refusedRoute !== undefined) {
59724
+ this.pendingStageRouteRegistrations.delete(refusedRouteId);
59725
+ refusedRoute.reject(new IntercomPendingStageRouteRefusedError(brokerMessage.reason));
59726
+ }
59662
59727
  this.disconnectError ??= refusal;
59663
59728
  this.failPending(this.disconnectError);
59664
59729
  this.socket?.destroy();
@@ -59784,6 +59849,21 @@ var init_client = __esm(() => {
59784
59849
  this.emit("pending_stage_route_revoked", { runId });
59785
59850
  break;
59786
59851
  }
59852
+ case "pending_stage_route_refused": {
59853
+ const { requestId, runId, code, reason } = brokerMessage;
59854
+ if (typeof requestId !== "string" || typeof runId !== "string" || typeof reason !== "string" || typeof code !== "string") {
59855
+ throw new Error("Invalid pending-stage route refusal");
59856
+ }
59857
+ const pending = this.pendingStageRouteRegistrations.get(requestId);
59858
+ if (!pending)
59859
+ return;
59860
+ this.pendingStageRouteRegistrations.delete(requestId);
59861
+ pending.reject(new IntercomPendingStageRouteRefusedError(reason, {
59862
+ connectionKept: true,
59863
+ ...PENDING_STAGE_ROUTE_REFUSAL_CODES.has(code) ? { code } : {}
59864
+ }));
59865
+ break;
59866
+ }
59787
59867
  case "pending_stage_route_registered": {
59788
59868
  const { requestId } = brokerMessage;
59789
59869
  if (typeof requestId !== "string")
@@ -60287,7 +60367,8 @@ var init_client = __esm(() => {
60287
60367
  stages,
60288
60368
  possibleStages,
60289
60369
  ...parent === undefined ? {} : { parent },
60290
- ...dormant === true ? { dormant: true } : {}
60370
+ ...dormant === true ? { dormant: true } : {},
60371
+ ...this.multiplexesPendingStageRoutes && this.brokerRefusesPendingStageRoutesInPlace ? { refusalKeepsConnection: true } : {}
60291
60372
  });
60292
60373
  } catch (error) {
60293
60374
  clearTimeout(timeout);
@@ -61021,120 +61102,165 @@ var init_reconnect_backoff = __esm(() => {
61021
61102
  BACKOFF_MS = [1000, 2000, 5000, 1e4, 30000];
61022
61103
  });
61023
61104
 
61024
- // dist/builtin/intercom/route-connect-gate.ts
61025
- function createConcurrencyGate(limit) {
61026
- let active = 0;
61027
- const waiting = [];
61028
- const release = () => {
61029
- const next = waiting.shift();
61030
- if (next === undefined)
61031
- active -= 1;
61032
- else
61033
- next();
61034
- };
61035
- return {
61036
- async run(operation) {
61037
- if (active < limit)
61038
- active += 1;
61039
- else
61040
- await new Promise((resolve) => waiting.push(resolve));
61041
- try {
61042
- return await operation();
61043
- } finally {
61044
- release();
61045
- }
61046
- }
61047
- };
61048
- }
61049
- var ROUTE_CLIENT_CONNECT_CONCURRENCY = 4;
61050
-
61051
61105
  // dist/builtin/intercom/workflow-route-owner.ts
61052
61106
  import { randomUUID as randomUUID14 } from "node:crypto";
61053
61107
  import { createEventBus, getBuiltinPackagePaths } from "@bastani/atomic";
61108
+ function closedError() {
61109
+ return new Error("Workflow Intercom route owner closed");
61110
+ }
61054
61111
  function createWorkflowRouteOwner(cwd) {
61055
61112
  const events = createEventBus();
61056
61113
  const config = loadConfig();
61057
61114
  const id = randomUUID14();
61058
61115
  const routes = new Map;
61059
- const gate = createConcurrencyGate(ROUTE_CLIENT_CONNECT_CONCURRENCY);
61116
+ const connections = new Map;
61117
+ const isCurrent = (state) => routes.get(state.route.runId) === state;
61118
+ const connectionKey = (state) => state.isolated ? state.route.runId : state.route.group;
61119
+ function connectionOf(state) {
61120
+ const key = connectionKey(state);
61121
+ let connection = connections.get(key);
61122
+ if (connection === undefined) {
61123
+ connection = { key, group: state.route.group, acknowledged: false, routes: new Map };
61124
+ connections.set(key, connection);
61125
+ }
61126
+ connection.routes.set(state.route.runId, state);
61127
+ return connection;
61128
+ }
61129
+ function detach(state) {
61130
+ const key = connectionKey(state);
61131
+ const connection = connections.get(key);
61132
+ connection?.routes.delete(state.route.runId);
61133
+ if (!state.isolated || connection === undefined)
61134
+ return;
61135
+ connections.delete(key);
61136
+ connection.client?.disconnect().catch(() => {});
61137
+ }
61138
+ function settle(state, registered) {
61139
+ state.registered = registered;
61140
+ if (!registered && isCurrent(state))
61141
+ return;
61142
+ for (const waiter of state.waiters)
61143
+ waiter(registered);
61144
+ state.waiters.clear();
61145
+ }
61060
61146
  function scheduleRetry(state) {
61061
- if (state.closed || state.retry !== undefined)
61147
+ if (!isCurrent(state) || state.retry !== undefined)
61062
61148
  return;
61063
61149
  state.retry = setTimeout(() => {
61064
61150
  state.retry = undefined;
61065
- publish(state, state.route).catch(() => scheduleRetry(state));
61151
+ publish(state, state.route).catch(() => {});
61066
61152
  }, jitteredReconnectDelayMs(state.attempt++));
61067
61153
  state.retry.unref();
61068
61154
  }
61069
- async function connect(state) {
61070
- if (state.client?.isConnected())
61071
- return state.client;
61072
- const [extensionDir] = getBuiltinPackagePaths({
61073
- workflows: false,
61074
- subagents: false,
61075
- mcp: false,
61076
- "web-access": false
61077
- });
61078
- if (extensionDir === undefined)
61079
- throw new Error("Intercom builtin package is unavailable");
61080
- await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
61081
- if (state.closed)
61082
- throw new Error("Workflow Intercom route owner closed");
61083
- const client = new IntercomClient(`${id}:pending-stage-route:${state.route.runId}`);
61084
- state.client = client;
61085
- client.on("error", () => {});
61086
- client.on("disconnected", () => {
61087
- if (state.client !== client)
61088
- return;
61089
- state.client = undefined;
61090
- scheduleRetry(state);
61091
- });
61092
- client.on("pending_stage_message", (request) => {
61093
- if (state.closed || state.client !== client)
61094
- return;
61095
- const event = { ...request, handled: false };
61096
- events.emit("atomic:workflow-pending-stage-message", event);
61097
- const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
61098
- completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
61099
- });
61100
- client.on("sticky_live_delivered", (notice) => {
61101
- if (state.closed || state.client !== client)
61102
- return;
61103
- events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
61104
- });
61105
- await client.connect({
61106
- cwd,
61107
- model: "workflow",
61108
- pid: process.pid,
61109
- startedAt: Date.now(),
61110
- lastActivity: Date.now(),
61111
- group: state.route.group,
61112
- groups: [state.route.group],
61113
- recipientPurpose: "control"
61155
+ function drop(connection, client) {
61156
+ if (connection.client === client) {
61157
+ connection.client = undefined;
61158
+ connection.acknowledged = false;
61159
+ for (const state of connection.routes.values()) {
61160
+ settle(state, false);
61161
+ scheduleRetry(state);
61162
+ }
61163
+ }
61164
+ client.disconnect().catch(() => {});
61165
+ }
61166
+ function connect(connection) {
61167
+ if (connection.client?.isConnected())
61168
+ return Promise.resolve(connection.client);
61169
+ connection.connecting ??= (async () => {
61170
+ const [extensionDir] = getBuiltinPackagePaths({
61171
+ workflows: false,
61172
+ subagents: false,
61173
+ mcp: false,
61174
+ "web-access": false
61175
+ });
61176
+ if (extensionDir === undefined)
61177
+ throw new Error("Intercom builtin package is unavailable");
61178
+ await spawnBrokerIfNeeded(config.brokerCommand, config.brokerArgs, undefined, extensionDir);
61179
+ if (connections.get(connection.key) !== connection)
61180
+ throw closedError();
61181
+ const client = new IntercomClient(`${id}:pending-stage-route:${connection.key}`, {
61182
+ multiplexesPendingStageRoutes: true
61183
+ });
61184
+ connection.client = client;
61185
+ connection.acknowledged = false;
61186
+ client.on("error", () => {});
61187
+ client.on("disconnected", () => drop(connection, client));
61188
+ client.on("pending_stage_message", (request) => {
61189
+ if (connection.client !== client)
61190
+ return;
61191
+ const event = { ...request, handled: false };
61192
+ events.emit("atomic:workflow-pending-stage-message", event);
61193
+ const completion = event.handled && event.completion !== undefined ? event.completion : Promise.resolve({ outcome: "refused", reason: "Session not found" });
61194
+ completion.catch((error) => ({ outcome: "refused", reason: error.message })).then((result) => client.respondPendingStageMessage(request.requestId, result)).catch(() => {});
61195
+ });
61196
+ client.on("sticky_live_delivered", (notice) => {
61197
+ if (connection.client !== client)
61198
+ return;
61199
+ events.emit("atomic:workflow-sticky-live-delivered", { ...notice, handled: false });
61200
+ });
61201
+ try {
61202
+ await client.connect({
61203
+ cwd,
61204
+ model: "workflow",
61205
+ pid: process.pid,
61206
+ startedAt: Date.now(),
61207
+ lastActivity: Date.now(),
61208
+ group: connection.group,
61209
+ groups: [connection.group],
61210
+ recipientPurpose: "control"
61211
+ });
61212
+ if (connections.get(connection.key) !== connection)
61213
+ throw closedError();
61214
+ } catch (error) {
61215
+ drop(connection, client);
61216
+ throw error;
61217
+ }
61218
+ for (const state of connection.routes.values()) {
61219
+ if (state.registered || state.retry === undefined)
61220
+ continue;
61221
+ clearTimeout(state.retry);
61222
+ state.retry = undefined;
61223
+ publish(state, state.route).catch(() => {});
61224
+ }
61225
+ return client;
61226
+ })().finally(() => {
61227
+ connection.connecting = undefined;
61114
61228
  });
61115
- return client;
61229
+ return connection.connecting;
61116
61230
  }
61117
61231
  function publish(state, route) {
61118
- const completion = state.tail.catch(() => {}).then(() => gate.run(async () => {
61119
- if (state.closed)
61120
- throw new Error("Workflow Intercom route owner closed");
61121
- const establishedClient = state.client?.isConnected() ? state.client : undefined;
61232
+ const completion = state.tail.catch(() => {}).then(async () => {
61233
+ if (!isCurrent(state))
61234
+ throw closedError();
61235
+ const connection = connectionOf(state);
61236
+ const established = connection.acknowledged && connection.client?.isConnected() ? connection.client : undefined;
61237
+ let client;
61122
61238
  try {
61123
- const client = await connect(state);
61239
+ client = await connect(connection);
61124
61240
  await client.registerPendingStageRouteAcknowledged(route.runId, route.group, route.capability, route.stages, route.possibleStages, route.parent);
61241
+ if (connection.client !== client || !isCurrent(state))
61242
+ return;
61243
+ connection.acknowledged = true;
61125
61244
  state.attempt = 0;
61126
61245
  clearTimeout(state.retry);
61127
61246
  state.retry = undefined;
61247
+ settle(state, true);
61128
61248
  } catch (error) {
61129
- const client = state.client;
61130
- if (client !== establishedClient || !client?.isConnected()) {
61131
- state.client = undefined;
61132
- await client?.disconnect();
61249
+ if (error instanceof IntercomPendingStageRouteRefusedError && error.connectionKept) {
61250
+ scheduleRetry(state);
61251
+ throw error;
61133
61252
  }
61253
+ if (error instanceof IntercomPendingStageRouteRefusedError && !state.isolated && isCurrent(state)) {
61254
+ detach(state);
61255
+ state.isolated = true;
61256
+ connectionOf(state);
61257
+ }
61258
+ if (client !== undefined && (client !== established || !client.isConnected()))
61259
+ drop(connection, client);
61134
61260
  scheduleRetry(state);
61135
61261
  throw error;
61136
61262
  }
61137
- }));
61263
+ });
61138
61264
  state.tail = completion;
61139
61265
  completion.catch(() => {});
61140
61266
  return completion;
@@ -61143,10 +61269,15 @@ function createWorkflowRouteOwner(cwd) {
61143
61269
  const route = payload;
61144
61270
  let state = routes.get(route.runId);
61145
61271
  if (state === undefined) {
61146
- state = { route, tail: Promise.resolve(), attempt: 0, closed: false };
61272
+ state = { route, tail: Promise.resolve(), attempt: 0, registered: false, isolated: false, waiters: new Set };
61147
61273
  routes.set(route.runId, state);
61274
+ } else if (state.route.group !== route.group) {
61275
+ detach(state);
61276
+ state.isolated = false;
61277
+ settle(state, false);
61148
61278
  }
61149
61279
  state.route = route;
61280
+ connectionOf(state);
61150
61281
  route.completion = publish(state, route);
61151
61282
  });
61152
61283
  events.on("atomic:workflow-pending-stage-undeliverable", (payload) => {
@@ -61157,7 +61288,7 @@ function createWorkflowRouteOwner(cwd) {
61157
61288
  event.handled = true;
61158
61289
  event.completion = (async () => {
61159
61290
  await publish(state, state.route);
61160
- const client = state.client;
61291
+ const client = connections.get(connectionKey(state))?.client;
61161
61292
  if (client === undefined || !client.isConnected())
61162
61293
  return false;
61163
61294
  const text = `Pending workflow stage could not receive intercom message: ${event.reason}`;
@@ -61167,18 +61298,33 @@ function createWorkflowRouteOwner(cwd) {
61167
61298
  });
61168
61299
  return {
61169
61300
  events,
61301
+ registered(runId) {
61302
+ const state = routes.get(runId);
61303
+ if (state === undefined)
61304
+ return Promise.resolve(false);
61305
+ if (state.registered)
61306
+ return Promise.resolve(true);
61307
+ return new Promise((resolve) => state.waiters.add(resolve));
61308
+ },
61170
61309
  async closeGroup(group) {
61171
61310
  const closing = [...routes.values()].filter((state) => state.route.group === group);
61311
+ const closingConnections = [...connections.values()].filter((connection) => connection.group === group);
61312
+ for (const connection of closingConnections) {
61313
+ connections.delete(connection.key);
61314
+ connection.routes.clear();
61315
+ }
61172
61316
  for (const state of closing) {
61173
- state.closed = true;
61174
- clearTimeout(state.retry);
61175
61317
  routes.delete(state.route.runId);
61318
+ clearTimeout(state.retry);
61319
+ state.retry = undefined;
61320
+ settle(state, false);
61176
61321
  }
61177
- await Promise.all(closing.map(async (state) => {
61178
- await state.client?.disconnect();
61179
- await state.tail.catch(() => {});
61180
- await state.client?.disconnect();
61322
+ await Promise.all(closingConnections.map(async (connection) => {
61323
+ await connection.client?.disconnect();
61324
+ await connection.connecting?.catch(() => {});
61181
61325
  }));
61326
+ await Promise.all(closing.map((state) => state.tail.catch(() => {})));
61327
+ await Promise.all(closingConnections.map((connection) => connection.client?.disconnect()));
61182
61328
  }
61183
61329
  };
61184
61330
  }
@@ -61297,7 +61443,7 @@ var init_workflow_infrastructure_error = __esm(() => {
61297
61443
  [/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
61298
61444
  [/Local worker retry limit/, "worker_retry_limit"],
61299
61445
  [/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
61300
- [/Worker exited before delivery acknowledgement/, "worker_lost"],
61446
+ [/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
61301
61447
  [/Stale work lease/, "stale_work_lease"],
61302
61448
  [/Intercom runtime no longer active/, "intercom_unavailable"],
61303
61449
  [/Storage unavailable/, "storage_unavailable"]
@@ -61313,12 +61459,925 @@ var init_workflow_infrastructure_error = __esm(() => {
61313
61459
  };
61314
61460
  });
61315
61461
 
61462
+ // dist/builtin/workflows/src/workers/coordinator-protocol.ts
61463
+ var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
61464
+
61316
61465
  // dist/builtin/workflows/src/workers/host-options.ts
61317
61466
  var hostWorkerBindings;
61318
61467
  var init_host_options = __esm(() => {
61319
61468
  hostWorkerBindings = Symbol("host-worker-bindings");
61320
61469
  });
61321
61470
 
61471
+ // dist/builtin/workflows/src/durable/journal/sql-pool-usage.ts
61472
+ function validate(report) {
61473
+ validateText("key", report.key);
61474
+ if (report.key === "")
61475
+ throw new Error("Pool usage key must not be empty");
61476
+ for (const field of POOL_USAGE_FIELDS) {
61477
+ const value2 = report.usage[field];
61478
+ if (typeof value2 !== "number" || !Number.isFinite(value2) || value2 < 0)
61479
+ throw new Error(`Pool usage ${field} must be a finite non-negative number`);
61480
+ }
61481
+ }
61482
+
61483
+ class SqlPoolUsage {
61484
+ transaction;
61485
+ journal;
61486
+ table;
61487
+ constructor(transaction, journal) {
61488
+ this.transaction = transaction;
61489
+ this.journal = journal;
61490
+ this.table = poolUsageTable(journal.runs);
61491
+ }
61492
+ static async open(transaction, journal) {
61493
+ const usage = new SqlPoolUsage(transaction, journal);
61494
+ await transaction((tx) => tx.exec(poolUsageSchema(journal.runs)));
61495
+ return usage;
61496
+ }
61497
+ async totalIn(tx, root) {
61498
+ const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
61499
+ FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
61500
+ return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
61501
+ }
61502
+ async report(lease, token, reports) {
61503
+ validateText("runId", lease.runId);
61504
+ validateText("holderId", lease.holderId);
61505
+ for (const report of reports)
61506
+ validate(report);
61507
+ const snapshot = reports.map((report) => ({ key: report.key, usage: { ...report.usage } }));
61508
+ return this.transaction(async (tx) => {
61509
+ const authority = await tx.get(`SELECT l.epoch, l.holder_id, l.renewed_at, p.timeout, ${STORE_NOW3} AS now FROM ${this.journal.leases} l
61510
+ LEFT JOIN ${this.journal.runs}_pool p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
61511
+ if (!authority || authority.timeout === null || authority.epoch !== lease.epoch || authority.holder_id !== lease.holderId || isWorkflowLeaseExpired(Number(authority.renewed_at), Number(authority.timeout), Number(authority.now)))
61512
+ throw new StaleLeaseError(lease.runId, lease.epoch, Number(authority?.epoch ?? 0));
61513
+ const work = await tx.get(`SELECT * FROM ${this.journal.runs}_work WHERE id = ? AND root = ?`, token.id, lease.runId);
61514
+ if (work?.state !== "leased" && work?.state !== "completed" || work.epoch !== token.epoch || work.epoch !== lease.epoch || work.generation !== token.generation || work.worker !== token.worker || work.state === "leased" && Number(work.expires_at) < Number(authority.now))
61515
+ throw new Error("Stale work lease");
61516
+ for (const { key, usage } of snapshot)
61517
+ await upsertPoolUsage(tx, this.journal.runs, token.id, token.generation, key, usage);
61518
+ return this.totalIn(tx, lease.runId);
61519
+ });
61520
+ }
61521
+ total(root) {
61522
+ return this.transaction((tx) => this.totalIn(tx, root));
61523
+ }
61524
+ }
61525
+ async function openSqlPoolUsage(journal) {
61526
+ const storage = sqlPoolStorage(journal);
61527
+ if (!storage)
61528
+ throw new Error("Worker pool requires a SQL run journal");
61529
+ await openSqlPoolQueue(journal);
61530
+ return SqlPoolUsage.open(storage.transaction, storage.tables);
61531
+ }
61532
+ var STORE_NOW3 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
61533
+ var init_sql_pool_usage = __esm(() => {
61534
+ init_internal();
61535
+ init_pool_stage_usage();
61536
+ init_sql_pool_queue();
61537
+ init_types();
61538
+ init_values();
61539
+ });
61540
+
61541
+ // dist/builtin/workflows/src/workers/pool-coordinator.ts
61542
+ import { fork } from "node:child_process";
61543
+ import { randomUUID as randomUUID15 } from "node:crypto";
61544
+ import { createWriteStream } from "node:fs";
61545
+ import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
61546
+ import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
61547
+ import { setTimeout as delay2 } from "node:timers/promises";
61548
+ function workerLogDirectory(journalPath, root) {
61549
+ return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
61550
+ }
61551
+ function token(work) {
61552
+ return {
61553
+ id: work.id,
61554
+ generation: work.generation,
61555
+ worker: work.worker,
61556
+ epoch: work.epoch,
61557
+ renewedAt: work.renewedAt,
61558
+ expiresAt: work.expiresAt
61559
+ };
61560
+ }
61561
+ function outside(runId) {
61562
+ return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
61563
+ }
61564
+ function isLiveStage(stage) {
61565
+ return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
61566
+ }
61567
+ function projectRootRun(run) {
61568
+ return {
61569
+ id: run.id,
61570
+ name: run.name,
61571
+ status: run.status,
61572
+ startedAt: run.startedAt,
61573
+ stages: run.stages.map((stage) => ({
61574
+ id: stage.id,
61575
+ name: stage.name,
61576
+ status: stage.status,
61577
+ parentIds: [],
61578
+ toolEvents: [],
61579
+ ...stage.replayKey === undefined ? {} : { replayKey: stage.replayKey },
61580
+ ...stage.intercomGroup === undefined ? {} : { intercomGroup: stage.intercomGroup },
61581
+ ...stage.workflowChildRun === undefined ? {} : { workflowChildRun: stage.workflowChildRun },
61582
+ ...stage.workflowChild === undefined ? {} : { workflowChild: { ...stage.workflowChild, outputs: {} } }
61583
+ }))
61584
+ };
61585
+ }
61586
+
61587
+ class PooledRoutes {
61588
+ root;
61589
+ owner;
61590
+ mirror = createStore();
61591
+ owners = new Map;
61592
+ attempts = new Map;
61593
+ activated = false;
61594
+ bridge;
61595
+ constructor(root, owner) {
61596
+ this.root = root;
61597
+ this.owner = owner;
61598
+ }
61599
+ static async open(root, cwd, rootRun, forward) {
61600
+ await Promise.resolve().then(() => init_workflow_route_owner());
61601
+ const owner = createWorkflowRouteOwner(cwd);
61602
+ const routes = new PooledRoutes(root, owner);
61603
+ if (rootRun !== undefined)
61604
+ routes.updateRoot(rootRun);
61605
+ routes.bridge = registerPendingStageIntercomBridge({
61606
+ events: {
61607
+ emit: (event, payload) => owner.events.emit(event, payload),
61608
+ on: (event, listener) => event === PENDING_STAGE_MESSAGE_EVENT2 || event === STICKY_LIVE_DELIVERED_EVENT2 ? undefined : owner.events.on(event, listener)
61609
+ }
61610
+ }, routes.mirror, (runId) => routes.activated && routes.owners.has(runId));
61611
+ for (const [event, kind] of [
61612
+ [PENDING_STAGE_MESSAGE_EVENT2, "pending-stage-message"],
61613
+ [STICKY_LIVE_DELIVERED_EVENT2, "sticky-live-delivered"]
61614
+ ])
61615
+ owner.events.on(event, (payload) => {
61616
+ const claimed = payload;
61617
+ if (claimed.handled !== false)
61618
+ return;
61619
+ claimed.handled = true;
61620
+ const { handled: _handled, completion: _completion, ...forwarded } = claimed;
61621
+ claimed.completion = forward({ kind, payload: forwarded });
61622
+ });
61623
+ return routes;
61624
+ }
61625
+ updateRoot(rootRun) {
61626
+ this.mirror.recordRunStart({ ...rootRun, inputs: {}, toolNodes: [], rootRunId: this.root });
61627
+ }
61628
+ accept(entries) {
61629
+ this.mirror.batch(() => {
61630
+ for (const { work, runs } of entries) {
61631
+ let attempt = this.attempts.get(work.id);
61632
+ if (attempt !== undefined && work.generation < attempt.generation)
61633
+ continue;
61634
+ if (attempt === undefined || attempt.generation !== work.generation) {
61635
+ if (attempt !== undefined)
61636
+ this.release(attempt);
61637
+ attempt = { generation: work.generation, runs: attempt?.runs ?? new Set, fenced: false };
61638
+ this.attempts.set(work.id, attempt);
61639
+ }
61640
+ if (attempt.fenced)
61641
+ continue;
61642
+ for (const run of runs) {
61643
+ attempt.runs.add(run.id);
61644
+ this.owners.set(run.id, work.id);
61645
+ const { pendingStageMessages: _messages, ...published } = run;
61646
+ this.mirror.recordRunStart({ ...published, rootRunId: this.root });
61647
+ if (run.stages.some((stage) => stage.pendingStageDeliveryAvailable === true))
61648
+ this.activated = true;
61649
+ }
61650
+ }
61651
+ });
61652
+ }
61653
+ fence(work) {
61654
+ const attempt = this.attempts.get(work.id);
61655
+ if (attempt === undefined || attempt.generation > work.generation)
61656
+ return;
61657
+ if (attempt.generation < work.generation) {
61658
+ this.release(attempt);
61659
+ this.attempts.set(work.id, { generation: work.generation, runs: attempt.runs, fenced: true });
61660
+ return;
61661
+ }
61662
+ this.release(attempt);
61663
+ attempt.fenced = true;
61664
+ }
61665
+ async route(work, runId) {
61666
+ this.requireOwned(work, runId);
61667
+ const readiness = workflowPendingStageRouteReady(this.mirror, runId);
61668
+ await readiness?.completion.catch(() => {});
61669
+ readiness?.assertCurrent();
61670
+ const waiting = new AbortController;
61671
+ await Promise.race([
61672
+ this.owner.registered(runId),
61673
+ delay2(ROUTE_REGISTRATION_WAIT_MS, undefined, { signal: waiting.signal, ref: false }).catch(() => {})
61674
+ ]);
61675
+ waiting.abort();
61676
+ this.requireOwned(work, runId);
61677
+ return workflowPendingStageRouteCapability(this.mirror, runId);
61678
+ }
61679
+ async undeliverable(payload) {
61680
+ const event = { ...payload, handled: false };
61681
+ this.owner.events.emit(PENDING_STAGE_UNDELIVERABLE_EVENT2, event);
61682
+ return event.handled && await event.completion === true;
61683
+ }
61684
+ async close() {
61685
+ const bridge = this.bridge;
61686
+ if (bridge === undefined)
61687
+ return;
61688
+ this.bridge = undefined;
61689
+ try {
61690
+ await bridge.flush();
61691
+ } finally {
61692
+ bridge.refresh();
61693
+ bridge();
61694
+ await this.owner.closeGroup(workflowInvocationIntercomGroup(this.root));
61695
+ }
61696
+ }
61697
+ requireOwned(work, runId) {
61698
+ const attempt = this.attempts.get(work.id);
61699
+ if (attempt === undefined || attempt.generation !== work.generation || attempt.fenced)
61700
+ throw new Error(STALE_WORK_LEASE);
61701
+ if (!attempt.runs.has(runId))
61702
+ throw outside(runId);
61703
+ }
61704
+ release(attempt) {
61705
+ this.mirror.batch(() => {
61706
+ for (const id of attempt.runs) {
61707
+ rotateWorkflowPendingStageRouteCapability(this.mirror, id);
61708
+ const run = this.mirror.runs().find((candidate) => candidate.id === id);
61709
+ if (run?.stages.some(isLiveStage) !== true)
61710
+ continue;
61711
+ this.mirror.recordRunStart({
61712
+ ...run,
61713
+ stages: run.stages.map((stage) => {
61714
+ if (!isLiveStage(stage))
61715
+ return stage;
61716
+ const { sessionId: _session, sessionFile: _file, ...rest } = stage;
61717
+ return { ...rest, attachable: false, attached: false };
61718
+ })
61719
+ });
61720
+ }
61721
+ });
61722
+ }
61723
+ }
61724
+
61725
+ class PoolCoordinator {
61726
+ port;
61727
+ children = new Set;
61728
+ incarnations = new Map;
61729
+ readyWorkers = new Set;
61730
+ logs = new Map;
61731
+ cleanups = new Set;
61732
+ controlAcks = new Map;
61733
+ lastHeard = new Map;
61734
+ pendingActivity = new Map;
61735
+ batches = new Map;
61736
+ publicationFailures = new Map;
61737
+ requests = new Map;
61738
+ deliveryRequests = new Map;
61739
+ inflight = new Set;
61740
+ lastTick = performance.now();
61741
+ stalledMs = 0;
61742
+ ticker;
61743
+ recoveryTimer;
61744
+ flushTimer;
61745
+ lastFlushAt = 0;
61746
+ batchSequence = 0;
61747
+ requestSequence = 0;
61748
+ usageChanged = false;
61749
+ budgetBlocked = false;
61750
+ desiredControl;
61751
+ root;
61752
+ lease;
61753
+ options;
61754
+ journal;
61755
+ queue;
61756
+ usageLedger;
61757
+ usageTotal;
61758
+ admission;
61759
+ routes;
61760
+ publishing = false;
61761
+ sources = [];
61762
+ logDirectory;
61763
+ starting;
61764
+ recovering;
61765
+ nextRecoveryAt = 0;
61766
+ cleanupFailure;
61767
+ closed = false;
61768
+ closing;
61769
+ replacements = 0;
61770
+ constructor(port) {
61771
+ this.port = port;
61772
+ port.onMessage((message) => this.receive(message));
61773
+ this.ticker = setInterval(() => this.observeResponsiveness(), TICK_INTERVAL_MS).unref();
61774
+ }
61775
+ async shutdown() {
61776
+ await this.closing?.catch(() => {});
61777
+ while (this.inflight.size > 0)
61778
+ await Promise.allSettled([...this.inflight]);
61779
+ await this.journal?.close().catch(() => {});
61780
+ }
61781
+ receive(message) {
61782
+ if (message.kind === "request-reply") {
61783
+ const pending = this.requests.get(message.id);
61784
+ this.requests.delete(message.id);
61785
+ if (message.error !== undefined)
61786
+ pending?.reject(new Error(message.error));
61787
+ else
61788
+ pending?.resolve(message.value);
61789
+ return;
61790
+ }
61791
+ if ("id" in message) {
61792
+ const command = message;
61793
+ const handled = this.execute(command).then((value2) => this.port.send({ kind: "reply", id: command.id, value: value2 }), (error) => this.port.send({ kind: "reply", id: command.id, error: error.message }));
61794
+ this.inflight.add(handled);
61795
+ handled.finally(() => this.inflight.delete(handled));
61796
+ return;
61797
+ }
61798
+ this.notice(message);
61799
+ }
61800
+ notice(notice) {
61801
+ if (notice.kind === "budget")
61802
+ this.budgetBlocked = notice.blocked;
61803
+ else if (notice.kind === "root")
61804
+ this.routes?.updateRoot(notice.rootRun);
61805
+ else {
61806
+ const batch = this.batches.get(notice.batch);
61807
+ this.batches.delete(notice.batch);
61808
+ const rejected = new Map(notice.rejected.map((entry) => [entry.workId, entry]));
61809
+ for (const entry of rejected.values())
61810
+ this.publicationFailures.set(entry.workId, { generation: entry.generation, error: entry.error });
61811
+ this.routes?.accept((batch?.entries ?? []).filter((entry) => rejected.get(entry.work.id)?.generation !== entry.work.generation));
61812
+ batch?.settle();
61813
+ }
61814
+ }
61815
+ async execute(command) {
61816
+ switch (command.kind) {
61817
+ case "prepare":
61818
+ return this.prepare(command.root, command.lease, command.options, command.control);
61819
+ case "control":
61820
+ return this.control(command.state);
61821
+ case "start":
61822
+ this.starting ??= this.start(command.sources, command.cwd, command.publish, command.rootRun);
61823
+ return this.starting;
61824
+ case "enqueue":
61825
+ return this.enqueue(command);
61826
+ case "transport":
61827
+ return this.transport(command.request, command);
61828
+ case "undeliverable":
61829
+ return await this.routes?.undeliverable(command.payload) ?? false;
61830
+ case "close":
61831
+ this.closing ??= this.close();
61832
+ return this.closing;
61833
+ }
61834
+ }
61835
+ async prepare(root, lease, options, control) {
61836
+ if (this.closed)
61837
+ throw new Error("Local worker pool is closed");
61838
+ if (this.root !== undefined && this.root !== root)
61839
+ throw new Error("Local worker pool belongs to another root");
61840
+ this.root = root;
61841
+ this.options = options;
61842
+ this.desiredControl ??= control;
61843
+ this.journal ??= await resolveRunJournal({ journal: { path: options.journalPath } });
61844
+ this.usageLedger ??= await openSqlPoolUsage(this.journal);
61845
+ this.usageTotal = await this.usageLedger.total(root);
61846
+ this.lease = lease;
61847
+ const queue = await openSqlPoolQueue(this.journal);
61848
+ await queue.initialize(lease, COORDINATOR_LEASE_MS);
61849
+ await queue.releasePreviousIncarnations(lease);
61850
+ for (;; ) {
61851
+ const intended = this.desiredControl ?? "running";
61852
+ await queue.setControl(lease, intended);
61853
+ if (intended === (this.desiredControl ?? "running"))
61854
+ break;
61855
+ }
61856
+ this.queue = queue;
61857
+ return this.usageTotal;
61858
+ }
61859
+ async control(state) {
61860
+ this.desiredControl = state;
61861
+ if (!this.queue || !this.lease)
61862
+ return;
61863
+ const control = await this.queue.setControl(this.lease, state);
61864
+ const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
61865
+ while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
61866
+ if (this.desiredControl !== state)
61867
+ return;
61868
+ if (Date.now() >= deadline) {
61869
+ if (state === "quit")
61870
+ break;
61871
+ throw new Error("Worker control acknowledgement unresolved");
61872
+ }
61873
+ await delay2(25);
61874
+ }
61875
+ this.flush();
61876
+ }
61877
+ async start(sources, cwd, publish, rootRun) {
61878
+ if (this.closed)
61879
+ throw new Error("Local worker pool is closed");
61880
+ const root = this.root;
61881
+ if (!this.queue || !this.lease || !this.options || root === undefined || this.lease.runId !== root)
61882
+ throw new Error("Local worker pool requires root preparation");
61883
+ this.sources = sources;
61884
+ this.publishing = publish;
61885
+ if (publish) {
61886
+ this.routes = await PooledRoutes.open(root, cwd, rootRun, (request) => this.request(request));
61887
+ if (this.closed) {
61888
+ await this.routes.close();
61889
+ throw new Error("Local worker pool is closed");
61890
+ }
61891
+ }
61892
+ if (this.options.admission)
61893
+ this.admission = await SqlResourceAdmission.open(this.journal, {
61894
+ ...this.options.admission,
61895
+ root,
61896
+ holder: "coordinator"
61897
+ });
61898
+ this.logDirectory = workerLogDirectory(this.options.journalPath, root);
61899
+ await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
61900
+ return;
61901
+ });
61902
+ this.recoveryTimer = setInterval(() => {
61903
+ this.recoverExpired().catch((error) => this.fail(error));
61904
+ }, RECOVERY_INTERVAL_MS).unref();
61905
+ const workers = Array.from({ length: this.options.workers }, () => this.spawn());
61906
+ try {
61907
+ await Promise.any(workers);
61908
+ } catch (error) {
61909
+ throw error.errors[0];
61910
+ }
61911
+ }
61912
+ async enqueue(command) {
61913
+ if (!this.queue || !this.lease || this.root === undefined)
61914
+ throw new Error("Local worker pool requires root preparation");
61915
+ if (command.rootRun !== undefined)
61916
+ this.routes?.updateRoot(command.rootRun);
61917
+ const work = await this.queue.enqueue(this.lease, command.descriptor, command.boundary);
61918
+ const result = await this.queue.result(this.root, work.id);
61919
+ return { work, ...result === undefined ? {} : { result } };
61920
+ }
61921
+ async transport(request, payload) {
61922
+ const target = this.deliveryRequests.get(request);
61923
+ if (target === undefined || !this.incarnations.has(target.slot.worker))
61924
+ throw new Error("Worker exited before delivery acknowledgement");
61925
+ const { slot, workerRequest } = target;
61926
+ await new Promise((resolve, reject) => {
61927
+ const id = randomUUID15();
61928
+ slot.deliveries.set(id, { resolve, reject });
61929
+ slot.child.send({ id: workerRequest, delivery: { id, from: payload.from, message: payload.message } }, (error) => {
61930
+ if (error) {
61931
+ slot.deliveries.delete(id);
61932
+ reject(error);
61933
+ }
61934
+ });
61935
+ });
61936
+ }
61937
+ request(body) {
61938
+ const id = ++this.requestSequence;
61939
+ return new Promise((resolve, reject) => {
61940
+ this.requests.set(id, { resolve, reject });
61941
+ this.port.send({ kind: "request", id, request: body });
61942
+ });
61943
+ }
61944
+ fail(error) {
61945
+ this.port.send({ kind: "failure", error: error.message });
61946
+ }
61947
+ publish(work, runs) {
61948
+ const failure = this.publicationFailures.get(work.id);
61949
+ if (failure?.generation === work.generation)
61950
+ throw new Error(failure.error);
61951
+ if (!this.publishing)
61952
+ return;
61953
+ this.pendingActivity.set(work.id, { work: token(work), runs });
61954
+ this.scheduleFlush();
61955
+ }
61956
+ scheduleFlush() {
61957
+ if (this.flushTimer !== undefined || this.closed)
61958
+ return;
61959
+ const wait = Math.max(0, this.lastFlushAt + POOL_SUMMARY_INTERVAL_MS - performance.now());
61960
+ this.flushTimer = setTimeout(() => {
61961
+ this.flushTimer = undefined;
61962
+ if (this.batches.size > 0) {
61963
+ this.lastFlushAt = performance.now();
61964
+ this.scheduleFlush();
61965
+ return;
61966
+ }
61967
+ this.flush();
61968
+ }, wait);
61969
+ }
61970
+ flush() {
61971
+ clearTimeout(this.flushTimer);
61972
+ this.flushTimer = undefined;
61973
+ if (this.pendingActivity.size === 0 && !this.usageChanged) {
61974
+ const last = [...this.batches.values()].at(-1);
61975
+ return last === undefined ? Promise.resolve() : new Promise((resolve) => this.chainSettle(last, resolve));
61976
+ }
61977
+ this.lastFlushAt = performance.now();
61978
+ const batch = ++this.batchSequence;
61979
+ const entries = [...this.pendingActivity.values()];
61980
+ this.pendingActivity.clear();
61981
+ const usage = this.usageChanged ? this.usageTotal : undefined;
61982
+ this.usageChanged = false;
61983
+ const settled = new Promise((resolve) => this.batches.set(batch, { entries, settle: resolve }));
61984
+ this.port.send({ kind: "activity", batch, entries, ...usage === undefined ? {} : { usage } });
61985
+ return settled;
61986
+ }
61987
+ chainSettle(batch, resolve) {
61988
+ const settle = batch.settle;
61989
+ batch.settle = () => {
61990
+ settle();
61991
+ resolve();
61992
+ };
61993
+ }
61994
+ noteUsage(usage) {
61995
+ this.usageTotal = usage;
61996
+ this.usageChanged = true;
61997
+ this.scheduleFlush();
61998
+ }
61999
+ async sendResult(workId) {
62000
+ const result = await this.queue.result(this.root, workId);
62001
+ if (result === undefined)
62002
+ return false;
62003
+ this.flush();
62004
+ this.port.send({
62005
+ kind: "result",
62006
+ workId,
62007
+ result,
62008
+ ...this.usageTotal === undefined ? {} : { usage: this.usageTotal }
62009
+ });
62010
+ return true;
62011
+ }
62012
+ fence(work) {
62013
+ this.pendingActivity.delete(work.id);
62014
+ this.routes?.fence(work);
62015
+ this.port.send({ kind: "fence", work: { id: work.id, generation: work.generation } });
62016
+ }
62017
+ spawn() {
62018
+ if (this.closed)
62019
+ return Promise.reject(new Error("Local worker pool is closed"));
62020
+ const root = this.root;
62021
+ const options = this.options;
62022
+ const child = fork(options.workerEntry ?? process.argv[1], [
62023
+ JSON.stringify({
62024
+ journalPath: options.journalPath,
62025
+ root,
62026
+ definitions: this.sources,
62027
+ admission: options.admission
62028
+ })
62029
+ ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62030
+ const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62031
+ this.children.add(child);
62032
+ if (child.pid !== undefined)
62033
+ this.port.send({ kind: "spawned", pid: child.pid });
62034
+ const worker = randomUUID15();
62035
+ this.incarnations.set(worker, child);
62036
+ const slot = { worker, child, deliveries: new Map };
62037
+ let ready = false;
62038
+ let serial = Promise.resolve();
62039
+ const logPath = join28(this.logDirectory, `${worker}.log`);
62040
+ this.logs.set(child, logPath);
62041
+ let logFailure;
62042
+ const log = createWriteStream(logPath, { flags: "a" });
62043
+ log.on("error", (error) => {
62044
+ logFailure = error.message;
62045
+ });
62046
+ let stderr = "";
62047
+ child.stderr?.on("data", (chunk) => {
62048
+ stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62049
+ });
62050
+ child.stderr?.on("error", (error) => {
62051
+ logFailure ??= error.message;
62052
+ log.end();
62053
+ });
62054
+ child.stderr?.pipe(log);
62055
+ const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62056
+ const startupTimeoutMs = options.startupTimeoutMs;
62057
+ let timedOut = false;
62058
+ let terminated = false;
62059
+ const terminate = () => {
62060
+ if (terminated)
62061
+ return;
62062
+ terminated = true;
62063
+ child.kill("SIGKILL");
62064
+ };
62065
+ const replied = (error) => {
62066
+ if (error)
62067
+ terminate();
62068
+ };
62069
+ return new Promise((resolve, reject) => {
62070
+ const startup = setTimeout(() => {
62071
+ timedOut = true;
62072
+ terminate();
62073
+ }, startupTimeoutMs);
62074
+ child.on("error", (error) => {
62075
+ clearTimeout(startup);
62076
+ reject(error);
62077
+ terminate();
62078
+ });
62079
+ const closed = new Promise((resolve) => child.once("close", () => resolve()));
62080
+ child.once("exit", () => {
62081
+ Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62082
+ this.children.delete(child);
62083
+ this.logs.delete(child);
62084
+ if (child.pid !== undefined)
62085
+ this.port.send({ kind: "exited", pid: child.pid });
62086
+ clearTimeout(startup);
62087
+ this.incarnations.delete(worker);
62088
+ this.readyWorkers.delete(worker);
62089
+ this.controlAcks.delete(worker);
62090
+ this.lastHeard.delete(worker);
62091
+ if (slot.work)
62092
+ this.fence(slot.work);
62093
+ for (const delivery of slot.deliveries.values())
62094
+ delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62095
+ slot.deliveries.clear();
62096
+ const cleanup = disconnected.then(() => serial).then(async () => {
62097
+ try {
62098
+ if (await this.admission?.releaseStoppedSessions(worker))
62099
+ throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62100
+ } catch (error) {
62101
+ this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62102
+ throw this.cleanupFailure;
62103
+ }
62104
+ }).then(async () => {
62105
+ if (this.closed || !ready || this.desiredControl === "quit")
62106
+ return;
62107
+ let work = slot.work;
62108
+ if (work && await this.sendResult(work.id)) {
62109
+ work = undefined;
62110
+ slot.work = undefined;
62111
+ this.replacements = 0;
62112
+ }
62113
+ if (this.closed)
62114
+ return;
62115
+ if (work)
62116
+ await this.queue.releaseExited(this.lease, work);
62117
+ const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62118
+ if (exhausted)
62119
+ this.fail(diagnosed("Local worker retry limit"));
62120
+ else
62121
+ await this.spawn();
62122
+ }).catch((error) => {
62123
+ this.fail(error);
62124
+ });
62125
+ this.cleanups.add(cleanup);
62126
+ cleanup.finally(() => this.cleanups.delete(cleanup));
62127
+ });
62128
+ child.on("message", (message) => {
62129
+ this.observeResponsiveness();
62130
+ this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
62131
+ if (message.kind === "delivery-ack") {
62132
+ const pending = slot.deliveries.get(message.deliveryId);
62133
+ slot.deliveries.delete(message.deliveryId);
62134
+ if (message.error)
62135
+ pending?.reject(new Error(message.error));
62136
+ else
62137
+ pending?.resolve();
62138
+ return;
62139
+ }
62140
+ let routeReady;
62141
+ const handled = serial.then(async () => {
62142
+ if (message.kind === "ready") {
62143
+ if (terminated)
62144
+ return;
62145
+ ready = true;
62146
+ this.readyWorkers.add(worker);
62147
+ clearTimeout(startup);
62148
+ resolve();
62149
+ return;
62150
+ }
62151
+ await this.starting;
62152
+ const queue = this.queue;
62153
+ const lease = this.lease;
62154
+ if (message.kind === "control")
62155
+ return queue.control(lease);
62156
+ if (message.kind === "controlled") {
62157
+ const control = await queue.control(lease);
62158
+ if (message.revision > control.revision)
62159
+ throw new Error("Unknown control revision");
62160
+ this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62161
+ return;
62162
+ }
62163
+ if (message.kind === "claim") {
62164
+ if (slot.work)
62165
+ throw new Error("Worker already holds an attempt");
62166
+ if (this.budgetBlocked)
62167
+ return;
62168
+ await this.recoverExpired();
62169
+ slot.work = await queue.claim(lease, worker, this.sources.map((source) => ({
62170
+ definition: `${source.path}#${source.exportName}`,
62171
+ digest: source.digest
62172
+ })), options.leaseMs);
62173
+ return slot.work;
62174
+ }
62175
+ let work = slot.work;
62176
+ if (!work)
62177
+ throw new Error("Worker has no active attempt");
62178
+ if (message.kind === "renew" || work.expiresAt - Date.now() < options.leaseMs / 2) {
62179
+ work = await queue.extend(lease, work, options.leaseMs);
62180
+ slot.work = work;
62181
+ }
62182
+ if (message.kind === "activity")
62183
+ return this.publish(work, message.runs);
62184
+ if (message.kind === "route") {
62185
+ if (this.routes === undefined)
62186
+ return;
62187
+ await this.flush();
62188
+ this.assertPublished(work);
62189
+ routeReady = this.routes.route(work, message.runId);
62190
+ return;
62191
+ }
62192
+ if (message.kind === "delivery-ready") {
62193
+ if (!this.publishing)
62194
+ return;
62195
+ await this.flush();
62196
+ this.assertPublished(work);
62197
+ return this.request({
62198
+ kind: "ready",
62199
+ work: token(work),
62200
+ runId: message.runId,
62201
+ stageId: message.stageId,
62202
+ stageName: message.stageName,
62203
+ ...message.sessionId === undefined ? {} : { sessionId: message.sessionId }
62204
+ });
62205
+ }
62206
+ if (message.kind === "delivery") {
62207
+ if (!message.recipient)
62208
+ throw new Error("Worker delivery requires a session identity");
62209
+ if (!this.publishing)
62210
+ return;
62211
+ await this.flush();
62212
+ this.assertPublished(work);
62213
+ const id = ++this.requestSequence;
62214
+ this.deliveryRequests.set(id, { slot, workerRequest: message.id });
62215
+ try {
62216
+ return await new Promise((resolve, reject) => {
62217
+ this.requests.set(id, { resolve, reject });
62218
+ this.port.send({
62219
+ kind: "request",
62220
+ id,
62221
+ request: {
62222
+ kind: "deliver",
62223
+ work: token(work),
62224
+ runId: message.runId,
62225
+ stageId: message.stageId,
62226
+ stageName: message.stageName,
62227
+ recipient: message.recipient
62228
+ }
62229
+ });
62230
+ });
62231
+ } finally {
62232
+ this.deliveryRequests.delete(id);
62233
+ }
62234
+ }
62235
+ if (message.kind === "renew")
62236
+ return;
62237
+ if (message.kind === "commit") {
62238
+ const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62239
+ this.noteUsage(await this.usageLedger.total(root));
62240
+ return receipt;
62241
+ }
62242
+ if (message.kind === "usage") {
62243
+ this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
62244
+ return;
62245
+ }
62246
+ if (message.kind === "budget") {
62247
+ await queue.assertCurrent(lease, work);
62248
+ if (this.budgetBlocked)
62249
+ throw new Error("Root workflow budget exhausted");
62250
+ return queue.control(lease);
62251
+ }
62252
+ if (message.kind !== "complete")
62253
+ throw new Error("Invalid worker operation");
62254
+ await queue.complete(lease, work, message.result);
62255
+ this.replacements = 0;
62256
+ slot.work = undefined;
62257
+ await this.sendResult(work.id);
62258
+ });
62259
+ const response = handled.then((value2) => routeReady ?? value2);
62260
+ if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62261
+ serial = handled.then(() => {}, () => {});
62262
+ response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62263
+ });
62264
+ });
62265
+ }
62266
+ assertPublished(work) {
62267
+ const failure = this.publicationFailures.get(work.id);
62268
+ if (failure?.generation === work.generation)
62269
+ throw new Error(failure.error);
62270
+ }
62271
+ recoverExpired() {
62272
+ if (this.recovering)
62273
+ return this.recovering;
62274
+ const root = this.root;
62275
+ if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
62276
+ return Promise.resolve();
62277
+ const queue = this.queue;
62278
+ const lease = this.lease;
62279
+ const leaseMs = this.options.leaseMs;
62280
+ this.recovering = (async () => {
62281
+ await new Promise((resolve) => setImmediate(resolve));
62282
+ await this.admission?.reconcile();
62283
+ const expiredBefore = Date.now();
62284
+ for (const work of await queue.inspect(root)) {
62285
+ if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62286
+ continue;
62287
+ const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62288
+ if (child && work.worker !== null && this.isLive(work.worker, child)) {
62289
+ await queue.extend(lease, work, leaseMs).catch((error) => {
62290
+ if (error.message !== STALE_WORK_LEASE)
62291
+ throw error;
62292
+ });
62293
+ continue;
62294
+ }
62295
+ if (child && !await this.stop(child).then(() => true, () => false))
62296
+ continue;
62297
+ await queue.expire(lease, work);
62298
+ }
62299
+ })().finally(() => {
62300
+ this.recovering = undefined;
62301
+ this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62302
+ });
62303
+ return this.recovering;
62304
+ }
62305
+ observeResponsiveness() {
62306
+ const now = performance.now();
62307
+ const gap = now - this.lastTick;
62308
+ if (gap > STALL_THRESHOLD_MS)
62309
+ this.stalledMs += gap - TICK_INTERVAL_MS;
62310
+ this.lastTick = now;
62311
+ }
62312
+ isLive(worker, child) {
62313
+ this.observeResponsiveness();
62314
+ const heard = this.lastHeard.get(worker);
62315
+ return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
62316
+ }
62317
+ stop(child) {
62318
+ if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62319
+ return Promise.resolve();
62320
+ return new Promise((resolve, reject) => {
62321
+ const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62322
+ child.once("exit", () => {
62323
+ clearTimeout(timeout);
62324
+ resolve();
62325
+ });
62326
+ child.kill("SIGKILL");
62327
+ });
62328
+ }
62329
+ async close() {
62330
+ this.closed = true;
62331
+ clearInterval(this.ticker);
62332
+ clearInterval(this.recoveryTimer);
62333
+ let failure;
62334
+ let stopping = [];
62335
+ try {
62336
+ await this.recovering?.catch(() => {});
62337
+ stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62338
+ await Promise.all(stopping.map(({ child }) => this.stop(child)));
62339
+ await Promise.all([...this.cleanups]);
62340
+ if (this.cleanupFailure)
62341
+ throw this.cleanupFailure;
62342
+ } catch (error) {
62343
+ failure = error;
62344
+ }
62345
+ if (failure === undefined) {
62346
+ const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62347
+ await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62348
+ if (this.logDirectory !== undefined)
62349
+ await rmdir(this.logDirectory).catch(() => {});
62350
+ }
62351
+ this.flush();
62352
+ try {
62353
+ await this.routes?.close();
62354
+ } catch (error) {
62355
+ failure ??= error;
62356
+ }
62357
+ for (const batch of this.batches.values())
62358
+ batch.settle();
62359
+ this.batches.clear();
62360
+ for (const pending of this.requests.values())
62361
+ pending.reject(new Error("Local worker pool is closed"));
62362
+ this.requests.clear();
62363
+ if (failure !== undefined)
62364
+ throw failure;
62365
+ }
62366
+ }
62367
+ var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
62368
+ var init_pool_coordinator = __esm(() => {
62369
+ init_resolution();
62370
+ init_sql_pool_queue();
62371
+ init_sql_pool_usage();
62372
+ init_pending_stage_intercom();
62373
+ init_intercom_group();
62374
+ init_pending_stage_route_capability();
62375
+ init_pending_stage_route_readiness();
62376
+ init_store();
62377
+ init_resource_admission();
62378
+ COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
62379
+ });
62380
+
61322
62381
  // dist/builtin/workflows/src/workers/protocol.ts
61323
62382
  var WORKER_HEARTBEAT_INTERVAL_MS = 1000;
61324
62383
 
@@ -61566,21 +62625,26 @@ class PoolStageHandoff {
61566
62625
  store;
61567
62626
  queue;
61568
62627
  lease;
62628
+ claimRun;
61569
62629
  attempts = new Map;
61570
62630
  owners = new Map;
61571
62631
  publications = new Map;
61572
62632
  drains = new Map;
61573
- constructor(backend, store, queue, lease) {
62633
+ constructor(backend, store, queue, lease, claimRun) {
61574
62634
  this.backend = backend;
61575
62635
  this.store = store;
61576
62636
  this.queue = queue;
61577
62637
  this.lease = lease;
62638
+ this.claimRun = claimRun;
61578
62639
  }
61579
62640
  publish(work, runs) {
61580
62641
  const previous = this.publications.get(work.id) ?? Promise.resolve();
61581
62642
  const publication = previous.catch(() => {
61582
62643
  return;
61583
- }).then(() => this.apply(work, runs));
62644
+ }).then(async () => {
62645
+ const commit = await this.prepare(work, runs);
62646
+ this.store.batch(commit);
62647
+ });
61584
62648
  const settled = publication.catch(() => {
61585
62649
  return;
61586
62650
  });
@@ -61591,6 +62655,21 @@ class PoolStageHandoff {
61591
62655
  });
61592
62656
  return publication;
61593
62657
  }
62658
+ async publishBatch(publications) {
62659
+ const prepared = [];
62660
+ for (const { work, runs } of publications)
62661
+ prepared.push(await this.prepare(work, runs).catch((error) => error));
62662
+ return this.store.batch(() => prepared.map((commit) => {
62663
+ if (commit instanceof Error)
62664
+ return commit;
62665
+ try {
62666
+ commit();
62667
+ return;
62668
+ } catch (error) {
62669
+ return error instanceof Error ? error : new Error(String(error));
62670
+ }
62671
+ }));
62672
+ }
61594
62673
  async route(work, runId) {
61595
62674
  this.requireOwned(await this.begin(work), runId);
61596
62675
  const readiness = workflowPendingStageRouteReady(this.store, runId);
@@ -61641,7 +62720,7 @@ class PoolStageHandoff {
61641
62720
  const result = await this.queue.result(root, work.id);
61642
62721
  if (result === undefined)
61643
62722
  return;
61644
- await this.backend.hydrateWorkflow(root);
62723
+ await this.backend.catchUpWorkflow(root);
61645
62724
  const terminals = new Map;
61646
62725
  for (const checkpoint of this.backend.listCheckpoints(root)) {
61647
62726
  const boundary = checkpoint.kind === "stage" ? checkpoint.topology?.boundary : undefined;
@@ -61651,6 +62730,9 @@ class PoolStageHandoff {
61651
62730
  if (child?.runId === boundary.child.runId)
61652
62731
  terminals.set(child.runId, { status: child.status, outputs: child.outputs });
61653
62732
  }
62733
+ this.store.batch(() => this.settle(work, result, terminals));
62734
+ }
62735
+ settle(work, result, terminals) {
61654
62736
  for (const run of this.store.runs()) {
61655
62737
  if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
61656
62738
  continue;
@@ -61674,22 +62756,31 @@ class PoolStageHandoff {
61674
62756
  if (current !== undefined && current.generation > work.generation)
61675
62757
  return;
61676
62758
  const attempt = current?.generation === work.generation ? current : this.start(work, current);
61677
- this.release(attempt);
62759
+ this.store.batch(() => this.release(attempt));
61678
62760
  attempt.fenced = true;
61679
62761
  }
61680
- async apply(work, runs) {
62762
+ async prepare(work, runs) {
61681
62763
  const subtree = this.subtree(work, runs);
61682
62764
  const root = this.lease.runId;
61683
- const known = new Map(this.store.runs().map((run) => [run.id, run]));
61684
- for (const run of subtree) {
61685
- const existing = known.get(run.id);
61686
- const owner = this.owners.get(run.id);
61687
- if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
61688
- throw outside(run.id);
61689
- }
61690
- if (subtree.some((run) => known.get(run.id) === undefined))
61691
- await this.backend.hydrateWorkflow(root);
62765
+ const owned = (known) => {
62766
+ for (const run of subtree) {
62767
+ const existing = known.get(run.id);
62768
+ const owner = this.owners.get(run.id);
62769
+ if (owner !== undefined && owner !== work.id || owner === undefined && existing !== undefined && (existing.parentRunId !== run.parentRunId || existing.parentStageId !== run.parentStageId))
62770
+ throw outside2(run.id);
62771
+ }
62772
+ };
62773
+ const before = new Map(this.store.runs().map((run) => [run.id, run]));
62774
+ owned(before);
62775
+ if (subtree.some((run) => before.get(run.id) === undefined))
62776
+ await this.backend.catchUpWorkflow(root);
61692
62777
  await this.queue.assertCurrent(this.lease, work);
62778
+ return () => this.commit(work, subtree, owned);
62779
+ }
62780
+ commit(work, subtree, owned) {
62781
+ const root = this.lease.runId;
62782
+ const known = new Map(this.store.runs().map((run) => [run.id, run]));
62783
+ owned(known);
61693
62784
  const attempt = this.enter(work);
61694
62785
  const execution = {
61695
62786
  worker: work.worker ?? "",
@@ -61701,11 +62792,13 @@ class PoolStageHandoff {
61701
62792
  ...run,
61702
62793
  rootRunId: root,
61703
62794
  pendingStageMessages: [
61704
- ...this.store.runs().find((current) => current.id === run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
62795
+ ...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
61705
62796
  ],
61706
62797
  stages: run.stages.map((stage) => ({ ...stage, workerExecution: execution }))
61707
62798
  }));
61708
62799
  const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
62800
+ for (const run of published)
62801
+ this.claimRun?.(run.id);
61709
62802
  for (const run of published) {
61710
62803
  attempt.runs.add(run.id);
61711
62804
  this.owners.set(run.id, work.id);
@@ -61730,7 +62823,7 @@ class PoolStageHandoff {
61730
62823
  const byId = new Map;
61731
62824
  for (const run of runs) {
61732
62825
  if (run.id === root || byId.has(run.id))
61733
- throw outside(run.id);
62826
+ throw outside2(run.id);
61734
62827
  byId.set(run.id, run);
61735
62828
  }
61736
62829
  if (runs.length === 0)
@@ -61743,25 +62836,25 @@ class PoolStageHandoff {
61743
62836
  const top = byId.get(work.childRunId);
61744
62837
  if (top !== undefined) {
61745
62838
  if (top.parentRunId !== root || top.parentStageId !== work.boundaryId)
61746
- throw outside(top.id);
62839
+ throw outside2(top.id);
61747
62840
  const pending = [top];
61748
62841
  for (let run = pending.shift();run !== undefined; run = pending.shift()) {
61749
62842
  if (run.rootRunId !== undefined && run.rootRunId !== root)
61750
- throw outside(run.id);
62843
+ throw outside2(run.id);
61751
62844
  ordered.push(run);
61752
62845
  for (const child of runs) {
61753
62846
  if (child.parentRunId !== run.id)
61754
62847
  continue;
61755
62848
  const parentStage = run.stages.find((stage) => stage.id === child.parentStageId);
61756
62849
  if (authoritativeWorkflowChildRunId(parentStage) !== child.id)
61757
- throw outside(child.id);
62850
+ throw outside2(child.id);
61758
62851
  pending.push(child);
61759
62852
  }
61760
62853
  }
61761
62854
  }
61762
62855
  const stray = runs.find((run) => !ordered.includes(run));
61763
62856
  if (stray !== undefined)
61764
- throw outside(stray.id);
62857
+ throw outside2(stray.id);
61765
62858
  return ordered;
61766
62859
  }
61767
62860
  async begin(work) {
@@ -61816,7 +62909,7 @@ class PoolStageHandoff {
61816
62909
  }
61817
62910
  requireOwned(attempt, runId) {
61818
62911
  if (!attempt.runs.has(runId))
61819
- throw outside(runId);
62912
+ throw outside2(runId);
61820
62913
  }
61821
62914
  delivery(attempt, work, runId, stageId, stageName) {
61822
62915
  const key = JSON.stringify([runId, stageId]);
@@ -61834,7 +62927,7 @@ class PoolStageHandoff {
61834
62927
  return delivery;
61835
62928
  }
61836
62929
  }
61837
- function outside(runId) {
62930
+ function outside2(runId) {
61838
62931
  return new Error(`Pool stage publication is outside the work subtree: ${runId}`);
61839
62932
  }
61840
62933
  function isLive(stage) {
@@ -61859,33 +62952,122 @@ var init_stage_handoff = __esm(() => {
61859
62952
  });
61860
62953
 
61861
62954
  // dist/builtin/workflows/src/workers/local-pool.ts
61862
- import { fork } from "node:child_process";
61863
- import { randomUUID as randomUUID15 } from "node:crypto";
61864
- import { createWriteStream } from "node:fs";
61865
- import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
61866
- import { dirname as dirname12, join as join28, resolve as resolve11 } from "node:path";
61867
- import { setTimeout as delay2 } from "node:timers/promises";
62955
+ import { fork as fork2 } from "node:child_process";
62956
+ import { resolve as resolve12 } from "node:path";
62957
+ import { setTimeout as delay3 } from "node:timers/promises";
61868
62958
  import { fileURLToPath as fileURLToPath5 } from "node:url";
61869
62959
  import { isBunBinary as isBunBinary2 } from "@bastani/atomic";
61870
- function workerLogDirectory(journalPath, root) {
61871
- return join28(dirname12(resolve11(journalPath)), "worker-logs", encodeURIComponent(root).replaceAll(".", "%2E"));
62960
+ function defaultWorkerEntry() {
62961
+ return isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url));
62962
+ }
62963
+ function launchCoordinatorProcess() {
62964
+ const child = fork2(defaultWorkerEntry(), [JSON.stringify({ role: POOL_COORDINATOR_ROLE })], {
62965
+ stdio: ["ignore", "ignore", "pipe", "ipc"],
62966
+ execArgv: []
62967
+ });
62968
+ let stderr = "";
62969
+ child.stderr?.on("data", (chunk) => {
62970
+ stderr = (stderr + chunk.toString()).slice(-COORDINATOR_STDERR_TAIL_CHARACTERS);
62971
+ });
62972
+ const listeners = [];
62973
+ let lostReason;
62974
+ const lose = (reason) => {
62975
+ if (lostReason !== undefined)
62976
+ return;
62977
+ lostReason = reason;
62978
+ for (const listener of listeners)
62979
+ listener(reason);
62980
+ };
62981
+ const exited = new Promise((resolve) => child.once("exit", () => resolve()));
62982
+ child.on("error", lose);
62983
+ child.once("exit", (code, signal) => lose(new Error(`Local worker pool coordinator exited (${signal ?? `code ${code}`})${stderr ? `: ${stderr.trim()}` : ""}`)));
62984
+ return {
62985
+ pid: child.pid,
62986
+ port: {
62987
+ send(message) {
62988
+ if (lostReason !== undefined || !child.connected)
62989
+ return;
62990
+ child.send(message, (error) => {
62991
+ if (error)
62992
+ lose(error);
62993
+ });
62994
+ },
62995
+ onMessage(listener) {
62996
+ child.on("message", (message) => listener(message));
62997
+ },
62998
+ onClose(listener) {
62999
+ if (lostReason !== undefined)
63000
+ listener(lostReason);
63001
+ else
63002
+ listeners.push(listener);
63003
+ },
63004
+ async close() {
63005
+ if (child.connected)
63006
+ child.disconnect();
63007
+ if (child.exitCode !== null || child.signalCode !== null)
63008
+ return;
63009
+ const timedOut = await Promise.race([exited.then(() => false), delay3(COORDINATOR_EXIT_GRACE_MS, true)]);
63010
+ if (timedOut) {
63011
+ child.kill("SIGKILL");
63012
+ await exited;
63013
+ }
63014
+ }
63015
+ }
63016
+ };
63017
+ }
63018
+ function launchCoordinatorInProcess() {
63019
+ const mainListeners = [];
63020
+ const coordinatorListeners = [];
63021
+ const copy = (message) => JSON.parse(JSON.stringify(message));
63022
+ const coordinator = new PoolCoordinator({
63023
+ send(message) {
63024
+ const copied = copy(message);
63025
+ setImmediate(() => {
63026
+ for (const listener of mainListeners)
63027
+ listener(copied);
63028
+ });
63029
+ },
63030
+ onMessage(listener) {
63031
+ coordinatorListeners.push(listener);
63032
+ },
63033
+ onClose() {},
63034
+ async close() {}
63035
+ });
63036
+ return {
63037
+ pid: process.pid,
63038
+ port: {
63039
+ send(message) {
63040
+ const copied = copy(message);
63041
+ setImmediate(() => {
63042
+ for (const listener of coordinatorListeners)
63043
+ listener(copied);
63044
+ });
63045
+ },
63046
+ onMessage(listener) {
63047
+ mainListeners.push(listener);
63048
+ },
63049
+ onClose() {},
63050
+ async close() {
63051
+ coordinator.shutdown();
63052
+ }
63053
+ }
63054
+ };
61872
63055
  }
61873
63056
 
61874
63057
  class LocalWorkerPool {
61875
63058
  backend;
61876
63059
  options;
61877
- children = new Set;
61878
- incarnations = new Map;
61879
- readyWorkers = new Set;
61880
- logs = new Map;
61881
- logDirectory;
61882
- cleanups = new Set;
61883
- recovering;
61884
- nextRecoveryAt = 0;
61885
- controlAcks = new Map;
61886
- lastHeard = new Map;
61887
- lastTick = performance.now();
61888
- rootStalledMs = 0;
63060
+ pids = new Set;
63061
+ coordinator;
63062
+ coordinatorLost;
63063
+ commands = new Map;
63064
+ commandSequence = 0;
63065
+ inbound = Promise.resolve();
63066
+ works = new Map;
63067
+ attempts = new Map;
63068
+ results = new Map;
63069
+ changed = Promise.withResolvers();
63070
+ waiting = 0;
61889
63071
  ticker;
61890
63072
  leaseMs;
61891
63073
  desiredControl;
@@ -61893,18 +63075,17 @@ class LocalWorkerPool {
61893
63075
  store;
61894
63076
  handoff;
61895
63077
  releaseRoutes;
63078
+ routeClaims = new Map;
61896
63079
  sources = [];
61897
63080
  queue;
61898
63081
  lease;
61899
- admission;
61900
- usageLedger;
61901
63082
  usageTotal;
61902
63083
  rootBudget;
63084
+ budgetBlocked = false;
63085
+ rootSignature;
61903
63086
  starting;
61904
- cleanupFailure;
61905
63087
  failure;
61906
63088
  closed = false;
61907
- replacements = 0;
61908
63089
  releaseBinding;
61909
63090
  constructor(backend, options) {
61910
63091
  this.backend = backend;
@@ -61919,10 +63100,12 @@ class LocalWorkerPool {
61919
63100
  this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
61920
63101
  if (options.admission && options.rootId === undefined)
61921
63102
  workflowResourceAdmission.set(backend, options.admission);
61922
- this.ticker = setInterval(() => this.observeResponsiveness(), ROOT_TICK_INTERVAL_MS).unref();
61923
63103
  }
61924
63104
  get workerPids() {
61925
- return [...this.children].flatMap((child) => child.pid === undefined ? [] : [child.pid]);
63105
+ return [...this.pids];
63106
+ }
63107
+ get coordinatorPid() {
63108
+ return this.coordinator?.pid;
61926
63109
  }
61927
63110
  usage() {
61928
63111
  return this.usageTotal;
@@ -61930,47 +63113,39 @@ class LocalWorkerPool {
61930
63113
  async prepare(root, store) {
61931
63114
  if (this.closed)
61932
63115
  throw new Error("Local worker pool is closed");
61933
- if (!(this.backend.journal instanceof SqlRunJournal) || resolve11(this.options.journalPath) !== resolve11(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
63116
+ if (!(this.backend.journal instanceof SqlRunJournal) || resolve12(this.options.journalPath) !== resolve12(this.backend.journal.path) || sqlPoolStorage(this.backend.journal)?.tables.runs !== "wf_runs")
61934
63117
  throw new Error("Local workers require the same default-prefix SQLite journal");
61935
63118
  if (this.root !== undefined && this.root !== root)
61936
63119
  throw new Error("Local worker pool belongs to another root");
61937
63120
  this.root = root;
61938
63121
  if (store)
61939
63122
  this.store = store;
61940
- this.usageLedger ??= await openSqlPoolUsage(this.backend.journal);
61941
- this.usageTotal = await this.usageLedger.total(root);
61942
63123
  const state = await this.backend.journal.leaseState(root);
61943
63124
  if (!state || state.holderId !== this.backend.holderId)
61944
63125
  throw new Error("Local worker pool requires the root owner");
61945
63126
  this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
61946
- const queue = await openSqlPoolQueue(this.backend.journal);
61947
- await queue.initialize(this.lease, COORDINATOR_LEASE_MS);
61948
- await queue.releasePreviousIncarnations(this.lease);
61949
- for (;; ) {
61950
- const intended = this.desiredControl ?? "running";
61951
- await queue.setControl(this.lease, intended);
61952
- if (intended === (this.desiredControl ?? "running"))
61953
- break;
61954
- }
61955
- this.queue = queue;
63127
+ const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
63128
+ this.usageTotal = await this.command({
63129
+ kind: "prepare",
63130
+ root,
63131
+ lease: this.lease,
63132
+ options: {
63133
+ journalPath: this.options.journalPath,
63134
+ workers: this.options.workers,
63135
+ leaseMs: this.leaseMs,
63136
+ startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
63137
+ ...this.options.admission === undefined ? {} : { admission: this.options.admission },
63138
+ ...workerEntry === undefined ? {} : { workerEntry }
63139
+ },
63140
+ ...this.desiredControl === undefined ? {} : { control: this.desiredControl }
63141
+ });
63142
+ this.queue ??= await openSqlPoolQueue(this.backend.journal);
61956
63143
  }
61957
63144
  async control(state) {
61958
63145
  this.desiredControl = state;
61959
63146
  try {
61960
- if (!this.queue || !this.lease)
61961
- return;
61962
- const control = await this.queue.setControl(this.lease, state);
61963
- const deadline = Date.now() + CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS;
61964
- while ([...this.readyWorkers].some((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision)) {
61965
- if (this.desiredControl !== state)
61966
- return;
61967
- if (Date.now() >= deadline) {
61968
- if (state === "quit")
61969
- break;
61970
- throw new Error("Worker control acknowledgement unresolved");
61971
- }
61972
- await delay2(25);
61973
- }
63147
+ if (this.coordinator !== undefined)
63148
+ await this.command({ kind: "control", state });
61974
63149
  } finally {
61975
63150
  if (state === "quit")
61976
63151
  await this.close();
@@ -62016,290 +63191,61 @@ class LocalWorkerPool {
62016
63191
  throw new Error(`Worker definition is not importable: ${binding.exportName}`);
62017
63192
  return source;
62018
63193
  }));
62019
- if (this.store) {
62020
- this.releaseRoutes = await acquireStandaloneIntercom(this.store, root, String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd()));
63194
+ const cwd = String(this.backend.getWorkflow(root)?.invocationCwd ?? process.cwd());
63195
+ const store = this.store;
63196
+ if (store) {
63197
+ this.releaseRoutes = await acquireStandaloneIntercom(store, root, cwd);
62021
63198
  if (this.closed) {
62022
63199
  await this.releaseRoutes();
62023
63200
  this.releaseRoutes = undefined;
62024
63201
  throw new Error("Local worker pool is closed");
62025
63202
  }
62026
- this.handoff = new PoolStageHandoff(this.backend, this.store, this.queue, this.lease);
62027
- }
62028
- if (this.options.admission)
62029
- this.admission = await SqlResourceAdmission.open(this.backend.journal, {
62030
- ...this.options.admission,
62031
- root,
62032
- holder: "coordinator"
63203
+ const routeOwner = {
63204
+ notifyUndeliverable: async (payload) => !this.closed && await this.command({ kind: "undeliverable", payload }) === true
63205
+ };
63206
+ this.handoff = new PoolStageHandoff(this.backend, store, this.queueView(), this.lease, (runId) => {
63207
+ this.routeClaims.set(runId, claimExternalRunRoute(store, runId, routeOwner));
62033
63208
  });
62034
- this.logDirectory = workerLogDirectory(this.options.journalPath, root);
62035
- await mkdir6(this.logDirectory, { recursive: true }).catch(() => {
62036
- return;
62037
- });
62038
- const workers = Array.from({ length: this.options.workers }, () => this.spawn(root));
62039
- try {
62040
- await Promise.any(workers);
62041
- } catch (error) {
62042
- throw error.errors[0];
62043
63209
  }
62044
- }
62045
- spawn(root) {
62046
- if (this.closed)
62047
- return Promise.reject(new Error("Local worker pool is closed"));
62048
- const child = fork(isBunBinary2 ? INTERNAL_WORKFLOW_WORKER_ARG : fileURLToPath5(new URL("./local-worker-entry.mjs", import.meta.url)), [
62049
- JSON.stringify({
62050
- journalPath: this.options.journalPath,
62051
- root,
62052
- definitions: this.sources,
62053
- admission: this.options.admission
62054
- })
62055
- ], { stdio: ["ignore", "ignore", "pipe", "ipc"], execArgv: [] });
62056
- const disconnected = new Promise((resolve) => child.once("disconnect", () => resolve()));
62057
- this.children.add(child);
62058
- const worker = randomUUID15();
62059
- this.incarnations.set(worker, child);
62060
- let ready = false;
62061
- let serial = Promise.resolve();
62062
- let work;
62063
- const deliveries = new Map;
62064
- const logPath = join28(this.logDirectory, `${worker}.log`);
62065
- this.logs.set(child, logPath);
62066
- let logFailure;
62067
- const log = createWriteStream(logPath, { flags: "a" });
62068
- log.on("error", (error) => {
62069
- logFailure = error.message;
62070
- });
62071
- let stderr = "";
62072
- child.stderr?.on("data", (chunk) => {
62073
- stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARACTERS);
62074
- });
62075
- child.stderr?.on("error", (error) => {
62076
- logFailure ??= error.message;
62077
- log.end();
62078
- });
62079
- child.stderr?.pipe(log);
62080
- const diagnosed = (summary) => new Error(`${summary} (worker stderr log: ${logFailure === undefined ? logPath : `unavailable, ${logFailure}`})${stderr ? `: ${stderr}` : ""}`);
62081
- const startupTimeoutMs = this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
62082
- let timedOut = false;
62083
- let terminated = false;
62084
- const terminate = () => {
62085
- if (terminated)
62086
- return;
62087
- terminated = true;
62088
- child.kill("SIGKILL");
62089
- };
62090
- const replied = (error) => {
62091
- if (error)
62092
- terminate();
62093
- };
62094
- return new Promise((resolve, reject) => {
62095
- const startup = setTimeout(() => {
62096
- timedOut = true;
62097
- terminate();
62098
- }, startupTimeoutMs);
62099
- child.on("error", (error) => {
62100
- clearTimeout(startup);
62101
- reject(error);
62102
- terminate();
62103
- });
62104
- const closed = new Promise((resolve) => child.once("close", () => resolve()));
62105
- child.once("exit", () => {
62106
- Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
62107
- this.children.delete(child);
62108
- this.logs.delete(child);
62109
- clearTimeout(startup);
62110
- this.incarnations.delete(worker);
62111
- this.readyWorkers.delete(worker);
62112
- this.controlAcks.delete(worker);
62113
- this.lastHeard.delete(worker);
62114
- if (work)
62115
- this.handoff?.fence(work);
62116
- for (const delivery of deliveries.values())
62117
- delivery.reject(new Error("Worker exited before delivery acknowledgement"));
62118
- deliveries.clear();
62119
- const cleanup = disconnected.then(() => serial).then(async () => {
62120
- try {
62121
- if (await this.admission?.releaseStoppedSessions(worker))
62122
- throw new Error("Local worker process cleanup unconfirmed; permits quarantined");
62123
- } catch (error) {
62124
- this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
62125
- throw this.cleanupFailure;
62126
- }
62127
- }).then(async () => {
62128
- if (this.closed || !ready || this.desiredControl === "quit")
62129
- return;
62130
- if (work && await this.queue.result(root, work.id)) {
62131
- work = undefined;
62132
- this.replacements = 0;
62133
- }
62134
- if (this.closed)
62135
- return;
62136
- if (work)
62137
- await this.queue.releaseExited(this.lease, work);
62138
- const exhausted = work ? work.generation >= 3 : ++this.replacements > 3;
62139
- if (exhausted)
62140
- this.failure = diagnosed("Local worker retry limit");
62141
- else
62142
- await this.spawn(root);
62143
- }).catch((error) => {
62144
- this.failure = error;
62145
- });
62146
- this.cleanups.add(cleanup);
62147
- cleanup.finally(() => this.cleanups.delete(cleanup));
62148
- });
62149
- child.on("message", (message) => {
62150
- this.observeResponsiveness();
62151
- this.lastHeard.set(worker, { at: performance.now(), rootStalled: this.rootStalledMs });
62152
- if (message.kind === "delivery-ack") {
62153
- const pending = deliveries.get(message.deliveryId);
62154
- deliveries.delete(message.deliveryId);
62155
- if (message.error)
62156
- pending?.reject(new Error(message.error));
62157
- else
62158
- pending?.resolve();
62159
- return;
62160
- }
62161
- const response = serial.then(async () => {
62162
- if (message.kind === "ready") {
62163
- if (terminated)
62164
- return;
62165
- ready = true;
62166
- this.readyWorkers.add(worker);
62167
- clearTimeout(startup);
62168
- resolve();
62169
- return;
62170
- }
62171
- await this.starting;
62172
- const queue = this.queue;
62173
- const lease = this.lease;
62174
- if (message.kind === "control")
62175
- return queue.control(lease);
62176
- if (message.kind === "controlled") {
62177
- const control = await queue.control(lease);
62178
- if (message.revision > control.revision)
62179
- throw new Error("Unknown control revision");
62180
- this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
62181
- return;
62182
- }
62183
- if (message.kind === "claim") {
62184
- if (work)
62185
- throw new Error("Worker already holds an attempt");
62186
- const decision = this.rootBudget?.checkpoint();
62187
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62188
- return;
62189
- await this.recoverExpired(root);
62190
- work = await queue.claim(lease, worker, this.sources.map((source) => ({
62191
- definition: `${source.path}#${source.exportName}`,
62192
- digest: source.digest
62193
- })), this.leaseMs);
62194
- return work;
62195
- }
62196
- if (!work)
62197
- throw new Error("Worker has no active attempt");
62198
- if (message.kind === "renew" || work.expiresAt - Date.now() < this.leaseMs / 2)
62199
- work = await queue.extend(lease, work, this.leaseMs);
62200
- if (message.kind === "activity")
62201
- return this.handoff?.publish(work, message.runs);
62202
- if (message.kind === "route")
62203
- return this.handoff?.route(work, message.runId);
62204
- if (message.kind === "delivery-ready")
62205
- return this.handoff?.ready(work, message.runId, message.stageId, message.stageName, message.sessionId);
62206
- if (message.kind === "delivery") {
62207
- if (!message.recipient)
62208
- throw new Error("Worker delivery requires a session identity");
62209
- return this.handoff?.deliver(work, message.runId, message.stageId, message.stageName, message.recipient, (from, payload) => new Promise((resolve, reject) => {
62210
- const id = randomUUID15();
62211
- deliveries.set(id, { resolve, reject });
62212
- child.send({ id: message.id, delivery: { id, from, message: payload } }, (error) => {
62213
- if (error) {
62214
- deliveries.delete(id);
62215
- reject(error);
62216
- }
62217
- });
62218
- }));
62219
- }
62220
- if (message.kind === "renew")
62221
- return;
62222
- if (message.kind === "commit") {
62223
- const receipt = await queue.commit(lease, work, message.operation, message.mutation);
62224
- this.usageTotal = await this.usageLedger.total(root);
62225
- return receipt;
62226
- }
62227
- if (message.kind === "usage") {
62228
- this.usageTotal = await this.usageLedger.report(lease, work, message.reports);
62229
- return;
62230
- }
62231
- if (message.kind === "budget") {
62232
- await queue.assertCurrent(lease, work);
62233
- const decision = this.rootBudget?.checkpoint();
62234
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62235
- throw new Error("Root workflow budget exhausted");
62236
- return queue.control(lease);
62237
- }
62238
- if (message.kind !== "complete")
62239
- throw new Error("Invalid worker operation");
62240
- await queue.complete(lease, work, message.result);
62241
- this.replacements = 0;
62242
- work = undefined;
62243
- });
62244
- if (message.kind !== "delivery" && message.kind !== "delivery-ready")
62245
- serial = response.then(() => {}, () => {});
62246
- response.then((value2) => child.send({ id: message.id, value: value2 }, replied), (error) => child.send({ id: message.id, error: error.message }, replied));
62247
- });
63210
+ await this.command({
63211
+ kind: "start",
63212
+ sources: this.sources,
63213
+ cwd,
63214
+ publish: store !== undefined,
63215
+ ...this.rootRun()
62248
63216
  });
62249
63217
  }
62250
- recoverExpired(root) {
62251
- if (this.recovering)
62252
- return this.recovering;
62253
- if (this.closed || performance.now() < this.nextRecoveryAt)
62254
- return Promise.resolve();
62255
- this.recovering = (async () => {
62256
- await new Promise((resolve) => setImmediate(resolve));
62257
- await this.admission?.reconcile();
62258
- const expiredBefore = Date.now();
62259
- for (const work of await this.queue.inspect(root)) {
62260
- if (work.state !== "leased" || work.expiresAt >= expiredBefore)
62261
- continue;
62262
- const child = work.worker === null ? undefined : this.incarnations.get(work.worker);
62263
- if (child && work.worker !== null && this.isLive(work.worker, child)) {
62264
- await this.queue.extend(this.lease, work, this.leaseMs).catch((error) => {
62265
- if (error.message !== STALE_WORK_LEASE)
62266
- throw error;
62267
- });
62268
- continue;
62269
- }
62270
- if (child && !await this.stop(child).then(() => true, () => false))
62271
- continue;
62272
- await this.queue.expire(this.lease, work);
62273
- }
62274
- })().finally(() => {
62275
- this.recovering = undefined;
62276
- this.nextRecoveryAt = performance.now() + RECOVERY_INTERVAL_MS;
62277
- });
62278
- return this.recovering;
63218
+ queueView() {
63219
+ return {
63220
+ assertCurrent: async (_lease, work) => {
63221
+ const current = this.attempts.get(work.id);
63222
+ if (current === undefined || current.fenced || current.generation !== work.generation || current.worker !== work.worker || current.epoch !== work.epoch)
63223
+ throw new Error(STALE_WORK_LEASE);
63224
+ },
63225
+ withCurrentAppend: (lease, work, assertSession, action) => this.queue.withCurrentAppend(lease, work, assertSession, action),
63226
+ result: async (root, id) => this.results.get(id) ?? this.queue.result(root, id)
63227
+ };
62279
63228
  }
62280
- observeResponsiveness() {
62281
- const now = performance.now();
62282
- const gap = now - this.lastTick;
62283
- if (gap > ROOT_STALL_THRESHOLD_MS)
62284
- this.rootStalledMs += gap - ROOT_TICK_INTERVAL_MS;
62285
- this.lastTick = now;
63229
+ rootProjection() {
63230
+ const run = this.store?.runs().find((candidate) => candidate.id === this.root);
63231
+ return run === undefined ? undefined : projectRootRun(run);
62286
63232
  }
62287
- isLive(worker, child) {
62288
- this.observeResponsiveness();
62289
- const heard = this.lastHeard.get(worker);
62290
- return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.rootStalledMs - heard.rootStalled) < this.leaseMs;
63233
+ rootRun() {
63234
+ const rootRun = this.rootProjection();
63235
+ if (rootRun === undefined)
63236
+ return {};
63237
+ this.rootSignature = JSON.stringify(rootRun);
63238
+ return { rootRun };
62291
63239
  }
62292
- stop(child) {
62293
- if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
62294
- return Promise.resolve();
62295
- return new Promise((resolve, reject) => {
62296
- const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
62297
- child.once("exit", () => {
62298
- clearTimeout(timeout);
62299
- resolve();
62300
- });
62301
- child.kill("SIGKILL");
62302
- });
63240
+ refreshRootRun() {
63241
+ const rootRun = this.rootProjection();
63242
+ if (rootRun === undefined)
63243
+ return;
63244
+ const signature = JSON.stringify(rootRun);
63245
+ if (signature === this.rootSignature)
63246
+ return;
63247
+ this.rootSignature = signature;
63248
+ this.notice({ kind: "root", rootRun });
62303
63249
  }
62304
63250
  async run(definition, inputs, options) {
62305
63251
  try {
@@ -62332,7 +63278,7 @@ class LocalWorkerPool {
62332
63278
  const selectedSource = this.sources[index];
62333
63279
  if (await definitionDigest(selectedSource.path) !== selectedSource.digest)
62334
63280
  throw new Error("Worker definition compatibility error");
62335
- await this.backend.hydrateWorkflow(root);
63281
+ await this.backend.catchUpWorkflow(root);
62336
63282
  const boundary = this.backend.listCheckpoints(root).find((entry) => entry.kind === "stage" && entry.topology?.boundary?.event === "start" && entry.topology.stageId === options.parentRun.stageId);
62337
63283
  if (boundary?.kind !== "stage")
62338
63284
  throw new Error("Missing durable child boundary");
@@ -62343,39 +63289,232 @@ class LocalWorkerPool {
62343
63289
  config: options.config,
62344
63290
  executionMode: options.executionMode
62345
63291
  }));
62346
- const work = await this.queue.enqueue(this.lease, {
62347
- boundaryId: options.parentRun.stageId,
62348
- childRunId: options.runId,
62349
- scope: options.durableScope.scopePrefix,
62350
- definition: `${selectedSource.path}#${selectedSource.exportName}`,
62351
- digest: selectedSource.digest,
62352
- inputs,
62353
- execution
62354
- }, boundary);
62355
- for (;; ) {
62356
- options.signal?.throwIfAborted();
62357
- if (this.closed) {
63292
+ this.evaluateBudget();
63293
+ const enqueued = await this.command({
63294
+ kind: "enqueue",
63295
+ descriptor: {
63296
+ boundaryId: options.parentRun.stageId,
63297
+ childRunId: options.runId,
63298
+ scope: options.durableScope.scopePrefix,
63299
+ definition: `${selectedSource.path}#${selectedSource.exportName}`,
63300
+ digest: selectedSource.digest,
63301
+ inputs,
63302
+ execution
63303
+ },
63304
+ boundary,
63305
+ ...this.rootRun()
63306
+ });
63307
+ const work = enqueued.work;
63308
+ this.works.set(work.id, work);
63309
+ if (enqueued.result !== undefined && !this.results.has(work.id))
63310
+ this.results.set(work.id, enqueued.result);
63311
+ this.waiting++;
63312
+ this.ticker ??= setInterval(() => {
63313
+ this.evaluateBudget();
63314
+ this.signalChange();
63315
+ }, POOL_SUMMARY_INTERVAL_MS).unref();
63316
+ try {
63317
+ for (;; ) {
63318
+ options.signal?.throwIfAborted();
63319
+ if (this.closed) {
63320
+ if (this.failure)
63321
+ throw this.failure;
63322
+ if (this.desiredControl === "quit")
63323
+ throw new WorkflowGracefulQuitError(root, "worker pool teardown");
63324
+ throw new Error("Local worker pool is closed");
63325
+ }
63326
+ const result = this.results.get(work.id);
63327
+ if (result) {
63328
+ await this.inbound;
63329
+ await this.backend.catchUpWorkflow(root);
63330
+ await this.handoff?.complete(work);
63331
+ return result;
63332
+ }
62358
63333
  if (this.failure)
62359
63334
  throw this.failure;
62360
- if (this.desiredControl === "quit")
62361
- throw new WorkflowGracefulQuitError(root, "worker pool teardown");
62362
- throw new Error("Local worker pool is closed");
63335
+ if (this.budgetBlocked)
63336
+ await this.rootBudget.stopAtBoundaryAsync(definition.name);
63337
+ await this.nextChange(options.signal);
62363
63338
  }
62364
- const result = await this.queue.result(root, work.id);
62365
- if (result) {
62366
- await this.backend.hydrateWorkflow(root);
62367
- await this.handoff?.complete(work);
62368
- return result;
63339
+ } finally {
63340
+ if (--this.waiting === 0) {
63341
+ clearInterval(this.ticker);
63342
+ this.ticker = undefined;
63343
+ }
63344
+ }
63345
+ }
63346
+ evaluateBudget() {
63347
+ const decision = this.rootBudget?.checkpoint();
63348
+ const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
63349
+ if (blocked === this.budgetBlocked)
63350
+ return;
63351
+ this.budgetBlocked = blocked;
63352
+ this.notice({ kind: "budget", blocked });
63353
+ }
63354
+ signalChange() {
63355
+ const changed = this.changed;
63356
+ this.changed = Promise.withResolvers();
63357
+ changed.resolve();
63358
+ }
63359
+ nextChange(signal) {
63360
+ const changed = this.changed.promise;
63361
+ if (signal === undefined)
63362
+ return changed;
63363
+ return new Promise((resolve, reject) => {
63364
+ const abort = () => reject(signal.reason);
63365
+ signal.addEventListener("abort", abort, { once: true });
63366
+ changed.then(() => {
63367
+ signal.removeEventListener("abort", abort);
63368
+ resolve();
63369
+ });
63370
+ });
63371
+ }
63372
+ connect() {
63373
+ if (this.coordinator !== undefined)
63374
+ return this.coordinator;
63375
+ const coordinator = this.options.coordinator === "in-process" ? launchCoordinatorInProcess() : launchCoordinatorProcess();
63376
+ this.coordinator = coordinator;
63377
+ coordinator.port.onMessage((event) => {
63378
+ this.inbound = this.inbound.then(() => this.receive(event)).catch(() => {});
63379
+ });
63380
+ coordinator.port.onClose((reason) => {
63381
+ this.coordinatorLost = reason;
63382
+ this.inbound = this.inbound.then(() => this.lost(reason));
63383
+ });
63384
+ return coordinator;
63385
+ }
63386
+ lost(reason) {
63387
+ this.pids.clear();
63388
+ if (!this.closed)
63389
+ this.failure ??= reason;
63390
+ for (const pending of this.commands.values())
63391
+ pending.reject(reason);
63392
+ this.commands.clear();
63393
+ this.signalChange();
63394
+ }
63395
+ command(body) {
63396
+ const coordinator = this.connect();
63397
+ if (this.coordinatorLost !== undefined)
63398
+ return Promise.reject(this.coordinatorLost);
63399
+ const id = ++this.commandSequence;
63400
+ return new Promise((resolve, reject) => {
63401
+ this.commands.set(id, { resolve, reject });
63402
+ coordinator.port.send({ ...body, id });
63403
+ });
63404
+ }
63405
+ notice(notice) {
63406
+ this.coordinator?.port.send(notice);
63407
+ }
63408
+ async receive(event) {
63409
+ switch (event.kind) {
63410
+ case "reply": {
63411
+ const pending = this.commands.get(event.id);
63412
+ this.commands.delete(event.id);
63413
+ if (event.error !== undefined)
63414
+ pending?.reject(new Error(event.error));
63415
+ else
63416
+ pending?.resolve(event.value);
63417
+ return;
63418
+ }
63419
+ case "activity":
63420
+ return this.applyActivity(event.batch, event.entries, event.usage);
63421
+ case "spawned":
63422
+ this.pids.add(event.pid);
63423
+ return;
63424
+ case "exited":
63425
+ this.pids.delete(event.pid);
63426
+ return;
63427
+ case "fence": {
63428
+ const current = this.attempts.get(event.work.id);
63429
+ if (current?.generation === event.work.generation)
63430
+ current.fenced = true;
63431
+ this.handoff?.fence(event.work);
63432
+ return;
62369
63433
  }
62370
- if (this.failure)
62371
- throw this.failure;
62372
- const decision = this.rootBudget?.checkpoint();
62373
- if (decision?.kind === "exhausted" || decision?.kind === "wrap_up")
62374
- await this.rootBudget.stopAtBoundaryAsync(definition.name);
62375
- await this.recoverExpired(root);
62376
- await delay2(25, undefined, { signal: options.signal });
63434
+ case "result":
63435
+ if (event.usage !== undefined)
63436
+ this.usageTotal = event.usage;
63437
+ this.results.set(event.workId, event.result);
63438
+ this.signalChange();
63439
+ return;
63440
+ case "failure":
63441
+ this.failure ??= new Error(event.error);
63442
+ this.signalChange();
63443
+ return;
63444
+ case "request":
63445
+ this.serve(event.id, event.request);
63446
+ return;
62377
63447
  }
62378
63448
  }
63449
+ async applyActivity(batch, entries, usage) {
63450
+ if (usage !== undefined)
63451
+ this.usageTotal = usage;
63452
+ const rejected = [];
63453
+ const handoff = this.handoff;
63454
+ if (handoff !== undefined) {
63455
+ const publications = entries.flatMap((entry) => {
63456
+ const work = this.track(entry.work);
63457
+ if (work !== undefined)
63458
+ return [{ work, runs: entry.runs }];
63459
+ rejected.push({ workId: entry.work.id, generation: entry.work.generation, error: STALE_WORK_LEASE });
63460
+ return [];
63461
+ });
63462
+ const outcomes = await handoff.publishBatch(publications);
63463
+ outcomes.forEach((error, index) => {
63464
+ const { work } = publications[index];
63465
+ if (error !== undefined)
63466
+ rejected.push({ workId: work.id, generation: work.generation, error: error.message });
63467
+ });
63468
+ }
63469
+ this.notice({ kind: "published", batch, rejected });
63470
+ this.refreshRootRun();
63471
+ }
63472
+ track(attempt) {
63473
+ const work = this.works.get(attempt.id);
63474
+ if (work === undefined)
63475
+ return;
63476
+ const current = this.attempts.get(attempt.id);
63477
+ if (current === undefined || attempt.generation > current.generation)
63478
+ this.attempts.set(attempt.id, { ...attempt, fenced: false });
63479
+ return { ...work, ...attempt, state: "leased" };
63480
+ }
63481
+ async serve(id, request) {
63482
+ try {
63483
+ const value2 = await this.answer(id, request);
63484
+ this.coordinator?.port.send({ kind: "request-reply", id, ...value2 === undefined ? {} : { value: value2 } });
63485
+ } catch (error) {
63486
+ this.coordinator?.port.send({
63487
+ kind: "request-reply",
63488
+ id,
63489
+ error: error instanceof Error ? error.message : String(error)
63490
+ });
63491
+ }
63492
+ }
63493
+ async answer(id, request) {
63494
+ if (request.kind === "pending-stage-message" || request.kind === "sticky-live-delivered") {
63495
+ const router = this.store === undefined ? undefined : pendingStageMessageRouter(this.store);
63496
+ const event = {
63497
+ ...request.payload,
63498
+ handled: false
63499
+ };
63500
+ if (request.kind === "pending-stage-message")
63501
+ router?.route(event);
63502
+ else
63503
+ router?.sticky(event);
63504
+ if (event.handled && event.completion !== undefined)
63505
+ return await event.completion;
63506
+ return request.kind === "pending-stage-message" ? { outcome: "refused", reason: "Session not found" } : undefined;
63507
+ }
63508
+ const handoff = this.handoff;
63509
+ const work = this.track(request.work);
63510
+ if (handoff === undefined || work === undefined)
63511
+ throw new Error(STALE_WORK_LEASE);
63512
+ if (request.kind === "ready")
63513
+ return handoff.ready(work, request.runId, request.stageId, request.stageName, request.sessionId);
63514
+ return handoff.deliver(work, request.runId, request.stageId, request.stageName, request.recipient, async (from, message) => {
63515
+ await this.command({ kind: "transport", request: id, from, message });
63516
+ });
63517
+ }
62379
63518
  closing;
62380
63519
  close() {
62381
63520
  this.closing ??= this.dispose();
@@ -62384,27 +63523,28 @@ class LocalWorkerPool {
62384
63523
  async dispose() {
62385
63524
  this.closed = true;
62386
63525
  clearInterval(this.ticker);
63526
+ this.ticker = undefined;
62387
63527
  this.releaseBinding();
62388
63528
  if (this.options.rootId === undefined)
62389
63529
  workflowResourceAdmission.delete(this.backend);
63530
+ this.signalChange();
62390
63531
  let failure;
62391
- let stopping = [];
62392
- try {
62393
- await this.recovering?.catch(() => {});
62394
- stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
62395
- await Promise.all(stopping.map(({ child }) => this.stop(child)));
62396
- await Promise.all([...this.cleanups]);
62397
- if (this.cleanupFailure)
62398
- throw this.cleanupFailure;
62399
- } catch (error) {
62400
- failure = error;
62401
- }
62402
- if (failure === undefined) {
62403
- const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
62404
- await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
62405
- if (this.logDirectory !== undefined)
62406
- await rmdir(this.logDirectory).catch(() => {});
63532
+ const coordinator = this.coordinator;
63533
+ if (coordinator !== undefined) {
63534
+ if (this.coordinatorLost === undefined) {
63535
+ try {
63536
+ await this.command({ kind: "close" });
63537
+ } catch (error) {
63538
+ failure = error;
63539
+ }
63540
+ }
63541
+ await this.inbound;
63542
+ await coordinator.port.close();
63543
+ this.pids.clear();
62407
63544
  }
63545
+ for (const release of this.routeClaims.values())
63546
+ release();
63547
+ this.routeClaims.clear();
62408
63548
  try {
62409
63549
  await this.releaseRoutes?.();
62410
63550
  } catch (error) {
@@ -62414,27 +63554,26 @@ class LocalWorkerPool {
62414
63554
  throw failure;
62415
63555
  }
62416
63556
  }
62417
- var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, ROOT_TICK_INTERVAL_MS = 100, ROOT_STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000;
63557
+ var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
62418
63558
  var init_local_pool = __esm(() => {
62419
63559
  init_internal();
62420
63560
  init_sql_pool_queue();
62421
- init_sql_pool_usage();
62422
63561
  init_sql_run_journal();
62423
63562
  init_workflow_tool_abort();
62424
63563
  init_sdk_intercom();
63564
+ init_external_run_routes();
62425
63565
  init_workflow_infrastructure_error();
62426
63566
  init_admission_binding();
62427
63567
  init_definitions();
62428
63568
  init_dispatch();
62429
63569
  init_host_options();
62430
- init_resource_admission();
63570
+ init_pool_coordinator();
62431
63571
  init_stage_handoff();
62432
63572
  MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
62433
- COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
62434
63573
  });
62435
63574
 
62436
63575
  // dist/builtin/workflows/src/workers/public-pool.ts
62437
- import { resolve as resolve12 } from "node:path";
63576
+ import { resolve as resolve13 } from "node:path";
62438
63577
  import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
62439
63578
  async function restoreWorkerOptions(backend, options) {
62440
63579
  const source = options.continuation?.source.id ?? options.runId;
@@ -62473,11 +63612,11 @@ async function activatePublicPool(backend, options) {
62473
63612
  ...options.localWorkers,
62474
63613
  startupTimeoutMs: undefined,
62475
63614
  leaseMs: undefined,
62476
- definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve12(cwd, source.path) }))
63615
+ definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve13(cwd, source.path) }))
62477
63616
  };
62478
63617
  const admission = options.admission && {
62479
63618
  ...options.admission,
62480
- ...options.admission.userDatabase ? { userDatabase: resolve12(cwd, options.admission.userDatabase) } : {}
63619
+ ...options.admission.userDatabase ? { userDatabase: resolve13(cwd, options.admission.userDatabase) } : {}
62481
63620
  };
62482
63621
  const sources = localWorkers && await Promise.all(localWorkers.definitions.map(async (source) => ({ ...source, digest: await definitionDigest(source.path) })));
62483
63622
  const resources = JSON.parse(JSON.stringify({
@@ -63823,7 +64962,7 @@ function hasUsableContentBlock(block) {
63823
64962
 
63824
64963
  // dist/builtin/workflows/src/shared/stage-session-store.ts
63825
64964
  import { randomUUID as randomUUID17 } from "node:crypto";
63826
- import { dirname as dirname13, resolve as resolve13 } from "node:path";
64965
+ import { dirname as dirname13, resolve as resolve14 } from "node:path";
63827
64966
  import {
63828
64967
  JsonlSessionStore,
63829
64968
  resolveSessionStore,
@@ -63859,11 +64998,11 @@ async function forkStageSession(store, sessionId, cwd, options, destination = st
63859
64998
  async function migrateLegacyStageSession(store, sessionId, sessionFile) {
63860
64999
  if (sessionFile === undefined || await store.readHeader(sessionId) !== undefined)
63861
65000
  return;
63862
- const location = resolve13(sessionFile);
65001
+ const location = resolve14(sessionFile);
63863
65002
  const legacy = await JsonlSessionStore.open(dirname13(location), { exactDirectory: true, scanExisting: false });
63864
65003
  try {
63865
65004
  for await (const file of legacy.scanFiles()) {
63866
- if (resolve13(file.location) !== location || file.header.id !== sessionId)
65005
+ if (resolve14(file.location) !== location || file.header.id !== sessionId)
63867
65006
  continue;
63868
65007
  const snapshot = await file.read();
63869
65008
  if (await store.readHeader(sessionId) === undefined)
@@ -64025,22 +65164,37 @@ function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
64025
65164
  }
64026
65165
  function retainReachableRunGroups(grouped, rootRunId) {
64027
65166
  const reachable = new Set([rootRunId]);
64028
- const pending = [rootRunId];
65167
+ const settled = new Set;
65168
+ const pending = [{ runId: rootRunId, live: true }];
64029
65169
  while (pending.length > 0) {
64030
- const drafts = grouped.get(pending.pop());
65170
+ const { runId, live } = pending.pop();
65171
+ const drafts = grouped.get(runId);
64031
65172
  if (drafts === undefined)
64032
65173
  continue;
64033
65174
  for (const draft of drafts) {
64034
- const childRunId = childRunIdFromDraft(draft);
64035
- if (childRunId === undefined || reachable.has(childRunId) || !grouped.has(childRunId))
65175
+ const childRunId = workflowChildFromDraft(draft)?.runId ?? draft.topology?.boundary?.child.runId;
65176
+ if (childRunId === undefined || !grouped.has(childRunId))
64036
65177
  continue;
64037
- reachable.add(childRunId);
64038
- pending.push(childRunId);
65178
+ if (live && childRunIdFromDraft(draft) !== undefined) {
65179
+ if (reachable.has(childRunId))
65180
+ continue;
65181
+ reachable.add(childRunId);
65182
+ pending.push({ runId: childRunId, live: true });
65183
+ } else if (!settled.has(childRunId)) {
65184
+ settled.add(childRunId);
65185
+ pending.push({ runId: childRunId, live: false });
65186
+ }
64039
65187
  }
64040
65188
  }
64041
- for (const runId of grouped.keys())
64042
- if (!reachable.has(runId))
64043
- grouped.delete(runId);
65189
+ let complete = true;
65190
+ for (const runId of grouped.keys()) {
65191
+ if (reachable.has(runId))
65192
+ continue;
65193
+ if (!settled.has(runId))
65194
+ complete = false;
65195
+ grouped.delete(runId);
65196
+ }
65197
+ return complete;
64044
65198
  }
64045
65199
  function childRunIdFromDraft(draft) {
64046
65200
  const boundary = draft.topology?.boundary;
@@ -64462,9 +65616,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
64462
65616
  }
64463
65617
  for (const group of grouped.values())
64464
65618
  group.sort(compareDraftSourceOrder);
64465
- const groupedCount = grouped.size;
64466
- retainReachableRunGroups(grouped, rootRunId);
64467
- if (policy.failOnUnreachable && grouped.size !== groupedCount)
65619
+ if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
64468
65620
  return [];
64469
65621
  if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
64470
65622
  return [];
@@ -70122,9 +71274,9 @@ var init_stage_runner_controller = __esm(() => {
70122
71274
  });
70123
71275
 
70124
71276
  // dist/builtin/workflows/src/runs/foreground/stage-runner-output.ts
70125
- import { createHash as createHash4 } from "node:crypto";
71277
+ import { createHash as createHash5 } from "node:crypto";
70126
71278
  import { mkdir as mkdir7, writeFile as writeFile10 } from "node:fs/promises";
70127
- import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve14 } from "node:path";
71279
+ import { basename as basename5, dirname as dirname14, isAbsolute as isAbsolute11, join as join29, resolve as resolve15 } from "node:path";
70128
71280
  function normalizeMaxOutput2(maxOutput) {
70129
71281
  return {
70130
71282
  bytes: maxOutput?.bytes ?? DEFAULT_MAX_OUTPUT_BYTES,
@@ -70186,7 +71338,7 @@ function formatByteSize(bytes) {
70186
71338
  return `${value2.toFixed(1)} ${units[unitIndex]}`;
70187
71339
  }
70188
71340
  function transcriptPath(runId, outputPath) {
70189
- const digest = createHash4("sha256").update(outputPath).digest("hex").slice(0, 16);
71341
+ const digest = createHash5("sha256").update(outputPath).digest("hex").slice(0, 16);
70190
71342
  return join29(workflowArtifactRunPath(runId), "transcripts", `${digest}-${basename5(outputPath)}.transcript.md`);
70191
71343
  }
70192
71344
  function appendContent(lines, content) {
@@ -70232,7 +71384,7 @@ function renderTranscript(messages, fallbackOutput) {
70232
71384
  `);
70233
71385
  }
70234
71386
  function savedOutputReference(outputPath, fullOutput, transcriptPathValue, transcriptText, transcriptAvailable, warning) {
70235
- const absolutePath = resolve14(outputPath);
71387
+ const absolutePath = resolve15(outputPath);
70236
71388
  const bytes = Buffer.byteLength(fullOutput, "utf8");
70237
71389
  const lines = countLines(fullOutput);
70238
71390
  const transcriptBytes = Buffer.byteLength(transcriptText, "utf8");
@@ -70247,8 +71399,8 @@ function resolveOutputPath(output, runtimeCwd, requestedCwd) {
70247
71399
  return;
70248
71400
  if (isAbsolute11(output))
70249
71401
  return output;
70250
- const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve14(runtimeCwd, requestedCwd);
70251
- return resolve14(baseCwd, output);
71402
+ const baseCwd = requestedCwd === undefined ? runtimeCwd : isAbsolute11(requestedCwd) ? requestedCwd : resolve15(runtimeCwd, requestedCwd);
71403
+ return resolve15(baseCwd, output);
70252
71404
  }
70253
71405
  function degenerateWarning(fullOutput) {
70254
71406
  return fullOutput.trim().length === 0 ? "WARNING: the stage artifact is empty; search the companion transcript for this stage's work." : undefined;
@@ -71257,7 +72409,7 @@ async function run(def, inputs, opts = {}) {
71257
72409
  if (!isWorkflowDefinition(def))
71258
72410
  throw new Error(workflowDefinitionRequirementMessage("run(definition, inputs)", def));
71259
72411
  const depth = opts.depth ?? 0;
71260
- const maxDepth = opts.config?.maxDepth ?? 4;
72412
+ const maxDepth = opts.config?.maxDepth ?? 3;
71261
72413
  if (depth >= maxDepth) {
71262
72414
  return {
71263
72415
  runId: opts.runId ?? crypto.randomUUID(),