@bastani/atomic 0.9.44-alpha.5 → 0.9.44-alpha.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/dist/builtin/intercom/CHANGELOG.md +6 -0
  3. package/dist/builtin/intercom/broker/broker.ts +13 -1
  4. package/dist/builtin/intercom/index.bundle.mjs +41 -5
  5. package/dist/builtin/intercom/package.json +1 -1
  6. package/dist/builtin/intercom/types.ts +7 -0
  7. package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
  8. package/dist/builtin/mcp/package.json +1 -1
  9. package/dist/builtin/subagents/package.json +1 -1
  10. package/dist/builtin/web-access/package.json +1 -1
  11. package/dist/builtin/workflows/CHANGELOG.md +20 -0
  12. package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
  13. package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
  14. package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
  15. package/dist/builtin/workflows/builtin/goal.js +2 -2
  16. package/dist/builtin/workflows/builtin/index.js +3 -3
  17. package/dist/builtin/workflows/builtin/ralph.js +2 -2
  18. package/dist/builtin/workflows/package.json +1 -1
  19. package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
  20. package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
  21. package/dist/builtin/workflows/src/extension/index.bundle.mjs +2514 -741
  22. package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2384 -727
  23. package/dist/builtin/workflows/src/index.js +2383 -727
  24. package/dist/builtin/workflows/src/local-worker-entry.mjs +2384 -727
  25. package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
  26. package/dist/core/agent-session-custom-message-commit.js +16 -0
  27. package/dist/core/agent-session-custom-message-commit.js.map +1 -1
  28. package/dist/core/storage/node-sqlite.d.ts.map +1 -1
  29. package/dist/core/storage/node-sqlite.js +130 -13
  30. package/dist/core/storage/node-sqlite.js.map +1 -1
  31. package/docs/intercom/reference.md +2 -0
  32. package/docs/workflows/operations.md +1 -1
  33. package/npm-shrinkwrap.json +32 -32
  34. package/package.json +3 -3
@@ -172,13 +172,19 @@ function registerRunJournalVersions(journal, versions) {
172
172
  function runJournalVersions(journal) {
173
173
  return versionReaders.get(journal);
174
174
  }
175
+ function registerRunJournalAppendMany(journal, appendMany) {
176
+ appendManyWriters.set(journal, appendMany);
177
+ }
178
+ function runJournalAppendMany(journal) {
179
+ return appendManyWriters.get(journal);
180
+ }
175
181
  function registerSqlPoolStorage(journal, storage) {
176
182
  poolStorage.set(journal, storage);
177
183
  }
178
184
  function sqlPoolStorage(journal) {
179
185
  return poolStorage.get(journal);
180
186
  }
181
- var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
187
+ var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
182
188
  var init_internal = __esm(() => {
183
189
  init_types();
184
190
  NON_SNAPSHOT_EVENT_TYPES = [
@@ -187,6 +193,7 @@ var init_internal = __esm(() => {
187
193
  RUN_EVENT_TYPES.messageDelivered
188
194
  ];
189
195
  versionReaders = new WeakMap;
196
+ appendManyWriters = new WeakMap;
190
197
  poolStorage = new WeakMap;
191
198
  });
192
199
 
@@ -297,6 +304,204 @@ var init_resume_eligibility = () => {};
297
304
  // dist/builtin/workflows/src/durable/types.ts
298
305
  var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
299
306
 
307
+ // dist/builtin/workflows/src/durable/stage-topology-validation.ts
308
+ function immutableStageGroupError(stages) {
309
+ const occurrenceError = promptOccurrenceIdentityError(stages);
310
+ if (occurrenceError !== undefined)
311
+ return occurrenceError;
312
+ const byReplayKey = groupByDurableStageKey(stages);
313
+ for (const group of byReplayKey.values()) {
314
+ const firstName = group[0]?.name;
315
+ if (group.some((stage) => stage.name !== firstName))
316
+ return "stage name drift within one replay-key group";
317
+ const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
318
+ const first = topologies[0];
319
+ if (first === undefined)
320
+ continue;
321
+ if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
322
+ return "immutable stage identity drift within one replay-key group";
323
+ }
324
+ }
325
+ return;
326
+ }
327
+ function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
328
+ return directChildTopologyErrorFromRecords({
329
+ claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
330
+ direct: (claiming) => {
331
+ const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
332
+ return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
333
+ }
334
+ }, replayScope, expectedRun, allowedNodeIds);
335
+ }
336
+ function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
337
+ const prefix = `${replayScope}:`;
338
+ const claiming = records.claiming;
339
+ if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
340
+ return "child run topology exists outside its durable invocation scope";
341
+ }
342
+ if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
343
+ return "child checkpoint ownership is not reciprocal";
344
+ }
345
+ const direct = records.direct(claiming);
346
+ const immutableError = immutableStageGroupError(direct);
347
+ if (immutableError !== undefined)
348
+ return immutableError;
349
+ const merged = [];
350
+ for (const group of groupByDurableStageKey(direct).values()) {
351
+ const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
352
+ if (topology === undefined)
353
+ return "direct child stage group lacks owning-run topology";
354
+ if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
355
+ return "direct child stage topology lacks current source identity";
356
+ }
357
+ if (topology.stageId.length === 0)
358
+ return "direct child stage id is empty";
359
+ merged.push(topology);
360
+ }
361
+ const ids = new Set(allowedNodeIds);
362
+ const stageIds = new Set;
363
+ const orders = new Set;
364
+ for (const topology of merged) {
365
+ if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
366
+ return "direct child stage ids are not unique";
367
+ stageIds.add(topology.stageId);
368
+ ids.add(topology.stageId);
369
+ if (topology.sourceOrder !== undefined) {
370
+ if (orders.has(topology.sourceOrder))
371
+ return "direct child source orders are not unique";
372
+ orders.add(topology.sourceOrder);
373
+ }
374
+ }
375
+ if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
376
+ return "direct child stage references an unknown parent";
377
+ }
378
+ return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
379
+ }
380
+ function owningBoundaryIdError(stages, owningRunId) {
381
+ const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
382
+ const seen = new Map;
383
+ for (const start of starts) {
384
+ const stageId = start.topology.stageId;
385
+ if (stageId.length === 0)
386
+ return "owning-run boundary stage id is empty";
387
+ const prior = seen.get(stageId);
388
+ if (prior !== undefined && prior !== start.replayKey)
389
+ return "owning-run boundary stage id is aliased";
390
+ seen.set(stageId, start.replayKey);
391
+ }
392
+ return;
393
+ }
394
+ function sameRun(actual, expected) {
395
+ return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
396
+ }
397
+ function sameStrings(left, right) {
398
+ return left.length === right.length && left.every((value2, index) => value2 === right[index]);
399
+ }
400
+ function sameOptionalRun(actual, expected) {
401
+ if (actual === undefined || expected === undefined)
402
+ return actual === expected;
403
+ return sameRun(actual, expected);
404
+ }
405
+ function promptOccurrenceIdentityError(stages) {
406
+ const replayByOccurrence = new Map;
407
+ const occurrenceByReplayStage = new Map;
408
+ const stageByReplayOccurrence = new Map;
409
+ for (const stage of stages) {
410
+ const topology = stage.topology;
411
+ const occurrence = topology?.occurrenceKey;
412
+ if (topology === undefined || occurrence === undefined)
413
+ continue;
414
+ const priorReplay = replayByOccurrence.get(occurrence);
415
+ if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
416
+ return "prompt occurrence is bound to multiple replay keys";
417
+ }
418
+ replayByOccurrence.set(occurrence, stage.replayKey);
419
+ const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
420
+ const priorOccurrence = occurrenceByReplayStage.get(replayStage);
421
+ if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
422
+ return "prompt stage identity is bound to multiple occurrences";
423
+ }
424
+ occurrenceByReplayStage.set(replayStage, occurrence);
425
+ const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
426
+ const priorStage = stageByReplayOccurrence.get(replayOccurrence);
427
+ if (priorStage !== undefined && priorStage !== topology.stageId) {
428
+ return "prompt occurrence is bound to multiple stage identities";
429
+ }
430
+ stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
431
+ }
432
+ return;
433
+ }
434
+ function durableStageGroupKey(stage) {
435
+ const occurrence = stage.topology?.occurrenceKey;
436
+ return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
437
+ }
438
+ function groupByDurableStageKey(stages) {
439
+ const groups = new Map;
440
+ for (const stage of stages) {
441
+ const key = durableStageGroupKey(stage);
442
+ const group = groups.get(key) ?? [];
443
+ group.push(stage);
444
+ groups.set(key, group);
445
+ }
446
+ return groups;
447
+ }
448
+ function compositeKey(domain, ...parts) {
449
+ return JSON.stringify([domain, ...parts]);
450
+ }
451
+ function hasCycle(topologies) {
452
+ const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
453
+ const visiting = new Set;
454
+ const visited = new Set;
455
+ const cyclic = (stageId) => {
456
+ if (visiting.has(stageId))
457
+ return true;
458
+ if (visited.has(stageId))
459
+ return false;
460
+ visiting.add(stageId);
461
+ if ((parents.get(stageId) ?? []).some(cyclic))
462
+ return true;
463
+ visiting.delete(stageId);
464
+ visited.add(stageId);
465
+ return false;
466
+ };
467
+ return topologies.some((topology) => cyclic(topology.stageId));
468
+ }
469
+ var init_stage_topology_validation = () => {};
470
+
471
+ // dist/builtin/workflows/src/durable/workflow-child-result.ts
472
+ function parseWorkflowChildResult(value2) {
473
+ if (!isOrdinaryObject(value2))
474
+ return;
475
+ const workflow = value2.workflow;
476
+ const runId = value2.runId;
477
+ const status = value2.status;
478
+ const exited = value2.exited;
479
+ const outputs = value2.outputs;
480
+ const exitReason = value2.exitReason;
481
+ if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
482
+ return;
483
+ if (exited === false) {
484
+ if (status !== "completed" || exitReason !== undefined)
485
+ return;
486
+ return value2;
487
+ }
488
+ if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
489
+ return;
490
+ return value2;
491
+ }
492
+ function isWorkflowChildResult(value2) {
493
+ return parseWorkflowChildResult(value2) !== undefined;
494
+ }
495
+ function isWorkflowExitStatus(value2) {
496
+ return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
497
+ }
498
+ function isOrdinaryObject(value2) {
499
+ if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
500
+ return false;
501
+ const prototype = Object.getPrototypeOf(value2);
502
+ return prototype === Object.prototype || prototype === null;
503
+ }
504
+
300
505
  // dist/builtin/workflows/src/durable/durable-hash.ts
301
506
  import { createHash } from "node:crypto";
302
507
  function durableHash(value2) {
@@ -363,6 +568,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
363
568
  function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
364
569
  return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
365
570
  }
571
+
572
+ class StageReceipts {
573
+ exact = new Map;
574
+ anySession = new Map;
575
+ has(logicalRunId, sessionId, admission) {
576
+ return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
577
+ }
578
+ hasAnySession(logicalRunId, admission) {
579
+ return this.anySession.get(logicalRunId)?.has(admission) ?? false;
580
+ }
581
+ add(logicalRunId, sessionId, admission) {
582
+ let bySession = this.exact.get(logicalRunId);
583
+ if (bySession === undefined) {
584
+ bySession = new Map;
585
+ this.exact.set(logicalRunId, bySession);
586
+ }
587
+ let admissions = bySession.get(sessionId);
588
+ if (admissions === undefined) {
589
+ admissions = new Set;
590
+ bySession.set(sessionId, admissions);
591
+ }
592
+ if (admissions.has(admission))
593
+ return false;
594
+ admissions.add(admission);
595
+ return true;
596
+ }
597
+ addAnySession(logicalRunId, admission) {
598
+ let admissions = this.anySession.get(logicalRunId);
599
+ if (admissions === undefined) {
600
+ admissions = new Set;
601
+ this.anySession.set(logicalRunId, admissions);
602
+ }
603
+ if (admissions.has(admission))
604
+ return false;
605
+ admissions.add(admission);
606
+ return true;
607
+ }
608
+ }
609
+
610
+ class PendingStageReceiptIndex {
611
+ workflows = new Map;
612
+ add(workflowId, messageId, delivery, logicalRunId) {
613
+ let workflow = this.workflows.get(workflowId);
614
+ if (workflow === undefined) {
615
+ workflow = { byMessage: new Map, log: [] };
616
+ this.workflows.set(workflowId, workflow);
617
+ }
618
+ let byRun = workflow.byMessage.get(messageId);
619
+ if (byRun === undefined) {
620
+ byRun = new Map;
621
+ workflow.byMessage.set(messageId, byRun);
622
+ }
623
+ let byStage = byRun.get(delivery.runId);
624
+ if (byStage === undefined) {
625
+ byStage = new Map;
626
+ byRun.set(delivery.runId, byStage);
627
+ }
628
+ let receipts = byStage.get(delivery.stageId);
629
+ if (receipts === undefined) {
630
+ receipts = new StageReceipts;
631
+ byStage.set(delivery.stageId, receipts);
632
+ }
633
+ const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
634
+ let changed = added;
635
+ if (receipts.add(logicalRunId, undefined, undefined))
636
+ changed = true;
637
+ if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
638
+ changed = true;
639
+ if (changed)
640
+ workflow.log.push(messageId);
641
+ return added;
642
+ }
643
+ has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
644
+ const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
645
+ if (receipts === undefined)
646
+ return false;
647
+ if (anySession && delivery.admission === "context")
648
+ return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
649
+ return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
650
+ }
651
+ changesSince(workflowId, cursor) {
652
+ let workflow = this.workflows.get(workflowId);
653
+ if (workflow === undefined) {
654
+ workflow = { byMessage: new Map, log: [] };
655
+ this.workflows.set(workflowId, workflow);
656
+ }
657
+ const next = { log: workflow.log, length: workflow.log.length };
658
+ if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
659
+ return { cursor: next, messageIds: undefined };
660
+ return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
661
+ }
662
+ delete(workflowId) {
663
+ this.workflows.delete(workflowId);
664
+ }
665
+ clear() {
666
+ this.workflows.clear();
667
+ }
668
+ }
669
+ function nextVersion() {
670
+ lastVersion += 1;
671
+ return lastVersion;
672
+ }
673
+
674
+ class NestedSnapshotVersion {
675
+ rootWorkflowId;
676
+ current = nextVersion();
677
+ latestCompletedAt = Number.NEGATIVE_INFINITY;
678
+ replayGroups = new Map;
679
+ boundaryReplayKeys = new Set;
680
+ constructor(rootWorkflowId) {
681
+ this.rootWorkflowId = rootWorkflowId;
682
+ }
683
+ get version() {
684
+ return this.current;
685
+ }
686
+ record(checkpoint) {
687
+ if (!this.isInert(checkpoint))
688
+ this.current = nextVersion();
689
+ this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
690
+ if (checkpoint.kind !== "stage")
691
+ return;
692
+ if (checkpoint.topology?.boundary !== undefined)
693
+ this.boundaryReplayKeys.add(checkpoint.replayKey);
694
+ if (checkpoint.topology?.occurrenceKey !== undefined)
695
+ return;
696
+ const identity = stageIdentity(checkpoint);
697
+ const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
698
+ const summary = this.replayGroups.get(checkpoint.replayKey);
699
+ if (summary === undefined) {
700
+ this.replayGroups.set(checkpoint.replayKey, {
701
+ name: checkpoint.name,
702
+ identity,
703
+ consistent: true,
704
+ hasOutput: checkpoint.output !== undefined,
705
+ hasExitedChildOutput: exited
706
+ });
707
+ return;
708
+ }
709
+ summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
710
+ summary.hasOutput ||= checkpoint.output !== undefined;
711
+ summary.hasExitedChildOutput ||= exited;
712
+ }
713
+ isInert(checkpoint) {
714
+ if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
715
+ return false;
716
+ }
717
+ const summary = this.replayGroups.get(checkpoint.replayKey);
718
+ if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
719
+ return false;
720
+ }
721
+ const id = checkpoint.checkpointId;
722
+ for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
723
+ if (this.boundaryReplayKeys.has(id.slice(0, colon)))
724
+ return false;
725
+ }
726
+ return true;
727
+ }
728
+ isRootOwned(checkpoint) {
729
+ const runId = checkpoint.topology?.run?.runId;
730
+ return runId === undefined || runId === this.rootWorkflowId;
731
+ }
732
+ }
733
+ function stageIdentity(checkpoint) {
734
+ const topology = checkpoint.topology;
735
+ return JSON.stringify([
736
+ durableStageGroupKey(checkpoint),
737
+ topology?.version,
738
+ topology?.stageId,
739
+ topology?.parentIds,
740
+ topology?.sourceOrder,
741
+ topology?.run === undefined ? undefined : [
742
+ topology.run.runId,
743
+ topology.run.runName,
744
+ topology.run.parentRunId,
745
+ topology.run.parentStageId,
746
+ topology.run.rootRunId
747
+ ]
748
+ ]);
749
+ }
366
750
  function checkpointKey(c) {
367
751
  return `${c.kind}:${c.checkpointId}`;
368
752
  }
@@ -374,7 +758,7 @@ class JournalProjection {
374
758
  workflows = new Map;
375
759
  deletedWorkflowIds = new Set;
376
760
  pendingReceipts = new Map;
377
- stickyReceipts = new Map;
761
+ stickyReceipts = new PendingStageReceiptIndex;
378
762
  registerWorkflow(handle, authoritativeOwner = false) {
379
763
  this.deletedWorkflowIds.delete(handle.workflowId);
380
764
  const existing = this.workflows.get(handle.workflowId);
@@ -422,7 +806,8 @@ class JournalProjection {
422
806
  toolByHash: new Map,
423
807
  uiByHash: new Map,
424
808
  stageOutputByReplayKey: new Map,
425
- stageSessionByReplayKey: new Map
809
+ stageSessionByReplayKey: new Map,
810
+ nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
426
811
  });
427
812
  }
428
813
  applyWorkflowSnapshot(handle) {
@@ -448,39 +833,43 @@ class JournalProjection {
448
833
  return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
449
834
  }
450
835
  stickyDeliveryCounts = new Map;
836
+ stickyDeliveryCountVersions = new Map;
451
837
  getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
452
- return this.stickyDeliveryCounts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId])) ?? 0;
838
+ return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
839
+ }
840
+ pendingStageDeliveryCountVersion(workflowId) {
841
+ return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
842
+ }
843
+ bumpDeliveryCountVersion(workflowId) {
844
+ this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
453
845
  }
454
846
  async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
455
- let receipts = this.stickyReceipts.get(workflowId);
456
- if (receipts === undefined) {
457
- receipts = new Set;
458
- this.stickyReceipts.set(workflowId, receipts);
459
- }
460
- const receiptKey = pendingDeliveryReceiptKey(messageId, delivery, accounting?.messageRunId);
461
- const existing = receipts.has(receiptKey);
462
- receipts.add(receiptKey);
463
- receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
464
- if (delivery.admission === "context" && delivery.sessionId !== undefined)
465
- receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
466
- if (accounting !== undefined && !existing) {
467
- let counts = this.stickyDeliveryCounts.get(workflowId);
847
+ const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
848
+ if (accounting !== undefined && added) {
849
+ let byRun = this.stickyDeliveryCounts.get(workflowId);
850
+ if (byRun === undefined) {
851
+ byRun = new Map;
852
+ this.stickyDeliveryCounts.set(workflowId, byRun);
853
+ }
854
+ let counts = byRun.get(accounting.messageRunId);
468
855
  if (counts === undefined) {
469
856
  counts = new Map;
470
- this.stickyDeliveryCounts.set(workflowId, counts);
857
+ byRun.set(accounting.messageRunId, counts);
471
858
  }
472
- const key = JSON.stringify([accounting.messageRunId, messageId]);
473
- counts.set(key, Math.max(counts.get(key) ?? 0, accounting.deliveryCount));
859
+ const previous = counts.get(messageId) ?? 0;
860
+ counts.set(messageId, Math.max(previous, accounting.deliveryCount));
861
+ if (accounting.deliveryCount > previous)
862
+ this.bumpDeliveryCountVersion(workflowId);
474
863
  }
475
864
  }
476
865
  async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
477
- return [logicalRunId, undefined].some((logicalId) => this.stickyReceipts.get(workflowId)?.has(pendingDeliveryReceiptKey(messageId, delivery, logicalId)) ?? false);
866
+ return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
478
867
  }
479
868
  hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
480
- return [logicalRunId, undefined].some((logicalId) => {
481
- const name = anySession && delivery.admission === "context" ? `${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, logicalId)}:context-any-session` : pendingDeliveryReceiptKey(messageId, delivery, logicalId);
482
- return this.stickyReceipts.get(workflowId)?.has(name) ?? false;
483
- });
869
+ return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
870
+ }
871
+ pendingStageDeliveryReceiptChanges(workflowId, cursor) {
872
+ return this.stickyReceipts.changesSince(workflowId, cursor);
484
873
  }
485
874
  recordCheckpoint(recorded) {
486
875
  const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
@@ -499,6 +888,7 @@ class JournalProjection {
499
888
  if (rec.checkpoints.has(key))
500
889
  return;
501
890
  rec.checkpoints.set(key, currentCheckpoint);
891
+ rec.nestedSnapshotVersion.record(currentCheckpoint);
502
892
  if (currentCheckpoint.kind === "tool") {
503
893
  const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
504
894
  if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
@@ -556,6 +946,9 @@ class JournalProjection {
556
946
  getWorkflow(workflowId) {
557
947
  return this.workflows.get(workflowId)?.handle;
558
948
  }
949
+ nestedSnapshotVersion(workflowId) {
950
+ return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
951
+ }
559
952
  getLoadableWorkflow(workflowId) {
560
953
  return this.getWorkflow(workflowId);
561
954
  }
@@ -633,6 +1026,7 @@ class JournalProjection {
633
1026
  this.pendingReceipts.delete(workflowId);
634
1027
  this.stickyReceipts.delete(workflowId);
635
1028
  this.stickyDeliveryCounts.delete(workflowId);
1029
+ this.bumpDeliveryCountVersion(workflowId);
636
1030
  this.deletedWorkflowIds.add(workflowId);
637
1031
  }
638
1032
  isWorkflowLoadable(workflowId) {
@@ -643,6 +1037,8 @@ class JournalProjection {
643
1037
  this.pendingReceipts.clear();
644
1038
  this.stickyReceipts.clear();
645
1039
  this.stickyDeliveryCounts.clear();
1040
+ for (const workflowId of this.stickyDeliveryCountVersions.keys())
1041
+ this.bumpDeliveryCountVersion(workflowId);
646
1042
  this.deletedWorkflowIds.clear();
647
1043
  }
648
1044
  }
@@ -696,216 +1092,32 @@ function toResumableEntry(handle) {
696
1092
  updatedAt: handle.updatedAt
697
1093
  };
698
1094
  }
699
- function pendingDeliveryReceiptKey(messageId, delivery, logicalRunId) {
700
- const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
701
- return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
702
- }
703
- var SESSION_HEADER_READ_LIMIT_BYTES;
1095
+ var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
704
1096
  var init_backend = __esm(() => {
705
1097
  init_resume_eligibility();
1098
+ init_stage_topology_validation();
706
1099
  init_durable_hash();
707
1100
  SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
708
1101
  });
709
1102
 
710
- // dist/builtin/workflows/src/durable/stage-topology-validation.ts
711
- function immutableStageGroupError(stages) {
712
- const occurrenceError = promptOccurrenceIdentityError(stages);
713
- if (occurrenceError !== undefined)
714
- return occurrenceError;
715
- const byReplayKey = groupByDurableStageKey(stages);
716
- for (const group of byReplayKey.values()) {
717
- const firstName = group[0]?.name;
718
- if (group.some((stage) => stage.name !== firstName))
719
- return "stage name drift within one replay-key group";
720
- const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
721
- const first = topologies[0];
722
- if (first === undefined)
723
- continue;
724
- if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
725
- return "immutable stage identity drift within one replay-key group";
726
- }
727
- }
728
- return;
729
- }
730
- function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
731
- const prefix = `${replayScope}:`;
732
- const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
733
- if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
734
- return "child run topology exists outside its durable invocation scope";
735
- }
736
- if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
737
- return "child checkpoint ownership is not reciprocal";
738
- }
739
- const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
740
- const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
741
- const immutableError = immutableStageGroupError(direct);
742
- if (immutableError !== undefined)
743
- return immutableError;
744
- const merged = [];
745
- for (const group of groupByDurableStageKey(direct).values()) {
746
- const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
747
- if (topology === undefined)
748
- return "direct child stage group lacks owning-run topology";
749
- if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
750
- return "direct child stage topology lacks current source identity";
751
- }
752
- if (topology.stageId.length === 0)
753
- return "direct child stage id is empty";
754
- merged.push(topology);
755
- }
756
- const ids = new Set(allowedNodeIds);
757
- const stageIds = new Set;
758
- const orders = new Set;
759
- for (const topology of merged) {
760
- if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
761
- return "direct child stage ids are not unique";
762
- stageIds.add(topology.stageId);
763
- ids.add(topology.stageId);
764
- if (topology.sourceOrder !== undefined) {
765
- if (orders.has(topology.sourceOrder))
766
- return "direct child source orders are not unique";
767
- orders.add(topology.sourceOrder);
768
- }
769
- }
770
- if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
771
- return "direct child stage references an unknown parent";
772
- }
773
- return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
774
- }
775
- function owningBoundaryIdError(stages, owningRunId) {
776
- const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
777
- const seen = new Map;
778
- for (const start of starts) {
779
- const stageId = start.topology.stageId;
780
- if (stageId.length === 0)
781
- return "owning-run boundary stage id is empty";
782
- const prior = seen.get(stageId);
783
- if (prior !== undefined && prior !== start.replayKey)
784
- return "owning-run boundary stage id is aliased";
785
- seen.set(stageId, start.replayKey);
786
- }
787
- return;
788
- }
789
- function sameRun(actual, expected) {
790
- return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
791
- }
792
- function sameStrings(left, right) {
793
- return left.length === right.length && left.every((value2, index) => value2 === right[index]);
794
- }
795
- function sameOptionalRun(actual, expected) {
796
- if (actual === undefined || expected === undefined)
797
- return actual === expected;
798
- return sameRun(actual, expected);
799
- }
800
- function promptOccurrenceIdentityError(stages) {
801
- const replayByOccurrence = new Map;
802
- const occurrenceByReplayStage = new Map;
803
- const stageByReplayOccurrence = new Map;
804
- for (const stage of stages) {
805
- const topology = stage.topology;
806
- const occurrence = topology?.occurrenceKey;
807
- if (topology === undefined || occurrence === undefined)
808
- continue;
809
- const priorReplay = replayByOccurrence.get(occurrence);
810
- if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
811
- return "prompt occurrence is bound to multiple replay keys";
812
- }
813
- replayByOccurrence.set(occurrence, stage.replayKey);
814
- const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
815
- const priorOccurrence = occurrenceByReplayStage.get(replayStage);
816
- if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
817
- return "prompt stage identity is bound to multiple occurrences";
818
- }
819
- occurrenceByReplayStage.set(replayStage, occurrence);
820
- const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
821
- const priorStage = stageByReplayOccurrence.get(replayOccurrence);
822
- if (priorStage !== undefined && priorStage !== topology.stageId) {
823
- return "prompt occurrence is bound to multiple stage identities";
824
- }
825
- stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
826
- }
827
- return;
828
- }
829
- function durableStageGroupKey(stage) {
830
- const occurrence = stage.topology?.occurrenceKey;
831
- return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
832
- }
833
- function groupByDurableStageKey(stages) {
834
- const groups = new Map;
835
- for (const stage of stages) {
836
- const key = durableStageGroupKey(stage);
837
- const group = groups.get(key) ?? [];
838
- group.push(stage);
839
- groups.set(key, group);
840
- }
841
- return groups;
842
- }
843
- function compositeKey(domain, ...parts) {
844
- return JSON.stringify([domain, ...parts]);
845
- }
846
- function hasCycle(topologies) {
847
- const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
848
- const visiting = new Set;
849
- const visited = new Set;
850
- const cyclic = (stageId) => {
851
- if (visiting.has(stageId))
852
- return true;
853
- if (visited.has(stageId))
854
- return false;
855
- visiting.add(stageId);
856
- if ((parents.get(stageId) ?? []).some(cyclic))
857
- return true;
858
- visiting.delete(stageId);
859
- visited.add(stageId);
860
- return false;
861
- };
862
- return topologies.some((topology) => cyclic(topology.stageId));
863
- }
864
- var init_stage_topology_validation = () => {};
865
-
866
- // dist/builtin/workflows/src/durable/workflow-child-result.ts
867
- function parseWorkflowChildResult(value2) {
868
- if (!isOrdinaryObject(value2))
869
- return;
870
- const workflow = value2.workflow;
871
- const runId = value2.runId;
872
- const status = value2.status;
873
- const exited = value2.exited;
874
- const outputs = value2.outputs;
875
- const exitReason = value2.exitReason;
876
- if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
877
- return;
878
- if (exited === false) {
879
- if (status !== "completed" || exitReason !== undefined)
880
- return;
881
- return value2;
882
- }
883
- if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
884
- return;
885
- return value2;
886
- }
887
- function isWorkflowChildResult(value2) {
888
- return parseWorkflowChildResult(value2) !== undefined;
889
- }
890
- function isWorkflowExitStatus(value2) {
891
- return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
892
- }
893
- function isOrdinaryObject(value2) {
894
- if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
895
- return false;
896
- const prototype = Object.getPrototypeOf(value2);
897
- return prototype === Object.prototype || prototype === null;
898
- }
899
-
900
1103
  // dist/builtin/workflows/src/durable/boundary-lifecycle.ts
901
1104
  function resolveBoundaryLifecycle(start, stages, requireTerminal) {
902
1105
  const startTopology = start.topology;
903
1106
  const startBoundary = startTopology?.boundary;
904
- const startRun = startTopology?.run;
905
1107
  const related = stages.filter((stage) => {
906
1108
  const boundary = stage.topology?.boundary;
907
1109
  return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
908
1110
  });
1111
+ return resolveBoundaryLifecycleFromRecords(start, {
1112
+ related,
1113
+ outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
1114
+ }, requireTerminal);
1115
+ }
1116
+ function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
1117
+ const startTopology = start.topology;
1118
+ const startBoundary = startTopology?.boundary;
1119
+ const startRun = startTopology?.run;
1120
+ const related = records.related;
909
1121
  const starts = related.filter((stage) => stage.topology.boundary.event === "start");
910
1122
  if (starts.length !== 1 || starts[0] !== start)
911
1123
  return failure("boundary has no unique start record");
@@ -942,7 +1154,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
942
1154
  return failure("boundary has an illegal terminal transition");
943
1155
  }
944
1156
  const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
945
- const outputRecords = stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined);
1157
+ const outputRecords = records.outputRecords();
946
1158
  if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
947
1159
  return failure("boundary replay key has conflicting output records");
948
1160
  }
@@ -1683,9 +1895,19 @@ function* invoke(action) {
1683
1895
  } catch (error) {
1684
1896
  if (error instanceof JournalError || error instanceof StorageError)
1685
1897
  throw error;
1686
- throw new StorageUnavailableError({ adapter: "custom", location: "custom journal" }, error instanceof Error ? error : new Error(String(error)));
1898
+ throw customJournalUnavailable(error);
1687
1899
  }
1688
1900
  }
1901
+ function customJournalUnavailable(failure) {
1902
+ const cause = failure instanceof Error ? failure : new Error(String(failure));
1903
+ const code = hasErrorCode(cause) ? cause.code : undefined;
1904
+ const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
1905
+ return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
1906
+ }
1907
+ function hasErrorCode(error) {
1908
+ const code = error.code;
1909
+ return typeof code === "string" || typeof code === "number";
1910
+ }
1689
1911
  async function execute(operation) {
1690
1912
  let step = operation.next();
1691
1913
  while (!step.done) {
@@ -1713,6 +1935,8 @@ class JournalDurableBackend {
1713
1935
  executions = new Map;
1714
1936
  executionContext = new AsyncLocalStorage;
1715
1937
  pending = new Map;
1938
+ reconciledPendingStages = new Map;
1939
+ appendBatches = new Map;
1716
1940
  catalogQueue = Symbol("catalog");
1717
1941
  lost = new Map;
1718
1942
  prompts = new Map;
@@ -1745,7 +1969,7 @@ class JournalDurableBackend {
1745
1969
  await backend.close();
1746
1970
  if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
1747
1971
  throw error;
1748
- throw new StorageUnavailableError({ adapter: "custom", location: "custom journal" }, error instanceof Error ? error : new Error(String(error)));
1972
+ throw customJournalUnavailable(error);
1749
1973
  }
1750
1974
  }
1751
1975
  serialize(runId, action, requiresActiveExecution = true) {
@@ -1760,6 +1984,8 @@ class JournalDurableBackend {
1760
1984
  return this.enqueue(runId, action, admitted);
1761
1985
  }
1762
1986
  enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
1987
+ if (typeof runId === "string")
1988
+ this.appendBatches.delete(runId);
1763
1989
  const prior = this.pending.get(runId) ?? Promise.resolve();
1764
1990
  const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
1765
1991
  const settled = result.then(() => {}, () => {});
@@ -1774,6 +2000,7 @@ class JournalDurableBackend {
1774
2000
  if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
1775
2001
  return;
1776
2002
  const event = entry.event;
2003
+ let raisedCount;
1777
2004
  if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
1778
2005
  const checkpoint = event.payload;
1779
2006
  validateCheckpointInvocationFingerprint(checkpoint);
@@ -1782,7 +2009,10 @@ class JournalDurableBackend {
1782
2009
  this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
1783
2010
  } else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
1784
2011
  const payload = event.payload;
2012
+ const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
1785
2013
  this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
2014
+ if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
2015
+ raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
1786
2016
  } else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
1787
2017
  this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
1788
2018
  this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
@@ -1793,20 +2023,80 @@ class JournalDurableBackend {
1793
2023
  if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
1794
2024
  this.prompts.delete(entry.runId);
1795
2025
  }
1796
- const handle = this.projection.getWorkflow(entry.runId);
1797
- if (handle?.pendingStageMessages !== undefined) {
1798
- const messages = handle.pendingStageMessages.map((message) => {
1799
- if (message.sticky !== true)
1800
- return message;
1801
- const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
1802
- return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
1803
- });
1804
- if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
1805
- this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
1806
- }
2026
+ this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
1807
2027
  this.sequences.set(entry.runId, entry.seq);
1808
2028
  this.foldedAt.set(entry.runId, entry.at);
1809
2029
  }
2030
+ reconcileFoldedDeliveryCounts(runId, raisedCount) {
2031
+ const handle = this.projection.getWorkflow(runId);
2032
+ const list = handle?.pendingStageMessages;
2033
+ if (handle === undefined || list === undefined) {
2034
+ this.reconciledPendingStages.delete(runId);
2035
+ return;
2036
+ }
2037
+ let state = this.reconciledPendingStages.get(runId);
2038
+ let next;
2039
+ const raise = (index, count) => {
2040
+ const message = (next ?? list)[index];
2041
+ if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
2042
+ return;
2043
+ next ??= [...list];
2044
+ next[index] = { ...message, deliveryCount: count };
2045
+ };
2046
+ const reconcile = (index) => {
2047
+ const message = list[index];
2048
+ if (message.sticky !== true)
2049
+ return;
2050
+ raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
2051
+ };
2052
+ if (state?.list !== list) {
2053
+ const previous = state?.list ?? [];
2054
+ let aligned = list.length >= previous.length;
2055
+ const changed = [];
2056
+ for (let index = 0;index < list.length; index++) {
2057
+ const message = list[index];
2058
+ if (previous[index] === message)
2059
+ continue;
2060
+ if (index < previous.length && previous[index].id !== message.id)
2061
+ aligned = false;
2062
+ changed.push(index);
2063
+ }
2064
+ const reconciled = aligned ? undefined : new Set(previous);
2065
+ for (const index of changed)
2066
+ if (!reconciled?.has(list[index]))
2067
+ reconcile(index);
2068
+ const positions = aligned ? state?.positions : undefined;
2069
+ if (positions !== undefined) {
2070
+ for (let index = previous.length;index < list.length; index++) {
2071
+ const indexes = positions.get(list[index].id);
2072
+ if (indexes === undefined)
2073
+ positions.set(list[index].id, [index]);
2074
+ else
2075
+ indexes.push(index);
2076
+ }
2077
+ }
2078
+ state = { list, positions };
2079
+ }
2080
+ if (raisedCount !== undefined) {
2081
+ const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
2082
+ state.positions ??= messagePositions(list);
2083
+ for (const index of state.positions.get(raisedCount.messageId) ?? []) {
2084
+ const message = list[index];
2085
+ if (message.sticky === true && message.runId === raisedCount.messageRunId)
2086
+ raise(index, count);
2087
+ }
2088
+ }
2089
+ if (next === undefined) {
2090
+ this.reconciledPendingStages.set(runId, state);
2091
+ return;
2092
+ }
2093
+ this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
2094
+ const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
2095
+ this.reconciledPendingStages.set(runId, {
2096
+ list: applied,
2097
+ positions: applied.length === list.length ? state.positions : undefined
2098
+ });
2099
+ }
1810
2100
  summarize(runId, tail) {
1811
2101
  const event = tail.snapshot.event;
1812
2102
  const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
@@ -1830,6 +2120,7 @@ class JournalDurableBackend {
1830
2120
  *rebuild(runId) {
1831
2121
  const entries = yield* this.entries(runId);
1832
2122
  this.projection.deleteWorkflow(runId);
2123
+ this.reconciledPendingStages.delete(runId);
1833
2124
  this.sequences.delete(runId);
1834
2125
  this.foldedAt.delete(runId);
1835
2126
  this.summarized.delete(runId);
@@ -1925,9 +2216,9 @@ class JournalDurableBackend {
1925
2216
  }
1926
2217
  yield* this.write(runId, lease, event, options);
1927
2218
  }
1928
- *write(runId, lease, event, options) {
2219
+ *write(runId, lease, event, options, context = (action) => action()) {
1929
2220
  try {
1930
- const outcome = yield* invoke(() => this.journal.append(lease, event, options));
2221
+ const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
1931
2222
  const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
1932
2223
  if (outcome.entry.seq > nextSeq) {
1933
2224
  for (const entry of yield* this.entries(runId, nextSeq))
@@ -2111,13 +2402,80 @@ class JournalDurableBackend {
2111
2402
  return true;
2112
2403
  }
2113
2404
  async archivePendingStageMessage(runId, message) {
2114
- await this.serialize(runId, () => this.append(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, { key: JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]) }));
2405
+ await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
2115
2406
  }
2116
2407
  async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
2117
- await this.serialize(runId, () => this.append(runId, {
2408
+ await this.queueAppend(runId, {
2118
2409
  type: RUN_EVENT_TYPES.messageDelivered,
2119
2410
  payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
2120
- }));
2411
+ });
2412
+ }
2413
+ queueAppend(runId, event, key) {
2414
+ try {
2415
+ this.ensureOpen();
2416
+ this.assertExecutionActive();
2417
+ } catch (error) {
2418
+ return Promise.reject(error);
2419
+ }
2420
+ const open = this.appendBatches.get(runId);
2421
+ const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
2422
+ const queued = new Promise((resolve, reject) => {
2423
+ batch.push({
2424
+ event,
2425
+ ...key === undefined ? {} : { key },
2426
+ context: AsyncLocalStorage.snapshot(),
2427
+ resolve,
2428
+ reject
2429
+ });
2430
+ });
2431
+ if (batch !== open) {
2432
+ this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
2433
+ for (const append of batch)
2434
+ append.reject(error);
2435
+ });
2436
+ this.appendBatches.set(runId, batch);
2437
+ }
2438
+ return queued;
2439
+ }
2440
+ *appendBatch(runId, batch) {
2441
+ if (this.appendBatches.get(runId) === batch)
2442
+ this.appendBatches.delete(runId);
2443
+ const appendMany = runJournalAppendMany(this.journal);
2444
+ if (appendMany === undefined || batch.length === 1) {
2445
+ for (const append of batch) {
2446
+ try {
2447
+ const lease = yield* this.ownership(runId);
2448
+ yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
2449
+ append.resolve();
2450
+ } catch (error) {
2451
+ append.reject(error);
2452
+ }
2453
+ }
2454
+ return;
2455
+ }
2456
+ let settled = 0;
2457
+ try {
2458
+ const lease = yield* this.ownership(runId);
2459
+ const results = yield* invoke(() => appendMany(lease, batch));
2460
+ for (const result of results) {
2461
+ const append = batch[settled];
2462
+ if (result.status === "rejected") {
2463
+ append.reject(yield* this.mapOwnership(result.reason, runId));
2464
+ } else {
2465
+ const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
2466
+ if (result.value.entry.seq > nextSeq)
2467
+ for (const entry of yield* this.entries(runId, nextSeq))
2468
+ this.fold(entry);
2469
+ this.fold(result.value.entry);
2470
+ append.resolve();
2471
+ }
2472
+ settled++;
2473
+ }
2474
+ } catch (error) {
2475
+ const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
2476
+ for (const append of batch.slice(settled))
2477
+ append.reject(failure);
2478
+ }
2121
2479
  }
2122
2480
  getToolOutput(...args) {
2123
2481
  return this.projection.getToolOutput(...args);
@@ -2137,6 +2495,9 @@ class JournalDurableBackend {
2137
2495
  listCheckpoints(runId) {
2138
2496
  return this.projection.listCheckpoints(runId);
2139
2497
  }
2498
+ nestedSnapshotVersion(runId) {
2499
+ return this.projection.nestedSnapshotVersion(runId);
2500
+ }
2140
2501
  getWorkflow(runId) {
2141
2502
  return this.projection.getWorkflow(runId);
2142
2503
  }
@@ -2161,6 +2522,12 @@ class JournalDurableBackend {
2161
2522
  getPendingStageMessageDeliveryCount(...args) {
2162
2523
  return this.projection.getPendingStageMessageDeliveryCount(...args);
2163
2524
  }
2525
+ pendingStageDeliveryReceiptChanges(...args) {
2526
+ return this.projection.pendingStageDeliveryReceiptChanges(...args);
2527
+ }
2528
+ pendingStageDeliveryCountVersion(runId) {
2529
+ return this.projection.pendingStageDeliveryCountVersion(runId);
2530
+ }
2164
2531
  hasPendingStageDeliveryReceipt(...args) {
2165
2532
  return this.projection.hasPendingStageDeliveryReceipt(...args);
2166
2533
  }
@@ -2245,6 +2612,7 @@ class JournalDurableBackend {
2245
2612
  this.foldedAt.delete(runId);
2246
2613
  this.summarized.delete(runId);
2247
2614
  this.prompts.delete(runId);
2615
+ this.reconciledPendingStages.delete(runId);
2248
2616
  this.projection.deleteWorkflow(runId);
2249
2617
  }
2250
2618
  async deleteWorkflow(runId) {
@@ -2427,8 +2795,24 @@ class JournalDurableBackend {
2427
2795
  }
2428
2796
  }
2429
2797
  }
2798
+ function messagePositions(list) {
2799
+ const positions = new Map;
2800
+ list.forEach((message, index) => {
2801
+ const indexes = positions.get(message.id);
2802
+ if (indexes === undefined)
2803
+ positions.set(message.id, [index]);
2804
+ else
2805
+ indexes.push(index);
2806
+ });
2807
+ return positions;
2808
+ }
2430
2809
  function identity(message) {
2431
- return JSON.stringify([message.runId, message.id]);
2810
+ let key = entryIdentity.get(message);
2811
+ if (key === undefined) {
2812
+ key = JSON.stringify([message.runId, message.id]);
2813
+ entryIdentity.set(message, key);
2814
+ }
2815
+ return key;
2432
2816
  }
2433
2817
  function uniqueByIdentity(messages) {
2434
2818
  const result = new Map;
@@ -2602,7 +2986,7 @@ function decodePendingStageDeliveries(previous, messages) {
2602
2986
  function hasPendingStageDeliveryDelta(messages) {
2603
2987
  return messages?.some(isDeltaEncoded) === true;
2604
2988
  }
2605
- var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
2989
+ var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
2606
2990
  var init_journal_backend = __esm(() => {
2607
2991
  init_workflow_instance_ownership_error();
2608
2992
  init_backend();
@@ -2614,6 +2998,7 @@ var init_journal_backend = __esm(() => {
2614
2998
  init_types();
2615
2999
  PROCESS_HOLDER_ID = randomUUID();
2616
3000
  processExitCleanups = new Set;
3001
+ CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
2617
3002
  JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
2618
3003
  runId;
2619
3004
  holderId;
@@ -2660,6 +3045,7 @@ var init_journal_backend = __esm(() => {
2660
3045
  this.name = "JournalMessageScopeError";
2661
3046
  }
2662
3047
  };
3048
+ entryIdentity = new WeakMap;
2663
3049
  entryJson = new WeakMap;
2664
3050
  });
2665
3051
 
@@ -2787,6 +3173,7 @@ class SqlRunJournal {
2787
3173
  runVersions: () => this.runVersions(),
2788
3174
  snapshotTail: (runId) => this.snapshotTail(runId)
2789
3175
  });
3176
+ registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
2790
3177
  registerSqlPoolStorage(this, {
2791
3178
  transaction: (body) => this.operation(() => this.database.transaction(body)),
2792
3179
  withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
@@ -3051,38 +3438,76 @@ class SqlRunJournal {
3051
3438
  this.assertOpen();
3052
3439
  validateText("runId", lease.runId);
3053
3440
  validateText("holderId", lease.holderId);
3054
- if (options?.key !== undefined)
3055
- validateText("key", options.key);
3056
3441
  const owned = { ...lease };
3057
- const key = options?.key;
3442
+ const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
3058
3443
  const expected = options?.expectSeq;
3444
+ return this.operation(() => this.database.transaction(async (tx) => {
3445
+ await this.checkLease(tx, owned);
3446
+ return this.insert(tx, owned, prepared, expected);
3447
+ }));
3448
+ }
3449
+ async appendMany(lease, appends) {
3450
+ this.assertOpen();
3451
+ validateText("runId", lease.runId);
3452
+ validateText("holderId", lease.holderId);
3453
+ const owned = { ...lease };
3454
+ const prepared = appends.map((append) => {
3455
+ try {
3456
+ return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
3457
+ } catch (error) {
3458
+ return error;
3459
+ }
3460
+ });
3461
+ return this.operation(() => this.database.transaction(async (tx) => {
3462
+ await this.checkLease(tx, owned);
3463
+ const results = [];
3464
+ for (const append of prepared) {
3465
+ if (append instanceof Error) {
3466
+ results.push({ status: "rejected", reason: append });
3467
+ continue;
3468
+ }
3469
+ await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
3470
+ try {
3471
+ results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
3472
+ } catch (error) {
3473
+ await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
3474
+ results.push({ status: "rejected", reason: error });
3475
+ }
3476
+ await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
3477
+ }
3478
+ return results;
3479
+ }));
3480
+ }
3481
+ prepare(event, key, appendGuard) {
3482
+ if (key !== undefined)
3483
+ validateText("key", key);
3059
3484
  const type = event.type;
3060
3485
  const payload = this.snapshotJson(event.payload);
3061
- const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? this.appendGuard.getStore() : undefined;
3486
+ const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
3062
3487
  const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
3063
3488
  if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
3064
3489
  validateText("status", status);
3065
- return this.operation(() => this.database.transaction(async (tx) => {
3066
- await this.checkLease(tx, owned);
3067
- await guard?.(tx, owned.runId);
3068
- if (key !== undefined) {
3069
- const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
3070
- if (row)
3071
- return { status: "exists", entry: entryFromRow(row) };
3072
- }
3073
- const actual = await this.sequence(tx, owned.runId);
3074
- if (expected !== undefined && expected !== actual)
3075
- throw new JournalCasConflictError(owned.runId, expected, actual);
3076
- const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
3077
- if (payload instanceof StorageUnavailableError2)
3078
- throw payload;
3079
- await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
3080
- if (status !== undefined)
3081
- await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
3082
- const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
3083
- await guard?.(tx, owned.runId);
3084
- return { status: "appended", entry: entryFromRow(row) };
3085
- }));
3490
+ return { key, type, payload, guard, status };
3491
+ }
3492
+ async insert(tx, owned, { key, type, payload, guard, status }, expected) {
3493
+ await guard?.(tx, owned.runId);
3494
+ if (key !== undefined) {
3495
+ const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
3496
+ if (row)
3497
+ return { status: "exists", entry: entryFromRow(row) };
3498
+ }
3499
+ const actual = await this.sequence(tx, owned.runId);
3500
+ if (expected !== undefined && expected !== actual)
3501
+ throw new JournalCasConflictError(owned.runId, expected, actual);
3502
+ const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
3503
+ if (payload instanceof StorageUnavailableError2)
3504
+ throw payload;
3505
+ await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
3506
+ if (status !== undefined)
3507
+ await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
3508
+ const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
3509
+ await guard?.(tx, owned.runId);
3510
+ return { status: "appended", entry: entryFromRow(row) };
3086
3511
  }
3087
3512
  async* read(runId, fromSeq = 1) {
3088
3513
  this.assertOpen();
@@ -3150,7 +3575,7 @@ class SqlRunJournal {
3150
3575
  return this.closePromise;
3151
3576
  }
3152
3577
  }
3153
- var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
3578
+ var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
3154
3579
  var init_sql_run_journal = __esm(() => {
3155
3580
  init_internal();
3156
3581
  init_legacy_journal_tables();
@@ -3707,8 +4132,59 @@ function firstError(schema, value2) {
3707
4132
  }
3708
4133
  return;
3709
4134
  }
4135
+ function certainlySerializable(root, rootMustBeObject) {
4136
+ if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
4137
+ return false;
4138
+ const pending = [root];
4139
+ const ancestors = new Set;
4140
+ const exits = [undefined];
4141
+ while (pending.length > 0) {
4142
+ const exit = exits.pop();
4143
+ const value2 = pending.pop();
4144
+ if (exit !== undefined) {
4145
+ ancestors.delete(exit);
4146
+ continue;
4147
+ }
4148
+ if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
4149
+ continue;
4150
+ if (typeof value2 === "number") {
4151
+ if (!Number.isFinite(value2))
4152
+ return false;
4153
+ continue;
4154
+ }
4155
+ if (typeof value2 !== "object" || ancestors.has(value2))
4156
+ return false;
4157
+ if (Object.getOwnPropertySymbols(value2).length > 0)
4158
+ return false;
4159
+ const names = Object.getOwnPropertyNames(value2);
4160
+ let children;
4161
+ if (Array.isArray(value2)) {
4162
+ if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
4163
+ return false;
4164
+ children = [];
4165
+ for (let index = 0;index < value2.length; index++)
4166
+ children.push(value2[index]);
4167
+ } else {
4168
+ const proto = Object.getPrototypeOf(value2);
4169
+ if (proto !== Object.prototype && proto !== null)
4170
+ return false;
4171
+ const keys = Object.keys(value2);
4172
+ if (keys.length !== names.length)
4173
+ return false;
4174
+ children = keys.map((key) => value2[key]);
4175
+ }
4176
+ ancestors.add(value2);
4177
+ pending.push(value2);
4178
+ exits.push(value2);
4179
+ for (const child of children) {
4180
+ pending.push(child);
4181
+ exits.push(undefined);
4182
+ }
4183
+ }
4184
+ return true;
4185
+ }
3710
4186
  function workflowSerializableValidationError(value2, label) {
3711
- if (Value.Check(workflowSerializableValueSchema, value2))
4187
+ if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
3712
4188
  return;
3713
4189
  const issue = firstError(workflowSerializableValueSchema, value2);
3714
4190
  const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
@@ -3717,7 +4193,7 @@ function workflowSerializableValidationError(value2, label) {
3717
4193
  return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
3718
4194
  }
3719
4195
  function workflowSerializableObjectValidationError(value2, label) {
3720
- if (Value.Check(workflowSerializableObjectSchema, value2))
4196
+ if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
3721
4197
  return;
3722
4198
  const issue = firstError(workflowSerializableObjectSchema, value2);
3723
4199
  const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
@@ -6089,6 +6565,12 @@ class ScopedDurableBackend {
6089
6565
  getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
6090
6566
  return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
6091
6567
  }
6568
+ pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
6569
+ return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
6570
+ }
6571
+ pendingStageDeliveryCountVersion(_workflowId) {
6572
+ return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
6573
+ }
6092
6574
  async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
6093
6575
  return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
6094
6576
  }
@@ -8354,8 +8836,11 @@ var init_store_internal = __esm(() => {
8354
8836
  });
8355
8837
 
8356
8838
  // dist/builtin/workflows/src/shared/pending-stage-retention.ts
8839
+ function isSettledPendingStageMessage(entry) {
8840
+ return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
8841
+ }
8357
8842
  async function retainPendingStageMessages(messages, archive) {
8358
- const settled = messages.filter((entry) => entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined);
8843
+ const settled = messages.filter(isSettledPendingStageMessage);
8359
8844
  const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
8360
8845
  for (const entry of expired)
8361
8846
  await archive(entry);
@@ -8363,53 +8848,210 @@ async function retainPendingStageMessages(messages, archive) {
8363
8848
  }
8364
8849
  var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
8365
8850
 
8851
+ // dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
8852
+ class PendingStageTransitionLedger {
8853
+ compacted = [];
8854
+ settled = 0;
8855
+ stages = new Map;
8856
+ fixed = new WeakSet;
8857
+ reconciled = [];
8858
+ reconciledVersion;
8859
+ reconcile(backend, runId, messages) {
8860
+ if (backend.getPendingStageMessageDeliveryCount === undefined)
8861
+ return messages;
8862
+ const version = backend.pendingStageDeliveryCountVersion?.(runId);
8863
+ const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
8864
+ let recovered;
8865
+ for (let index = 0;index < messages.length; index++) {
8866
+ const entry = messages[index];
8867
+ if (entry.sticky !== true || base !== undefined && base[index] === entry)
8868
+ continue;
8869
+ const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
8870
+ if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
8871
+ continue;
8872
+ recovered ??= [...messages];
8873
+ recovered[index] = { ...entry, deliveryCount: count };
8874
+ }
8875
+ const result = recovered ?? messages;
8876
+ this.reconciled = result;
8877
+ this.reconciledVersion = version;
8878
+ return result;
8879
+ }
8880
+ async compact(backend, runId, messages, runs, archiveSettled) {
8881
+ const stageStatus = createStageStatusLookup(runs);
8882
+ const endedStages = [];
8883
+ const dirty = new Set;
8884
+ for (const [deliveryRunId, byStage] of this.stages) {
8885
+ for (const [stageId, entries] of byStage) {
8886
+ if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
8887
+ continue;
8888
+ endedStages.push([deliveryRunId, stageId]);
8889
+ for (const entry of entries)
8890
+ dirty.add(entry);
8891
+ }
8892
+ }
8893
+ const base = this.compacted;
8894
+ const output = [];
8895
+ const removed = [];
8896
+ const added = [];
8897
+ let settled = this.settled;
8898
+ const length = Math.max(base.length, messages.length);
8899
+ for (let index = 0;index < length; index++) {
8900
+ const previous = base[index];
8901
+ const entry = messages[index];
8902
+ if (previous === entry && !dirty.has(entry)) {
8903
+ output.push(entry);
8904
+ continue;
8905
+ }
8906
+ if (previous !== undefined) {
8907
+ removed.push(previous);
8908
+ if (isSettledPendingStageMessage(previous))
8909
+ settled--;
8910
+ }
8911
+ if (entry === undefined)
8912
+ continue;
8913
+ if (isSettledPendingStageMessage(entry))
8914
+ settled++;
8915
+ const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
8916
+ output.push(next);
8917
+ added.push(next);
8918
+ }
8919
+ let retained = output;
8920
+ if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
8921
+ retained = await retainPendingStageMessages(output, archiveSettled);
8922
+ settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
8923
+ }
8924
+ return {
8925
+ messages: retained,
8926
+ commit: () => {
8927
+ for (const [deliveryRunId, stageId] of endedStages) {
8928
+ const byStage = this.stages.get(deliveryRunId);
8929
+ byStage?.delete(stageId);
8930
+ if (byStage?.size === 0)
8931
+ this.stages.delete(deliveryRunId);
8932
+ }
8933
+ for (const entry of dirty)
8934
+ this.fixed.delete(entry);
8935
+ for (const entry of removed)
8936
+ this.unindex(entry);
8937
+ for (const entry of added)
8938
+ this.index(entry);
8939
+ this.compacted = retained;
8940
+ this.settled = settled;
8941
+ }
8942
+ };
8943
+ }
8944
+ index(entry) {
8945
+ this.fixed.add(entry);
8946
+ for (const delivery of entry.deliveries ?? []) {
8947
+ let byStage = this.stages.get(delivery.runId);
8948
+ if (byStage === undefined) {
8949
+ byStage = new Map;
8950
+ this.stages.set(delivery.runId, byStage);
8951
+ }
8952
+ let entries = byStage.get(delivery.stageId);
8953
+ if (entries === undefined) {
8954
+ entries = new Set;
8955
+ byStage.set(delivery.stageId, entries);
8956
+ }
8957
+ entries.add(entry);
8958
+ }
8959
+ }
8960
+ unindex(entry) {
8961
+ this.fixed.delete(entry);
8962
+ for (const delivery of entry.deliveries ?? []) {
8963
+ const byStage = this.stages.get(delivery.runId);
8964
+ const entries = byStage?.get(delivery.stageId);
8965
+ if (entries === undefined)
8966
+ continue;
8967
+ entries.delete(entry);
8968
+ if (entries.size > 0)
8969
+ continue;
8970
+ byStage.delete(delivery.stageId);
8971
+ if (byStage.size === 0)
8972
+ this.stages.delete(delivery.runId);
8973
+ }
8974
+ }
8975
+ }
8976
+ function isCompactingBackend(backend) {
8977
+ return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
8978
+ }
8979
+ function createStageStatusLookup(runs) {
8980
+ let runById;
8981
+ return (runId, stageId) => {
8982
+ if (runById === undefined) {
8983
+ runById = new Map;
8984
+ for (const run of runs)
8985
+ if (!runById.has(run.id))
8986
+ runById.set(run.id, run);
8987
+ }
8988
+ return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
8989
+ };
8990
+ }
8991
+ function isEndedStageStatus(status) {
8992
+ return status === "completed" || status === "skipped" || status === "failed";
8993
+ }
8994
+ async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
8995
+ if (entry.sticky !== true || entry.deliveries === undefined)
8996
+ return entry;
8997
+ const latest = new Map;
8998
+ entry.deliveries.forEach((delivery, index) => {
8999
+ latest.set(deliverySlot(delivery), index);
9000
+ });
9001
+ const deliveries = [];
9002
+ let archivedThrough = -1;
9003
+ for (let index = 0;index < entry.deliveries.length; index++) {
9004
+ const delivery = entry.deliveries[index];
9005
+ if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
9006
+ while (archivedThrough < index) {
9007
+ const prefixIndex = archivedThrough + 1;
9008
+ const receipt = entry.deliveries[prefixIndex];
9009
+ if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
9010
+ await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
9011
+ messageRunId: entry.runId,
9012
+ deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
9013
+ });
9014
+ archivedThrough = prefixIndex;
9015
+ }
9016
+ } else {
9017
+ deliveries.push(delivery);
9018
+ }
9019
+ }
9020
+ return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
9021
+ }
9022
+ function deliverySlot(delivery) {
9023
+ return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
9024
+ }
9025
+ var init_pending_stage_transition_ledger = () => {};
9026
+
8366
9027
  // dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
8367
9028
  function createPendingStageDeliveryStoreMethods(context) {
8368
- const transitions = new Map;
8369
- const serialize = async (runId, transition) => {
8370
- const previous = transitions.get(runId) ?? Promise.resolve();
8371
- const result = previous.catch(() => {
8372
- return;
8373
- }).then(transition);
8374
- const settled = result.then(() => {
8375
- return;
8376
- }, () => {
8377
- return;
8378
- });
8379
- transitions.set(runId, settled);
8380
- settled.finally(() => {
8381
- if (transitions.get(runId) === settled)
8382
- transitions.delete(runId);
8383
- });
8384
- return await result;
8385
- };
9029
+ const ledgers = new PendingStageLedgers;
9030
+ const batches = new PendingStageTransitionBatches(context, ledgers);
8386
9031
  return {
8387
9032
  async queueStageMessage(input, senderGroup, runGroup, backend) {
8388
- return await serialize(input.runId, async () => {
9033
+ return await batches.run(input.runId, backend, async (batch) => {
8389
9034
  const run = context.findRun(input.runId);
8390
9035
  if (run === undefined)
8391
9036
  return;
8392
9037
  const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
8393
9038
  const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
8394
- const result = queueStageMessage(receipt === undefined ? run.pendingStageMessages ?? [] : [...run.pendingStageMessages ?? [], receipt], input, senderGroup, runGroup, stageIdentity);
9039
+ const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
8395
9040
  if (result.ok && !result.deduplicated) {
8396
- const messages = await persistTransition(backend, input.runId, result.messages, context);
8397
- const live = context.findRun(input.runId) ?? run;
8398
- live.pendingStageMessages = [...messages];
8399
- context.bumpAndNotify();
9041
+ await batch.apply(backend, input.runId, result.messages);
8400
9042
  }
8401
- return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run).pendingStageMessages ?? [] } : result;
9043
+ return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
8402
9044
  });
8403
9045
  },
8404
9046
  async validateLiveStageMessage(input) {
8405
- return await serialize(input.runId, async () => {
9047
+ return await batches.run(input.runId, undefined, async (batch) => {
8406
9048
  const run = context.findRun(input.runId);
8407
9049
  if (run === undefined)
8408
9050
  return;
8409
9051
  const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
8410
9052
  const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
8411
9053
  const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
8412
- const result = queueStageMessage(receipt === undefined ? run.pendingStageMessages ?? [] : [...run.pendingStageMessages ?? [], receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
9054
+ const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
8413
9055
  if (!result.ok) {
8414
9056
  return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
8415
9057
  }
@@ -8431,77 +9073,65 @@ function createPendingStageDeliveryStoreMethods(context) {
8431
9073
  return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
8432
9074
  },
8433
9075
  async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
8434
- return await serialize(runId, async () => {
9076
+ return await batches.run(runId, backend, async (batch) => {
8435
9077
  const run = context.findRun(runId);
8436
9078
  if (run === undefined)
8437
9079
  return false;
8438
- const current = run.pendingStageMessages ?? [];
9080
+ const current = batch.messages(run);
8439
9081
  const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
8440
9082
  if (next === current)
8441
9083
  return false;
8442
- const messages = await persistTransition(backend, runId, next, context);
8443
- const live = context.findRun(runId) ?? run;
8444
- live.pendingStageMessages = [...messages];
8445
- context.bumpAndNotify();
9084
+ await batch.apply(backend, runId, next);
8446
9085
  return true;
8447
9086
  });
8448
9087
  },
8449
9088
  async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
8450
- return await serialize(runId, async () => {
9089
+ return await batches.run(runId, backend, async (batch) => {
8451
9090
  const run = context.findRun(runId);
8452
9091
  if (run === undefined)
8453
9092
  return false;
8454
- const current = run.pendingStageMessages ?? [];
9093
+ const current = batch.messages(run);
8455
9094
  const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
8456
9095
  if (next === current)
8457
9096
  return false;
8458
- const messages = await persistTransition(backend, runId, next, context);
8459
- const live = context.findRun(runId) ?? run;
8460
- live.pendingStageMessages = [...messages];
8461
- context.bumpAndNotify();
9097
+ await batch.apply(backend, runId, next);
8462
9098
  return true;
8463
9099
  });
8464
9100
  },
8465
9101
  async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
8466
- return await serialize(runId, async () => {
9102
+ return await batches.run(runId, backend, async (batch) => {
8467
9103
  const run = context.findRun(runId);
8468
9104
  if (run === undefined)
8469
9105
  return false;
8470
- const current = run.pendingStageMessages ?? [];
9106
+ const current = batch.messages(run);
8471
9107
  const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
8472
9108
  if (next === current)
8473
9109
  return false;
8474
- const messages = await persistTransition(backend, runId, next, context);
8475
- const live = context.findRun(runId) ?? run;
8476
- live.pendingStageMessages = [...messages];
8477
- context.bumpAndNotify();
9110
+ await batch.apply(backend, runId, next);
8478
9111
  return true;
8479
9112
  });
8480
9113
  },
8481
9114
  async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
8482
- return await serialize(input.runId, async () => {
9115
+ return await batches.run(input.runId, backend, async (batch) => {
8483
9116
  const run = context.findRun(input.runId);
8484
9117
  if (run === undefined)
8485
9118
  return;
8486
9119
  const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
8487
- const result = queueStickyStageMessage(receipt === undefined ? run.pendingStageMessages ?? [] : [...run.pendingStageMessages ?? [], receipt], input, senderGroup, runGroup);
9120
+ const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
8488
9121
  if (result.ok && !result.deduplicated) {
8489
- const messages = await persistTransition(backend, input.runId, result.messages, context);
8490
- const live = context.findRun(input.runId) ?? run;
8491
- live.pendingStageMessages = [...messages];
8492
- context.bumpAndNotify();
9122
+ await batch.apply(backend, input.runId, result.messages);
8493
9123
  }
8494
- return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run).pendingStageMessages ?? [] } : result;
9124
+ return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
8495
9125
  });
8496
9126
  },
8497
9127
  async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
8498
- return await serialize(runId, async () => {
9128
+ return await batches.run(runId, backend, async (batch) => {
8499
9129
  const run = context.findRun(runId);
8500
9130
  if (run === undefined)
8501
9131
  return false;
8502
9132
  if (records.length === 0)
8503
9133
  await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
8504
- let current = reconcileDeliveryCounts(backend, runId, run.pendingStageMessages ?? []);
9134
+ let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
8505
9135
  const unconfirmedRecords = [];
8506
9136
  for (const record of records) {
8507
9137
  const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
@@ -8509,35 +9139,26 @@ function createPendingStageDeliveryStoreMethods(context) {
8509
9139
  unconfirmedRecords.push(record);
8510
9140
  }
8511
9141
  }
8512
- current = reconcileDeliveryCounts(backend, runId, current);
8513
- if (current !== run.pendingStageMessages) {
8514
- run.pendingStageMessages = [...current];
8515
- context.bumpAndNotify();
8516
- }
9142
+ current = ledgers.for(run).reconcile(backend, runId, current);
9143
+ batch.reconciled(run, current);
8517
9144
  const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
8518
9145
  if (next === current)
8519
9146
  return false;
8520
- const messages = await persistTransition(backend, runId, next, context);
8521
- const live = context.findRun(runId) ?? run;
8522
- live.pendingStageMessages = [...messages];
8523
- context.bumpAndNotify();
9147
+ await batch.apply(backend, runId, next);
8524
9148
  return true;
8525
9149
  });
8526
9150
  },
8527
9151
  async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
8528
- return await serialize(runId, async () => {
9152
+ return await batches.run(runId, backend, async (batch) => {
8529
9153
  const run = context.findRun(runId);
8530
9154
  if (run === undefined)
8531
9155
  return false;
8532
9156
  await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
8533
- const current = reconcileDeliveryCounts(backend, runId, run.pendingStageMessages ?? []);
9157
+ const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
8534
9158
  const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
8535
9159
  if (next === current)
8536
9160
  return false;
8537
- const messages = await persistTransition(backend, runId, next, context);
8538
- const live = context.findRun(runId) ?? run;
8539
- live.pendingStageMessages = [...messages];
8540
- context.bumpAndNotify();
9161
+ await batch.apply(backend, runId, next);
8541
9162
  return true;
8542
9163
  });
8543
9164
  }
@@ -8555,78 +9176,186 @@ function resolvePendingStageIdentity(run, stageKey) {
8555
9176
  aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
8556
9177
  };
8557
9178
  }
8558
- async function persistTransition(backend, runId, messages, context) {
8559
- let retained = reconcileDeliveryCounts(backend, runId, messages);
8560
- try {
8561
- if (backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined) {
8562
- const compacted = [];
8563
- for (const entry of retained) {
8564
- if (entry.sticky !== true || entry.deliveries === undefined) {
8565
- compacted.push(entry);
8566
- continue;
8567
- }
8568
- const latest = new Map;
8569
- entry.deliveries.forEach((delivery, index) => {
8570
- latest.set(JSON.stringify([delivery.runId, delivery.stageId, delivery.admission]), index);
8571
- });
8572
- const deliveries = [];
8573
- let archivedThrough = -1;
8574
- for (let index = 0;index < entry.deliveries.length; index++) {
8575
- const delivery = entry.deliveries[index];
8576
- const stage = context.state.runs.find((run) => run.id === delivery.runId)?.stages.find((stage) => stage.id === delivery.stageId);
8577
- if (entry.status !== "queued" || stage?.status === "completed" || stage?.status === "skipped" || stage?.status === "failed" || latest.get(JSON.stringify([delivery.runId, delivery.stageId, delivery.admission])) !== index && delivery.sessionId !== undefined) {
8578
- while (archivedThrough < index) {
8579
- const prefixIndex = archivedThrough + 1;
8580
- await backend.archivePendingStageDeliveryReceipt(runId, entry.id, entry.deliveries[prefixIndex], {
8581
- messageRunId: entry.runId,
8582
- deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
8583
- });
8584
- archivedThrough = prefixIndex;
8585
- }
8586
- } else {
8587
- deliveries.push(delivery);
8588
- }
8589
- }
8590
- compacted.push(deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries });
9179
+
9180
+ class PendingStageTransitionBatches {
9181
+ queues = new Map;
9182
+ context;
9183
+ ledgers;
9184
+ constructor(context, ledgers) {
9185
+ this.context = context;
9186
+ this.ledgers = ledgers;
9187
+ }
9188
+ run(runId, backend, step) {
9189
+ return new Promise((resolve, reject) => {
9190
+ const transition = {
9191
+ backend,
9192
+ step,
9193
+ resolve: (value2) => resolve(value2),
9194
+ reject
9195
+ };
9196
+ const queue = this.queues.get(runId);
9197
+ if (queue !== undefined) {
9198
+ queue.push(transition);
9199
+ return;
9200
+ }
9201
+ const created = [transition];
9202
+ this.queues.set(runId, created);
9203
+ Promise.resolve().then(() => this.drain(runId, created));
9204
+ });
9205
+ }
9206
+ async drain(runId, queue) {
9207
+ while (queue.length > 0)
9208
+ await this.runBatch(runId, takeBatch(queue));
9209
+ this.queues.delete(runId);
9210
+ }
9211
+ async runBatch(runId, transitions) {
9212
+ const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
9213
+ const outcomes = [];
9214
+ for (const transition of transitions) {
9215
+ const changedBefore = batch.changed;
9216
+ try {
9217
+ const value2 = await transition.step(batch);
9218
+ outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
9219
+ } catch (error) {
9220
+ outcomes.push({ ok: false, error });
8591
9221
  }
8592
- retained = compacted;
8593
9222
  }
8594
- retained = compactStickyStageMessageDeliveries(retained, context.state.runs);
8595
- if (backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined) {
8596
- retained = await retainPendingStageMessages(retained, (entry) => backend.archivePendingStageMessage(runId, entry));
9223
+ let failure;
9224
+ try {
9225
+ await batch.persist();
9226
+ } catch (error) {
9227
+ failure = { error };
9228
+ }
9229
+ transitions.forEach((transition, index) => {
9230
+ const outcome = outcomes[index];
9231
+ if (!outcome.ok)
9232
+ transition.reject(outcome.error);
9233
+ else if (failure !== undefined && outcome.uncommitted)
9234
+ transition.reject(failure.error);
9235
+ else
9236
+ transition.resolve(outcome.value);
9237
+ });
9238
+ }
9239
+ }
9240
+ function takeBatch(queue) {
9241
+ let count = 1;
9242
+ let backend = queue[0].backend;
9243
+ while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
9244
+ const next = queue[count].backend;
9245
+ if (next !== undefined && backend !== undefined && next !== backend)
9246
+ break;
9247
+ backend ??= next;
9248
+ count++;
9249
+ }
9250
+ return queue.splice(0, count);
9251
+ }
9252
+
9253
+ class PendingStageTransitionBatch {
9254
+ changed = false;
9255
+ list;
9256
+ pending;
9257
+ context;
9258
+ ledgers;
9259
+ runId;
9260
+ constructor(context, ledgers, runId) {
9261
+ this.context = context;
9262
+ this.ledgers = ledgers;
9263
+ this.runId = runId;
9264
+ }
9265
+ messages(run) {
9266
+ return this.list ?? run.pendingStageMessages ?? [];
9267
+ }
9268
+ reconciled(run, messages) {
9269
+ if (this.changed) {
9270
+ this.list = messages;
9271
+ return;
8597
9272
  }
8598
- if (!await backend.persistPendingStageMessages(runId, retained)) {
8599
- throw new Error(`atomic-workflows: durable workflow ${runId} is unavailable for pending-stage persistence`);
9273
+ if (messages !== run.pendingStageMessages) {
9274
+ run.pendingStageMessages = [...messages];
9275
+ this.context.bumpAndNotify();
8600
9276
  }
8601
- return retained;
8602
- } finally {
8603
- const run = context.findRun(runId);
8604
- if (run !== undefined) {
8605
- const current = run.pendingStageMessages ?? [];
8606
- const recovered = reconcileDeliveryCounts(backend, runId, current);
8607
- if (recovered !== current) {
8608
- run.pendingStageMessages = [...recovered];
8609
- context.bumpAndNotify();
8610
- }
9277
+ }
9278
+ async apply(backend, runId, messages) {
9279
+ const run = this.context.findRun(runId);
9280
+ try {
9281
+ const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
9282
+ this.list = compacted.messages;
9283
+ this.changed = true;
9284
+ this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
9285
+ return compacted.messages;
9286
+ } catch (error) {
9287
+ if (this.changed && this.list !== undefined) {
9288
+ const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
9289
+ this.list = ledger.reconcile(backend, runId, this.list);
9290
+ } else
9291
+ reconcileLive(backend, runId, this.context, this.ledgers);
9292
+ throw error;
8611
9293
  }
8612
9294
  }
9295
+ async persist() {
9296
+ if (this.pending === undefined)
9297
+ return;
9298
+ const { backend, messages, commit, run } = this.pending;
9299
+ const published = this.list ?? messages;
9300
+ try {
9301
+ if (!await backend.persistPendingStageMessages(this.runId, messages)) {
9302
+ throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
9303
+ }
9304
+ commit?.();
9305
+ } finally {
9306
+ reconcileLive(backend, this.runId, this.context, this.ledgers);
9307
+ }
9308
+ const live = this.context.findRun(this.runId) ?? run;
9309
+ if (live !== undefined)
9310
+ live.pendingStageMessages = [...published];
9311
+ this.context.bumpAndNotify();
9312
+ }
9313
+ }
9314
+ async function compactTransition(backend, runId, messages, context, ledgers, run) {
9315
+ const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
9316
+ const reconciled = ledger.reconcile(backend, runId, messages);
9317
+ const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
9318
+ if (isCompactingBackend(backend)) {
9319
+ return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
9320
+ }
9321
+ let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
9322
+ if (archiveSettled !== undefined)
9323
+ retained = await retainPendingStageMessages(retained, archiveSettled);
9324
+ if (run !== undefined)
9325
+ ledgers.reset(run);
9326
+ return { messages: retained };
9327
+ }
9328
+ function reconcileLive(backend, runId, context, ledgers) {
9329
+ const current = context.findRun(runId);
9330
+ if (current === undefined)
9331
+ return;
9332
+ const messages = current.pendingStageMessages ?? [];
9333
+ const recovered = ledgers.for(current).reconcile(backend, runId, messages);
9334
+ if (recovered !== messages) {
9335
+ current.pendingStageMessages = [...recovered];
9336
+ context.bumpAndNotify();
9337
+ }
8613
9338
  }
8614
- function reconcileDeliveryCounts(backend, runId, messages) {
8615
- let changed = false;
8616
- const recovered = messages.map((entry) => {
8617
- if (entry.sticky !== true)
8618
- return entry;
8619
- const count = backend.getPendingStageMessageDeliveryCount?.(runId, entry.id, entry.runId) ?? 0;
8620
- if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
8621
- return entry;
8622
- changed = true;
8623
- return { ...entry, deliveryCount: count };
8624
- });
8625
- return changed ? recovered : messages;
9339
+
9340
+ class PendingStageLedgers {
9341
+ ledgers = new WeakMap;
9342
+ for(run) {
9343
+ let ledger = this.ledgers.get(run);
9344
+ if (ledger === undefined) {
9345
+ ledger = new PendingStageTransitionLedger;
9346
+ this.ledgers.set(run, ledger);
9347
+ }
9348
+ return ledger;
9349
+ }
9350
+ reset(run) {
9351
+ this.ledgers.delete(run);
9352
+ }
8626
9353
  }
9354
+ var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
8627
9355
  var init_store_pending_stage_delivery_methods = __esm(() => {
8628
9356
  init_run_owner_backend();
8629
9357
  init_pending_stage_delivery();
9358
+ init_pending_stage_transition_ledger();
8630
9359
  });
8631
9360
 
8632
9361
  // dist/builtin/workflows/src/shared/store-prompt-methods.ts
@@ -57620,6 +58349,9 @@ var init_workflow_stage_target = __esm(() => {
57620
58349
  });
57621
58350
 
57622
58351
  // dist/builtin/workflows/src/shared/pending-stage-status.ts
58352
+ function isAddressableStageStatus(status) {
58353
+ return ADDRESSABLE_STAGE_STATUSES.has(status);
58354
+ }
57623
58355
  function workflowBoundarySegments(runs, runId) {
57624
58356
  return createWorkflowBoundarySegmentsResolver(runs)(runId);
57625
58357
  }
@@ -57742,44 +58474,412 @@ function createStagePathPatternIndex(patterns) {
57742
58474
  }
57743
58475
  };
57744
58476
  }
58477
+ var ADDRESSABLE_STAGE_STATUSES;
57745
58478
  var init_pending_stage_status = __esm(() => {
57746
58479
  init_store_internal();
57747
58480
  init_workflow_stage_target();
58481
+ ADDRESSABLE_STAGE_STATUSES = new Set([
58482
+ "pending",
58483
+ "running",
58484
+ "awaiting_input",
58485
+ "paused",
58486
+ "blocked"
58487
+ ]);
57748
58488
  });
57749
58489
 
57750
- // dist/builtin/workflows/src/shared/sticky-stage-delivery-status.ts
57751
- function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
57752
- const root = runs.find((run) => run.id === rootRunId);
57753
- const entries = (root?.pendingStageMessages ?? []).filter((entry) => entry.sticky === true);
57754
- const result = [];
57755
- const patternByTarget = new Map;
57756
- const patterns = [];
57757
- const entriesByPattern = [];
57758
- const ownEntries = [];
57759
- entries.forEach((entry, index) => {
58490
+ // dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
58491
+ function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
58492
+ if (states.length === 0)
58493
+ return { deliveryStates: states };
58494
+ const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
58495
+ counts.set("delivered", 0);
58496
+ for (const delivery of states)
58497
+ counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
58498
+ const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
58499
+ if (states.length <= limit)
58500
+ return { deliveryStates: states, deliveryStateCounts };
58501
+ const exemplars = [];
58502
+ for (const state of UNRESOLVED_STATES) {
58503
+ for (const delivery of states) {
58504
+ if (exemplars.length === limit)
58505
+ break;
58506
+ if (delivery.state === state)
58507
+ exemplars.push(delivery);
58508
+ }
58509
+ }
58510
+ for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
58511
+ const delivery = states[index];
58512
+ if (delivery.state === "delivered")
58513
+ exemplars.push(delivery);
58514
+ }
58515
+ return {
58516
+ deliveryStates: exemplars,
58517
+ deliveryStateCounts,
58518
+ omittedDeliveryStates: states.length - exemplars.length
58519
+ };
58520
+ }
58521
+ var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
58522
+ var init_sticky_delivery_state_bound = __esm(() => {
58523
+ UNRESOLVED_STATES = [
58524
+ "queued",
58525
+ "receipt-unverified",
58526
+ "session-replaced",
58527
+ "delivery-unavailable",
58528
+ "root-unresolved",
58529
+ "stage-terminal"
58530
+ ];
58531
+ });
58532
+
58533
+ // dist/builtin/workflows/src/shared/possible-stage-rows.ts
58534
+ function boundPossibleStageDeliveryStates(states) {
58535
+ const bounded = boundStickyDeliveryStates(states);
58536
+ return {
58537
+ ...bounded,
58538
+ deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
58539
+ };
58540
+ }
58541
+
58542
+ class WorkflowTopologyIndex {
58543
+ lineage = [];
58544
+ runById = new Map;
58545
+ roots = new Map;
58546
+ hops = new Map;
58547
+ refresh(runs) {
58548
+ const runById = new Map(runs.map((run) => [run.id, run]));
58549
+ const lineage = [];
58550
+ for (const run of runs) {
58551
+ lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
58552
+ if (run.parentRunId === undefined)
58553
+ continue;
58554
+ const parent = runById.get(run.parentRunId);
58555
+ const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
58556
+ lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
58557
+ }
58558
+ if (sameValues(lineage, this.lineage))
58559
+ return;
58560
+ this.lineage = lineage;
58561
+ this.runById = runById;
58562
+ this.roots.clear();
58563
+ this.hops.clear();
58564
+ }
58565
+ rootOf(runId) {
58566
+ if (this.roots.has(runId))
58567
+ return this.roots.get(runId);
58568
+ const root = reciprocalWorkflowRootRunId(this.runById, runId);
58569
+ this.roots.set(runId, root);
58570
+ return root;
58571
+ }
58572
+ hopsOf(runId) {
58573
+ if (this.hops.has(runId))
58574
+ return this.hops.get(runId);
58575
+ const hops = boundaryHops(this.runById, runId);
58576
+ this.hops.set(runId, hops);
58577
+ return hops;
58578
+ }
58579
+ }
58580
+ function boundaryHops(runById, runId) {
58581
+ const hops = [];
58582
+ let current = runById.get(runId);
58583
+ while (current !== undefined && current.parentRunId !== undefined) {
58584
+ const parent = runById.get(current.parentRunId);
58585
+ const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
58586
+ if (parent === undefined || boundary === undefined)
58587
+ return;
58588
+ hops.unshift({ name: boundary.name, runId: current.id });
58589
+ current = parent;
58590
+ }
58591
+ return current === undefined ? undefined : hops;
58592
+ }
58593
+ function sameValues(left, right) {
58594
+ if (left.length !== right.length)
58595
+ return false;
58596
+ for (let index = 0;index < left.length; index++)
58597
+ if (left[index] !== right[index])
58598
+ return false;
58599
+ return true;
58600
+ }
58601
+
58602
+ class PossibleStageRowsIndex {
58603
+ rootRunId;
58604
+ topology;
58605
+ targets = new Map;
58606
+ patterns = [];
58607
+ targetOf = new Map;
58608
+ targetsByMessage = new Map;
58609
+ list = [];
58610
+ stageFields = new WeakMap;
58611
+ backend;
58612
+ receiptCursor;
58613
+ scanMatches = new Map;
58614
+ rowCache = new Map;
58615
+ boundedStates = new WeakMap;
58616
+ deliveriesByEntry = new WeakMap;
58617
+ statusCache = new WeakMap;
58618
+ receiptGeneration = 0;
58619
+ receiptEpochs = new Map;
58620
+ constructor(rootRunId, topology) {
58621
+ this.rootRunId = rootRunId;
58622
+ this.topology = topology;
58623
+ }
58624
+ rows(runs, rootRun, availableBackend) {
58625
+ if (isTerminalRunStatus(rootRun.status))
58626
+ return [];
58627
+ this.refreshEntries(rootRun.pendingStageMessages ?? []);
58628
+ const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
58629
+ const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
58630
+ this.refreshReceipts(backend);
58631
+ this.refreshStages(runs);
58632
+ for (const state of this.patterns)
58633
+ if (state.dirty)
58634
+ this.recompute(state, backend);
58635
+ return this.assemble(runs, rootRun);
58636
+ }
58637
+ targetFor(entry) {
57760
58638
  const target = entry.targetPath ?? entry.stageKey;
57761
- let pattern = patternByTarget.get(target);
57762
- if (pattern === undefined) {
58639
+ let state = this.targets.get(target);
58640
+ if (state === undefined) {
57763
58641
  const parsed = parseWorkflowStageTarget(target);
57764
- pattern = parsed?.rootRunId === rootRunId ? patterns.push(parsed.segments) - 1 : -1;
57765
- if (pattern >= 0)
57766
- entriesByPattern.push([]);
57767
- patternByTarget.set(target, pattern);
58642
+ const owned = parsed?.rootRunId === this.rootRunId;
58643
+ state = {
58644
+ target,
58645
+ segments: owned ? parsed.segments : undefined,
58646
+ pattern: owned ? this.patterns.length : -1,
58647
+ entries: [],
58648
+ positions: [],
58649
+ queued: 0,
58650
+ stages: [],
58651
+ statuses: [],
58652
+ dirty: true
58653
+ };
58654
+ this.targets.set(target, state);
58655
+ if (owned)
58656
+ this.patterns.push(state);
57768
58657
  }
57769
- if (pattern < 0)
58658
+ return state;
58659
+ }
58660
+ refreshEntries(list) {
58661
+ const previous = this.list;
58662
+ if (list === previous)
57770
58663
  return;
57771
- entriesByPattern[pattern].push(index);
57772
- ownEntries.push(index);
57773
- });
57774
- if (ownEntries.length === 0)
57775
- return result;
57776
- const patternIndex = createStagePathPatternIndex(patterns);
57777
- const deliveriesByEntry = new Map;
57778
- const entryReceipts = (index, runId, stageId) => {
57779
- let byStage = deliveriesByEntry.get(index);
58664
+ this.list = list;
58665
+ let incremental = list.length >= previous.length;
58666
+ const replaced = [];
58667
+ for (let index = 0;incremental && index < previous.length; index++) {
58668
+ if (list[index] === previous[index])
58669
+ continue;
58670
+ const before = previous[index];
58671
+ const after = list[index];
58672
+ if (before.sticky !== true && after.sticky !== true)
58673
+ continue;
58674
+ if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
58675
+ incremental = false;
58676
+ break;
58677
+ }
58678
+ replaced.push(index);
58679
+ }
58680
+ if (!incremental) {
58681
+ this.rebuildEntries(list);
58682
+ return;
58683
+ }
58684
+ for (const index of replaced) {
58685
+ const before = previous[index];
58686
+ const after = list[index];
58687
+ const state = this.targetOf.get(before);
58688
+ const slot = state.entries.indexOf(before);
58689
+ state.entries[slot] = after;
58690
+ this.targetOf.delete(before);
58691
+ this.targetOf.set(after, state);
58692
+ state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
58693
+ if (before.id !== after.id)
58694
+ this.rebuildMessageIndex();
58695
+ state.dirty = true;
58696
+ }
58697
+ for (let index = previous.length;index < list.length; index++)
58698
+ this.addEntry(list[index], index);
58699
+ }
58700
+ addEntry(entry, position) {
58701
+ if (entry.sticky !== true)
58702
+ return;
58703
+ const state = this.targetFor(entry);
58704
+ state.entries.push(entry);
58705
+ state.positions.push(position);
58706
+ if (entry.status === "queued")
58707
+ state.queued++;
58708
+ state.dirty = true;
58709
+ this.targetOf.set(entry, state);
58710
+ let targets = this.targetsByMessage.get(entry.id);
58711
+ if (targets === undefined) {
58712
+ targets = new Set;
58713
+ this.targetsByMessage.set(entry.id, targets);
58714
+ }
58715
+ targets.add(state);
58716
+ }
58717
+ rebuildEntries(list) {
58718
+ const before = new Map;
58719
+ for (const state of this.targets.values()) {
58720
+ before.set(state, state.entries);
58721
+ state.entries = [];
58722
+ state.positions = [];
58723
+ state.queued = 0;
58724
+ }
58725
+ this.targetOf.clear();
58726
+ this.targetsByMessage.clear();
58727
+ list.forEach((entry, position) => {
58728
+ this.addEntry(entry, position);
58729
+ });
58730
+ for (const state of this.targets.values()) {
58731
+ const previous = before.get(state);
58732
+ state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
58733
+ }
58734
+ }
58735
+ rebuildMessageIndex() {
58736
+ this.targetsByMessage.clear();
58737
+ for (const [entry, state] of this.targetOf) {
58738
+ let targets = this.targetsByMessage.get(entry.id);
58739
+ if (targets === undefined) {
58740
+ targets = new Set;
58741
+ this.targetsByMessage.set(entry.id, targets);
58742
+ }
58743
+ targets.add(state);
58744
+ }
58745
+ }
58746
+ refreshReceipts(backend) {
58747
+ if (backend !== this.backend) {
58748
+ this.backend = backend;
58749
+ this.receiptCursor = undefined;
58750
+ this.invalidateReceipts();
58751
+ }
58752
+ if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
58753
+ return;
58754
+ const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
58755
+ if (changes === undefined) {
58756
+ this.invalidateReceipts();
58757
+ return;
58758
+ }
58759
+ const firstRead = this.receiptCursor === undefined;
58760
+ this.receiptCursor = changes.cursor;
58761
+ if (changes.messageIds === undefined) {
58762
+ if (!firstRead)
58763
+ this.invalidateReceipts();
58764
+ return;
58765
+ }
58766
+ for (const messageId of changes.messageIds) {
58767
+ this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
58768
+ for (const state of this.targetsByMessage.get(messageId) ?? [])
58769
+ state.dirty = true;
58770
+ }
58771
+ }
58772
+ invalidateReceipts() {
58773
+ this.receiptGeneration++;
58774
+ this.receiptEpochs.clear();
58775
+ for (const state of this.patterns)
58776
+ state.dirty = true;
58777
+ }
58778
+ refreshStages(runs) {
58779
+ const current = new Map;
58780
+ let ordinal = 0;
58781
+ for (const run of runs) {
58782
+ const root = this.topology.rootOf(run.id);
58783
+ if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
58784
+ continue;
58785
+ const resolved = root === this.rootRunId;
58786
+ const hops = this.topology.hopsOf(run.id);
58787
+ for (const stage of run.stages) {
58788
+ if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
58789
+ continue;
58790
+ const fields = this.fieldsOf(run, stage, resolved, hops);
58791
+ const status = { run, stage, resolved, hops, ordinal: ordinal++ };
58792
+ for (const state of fields.matched) {
58793
+ const stages = current.get(state);
58794
+ if (stages === undefined)
58795
+ current.set(state, [status]);
58796
+ else
58797
+ stages.push(status);
58798
+ }
58799
+ }
58800
+ }
58801
+ for (const state of this.patterns) {
58802
+ const stages = current.get(state) ?? [];
58803
+ if (stages.length !== state.stages.length || stages.some((status, index) => {
58804
+ const previous = state.stages[index];
58805
+ return previous.stage !== status.stage || previous.run !== status.run;
58806
+ }))
58807
+ state.dirty = true;
58808
+ state.stages = stages;
58809
+ }
58810
+ }
58811
+ fieldsOf(run, stage, resolved, hops) {
58812
+ let fields = this.stageFields.get(stage);
58813
+ const matchable = resolved && hops !== undefined;
58814
+ let changed = false;
58815
+ if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
58816
+ const previous = fields;
58817
+ const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
58818
+ fields = {
58819
+ run,
58820
+ status: stage.status,
58821
+ error: stage.error,
58822
+ sessionId: stage.sessionId,
58823
+ pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
58824
+ resolved,
58825
+ hops,
58826
+ id: stage.id,
58827
+ name: stage.name,
58828
+ tested: sameMatching ? previous.tested : 0,
58829
+ matched: sameMatching ? previous.matched : []
58830
+ };
58831
+ this.stageFields.set(stage, fields);
58832
+ changed = true;
58833
+ }
58834
+ if (fields.tested < this.patterns.length) {
58835
+ const added = this.patterns.slice(fields.tested);
58836
+ if (!matchable)
58837
+ fields.matched.push(...added);
58838
+ else {
58839
+ const index = createStagePathPatternIndex(added.map((state) => state.segments));
58840
+ for (const position of index.matching(hops, [stage.id, stage.name]))
58841
+ fields.matched.push(added[position]);
58842
+ }
58843
+ fields.tested = this.patterns.length;
58844
+ }
58845
+ if (changed)
58846
+ for (const state of fields.matched)
58847
+ state.dirty = true;
58848
+ return fields;
58849
+ }
58850
+ recompute(state, backend) {
58851
+ state.dirty = false;
58852
+ const statuses = [];
58853
+ for (const { run, stage, resolved, hops } of state.stages) {
58854
+ const fields = this.stageFields.get(stage);
58855
+ for (const entry of state.entries) {
58856
+ let byStage = this.statusCache.get(entry);
58857
+ if (byStage === undefined) {
58858
+ byStage = new Map;
58859
+ this.statusCache.set(entry, byStage);
58860
+ }
58861
+ const epoch = this.receiptEpochs.get(entry.id) ?? 0;
58862
+ let cached = byStage.get(stage);
58863
+ if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
58864
+ cached = {
58865
+ fields,
58866
+ generation: this.receiptGeneration,
58867
+ epoch,
58868
+ status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
58869
+ };
58870
+ byStage.set(stage, cached);
58871
+ }
58872
+ statuses.push(cached.status);
58873
+ }
58874
+ }
58875
+ if (!sameStatuses(statuses, state.statuses))
58876
+ state.statuses = statuses;
58877
+ }
58878
+ receiptsOf(entry, runId, stageId) {
58879
+ let byStage = this.deliveriesByEntry.get(entry);
57780
58880
  if (byStage === undefined) {
57781
58881
  byStage = new Map;
57782
- for (const delivery of entries[index].deliveries ?? []) {
58882
+ for (const delivery of entry.deliveries ?? []) {
57783
58883
  const key = `${delivery.runId}\x00${delivery.stageId}`;
57784
58884
  const list = byStage.get(key);
57785
58885
  if (list === undefined)
@@ -57787,75 +58887,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
57787
58887
  else
57788
58888
  list.push(delivery);
57789
58889
  }
57790
- deliveriesByEntry.set(index, byStage);
58890
+ this.deliveriesByEntry.set(entry, byStage);
57791
58891
  }
57792
58892
  return byStage.get(`${runId}\x00${stageId}`) ?? [];
57793
- };
57794
- availableBackend = availableRunDurableBackend(runs, rootRunId) ?? availableBackend;
57795
- const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, runs, rootRunId);
57796
- const runById = new Map(runs.map((run) => [run.id, run]));
57797
- const hopsFor = createWorkflowBoundaryHopsResolver(runs);
57798
- for (const run of runs) {
57799
- const durableRootRunId = reciprocalWorkflowRootRunId(runById, run.id);
57800
- if (run.id !== rootRunId && run.rootRunId !== rootRunId && durableRootRunId !== rootRunId)
57801
- continue;
57802
- const resolved = durableRootRunId === rootRunId;
57803
- const hops = hopsFor(run.id);
57804
- for (const stage of run.stages) {
57805
- if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
58893
+ }
58894
+ status(entry, run, stage, matchable, backend) {
58895
+ const target = entry.targetPath ?? entry.stageKey;
58896
+ const receipts = this.receiptsOf(entry, run.id, stage.id);
58897
+ const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
58898
+ let state;
58899
+ let reason;
58900
+ if (!matchable) {
58901
+ state = "root-unresolved";
58902
+ reason = "Repair reciprocal parent/boundary ownership before startup";
58903
+ } else if (delivered) {
58904
+ state = "delivered";
58905
+ reason = "Startup context admission confirmed; sticky subscription remains for future stages";
58906
+ } else if (["completed", "failed", "skipped"].includes(stage.status)) {
58907
+ state = "stage-terminal";
58908
+ reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
58909
+ } else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
58910
+ state = "receipt-unverified";
58911
+ reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
58912
+ } else if (stage.pendingStageDeliveryAvailable !== true) {
58913
+ state = "delivery-unavailable";
58914
+ reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
58915
+ } else if (receipts.length > 0) {
58916
+ state = "session-replaced";
58917
+ reason = "Only a previous session has a receipt; replacement must drain before its first turn";
58918
+ } else {
58919
+ state = "queued";
58920
+ reason = "No delivery receipt; stage startup must drain before its first turn";
58921
+ }
58922
+ return {
58923
+ messageId: entry.message.id,
58924
+ target,
58925
+ runId: run.id,
58926
+ stageId: stage.id,
58927
+ ...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
58928
+ state,
58929
+ reason
58930
+ };
58931
+ }
58932
+ assemble(runs, rootRun) {
58933
+ const rows = [];
58934
+ const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
58935
+ let liveStageMatches;
58936
+ const rowTargets = new Set;
58937
+ for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
58938
+ const rowSegments = splitStagePathSegments(scanEntry);
58939
+ if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
57806
58940
  continue;
57807
- const matchedEntries = resolved && hops ? patternIndex.matching(hops, [stage.id, stage.name]).flatMap((pattern) => entriesByPattern[pattern]).sort((left, right) => left - right) : ownEntries;
57808
- for (const index of matchedEntries) {
57809
- const entry = entries[index];
57810
- const target = entry.targetPath ?? entry.stageKey;
57811
- const receipts = entryReceipts(index, run.id, stage.id);
57812
- const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
57813
- runId: run.id,
57814
- stageId: stage.id,
57815
- sessionId: stage.sessionId,
57816
- admission: "context"
57817
- }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
57818
- let state;
57819
- let reason;
57820
- if (!resolved || !hops) {
57821
- state = "root-unresolved";
57822
- reason = "Repair reciprocal parent/boundary ownership before startup";
57823
- } else if (delivered) {
57824
- state = "delivered";
57825
- reason = "Startup context admission confirmed; sticky subscription remains for future stages";
57826
- } else if (["completed", "failed", "skipped"].includes(stage.status)) {
57827
- state = "stage-terminal";
57828
- reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
57829
- } else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
57830
- state = "receipt-unverified";
57831
- reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
57832
- } else if (stage.pendingStageDeliveryAvailable !== true) {
57833
- state = "delivery-unavailable";
57834
- reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
57835
- } else if (receipts.length > 0) {
57836
- state = "session-replaced";
57837
- reason = "Only a previous session has a receipt; replacement must drain before its first turn";
57838
- } else {
57839
- state = "queued";
57840
- reason = "No delivery receipt; stage startup must drain before its first turn";
57841
- }
57842
- result.push({
57843
- messageId: entry.message.id,
57844
- target,
57845
- runId: run.id,
57846
- stageId: stage.id,
57847
- ...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
57848
- state,
57849
- reason
57850
- });
58941
+ if (!rowSegments.some((segment) => segment.includes("*"))) {
58942
+ liveStageMatches ??= this.materializedStageMatcher(runs);
58943
+ if (liveStageMatches(rowSegments))
58944
+ continue;
57851
58945
  }
58946
+ let matches = this.scanMatches.get(scanEntry);
58947
+ if (matches === undefined) {
58948
+ matches = new Map;
58949
+ this.scanMatches.set(scanEntry, matches);
58950
+ }
58951
+ let queuedCount = 0;
58952
+ for (const state of scanTargets) {
58953
+ let match = matches.get(state);
58954
+ if (match === undefined) {
58955
+ const segments = state.segments;
58956
+ match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
58957
+ matches.set(state, match);
58958
+ }
58959
+ if (match)
58960
+ queuedCount += state.queued;
58961
+ }
58962
+ const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
58963
+ rows.push({ slot: `s${position}`, target, queuedCount });
58964
+ rowTargets.add(target);
58965
+ }
58966
+ let broadcastCount = 0;
58967
+ for (const state of this.targets.values()) {
58968
+ if (state.segments?.length === 1 && state.segments[0] === "**")
58969
+ broadcastCount += state.queued;
58970
+ }
58971
+ const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
58972
+ rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
58973
+ rowTargets.add(broadcast);
58974
+ const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
58975
+ const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
58976
+ for (const state of statusTargets) {
58977
+ if (rowTargets.has(state.target))
58978
+ continue;
58979
+ rowTargets.add(state.target);
58980
+ rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
57852
58981
  }
58982
+ const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
58983
+ this.pruneRows(result);
58984
+ return result;
57853
58985
  }
57854
- return result;
58986
+ row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
58987
+ const cached = this.rowCache.get(slot);
58988
+ const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
58989
+ if (same && cached.statuses === deliveryStates)
58990
+ return cached.row;
58991
+ const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
58992
+ if (same && sameBoundedRow(cached.row, bounded)) {
58993
+ this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
58994
+ return cached.row;
58995
+ }
58996
+ const row = {
58997
+ target,
58998
+ queuedCount,
58999
+ ...deliveryOnly === true ? { deliveryOnly } : {},
59000
+ ...bounded
59001
+ };
59002
+ this.rowCache.set(slot, { row, statuses: deliveryStates });
59003
+ return row;
59004
+ }
59005
+ bounded(states) {
59006
+ let bounded = this.boundedStates.get(states);
59007
+ if (bounded === undefined) {
59008
+ bounded = boundPossibleStageDeliveryStates(states);
59009
+ this.boundedStates.set(states, bounded);
59010
+ }
59011
+ return bounded;
59012
+ }
59013
+ pruneRows(rows) {
59014
+ if (this.rowCache.size <= rows.length * 4 + 16)
59015
+ return;
59016
+ const kept = new Set(rows);
59017
+ for (const [key, cached] of this.rowCache)
59018
+ if (!kept.has(cached.row))
59019
+ this.rowCache.delete(key);
59020
+ }
59021
+ materializedStageMatcher(runs) {
59022
+ const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
59023
+ return (entrySegments) => {
59024
+ for (const run of invocationRuns) {
59025
+ const hops = this.topology.hopsOf(run.id);
59026
+ if (hops === undefined)
59027
+ continue;
59028
+ for (const stage of run.stages) {
59029
+ if (stage.pendingStageDeliveryAvailable !== true)
59030
+ continue;
59031
+ if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
59032
+ continue;
59033
+ }
59034
+ if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
59035
+ return true;
59036
+ }
59037
+ }
59038
+ return false;
59039
+ };
59040
+ }
59041
+ }
59042
+ function sameBoundedRow(row, bounded) {
59043
+ if (bounded === undefined)
59044
+ return row.deliveryStates === undefined;
59045
+ if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
59046
+ return false;
59047
+ const counts = Object.entries(row.deliveryStateCounts ?? {});
59048
+ const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
59049
+ if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
59050
+ return false;
59051
+ return sameStatuses(row.deliveryStates, bounded.deliveryStates);
59052
+ }
59053
+ function sameStatuses(left, right) {
59054
+ if (left.length !== right.length)
59055
+ return false;
59056
+ for (let index = 0;index < left.length; index++) {
59057
+ const a = left[index];
59058
+ const b = right[index];
59059
+ if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
59060
+ return false;
59061
+ }
59062
+ return true;
57855
59063
  }
57856
- var init_sticky_stage_delivery_status = __esm(() => {
59064
+ var init_possible_stage_rows = __esm(() => {
57857
59065
  init_run_owner_backend();
57858
59066
  init_pending_stage_status();
59067
+ init_sticky_delivery_state_bound();
59068
+ init_store_internal();
57859
59069
  init_workflow_stage_target();
57860
59070
  });
57861
59071
 
@@ -57920,11 +59130,32 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57920
59130
  announceIfPending();
57921
59131
  return routeCompletions.get(runId);
57922
59132
  });
59133
+ const topology = new WorkflowTopologyIndex;
59134
+ const possibleStageRowIndexes = new Map;
59135
+ const rowIdentities = new WeakMap;
59136
+ let nextRowIdentity = 0;
59137
+ const rowIdentity = (row) => {
59138
+ let identity = rowIdentities.get(row);
59139
+ if (identity === undefined) {
59140
+ identity = nextRowIdentity++;
59141
+ rowIdentities.set(row, identity);
59142
+ }
59143
+ return identity;
59144
+ };
59145
+ const possibleStageRowsIndex = (rootRunId) => {
59146
+ let index = possibleStageRowIndexes.get(rootRunId);
59147
+ if (index === undefined) {
59148
+ index = new PossibleStageRowsIndex(rootRunId, topology);
59149
+ possibleStageRowIndexes.set(rootRunId, index);
59150
+ }
59151
+ return index;
59152
+ };
57923
59153
  const announceRoutes = () => {
57924
59154
  if (disposed)
57925
59155
  return;
57926
59156
  const ownedRunIds = new Set;
57927
59157
  const runs = activeStore.runs();
59158
+ topology.refresh(runs);
57928
59159
  const runById = new Map(runs.map((run) => [run.id, run]));
57929
59160
  const childrenByBoundary = new Map;
57930
59161
  for (const run of runs) {
@@ -57939,7 +59170,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57939
59170
  for (const run of runs) {
57940
59171
  if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
57941
59172
  continue;
57942
- const rootRunId = reciprocalWorkflowRootRunId(runById, run.id);
59173
+ const rootRunId = topology.rootOf(run.id);
57943
59174
  if (rootRunId === undefined)
57944
59175
  continue;
57945
59176
  ownedRunIds.add(run.id);
@@ -57949,6 +59180,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57949
59180
  runId: parentRun.id,
57950
59181
  stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
57951
59182
  };
59183
+ let possibleStages;
59184
+ if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
59185
+ possibleStageRowIndexes.delete(rootRunId);
59186
+ possibleStages = [];
59187
+ } else if (run.id === rootRunId) {
59188
+ possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
59189
+ }
57952
59190
  const announcement = {
57953
59191
  runId: run.id,
57954
59192
  group: workflowInvocationIntercomGroup(rootRunId),
@@ -57960,7 +59198,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57960
59198
  stageName: stage.name,
57961
59199
  target: stageRouteTarget(runs, rootRunId, run.id, stage.id, resolveBoundarySegments),
57962
59200
  lifecycle: stage.sessionId === undefined && stage.sessionFile === undefined ? "pending" : "running",
57963
- routeEligible: stage.pendingStageDeliveryAvailable === true && (stage.status === "pending" || stage.status === "running" || stage.status === "awaiting_input" || stage.status === "paused" || stage.status === "blocked"),
59201
+ routeEligible: stage.pendingStageDeliveryAvailable === true && isAddressableStageStatus(stage.status),
57964
59202
  group: stage.intercomGroup ?? workflowInvocationIntercomGroup(rootRunId)
57965
59203
  })),
57966
59204
  ...nonAgentNodes(run).map((node) => ({
@@ -57973,11 +59211,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
57973
59211
  group: workflowInvocationIntercomGroup(rootRunId)
57974
59212
  }))
57975
59213
  ],
57976
- ...run.id === rootRunId ? {
57977
- possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
57978
- } : {}
59214
+ ...possibleStages === undefined ? {} : { possibleStages }
57979
59215
  };
57980
- const signature = JSON.stringify(announcement);
59216
+ const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
57981
59217
  if (announcedRoutes.get(run.id)?.signature === signature)
57982
59218
  continue;
57983
59219
  const entry = { signature };
@@ -58012,6 +59248,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
58012
59248
  if (!ownedRunIds.has(runId))
58013
59249
  routeCompletions.delete(runId);
58014
59250
  }
59251
+ for (const rootRunId of possibleStageRowIndexes.keys()) {
59252
+ if (!ownedRunIds.has(rootRunId))
59253
+ possibleStageRowIndexes.delete(rootRunId);
59254
+ }
58015
59255
  sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
58016
59256
  return;
58017
59257
  }).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
@@ -58172,102 +59412,6 @@ function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegmen
58172
59412
  const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
58173
59413
  return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
58174
59414
  }
58175
- function possibleStageRows(runs, rootRunId, rootRun, backend) {
58176
- if (isTerminalRunStatus(rootRun.status))
58177
- return [];
58178
- const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
58179
- const segmentsByTarget = new Map;
58180
- const stickySegments = (entry) => {
58181
- const target = entry.targetPath ?? entry.stageKey;
58182
- if (segmentsByTarget.has(target))
58183
- return segmentsByTarget.get(target);
58184
- const parsed = parseWorkflowStageTarget(target);
58185
- const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
58186
- segmentsByTarget.set(target, segments);
58187
- return segments;
58188
- };
58189
- const scanEntries = stickyQueued.filter((entry) => {
58190
- const segments = stickySegments(entry);
58191
- return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
58192
- });
58193
- const liveStageMatches = materializedStageMatcher(runs, rootRunId);
58194
- const rows = [];
58195
- for (const scanEntry of rootRun.possibleStages ?? []) {
58196
- const rowSegments = splitStagePathSegments(scanEntry);
58197
- if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
58198
- continue;
58199
- if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
58200
- continue;
58201
- }
58202
- rows.push({
58203
- target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
58204
- queuedCount: scanEntries.filter((entry) => {
58205
- const segments = stickySegments(entry);
58206
- if (segments === undefined)
58207
- return false;
58208
- return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
58209
- }).length
58210
- });
58211
- }
58212
- rows.push({
58213
- target: formatWorkflowStageTarget(rootRunId, "**"),
58214
- queuedCount: stickyQueued.filter((entry) => {
58215
- const segments = stickySegments(entry);
58216
- return segments !== undefined && segments.length === 1 && segments[0] === "**";
58217
- }).length
58218
- });
58219
- const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
58220
- const statusesByTarget = new Map;
58221
- for (const status of statuses) {
58222
- const forTarget = statusesByTarget.get(status.target);
58223
- if (forTarget === undefined)
58224
- statusesByTarget.set(status.target, [status]);
58225
- else
58226
- forTarget.push(status);
58227
- }
58228
- const queuedByTarget = new Map;
58229
- for (const entry of stickyQueued) {
58230
- const target = entry.targetPath ?? entry.stageKey;
58231
- queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
58232
- }
58233
- const rowTargets = new Set(rows.map((row) => row.target));
58234
- for (const target of statusesByTarget.keys()) {
58235
- if (rowTargets.has(target))
58236
- continue;
58237
- rowTargets.add(target);
58238
- rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
58239
- }
58240
- for (let index = 0;index < rows.length; index++) {
58241
- const row = rows[index];
58242
- const deliveryStates = statusesByTarget.get(row.target);
58243
- if (deliveryStates !== undefined)
58244
- rows[index] = { ...row, deliveryStates };
58245
- }
58246
- return rows;
58247
- }
58248
- function materializedStageMatcher(runs, rootRunId) {
58249
- const runById = new Map(runs.map((run) => [run.id, run]));
58250
- const hopsFor = createWorkflowBoundaryHopsResolver(runs);
58251
- const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
58252
- return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
58253
- }
58254
- function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
58255
- for (const run of invocationRuns) {
58256
- const hops = hopsFor(run.id);
58257
- if (hops === undefined)
58258
- continue;
58259
- for (const stage of run.stages) {
58260
- if (stage.pendingStageDeliveryAvailable !== true)
58261
- continue;
58262
- if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
58263
- continue;
58264
- }
58265
- if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
58266
- return true;
58267
- }
58268
- }
58269
- return false;
58270
- }
58271
59415
  function isPendingStageMessageEvent(value2) {
58272
59416
  if (typeof value2 !== "object" || value2 === null)
58273
59417
  return false;
@@ -58557,7 +59701,7 @@ function knownUninitializedStage(run, stageKey) {
58557
59701
  if (candidates.length !== 1)
58558
59702
  return;
58559
59703
  const stage = candidates[0];
58560
- return (stage.status === "pending" || stage.status === "running") && stage.sessionId === undefined && stage.sessionFile === undefined ? stage : undefined;
59704
+ return isAddressableStageStatus(stage.status) && stage.sessionId === undefined && stage.sessionFile === undefined ? stage : undefined;
58561
59705
  }
58562
59706
  function pendingStageDestination(run, entry) {
58563
59707
  if (entry.stageId !== undefined) {
@@ -58651,7 +59795,7 @@ var init_pending_stage_intercom = __esm(() => {
58651
59795
  init_pending_stage_route_capability();
58652
59796
  init_pending_stage_route_readiness();
58653
59797
  init_pending_stage_status();
58654
- init_sticky_stage_delivery_status();
59798
+ init_possible_stage_rows();
58655
59799
  init_store_internal();
58656
59800
  init_workflow_stage_target();
58657
59801
  });
@@ -61481,9 +62625,146 @@ var init_definitions = __esm(() => {
61481
62625
  workerDefinitionSources = new WeakMap;
61482
62626
  });
61483
62627
 
62628
+ // dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
62629
+ function push(map, key, value2) {
62630
+ if (key === undefined)
62631
+ return;
62632
+ const list = map.get(key);
62633
+ if (list)
62634
+ list.push(value2);
62635
+ else
62636
+ map.set(key, [value2]);
62637
+ }
62638
+
62639
+ class Partitions {
62640
+ byRun = new Map;
62641
+ startsByChild = new Map;
62642
+ stagesByGroup = new Map;
62643
+ stagesByReplayKey = new Map;
62644
+ boundariesByStageId = new Map;
62645
+ boundariesByChild = new Map;
62646
+ add(entry) {
62647
+ const checkpoint = entry.checkpoint;
62648
+ push(this.byRun, checkpoint.topology?.run?.runId, entry);
62649
+ if (checkpoint.kind !== "stage")
62650
+ return;
62651
+ const stage = entry;
62652
+ push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
62653
+ push(this.stagesByReplayKey, checkpoint.replayKey, stage);
62654
+ const boundary = checkpoint.topology?.boundary;
62655
+ if (!boundary)
62656
+ return;
62657
+ push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
62658
+ push(this.boundariesByChild, boundary.child?.runId, stage);
62659
+ if (boundary.event === "start")
62660
+ push(this.startsByChild, boundary.child?.runId, stage);
62661
+ }
62662
+ }
62663
+ function inKeyOrder(entries) {
62664
+ return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
62665
+ }
62666
+
62667
+ class PoolCheckpointView {
62668
+ base;
62669
+ from;
62670
+ delta;
62671
+ constructor(base, from, delta) {
62672
+ this.base = base;
62673
+ this.from = from;
62674
+ this.delta = delta;
62675
+ }
62676
+ entries(partition, key) {
62677
+ const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
62678
+ return [...committed, ...this.delta[partition].get(key) ?? []];
62679
+ }
62680
+ }
62681
+
62682
+ class PoolCheckpointRead {
62683
+ index;
62684
+ root;
62685
+ pending;
62686
+ constructor(index, root) {
62687
+ this.index = index;
62688
+ this.root = root;
62689
+ }
62690
+ read(tx, events) {
62691
+ return this.index.read(tx, events, this.root, (apply) => {
62692
+ this.pending = apply;
62693
+ });
62694
+ }
62695
+ commit() {
62696
+ this.pending?.();
62697
+ this.pending = undefined;
62698
+ }
62699
+ }
62700
+
62701
+ class PoolCheckpointIndex {
62702
+ roots = new Map;
62703
+ begin(root) {
62704
+ return new PoolCheckpointRead(this, root);
62705
+ }
62706
+ async read(tx, events, root, onCommit) {
62707
+ const current = this.roots.get(root);
62708
+ let base = current;
62709
+ const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
62710
+ if (base && base.seq > 0) {
62711
+ const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
62712
+ if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
62713
+ base = undefined;
62714
+ }
62715
+ const from = base?.seq ?? 0;
62716
+ const to = head ? Number(head.seq) : 0;
62717
+ const delta = new Partitions;
62718
+ const added = [];
62719
+ if (to > from) {
62720
+ const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
62721
+ for (const row of rows) {
62722
+ const checkpoint = JSON.parse(String(row.payload));
62723
+ if (checkpoint.kind === "ui")
62724
+ continue;
62725
+ const entry = {
62726
+ seq: Number(row.seq),
62727
+ order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
62728
+ checkpoint
62729
+ };
62730
+ delta.add(entry);
62731
+ added.push(entry);
62732
+ }
62733
+ }
62734
+ onCommit(() => {
62735
+ const latest = this.roots.get(root);
62736
+ if (!base) {
62737
+ if (latest !== current)
62738
+ return;
62739
+ const fresh = {
62740
+ seq: to,
62741
+ key: head?.key ?? null,
62742
+ at: Number(head?.at ?? 0),
62743
+ partitions: delta
62744
+ };
62745
+ this.roots.set(root, fresh);
62746
+ return;
62747
+ }
62748
+ if (latest !== base || to <= base.seq)
62749
+ return;
62750
+ for (const entry of added)
62751
+ if (entry.seq > base.seq)
62752
+ base.partitions.add(entry);
62753
+ base.seq = to;
62754
+ base.key = head?.key ?? null;
62755
+ base.at = Number(head?.at ?? 0);
62756
+ });
62757
+ return new PoolCheckpointView(base, from, delta);
62758
+ }
62759
+ }
62760
+ var init_pool_checkpoint_index = __esm(() => {
62761
+ init_stage_topology_validation();
62762
+ init_types();
62763
+ });
62764
+
61484
62765
  // dist/builtin/workflows/src/durable/journal/pool-mutations.ts
61485
62766
  import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
61486
- async function validatePoolMutation(tx, journal, root, work, mutation) {
62767
+ async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
61487
62768
  if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
61488
62769
  throw new Error("Invalid pool mutation");
61489
62770
  const checkpoint = mutation.checkpoint;
@@ -61503,42 +62784,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
61503
62784
  }
61504
62785
  if (checkpoint.kind === "ui")
61505
62786
  return;
61506
- const rows = await tx.all(`SELECT payload FROM ${journal.events} WHERE run_id = ? AND type = ?`, root, RUN_EVENT_TYPES.checkpointRecorded);
61507
- const records = rows.map((row) => JSON.parse(String(row.payload)));
61508
- const stages = records.filter((entry) => entry.kind === "stage");
62787
+ const view = await checkpoints.read(tx, journal.events);
61509
62788
  const topology = checkpoint.topology;
61510
62789
  const run = topology?.run;
61511
62790
  if (!topology || !run || run.rootRunId !== root || run.runId === root)
61512
62791
  throw new Error("Pool topology scope mismatch");
61513
- const owner = stages.find((entry) => entry.topology?.boundary?.event === "start" && entry.topology.boundary.child.runId === run.runId)?.topology?.boundary;
62792
+ const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
61514
62793
  if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
61515
62794
  throw new Error("Pool topology scope mismatch");
61516
62795
  if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
61517
62796
  throw new Error("Pool session scope mismatch");
62797
+ const runEntries = view.entries("byRun", run.runId);
61518
62798
  if (checkpoint.kind === "tool") {
61519
62799
  const tool = checkpoint.topology;
61520
- const peers = records.filter((entry) => entry.kind !== "ui" && entry.topology?.run?.runId === run.runId);
62800
+ const peers = runEntries.map((entry) => entry.checkpoint);
61521
62801
  const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
61522
62802
  if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
61523
62803
  throw new Error("Pool tool topology scope mismatch");
61524
62804
  return;
61525
62805
  }
61526
- const next = [...stages.filter((entry) => entry.checkpointId !== checkpoint.checkpointId), checkpoint];
61527
- const tools = new Set(records.flatMap((entry) => entry.kind === "tool" && entry.topology?.run?.runId === run.runId ? [entry.topology.nodeId] : []));
62806
+ const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
62807
+ const relevant = [...runStages];
62808
+ const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
62809
+ for (const group of groups)
62810
+ relevant.push(...view.entries("stagesByGroup", group));
62811
+ const boundary = checkpoint.topology?.boundary;
62812
+ let start;
62813
+ if (boundary) {
62814
+ const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
62815
+ start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
62816
+ relevant.push(...replayed);
62817
+ if (start) {
62818
+ const startTopology = start.topology;
62819
+ relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
62820
+ relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
62821
+ }
62822
+ }
62823
+ const next = [...stagesExcept(relevant, checkpoint), checkpoint];
62824
+ const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
61528
62825
  const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
61529
62826
  if (stageError)
61530
62827
  throw new Error(`Pool topology scope: ${stageError}`);
61531
- const boundary = checkpoint.topology?.boundary;
61532
62828
  if (!boundary) {
61533
62829
  if (parseWorkflowChildResult(checkpoint.output))
61534
62830
  throw new Error("Pool boundary output without ownership");
61535
62831
  return;
61536
62832
  }
61537
- const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
61538
62833
  const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
61539
62834
  if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
61540
62835
  throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
61541
62836
  }
62837
+ function stagesExcept(entries, checkpoint) {
62838
+ return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
62839
+ }
61542
62840
  async function sessionStore(tx, prefix) {
61543
62841
  const database = {
61544
62842
  descriptor: { adapter: "pool-transaction", location: prefix },
@@ -61587,20 +62885,52 @@ var init_pool_mutations = __esm(() => {
61587
62885
  init_boundary_lifecycle();
61588
62886
  init_boundary_topology();
61589
62887
  init_stage_topology_validation();
61590
- init_types();
62888
+ init_pool_checkpoint_index();
61591
62889
  });
61592
62890
 
61593
62891
  // dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
61594
62892
  function poolUsageTable(runs) {
61595
62893
  return `${runs}_pool_usage`;
61596
62894
  }
62895
+ function poolUsageTotalsTable(runs) {
62896
+ return `${runs}_pool_usage_totals`;
62897
+ }
62898
+ function addToTotals(totals, runs, work, sign, row) {
62899
+ const assignments = POOL_USAGE_FIELDS.map((field) => {
62900
+ const total = field.toLowerCase();
62901
+ const value2 = `(${sign}${row}.${total})`;
62902
+ return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
62903
+ });
62904
+ return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
62905
+ }
61597
62906
  function poolUsageSchema(runs) {
61598
- return `CREATE TABLE IF NOT EXISTS ${poolUsageTable(runs)}(
62907
+ const usage = poolUsageTable(runs);
62908
+ const totals = poolUsageTotalsTable(runs);
62909
+ const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
62910
+ return `CREATE TABLE IF NOT EXISTS ${usage}(
61599
62911
  work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
61600
62912
  generation INTEGER NOT NULL, key TEXT NOT NULL,
61601
62913
  input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
61602
62914
  PRIMARY KEY(work, generation, key)
61603
- )`;
62915
+ );
62916
+ CREATE TABLE IF NOT EXISTS ${totals}(
62917
+ root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
62918
+ ${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
62919
+ );
62920
+ CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
62921
+ ${addToTotals(totals, runs, "NEW.work", "", "NEW")}
62922
+ END;
62923
+ CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
62924
+ WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
62925
+ ${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
62926
+ ${addToTotals(totals, runs, "NEW.work", "", "NEW")}
62927
+ END;
62928
+ CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
62929
+ DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
62930
+ END;
62931
+ CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
62932
+ DELETE FROM ${totals} WHERE root = OLD.root;
62933
+ END;`;
61604
62934
  }
61605
62935
  function poolStageUsageKey(runId, stageId) {
61606
62936
  return `stage:${runId}:${stageId}`;
@@ -61627,7 +62957,7 @@ var init_pool_stage_usage = __esm(() => {
61627
62957
 
61628
62958
  // dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
61629
62959
  import { createHash as createHash4, randomUUID as randomUUID15 } from "node:crypto";
61630
- import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
62960
+ import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
61631
62961
  function requestIdentity(request) {
61632
62962
  return createHash4("sha256").update(request).digest("hex");
61633
62963
  }
@@ -61666,6 +62996,7 @@ class SqlPoolQueue {
61666
62996
  operations;
61667
62997
  transaction;
61668
62998
  journal;
62999
+ checkpoints = new PoolCheckpointIndex;
61669
63000
  constructor(storage) {
61670
63001
  this.storage = storage;
61671
63002
  const { transaction, tables: journal } = storage;
@@ -61777,17 +63108,23 @@ class SqlPoolQueue {
61777
63108
  lease = snapshotLease(lease);
61778
63109
  token = { ...token };
61779
63110
  return this.storage.withAppendGuard(async (tx, runId) => {
61780
- if (runId !== lease.runId)
61781
- throw new Error("Pool delivery cannot write another root");
61782
- const { now } = await this.authority(tx, lease).catch((error) => {
61783
- if (error instanceof StaleLeaseError)
61784
- throw new Error("Stale pool coordinator authority", { cause: error });
61785
- throw error;
61786
- });
61787
- const row = await this.current(tx, lease, token);
61788
- if (row.epoch !== lease.epoch || row.expires_at < now)
61789
- throw new Error(STALE_WORK_LEASE);
61790
- assertSession();
63111
+ try {
63112
+ if (runId !== lease.runId)
63113
+ throw new Error("Pool delivery cannot write another root");
63114
+ await this.authority(tx, lease).catch((error) => {
63115
+ if (error instanceof StaleLeaseError)
63116
+ throw new Error("Stale pool coordinator authority", { cause: error });
63117
+ throw error;
63118
+ });
63119
+ const row = await this.current(tx, lease, token);
63120
+ if (row.epoch !== lease.epoch)
63121
+ throw new Error(STALE_WORK_LEASE);
63122
+ assertSession();
63123
+ } catch (error) {
63124
+ if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
63125
+ throw error;
63126
+ throw new PoolAppendRefusedError(error.message, { cause: error });
63127
+ }
61791
63128
  }, action);
61792
63129
  }
61793
63130
  async assertCurrent(lease, token) {
@@ -61900,7 +63237,8 @@ class SqlPoolQueue {
61900
63237
  const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
61901
63238
  const identity = requestIdentity(request);
61902
63239
  mutation = JSON.parse(request).mutation;
61903
- return this.transaction(async (tx) => {
63240
+ const checkpoints = this.checkpoints.begin(lease.runId);
63241
+ const receipt = await this.transaction(async (tx) => {
61904
63242
  const { now, status } = await this.authority(tx, lease);
61905
63243
  const row = await this.current(tx, lease, token);
61906
63244
  if (row.epoch !== lease.epoch)
@@ -61915,24 +63253,26 @@ class SqlPoolQueue {
61915
63253
  throw new Error(STALE_WORK_LEASE);
61916
63254
  if (!await this.admitsAttemptWrites(tx, lease, status))
61917
63255
  throw new Error("Pool root is not running");
61918
- const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
63256
+ const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
61919
63257
  await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
61920
63258
  return receipt;
61921
63259
  });
63260
+ checkpoints.commit();
63261
+ return receipt;
61922
63262
  }
61923
63263
  async appendSession(tx, row, mutation, now) {
61924
63264
  return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
61925
63265
  }
61926
- async recordCheckpoint(tx, lease, token, row, mutation, now) {
63266
+ async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
61927
63267
  const checkpoint = mutation.checkpoint;
61928
- await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
63268
+ await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
61929
63269
  if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
61930
63270
  throw new Error("Pool checkpoint timestamp is in the future");
61931
63271
  assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
61932
63272
  const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
61933
63273
  const payload = serializeJson(checkpoint);
61934
63274
  const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
61935
- if (existing && existing.payload !== payload)
63275
+ if (existing && !isRecordedCheckpoint(existing.payload, payload))
61936
63276
  throw new Error("Pool checkpoint identity conflict");
61937
63277
  if (existing && mutation.kind !== "checkpoint")
61938
63278
  throw new Error("Pool session checkpoint identity conflict");
@@ -62030,22 +63370,37 @@ class SqlPoolQueue {
62030
63370
  return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
62031
63371
  }
62032
63372
  }
63373
+ function isRecordedCheckpoint(stored, payload) {
63374
+ if (stored === payload)
63375
+ return true;
63376
+ const { completedAt } = JSON.parse(stored);
63377
+ if (typeof completedAt !== "number")
63378
+ return false;
63379
+ return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
63380
+ }
62033
63381
  function openSqlPoolQueue(journal) {
62034
63382
  const storage = sqlPoolStorage(journal);
62035
63383
  if (!storage)
62036
63384
  return Promise.reject(new Error("Worker pool requires a SQL run journal"));
62037
63385
  return SqlPoolQueue.open(storage);
62038
63386
  }
62039
- var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
63387
+ var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
62040
63388
  var init_sql_pool_queue = __esm(() => {
62041
63389
  init_serializable();
62042
63390
  init_boundary_topology();
62043
63391
  init_durable_hash();
62044
63392
  init_internal();
63393
+ init_pool_checkpoint_index();
62045
63394
  init_pool_mutations();
62046
63395
  init_pool_stage_usage();
62047
63396
  init_types();
62048
63397
  init_values();
63398
+ PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
63399
+ constructor(message, options) {
63400
+ super(message, options);
63401
+ this.name = "PoolAppendRefusedError";
63402
+ }
63403
+ };
62049
63404
  });
62050
63405
 
62051
63406
  // dist/builtin/workflows/src/workers/coordinator-protocol.ts
@@ -62068,15 +63423,22 @@ function validate(report) {
62068
63423
  throw new Error(`Pool usage ${field} must be a finite non-negative number`);
62069
63424
  }
62070
63425
  }
63426
+ function addCompensated(total, value2) {
63427
+ const sum = total.sum + value2;
63428
+ total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
63429
+ total.sum = sum;
63430
+ }
62071
63431
 
62072
63432
  class SqlPoolUsage {
62073
63433
  transaction;
62074
63434
  journal;
62075
63435
  table;
63436
+ totals;
62076
63437
  constructor(transaction, journal) {
62077
63438
  this.transaction = transaction;
62078
63439
  this.journal = journal;
62079
63440
  this.table = poolUsageTable(journal.runs);
63441
+ this.totals = poolUsageTotalsTable(journal.runs);
62080
63442
  }
62081
63443
  static async open(transaction, journal) {
62082
63444
  const usage = new SqlPoolUsage(transaction, journal);
@@ -62084,9 +63446,27 @@ class SqlPoolUsage {
62084
63446
  return usage;
62085
63447
  }
62086
63448
  async totalIn(tx, root) {
63449
+ const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
63450
+ const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
63451
+ if (kept)
63452
+ return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
63453
+ field,
63454
+ Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
63455
+ ]));
62087
63456
  const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
62088
63457
  FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
62089
- return Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
63458
+ const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
63459
+ if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
63460
+ return total;
63461
+ const sums = columns.map(() => ({ sum: 0, error: 0 }));
63462
+ const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
63463
+ FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
63464
+ for (const usage of rows)
63465
+ for (const [index, column] of columns.entries())
63466
+ addCompensated(sums[index], Number(usage[column]));
63467
+ await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
63468
+ VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
63469
+ return total;
62090
63470
  }
62091
63471
  async report(lease, token, reports) {
62092
63472
  validateText("runId", lease.runId);
@@ -63002,15 +64382,74 @@ function isWorkflowPendingStageDeliveryFailure(error) {
63002
64382
  return true;
63003
64383
  return error?.code === "pending_stage_delivery_failed";
63004
64384
  }
63005
- function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action()) {
64385
+ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action(), options = {}) {
63006
64386
  let resolveReady;
63007
64387
  let rejectReady;
63008
64388
  let readyPromise;
63009
64389
  let drainError;
63010
64390
  let terminalError;
63011
64391
  let drainPromise;
64392
+ let drainedSessionId;
63012
64393
  let currentRecipient;
63013
- const hasUndelivered = (sessionId) => activeStore.pendingStageMessagesFor(runId, stageId).length > 0 || stageId !== stageName && activeStore.pendingStageMessagesFor(runId, stageName).length > 0 || stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).length > 0;
64394
+ let lastProgressAt = Date.now();
64395
+ let stallReported = false;
64396
+ const progress = () => {
64397
+ lastProgressAt = Date.now();
64398
+ };
64399
+ const progressingCommit = async (action) => {
64400
+ try {
64401
+ return await commit(action);
64402
+ } finally {
64403
+ progress();
64404
+ }
64405
+ };
64406
+ const reportStall = (sessionId) => {
64407
+ if (stallReported)
64408
+ return;
64409
+ stallReported = true;
64410
+ const undelivered = [
64411
+ ...activeStore.pendingStageMessagesFor(runId, stageId),
64412
+ ...stageId === stageName ? [] : activeStore.pendingStageMessagesFor(runId, stageName)
64413
+ ].map((entry) => `${entry.message.id} (direct → ${entry.stageKey})`).concat(stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).map((entry) => `${entry.message.id} (sticky → ${entry.targetPath ?? entry.stageKey})`));
64414
+ const listed = undelivered.slice(0, 10).join(", ");
64415
+ const more = undelivered.length > 10 ? `, and ${undelivered.length - 10} more` : "";
64416
+ const message = `atomic-workflows: stage "${stageName}" (${stageId}) is starting without its queued Intercom messages: ` + `no delivery progress for ${PENDING_STAGE_READY_STALL_MS / 1000}s; ` + `drain ${drainPromise === undefined ? "not running" : "running but hung"}; ` + `undelivered: ${listed.length === 0 ? "none" : `${listed}${more}`}. ` + "They stay queued and arrive live or on the stage's next drain.";
64417
+ try {
64418
+ (options.reportStall ?? console.warn)(message);
64419
+ } catch {}
64420
+ };
64421
+ const watchStall = (pending, sessionId, onStall) => {
64422
+ let timer;
64423
+ let settled = false;
64424
+ const check = () => {
64425
+ if (settled)
64426
+ return;
64427
+ const remaining = lastProgressAt + PENDING_STAGE_READY_STALL_MS - Date.now();
64428
+ if (remaining > 0) {
64429
+ timer = setTimeout(check, remaining);
64430
+ timer.unref?.();
64431
+ return;
64432
+ }
64433
+ reportStall(sessionId);
64434
+ onStall();
64435
+ };
64436
+ const stop = () => {
64437
+ settled = true;
64438
+ clearTimeout(timer);
64439
+ };
64440
+ pending.then(stop, stop);
64441
+ check();
64442
+ };
64443
+ const boundedDrain = (drain, sessionId) => {
64444
+ const waiting = new Promise((resolve, reject) => {
64445
+ drain.then(resolve, reject);
64446
+ watchStall(drain, sessionId, resolve);
64447
+ });
64448
+ waiting.catch(() => {});
64449
+ return waiting;
64450
+ };
64451
+ const hasUndeliveredExact = () => activeStore.pendingStageMessagesFor(runId, stageId).length > 0 || stageId !== stageName && activeStore.pendingStageMessagesFor(runId, stageName).length > 0;
64452
+ const hasUndelivered = (sessionId) => hasUndeliveredExact() || stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).length > 0;
63014
64453
  const pendingReady = () => {
63015
64454
  if (readyPromise === undefined) {
63016
64455
  readyPromise = new Promise((resolve, reject) => {
@@ -63020,35 +64459,64 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
63020
64459
  }
63021
64460
  return readyPromise;
63022
64461
  };
64462
+ let followUp;
64463
+ const startDrain = (deliver, recipient) => {
64464
+ if (terminalError !== undefined)
64465
+ return Promise.resolve();
64466
+ drainError = undefined;
64467
+ progress();
64468
+ const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, async (from, message) => {
64469
+ progress();
64470
+ await deliver(from, message);
64471
+ progress();
64472
+ }, progressingCommit, recipient);
64473
+ const drain = attempt.then(() => {
64474
+ progress();
64475
+ drainedSessionId = recipient?.sessionId;
64476
+ resolveReady?.();
64477
+ readyPromise = undefined;
64478
+ resolveReady = undefined;
64479
+ rejectReady = undefined;
64480
+ if (drainPromise === drain)
64481
+ drainPromise = undefined;
64482
+ }, (error) => {
64483
+ progress();
64484
+ drainError = error;
64485
+ rejectReady?.(error);
64486
+ readyPromise = undefined;
64487
+ resolveReady = undefined;
64488
+ rejectReady = undefined;
64489
+ if (drainPromise === drain)
64490
+ drainPromise = undefined;
64491
+ throw error;
64492
+ });
64493
+ drainPromise = drain;
64494
+ return boundedDrain(drain, recipient?.sessionId);
64495
+ };
64496
+ const requestDrain = (deliver, recipient) => {
64497
+ const running = drainPromise;
64498
+ if (running === undefined)
64499
+ return startDrain(deliver, recipient);
64500
+ if (followUp !== undefined) {
64501
+ followUp.deliver = deliver;
64502
+ followUp.recipient = recipient;
64503
+ return boundedDrain(followUp.settled.promise, recipient?.sessionId);
64504
+ }
64505
+ const next = { deliver, recipient, settled: Promise.withResolvers() };
64506
+ next.settled.promise.catch(() => {});
64507
+ followUp = next;
64508
+ running.catch(() => {}).then(() => {
64509
+ if (followUp === next)
64510
+ followUp = undefined;
64511
+ return requestDrain(next.deliver, next.recipient);
64512
+ }).then(next.settled.resolve, next.settled.reject);
64513
+ return boundedDrain(next.settled.promise, recipient?.sessionId);
64514
+ };
63023
64515
  return {
63024
64516
  routeCapability: workflowPendingStageRouteCapability(activeStore, runId),
63025
64517
  deliverPending(deliver, recipient) {
63026
64518
  currentRecipient = recipient;
63027
- if (drainPromise !== undefined)
63028
- return drainPromise;
63029
- if (terminalError !== undefined)
63030
- return Promise.resolve();
63031
- drainError = undefined;
63032
- const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, deliver, commit, recipient);
63033
- const drain = attempt.then(() => {
63034
- resolveReady?.();
63035
- readyPromise = undefined;
63036
- resolveReady = undefined;
63037
- rejectReady = undefined;
63038
- if (drainPromise === drain)
63039
- drainPromise = undefined;
63040
- }, (error) => {
63041
- drainError = error;
63042
- rejectReady?.(error);
63043
- readyPromise = undefined;
63044
- resolveReady = undefined;
63045
- rejectReady = undefined;
63046
- if (drainPromise === drain)
63047
- drainPromise = undefined;
63048
- throw error;
63049
- });
63050
- drainPromise = drain;
63051
- return drainPromise;
64519
+ return requestDrain(deliver, recipient);
63052
64520
  },
63053
64521
  ready(sessionId) {
63054
64522
  if (!hasUndelivered(sessionId)) {
@@ -63058,6 +64526,8 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
63058
64526
  return Promise.reject(terminalError);
63059
64527
  if (drainError !== undefined)
63060
64528
  return Promise.reject(drainError);
64529
+ if (drainPromise === undefined && sessionId !== undefined && drainedSessionId === sessionId && !hasUndeliveredExact())
64530
+ return;
63061
64531
  const ready = pendingReady();
63062
64532
  if (sessionId !== undefined && activeStore.pendingStageMessagesFor(runId, stageId).length === 0 && activeStore.pendingStageMessagesFor(runId, stageName).length === 0) {
63063
64533
  const entries = stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, { sessionId });
@@ -63081,6 +64551,14 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
63081
64551
  rejectReady?.(error);
63082
64552
  });
63083
64553
  }
64554
+ watchStall(ready, sessionId, () => {
64555
+ if (readyPromise !== ready)
64556
+ return;
64557
+ resolveReady?.();
64558
+ readyPromise = undefined;
64559
+ resolveReady = undefined;
64560
+ rejectReady = undefined;
64561
+ });
63084
64562
  return ready;
63085
64563
  },
63086
64564
  fail(reason) {
@@ -63208,7 +64686,7 @@ function toPendingStageSender(entry) {
63208
64686
  lastActivity: entry.from.lastActivity ?? entry.message.timestamp
63209
64687
  };
63210
64688
  }
63211
- var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError;
64689
+ var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError, PENDING_STAGE_READY_STALL_MS = 60000;
63212
64690
  var init_pending_stage_delivery2 = __esm(() => {
63213
64691
  init_run_owner_backend();
63214
64692
  init_store_backend();
@@ -65630,10 +67108,120 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
65630
67108
  }
65631
67109
  var init_stage_session_store = () => {};
65632
67110
 
67111
+ // dist/builtin/workflows/src/durable/completed-catalog-index.ts
67112
+ class CheckpointCatalogIndex {
67113
+ checkpoints;
67114
+ stages;
67115
+ positions = new Map;
67116
+ groupKeys = new Map;
67117
+ groups = new Map;
67118
+ stagesByRunId = new Map;
67119
+ boundariesByReplayKey = new Map;
67120
+ boundariesByStageId = new Map;
67121
+ boundariesByChildRunId = new Map;
67122
+ outputsByReplayKey = new Map;
67123
+ evidence;
67124
+ constructor(checkpoints) {
67125
+ this.checkpoints = checkpoints;
67126
+ const stages = [];
67127
+ checkpoints.forEach((checkpoint, index) => {
67128
+ if (!this.positions.has(checkpoint))
67129
+ this.positions.set(checkpoint, index);
67130
+ if (checkpoint.kind !== "stage")
67131
+ return;
67132
+ stages.push(checkpoint);
67133
+ const key = durableStageGroupKey(checkpoint);
67134
+ this.groupKeys.set(checkpoint, key);
67135
+ append(this.groups, key, checkpoint);
67136
+ const runId = checkpoint.topology?.run?.runId;
67137
+ if (runId !== undefined)
67138
+ append(this.stagesByRunId, runId, checkpoint);
67139
+ if (checkpoint.output !== undefined)
67140
+ append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
67141
+ const boundary = checkpoint.topology?.boundary;
67142
+ if (boundary === undefined)
67143
+ return;
67144
+ append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
67145
+ append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
67146
+ append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
67147
+ });
67148
+ this.stages = stages;
67149
+ }
67150
+ sequence(checkpoint) {
67151
+ return (this.positions.get(checkpoint) ?? -1) + 1;
67152
+ }
67153
+ stageGroups() {
67154
+ return this.groups.values();
67155
+ }
67156
+ boundaryLifecycle(start, requireTerminal) {
67157
+ const topology = start.topology;
67158
+ const childRunId = topology?.boundary?.child.runId;
67159
+ const related = this.inCheckpointOrder([
67160
+ this.boundariesByReplayKey.get(start.replayKey),
67161
+ topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
67162
+ childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
67163
+ ]);
67164
+ return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
67165
+ }
67166
+ directChildRecords(childRunId) {
67167
+ return {
67168
+ claiming: this.stagesByRunId.get(childRunId) ?? [],
67169
+ direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
67170
+ };
67171
+ }
67172
+ childScopedEvidenceExists(boundaryReplayKey, childRunId) {
67173
+ this.evidence ??= scopedEvidenceIndex(this.checkpoints);
67174
+ return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
67175
+ }
67176
+ inCheckpointOrder(lists) {
67177
+ const present = lists.filter((list) => list !== undefined);
67178
+ if (present.length === 1)
67179
+ return present[0];
67180
+ return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
67181
+ }
67182
+ }
67183
+ function scopedEvidenceIndex(checkpoints) {
67184
+ const runIds = new Set;
67185
+ const identities = [];
67186
+ for (const checkpoint of checkpoints) {
67187
+ identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
67188
+ if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
67189
+ runIds.add(checkpoint.topology.run.runId);
67190
+ }
67191
+ }
67192
+ identities.sort(compareCodeUnits);
67193
+ return { runIds, identities };
67194
+ }
67195
+ function hasPrefix(sorted, prefix) {
67196
+ let low = 0;
67197
+ let high = sorted.length;
67198
+ while (low < high) {
67199
+ const middle = low + high >>> 1;
67200
+ if (compareCodeUnits(sorted[middle], prefix) < 0)
67201
+ low = middle + 1;
67202
+ else
67203
+ high = middle;
67204
+ }
67205
+ return low < sorted.length && sorted[low].startsWith(prefix);
67206
+ }
67207
+ function compareCodeUnits(left, right) {
67208
+ return left < right ? -1 : left > right ? 1 : 0;
67209
+ }
67210
+ function append(map, key, value2) {
67211
+ const list = map.get(key);
67212
+ if (list === undefined)
67213
+ map.set(key, [value2]);
67214
+ else
67215
+ list.push(value2);
67216
+ }
67217
+ var init_completed_catalog_index = __esm(() => {
67218
+ init_boundary_lifecycle();
67219
+ init_stage_topology_validation();
67220
+ });
67221
+
65633
67222
  // dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
65634
- function validBoundaryRecordSet(checkpoints, strict) {
65635
- const stages = checkpoints.filter((item) => item.kind === "stage");
65636
- const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
67223
+ function validBoundaryRecordSet(index, strict) {
67224
+ const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
65637
67225
  const startsByReplay = new Map;
65638
67226
  for (const checkpoint of boundaries) {
65639
67227
  if (checkpoint.topology.boundary.event !== "start")
@@ -65648,7 +67236,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
65648
67236
  return false;
65649
67237
  return [...startsByReplay.values()].every((starts) => {
65650
67238
  const start = starts[0];
65651
- if (boundaryTransitionError(start, stages, strict) !== undefined)
67239
+ if ("error" in index.boundaryLifecycle(start, strict))
65652
67240
  return false;
65653
67241
  return start.topology.boundary.invocationFingerprint !== undefined;
65654
67242
  });
@@ -65712,8 +67300,8 @@ function compareDraftSourceOrder(left, right) {
65712
67300
  function runTopologyFor(drafts, tools) {
65713
67301
  return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
65714
67302
  }
65715
- function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingChildren = false) {
65716
- const toolsFor = (runId) => tools.filter((checkpoint) => (checkpoint.topology?.run?.runId ?? rootRunId) === runId);
67303
+ function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
67304
+ const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
65717
67305
  const owners = new Map;
65718
67306
  const syntheticChildIds = new Set;
65719
67307
  for (const [parentRunId, drafts] of grouped) {
@@ -65724,12 +67312,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
65724
67312
  if (childRunId === parentRunId || owners.has(childRunId))
65725
67313
  return false;
65726
67314
  if (!grouped.has(childRunId)) {
65727
- if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(checkpoints, draft.replayKey, childRunId)) {
67315
+ if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
65728
67316
  syntheticChildIds.add(childRunId);
65729
67317
  owners.set(childRunId, draft);
65730
67318
  continue;
65731
67319
  }
65732
- if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(checkpoints, draft.replayKey, childRunId))
67320
+ if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
65733
67321
  return false;
65734
67322
  syntheticChildIds.add(childRunId);
65735
67323
  owners.set(childRunId, draft);
@@ -65770,13 +67358,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
65770
67358
  const boundary = draft.topology?.boundary;
65771
67359
  return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
65772
67360
  }
65773
- function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
65774
- const prefix = `${boundaryReplayKey}:`;
65775
- return checkpoints.some((checkpoint) => {
65776
- const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
65777
- return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
65778
- });
65779
- }
65780
67361
  function retainReachableRunGroups(grouped, rootRunId) {
65781
67362
  const reachable = new Set([rootRunId]);
65782
67363
  const settled = new Set;
@@ -65817,14 +67398,6 @@ function childRunIdFromDraft(draft) {
65817
67398
  return;
65818
67399
  return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
65819
67400
  }
65820
- function boundaryOwner(grouped, childRunId) {
65821
- for (const drafts of grouped.values()) {
65822
- const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
65823
- if (owner !== undefined)
65824
- return owner;
65825
- }
65826
- return;
65827
- }
65828
67401
  function childRunStatus(owner) {
65829
67402
  const child = workflowChildFromDraft(owner);
65830
67403
  if (child !== undefined)
@@ -65832,15 +67405,15 @@ function childRunStatus(owner) {
65832
67405
  const status = owner.topology?.boundary?.status;
65833
67406
  return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
65834
67407
  }
65835
- function mergeStageGroup(records, allStages, allCheckpoints) {
67408
+ function mergeStageGroup(records, index) {
65836
67409
  let merged;
65837
67410
  for (const record of records)
65838
- merged = mergeStageDraft(merged, record, allCheckpoints.indexOf(record) + 1);
67411
+ merged = mergeStageDraft(merged, record, index.sequence(record));
65839
67412
  const draft = merged;
65840
67413
  const start = records.find((record) => record.topology?.boundary?.event === "start");
65841
67414
  if (start === undefined)
65842
67415
  return draft;
65843
- const resolved = resolveBoundaryLifecycle(start, allStages, false);
67416
+ const resolved = index.boundaryLifecycle(start, false);
65844
67417
  if ("error" in resolved)
65845
67418
  return draft;
65846
67419
  const terminal = resolved.lifecycle.latestTerminal;
@@ -65903,9 +67476,7 @@ function workflowChildFromDraft(draft) {
65903
67476
  ...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
65904
67477
  };
65905
67478
  }
65906
- var init_completed_catalog_stage_groups = __esm(() => {
65907
- init_boundary_lifecycle();
65908
- });
67479
+ var init_completed_catalog_stage_groups = () => {};
65909
67480
 
65910
67481
  // dist/builtin/workflows/src/engine/run-tool-control-registry.ts
65911
67482
  function createToolControlRegistry() {
@@ -66110,11 +67681,68 @@ var init_resume_runtime = __esm(() => {
66110
67681
  function isReservedRunCheckpointHash(argsHash) {
66111
67682
  return [RUN_TIMING_CHECKPOINT_NAME, RUN_USAGE_CHECKPOINT_NAME, RUN_RESULT_CHECKPOINT_NAME].includes(argsHash) || argsHash === WORKFLOW_HEARTBEAT_ANCHOR_CHECKPOINT_NAME;
66112
67683
  }
66113
- function durableNestedRunSnapshots(backend, rootWorkflowId) {
67684
+ function durableNestedRunCatalog(backend, rootWorkflowId) {
66114
67685
  const handle = backend.getWorkflow(rootWorkflowId);
66115
67686
  if (handle === undefined)
66116
- return [];
66117
- return runSnapshotsFromCheckpoints(backend.listCheckpoints(rootWorkflowId), rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
67687
+ return EMPTY_NESTED_RUN_CATALOG;
67688
+ const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
67689
+ const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
67690
+ if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
67691
+ return cached.catalog;
67692
+ }
67693
+ const checkpoints = backend.listCheckpoints(rootWorkflowId);
67694
+ const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
67695
+ const catalog = nestedRunCatalog(runs, checkpoints);
67696
+ if (version !== undefined) {
67697
+ const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
67698
+ nestedRunCatalogs.set(backend, byRoot);
67699
+ byRoot.delete(rootWorkflowId);
67700
+ byRoot.set(rootWorkflowId, {
67701
+ version,
67702
+ name: handle.name,
67703
+ updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
67704
+ catalog
67705
+ });
67706
+ if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
67707
+ byRoot.delete(byRoot.keys().next().value);
67708
+ }
67709
+ return catalog;
67710
+ }
67711
+ function nestedRunCatalog(runs, checkpoints) {
67712
+ const positionsById = new Map;
67713
+ const childIdsByParent = new Map;
67714
+ runs.forEach((run, position) => {
67715
+ const positions = positionsById.get(run.id);
67716
+ if (positions === undefined)
67717
+ positionsById.set(run.id, [position]);
67718
+ else
67719
+ positions.push(position);
67720
+ if (run.parentRunId === undefined)
67721
+ return;
67722
+ const children = childIdsByParent.get(run.parentRunId);
67723
+ if (children === undefined)
67724
+ childIdsByParent.set(run.parentRunId, [run.id]);
67725
+ else
67726
+ children.push(run.id);
67727
+ });
67728
+ const intentionalExitStatuses = new Map;
67729
+ for (const checkpoint of checkpoints) {
67730
+ if (checkpoint.kind !== "stage")
67731
+ continue;
67732
+ const child = parseWorkflowChildResult(checkpoint.output);
67733
+ if (child?.exited === true)
67734
+ intentionalExitStatuses.set(child.runId, child.status);
67735
+ }
67736
+ return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
67737
+ }
67738
+ function readsFallbackCompletedAt(run) {
67739
+ return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
67740
+ }
67741
+ function cloneRunSnapshot(run) {
67742
+ const copy = structuredClone(run);
67743
+ for (const tool of copy.toolNodes)
67744
+ Object.freeze(tool.parentIds);
67745
+ return copy;
66118
67746
  }
66119
67747
  function checkpointDrafts(checkpoints) {
66120
67748
  const stageByReplayKey = new Map;
@@ -66138,11 +67766,8 @@ function checkpointDrafts(checkpoints) {
66138
67766
  tools: [...toolByHash.values()]
66139
67767
  };
66140
67768
  }
66141
- function completedToolNodes(checkpoints, runId, rootRunId, sourceIds) {
66142
- return checkpoints.flatMap((checkpoint) => {
66143
- const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
66144
- if (topologyRunId !== runId)
66145
- return [];
67769
+ function completedToolNodes(ownedCheckpoints, sourceIds) {
67770
+ return ownedCheckpoints.flatMap((checkpoint) => {
66146
67771
  const topology = checkpoint.topology;
66147
67772
  if (topology === undefined)
66148
67773
  return [];
@@ -66179,18 +67804,34 @@ function summarizeCompletedToolResult(value2) {
66179
67804
  return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
66180
67805
  }
66181
67806
  function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
66182
- if (!validBoundaryRecordSet(checkpoints, policy.requireTerminal))
67807
+ const index = new CheckpointCatalogIndex(checkpoints);
67808
+ if (!validBoundaryRecordSet(index, policy.requireTerminal))
66183
67809
  return [];
66184
- const stageRecords = checkpoints.filter((checkpoint) => checkpoint.kind === "stage");
67810
+ const stageRecords = index.stages;
66185
67811
  const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
66186
67812
  if (immutableStageGroupError(identityRecords) !== undefined)
66187
67813
  return [];
66188
- const recordsByGroup = groupByDurableStageKey(stageRecords);
66189
- const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
67814
+ const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
66190
67815
  const drafts = checkpointDrafts(checkpoints);
66191
67816
  const toolCheckpoints = drafts.tools;
66192
67817
  if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
66193
67818
  return [];
67819
+ const toolNodeIdsByClaimedRun = new Map;
67820
+ const toolsByRun = new Map;
67821
+ for (const checkpoint of toolCheckpoints) {
67822
+ const claimedRunId = checkpoint.topology?.run?.runId;
67823
+ if (claimedRunId !== undefined) {
67824
+ const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
67825
+ ids.add(checkpoint.topology.nodeId);
67826
+ toolNodeIdsByClaimedRun.set(claimedRunId, ids);
67827
+ }
67828
+ const ownerRunId = claimedRunId ?? rootRunId;
67829
+ const owned = toolsByRun.get(ownerRunId);
67830
+ if (owned === undefined)
67831
+ toolsByRun.set(ownerRunId, [checkpoint]);
67832
+ else
67833
+ owned.push(checkpoint);
67834
+ }
66194
67835
  for (const draft of candidates) {
66195
67836
  const boundary = draft.topology?.boundary;
66196
67837
  if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
@@ -66204,8 +67845,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
66204
67845
  parentStageId: boundary.child.parentStageId,
66205
67846
  rootRunId: boundary.child.rootRunId
66206
67847
  };
66207
- const childToolIds = new Set(toolCheckpoints.flatMap((checkpoint) => checkpoint.topology?.run?.runId === expectedChildRun.runId ? [checkpoint.topology.nodeId] : []));
66208
- if (directChildTopologyError(stageRecords, boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
67848
+ const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
67849
+ if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
66209
67850
  return [];
66210
67851
  }
66211
67852
  if (candidates.length === 0 && toolCheckpoints.length === 0) {
@@ -66233,7 +67874,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
66233
67874
  group.sort(compareDraftSourceOrder);
66234
67875
  if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
66235
67876
  return [];
66236
- if (!validateRunGroups(grouped, rootRunId, toolCheckpoints, checkpoints, policy.retainActive))
67877
+ if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
66237
67878
  return [];
66238
67879
  const syntheticChildren = [];
66239
67880
  const pendingChildren = [];
@@ -66247,7 +67888,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
66247
67888
  const child = workflowChildFromDraft(draft);
66248
67889
  if (child?.exited !== true || grouped.has(child.runId))
66249
67890
  continue;
66250
- if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(checkpoints, draft.replayKey, child.runId)) {
67891
+ if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
66251
67892
  return [];
66252
67893
  }
66253
67894
  syntheticChildren.push({ parentRunId, owner: draft, child });
@@ -66255,7 +67896,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
66255
67896
  }
66256
67897
  const idMaps = new Map;
66257
67898
  for (const [runId, runDrafts] of grouped) {
66258
- const ownedTools = toolCheckpoints.filter((checkpoint) => (checkpoint.topology?.run?.runId ?? rootRunId) === runId);
67899
+ const ownedTools = toolsByRun.get(runId) ?? [];
66259
67900
  if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
66260
67901
  return [];
66261
67902
  const ids = new Map;
@@ -66273,32 +67914,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
66273
67914
  });
66274
67915
  if (hasIdentityConflict)
66275
67916
  return [];
66276
- for (const checkpoint of toolCheckpoints) {
66277
- const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
67917
+ for (const checkpoint of ownedTools) {
66278
67918
  const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
66279
- if (ownerRunId === runId)
66280
- ids.set(sourceId, sourceId);
67919
+ ids.set(sourceId, sourceId);
66281
67920
  }
66282
67921
  idMaps.set(runId, ids);
66283
67922
  }
67923
+ const boundaryOwners = new Map;
67924
+ const boundarySourcesByParent = new Map;
67925
+ for (const [parentRunId, runDrafts] of grouped) {
67926
+ const sources = new Map;
67927
+ for (const draft of runDrafts) {
67928
+ const childRunId = childRunIdFromDraft(draft);
67929
+ if (childRunId === undefined)
67930
+ continue;
67931
+ if (!boundaryOwners.has(childRunId))
67932
+ boundaryOwners.set(childRunId, draft);
67933
+ if (!sources.has(childRunId))
67934
+ sources.set(childRunId, draft);
67935
+ }
67936
+ boundarySourcesByParent.set(parentRunId, sources);
67937
+ }
66284
67938
  const runs = [];
66285
67939
  const emittedRunIds = new Set;
66286
67940
  for (const [runId, runDrafts] of grouped) {
66287
67941
  const ids = idMaps.get(runId);
66288
- const ownedTools = toolCheckpoints.filter((checkpoint) => (checkpoint.topology?.run?.runId ?? rootRunId) === runId);
67942
+ const ownedTools = toolsByRun.get(runId) ?? [];
66289
67943
  const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
66290
67944
  if (hasUnknownParents)
66291
67945
  return [];
66292
67946
  const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
66293
- const toolNodes = completedToolNodes(toolCheckpoints, runId, rootRunId, ids);
67947
+ const toolNodes = completedToolNodes(ownedTools, ids);
66294
67948
  const run = runTopologyFor(runDrafts, ownedTools);
66295
67949
  const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
66296
67950
  const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
66297
- const owner = runId === rootRunId ? undefined : boundaryOwner(grouped, runId);
67951
+ const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
66298
67952
  const status = owner === undefined ? "completed" : childRunStatus(owner);
66299
67953
  const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
66300
67954
  const parentRunId = run?.parentRunId ?? rootRunId;
66301
- const boundarySourceId = grouped.get(parentRunId)?.find((draft) => childRunIdFromDraft(draft) === runId)?.topology?.stageId;
67955
+ const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
66302
67956
  const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
66303
67957
  const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
66304
67958
  const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
@@ -66434,10 +68088,11 @@ function emptyGraphRun(runId, runName, completedAt) {
66434
68088
  resumable: false
66435
68089
  };
66436
68090
  }
66437
- var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
68091
+ var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
66438
68092
  var init_completed_catalog = __esm(() => {
66439
68093
  init_stage_session_store();
66440
68094
  init_tool_payload_bounds();
68095
+ init_completed_catalog_index();
66441
68096
  init_completed_catalog_stage_groups();
66442
68097
  init_resume_eligibility();
66443
68098
  init_resume_runtime();
@@ -66446,6 +68101,13 @@ var init_completed_catalog = __esm(() => {
66446
68101
  init_workflow_heartbeat_anchor();
66447
68102
  init_stage_topology_validation();
66448
68103
  init_tool_outcome();
68104
+ nestedRunCatalogs = new WeakMap;
68105
+ EMPTY_NESTED_RUN_CATALOG = {
68106
+ runs: [],
68107
+ positionsById: new Map,
68108
+ childIdsByParent: new Map,
68109
+ intentionalExitStatuses: new Map
68110
+ };
66449
68111
  COMPLETED_RECONSTRUCTION = {
66450
68112
  requireTerminal: true,
66451
68113
  retainActive: false,
@@ -66460,29 +68122,23 @@ var init_completed_catalog = __esm(() => {
66460
68122
 
66461
68123
  // dist/builtin/workflows/src/durable/completed-subtree.ts
66462
68124
  function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
66463
- const nested = durableNestedRunSnapshots(backend, rootWorkflowId);
66464
- if (!nested.some((run) => run.id === childRunId))
68125
+ const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
68126
+ if (!catalog.positionsById.has(childRunId))
66465
68127
  return;
66466
- const intentionalExitStatuses = new Map;
66467
- for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
66468
- if (checkpoint.kind !== "stage")
66469
- continue;
66470
- const child = parseWorkflowChildResult(checkpoint.output);
66471
- if (child?.exited === true)
66472
- intentionalExitStatuses.set(child.runId, child.status);
66473
- }
66474
68128
  const included = new Set([childRunId]);
66475
- let priorSize = -1;
66476
- while (priorSize !== included.size) {
66477
- priorSize = included.size;
66478
- for (const run of nested) {
66479
- if (run.parentRunId !== undefined && included.has(run.parentRunId))
66480
- included.add(run.id);
68129
+ const pending = [childRunId];
68130
+ for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
68131
+ for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
68132
+ if (included.has(childId))
68133
+ continue;
68134
+ included.add(childId);
68135
+ pending.push(childId);
66481
68136
  }
66482
68137
  }
66483
- const subtree = nested.filter((run) => included.has(run.id));
66484
- const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, intentionalExitStatuses));
66485
- return complete ? subtree : undefined;
68138
+ const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
68139
+ const subtree = positions.map((position) => catalog.runs[position]);
68140
+ const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
68141
+ return complete ? subtree.map(cloneRunSnapshot) : undefined;
66486
68142
  }
66487
68143
  function isCompleteCachedRun(run, intentionalExitStatuses) {
66488
68144
  if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
@@ -66603,7 +68259,7 @@ function createDurableCachedStageRecorder(input) {
66603
68259
  const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
66604
68260
  kind: "stage",
66605
68261
  workflowId: input.run.id,
66606
- checkpointId: `stage-replay-meta:${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
68262
+ checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
66607
68263
  name,
66608
68264
  replayKey,
66609
68265
  completedAt: Date.now(),
@@ -74229,6 +75885,7 @@ async function runWorker(start) {
74229
75885
  "getStageOutput",
74230
75886
  "getStageSession",
74231
75887
  "listCheckpoints",
75888
+ "nestedSnapshotVersion",
74232
75889
  "getWorkflow",
74233
75890
  "getLoadableWorkflow",
74234
75891
  "toMetadata",