@bastani/atomic 0.9.44-alpha.4 → 0.9.44-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/builtin/intercom/CHANGELOG.md +12 -0
- package/dist/builtin/intercom/broker/broker.ts +13 -1
- package/dist/builtin/intercom/index.bundle.mjs +9 -1
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/types.ts +7 -0
- package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +26 -0
- package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
- package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
- package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +2457 -736
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2329 -730
- package/dist/builtin/workflows/src/index.js +2328 -730
- package/dist/builtin/workflows/src/local-worker-entry.mjs +2329 -730
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +16 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/storage/errors.d.ts +5 -1
- package/dist/core/storage/errors.d.ts.map +1 -1
- package/dist/core/storage/errors.js +8 -2
- package/dist/core/storage/errors.js.map +1 -1
- package/dist/core/storage/node-sqlite.d.ts +2 -1
- package/dist/core/storage/node-sqlite.d.ts.map +1 -1
- package/dist/core/storage/node-sqlite.js +213 -20
- package/dist/core/storage/node-sqlite.js.map +1 -1
- package/docs/intercom/reference.md +2 -0
- package/docs/workflows/operations.md +1 -1
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -172,13 +172,19 @@ function registerRunJournalVersions(journal, versions) {
|
|
|
172
172
|
function runJournalVersions(journal) {
|
|
173
173
|
return versionReaders.get(journal);
|
|
174
174
|
}
|
|
175
|
+
function registerRunJournalAppendMany(journal, appendMany) {
|
|
176
|
+
appendManyWriters.set(journal, appendMany);
|
|
177
|
+
}
|
|
178
|
+
function runJournalAppendMany(journal) {
|
|
179
|
+
return appendManyWriters.get(journal);
|
|
180
|
+
}
|
|
175
181
|
function registerSqlPoolStorage(journal, storage) {
|
|
176
182
|
poolStorage.set(journal, storage);
|
|
177
183
|
}
|
|
178
184
|
function sqlPoolStorage(journal) {
|
|
179
185
|
return poolStorage.get(journal);
|
|
180
186
|
}
|
|
181
|
-
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
|
|
187
|
+
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
|
|
182
188
|
var init_internal = __esm(() => {
|
|
183
189
|
init_types();
|
|
184
190
|
NON_SNAPSHOT_EVENT_TYPES = [
|
|
@@ -187,6 +193,7 @@ var init_internal = __esm(() => {
|
|
|
187
193
|
RUN_EVENT_TYPES.messageDelivered
|
|
188
194
|
];
|
|
189
195
|
versionReaders = new WeakMap;
|
|
196
|
+
appendManyWriters = new WeakMap;
|
|
190
197
|
poolStorage = new WeakMap;
|
|
191
198
|
});
|
|
192
199
|
|
|
@@ -297,6 +304,204 @@ var init_resume_eligibility = () => {};
|
|
|
297
304
|
// dist/builtin/workflows/src/durable/types.ts
|
|
298
305
|
var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
|
|
299
306
|
|
|
307
|
+
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
308
|
+
function immutableStageGroupError(stages) {
|
|
309
|
+
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
310
|
+
if (occurrenceError !== undefined)
|
|
311
|
+
return occurrenceError;
|
|
312
|
+
const byReplayKey = groupByDurableStageKey(stages);
|
|
313
|
+
for (const group of byReplayKey.values()) {
|
|
314
|
+
const firstName = group[0]?.name;
|
|
315
|
+
if (group.some((stage) => stage.name !== firstName))
|
|
316
|
+
return "stage name drift within one replay-key group";
|
|
317
|
+
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
318
|
+
const first = topologies[0];
|
|
319
|
+
if (first === undefined)
|
|
320
|
+
continue;
|
|
321
|
+
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
322
|
+
return "immutable stage identity drift within one replay-key group";
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
328
|
+
return directChildTopologyErrorFromRecords({
|
|
329
|
+
claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
|
|
330
|
+
direct: (claiming) => {
|
|
331
|
+
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
332
|
+
return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
333
|
+
}
|
|
334
|
+
}, replayScope, expectedRun, allowedNodeIds);
|
|
335
|
+
}
|
|
336
|
+
function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
337
|
+
const prefix = `${replayScope}:`;
|
|
338
|
+
const claiming = records.claiming;
|
|
339
|
+
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
340
|
+
return "child run topology exists outside its durable invocation scope";
|
|
341
|
+
}
|
|
342
|
+
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
343
|
+
return "child checkpoint ownership is not reciprocal";
|
|
344
|
+
}
|
|
345
|
+
const direct = records.direct(claiming);
|
|
346
|
+
const immutableError = immutableStageGroupError(direct);
|
|
347
|
+
if (immutableError !== undefined)
|
|
348
|
+
return immutableError;
|
|
349
|
+
const merged = [];
|
|
350
|
+
for (const group of groupByDurableStageKey(direct).values()) {
|
|
351
|
+
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
352
|
+
if (topology === undefined)
|
|
353
|
+
return "direct child stage group lacks owning-run topology";
|
|
354
|
+
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
355
|
+
return "direct child stage topology lacks current source identity";
|
|
356
|
+
}
|
|
357
|
+
if (topology.stageId.length === 0)
|
|
358
|
+
return "direct child stage id is empty";
|
|
359
|
+
merged.push(topology);
|
|
360
|
+
}
|
|
361
|
+
const ids = new Set(allowedNodeIds);
|
|
362
|
+
const stageIds = new Set;
|
|
363
|
+
const orders = new Set;
|
|
364
|
+
for (const topology of merged) {
|
|
365
|
+
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
366
|
+
return "direct child stage ids are not unique";
|
|
367
|
+
stageIds.add(topology.stageId);
|
|
368
|
+
ids.add(topology.stageId);
|
|
369
|
+
if (topology.sourceOrder !== undefined) {
|
|
370
|
+
if (orders.has(topology.sourceOrder))
|
|
371
|
+
return "direct child source orders are not unique";
|
|
372
|
+
orders.add(topology.sourceOrder);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
376
|
+
return "direct child stage references an unknown parent";
|
|
377
|
+
}
|
|
378
|
+
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
379
|
+
}
|
|
380
|
+
function owningBoundaryIdError(stages, owningRunId) {
|
|
381
|
+
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
382
|
+
const seen = new Map;
|
|
383
|
+
for (const start of starts) {
|
|
384
|
+
const stageId = start.topology.stageId;
|
|
385
|
+
if (stageId.length === 0)
|
|
386
|
+
return "owning-run boundary stage id is empty";
|
|
387
|
+
const prior = seen.get(stageId);
|
|
388
|
+
if (prior !== undefined && prior !== start.replayKey)
|
|
389
|
+
return "owning-run boundary stage id is aliased";
|
|
390
|
+
seen.set(stageId, start.replayKey);
|
|
391
|
+
}
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
function sameRun(actual, expected) {
|
|
395
|
+
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
396
|
+
}
|
|
397
|
+
function sameStrings(left, right) {
|
|
398
|
+
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
399
|
+
}
|
|
400
|
+
function sameOptionalRun(actual, expected) {
|
|
401
|
+
if (actual === undefined || expected === undefined)
|
|
402
|
+
return actual === expected;
|
|
403
|
+
return sameRun(actual, expected);
|
|
404
|
+
}
|
|
405
|
+
function promptOccurrenceIdentityError(stages) {
|
|
406
|
+
const replayByOccurrence = new Map;
|
|
407
|
+
const occurrenceByReplayStage = new Map;
|
|
408
|
+
const stageByReplayOccurrence = new Map;
|
|
409
|
+
for (const stage of stages) {
|
|
410
|
+
const topology = stage.topology;
|
|
411
|
+
const occurrence = topology?.occurrenceKey;
|
|
412
|
+
if (topology === undefined || occurrence === undefined)
|
|
413
|
+
continue;
|
|
414
|
+
const priorReplay = replayByOccurrence.get(occurrence);
|
|
415
|
+
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
416
|
+
return "prompt occurrence is bound to multiple replay keys";
|
|
417
|
+
}
|
|
418
|
+
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
419
|
+
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
420
|
+
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
421
|
+
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
422
|
+
return "prompt stage identity is bound to multiple occurrences";
|
|
423
|
+
}
|
|
424
|
+
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
425
|
+
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
426
|
+
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
427
|
+
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
428
|
+
return "prompt occurrence is bound to multiple stage identities";
|
|
429
|
+
}
|
|
430
|
+
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
431
|
+
}
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
function durableStageGroupKey(stage) {
|
|
435
|
+
const occurrence = stage.topology?.occurrenceKey;
|
|
436
|
+
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
437
|
+
}
|
|
438
|
+
function groupByDurableStageKey(stages) {
|
|
439
|
+
const groups = new Map;
|
|
440
|
+
for (const stage of stages) {
|
|
441
|
+
const key = durableStageGroupKey(stage);
|
|
442
|
+
const group = groups.get(key) ?? [];
|
|
443
|
+
group.push(stage);
|
|
444
|
+
groups.set(key, group);
|
|
445
|
+
}
|
|
446
|
+
return groups;
|
|
447
|
+
}
|
|
448
|
+
function compositeKey(domain, ...parts) {
|
|
449
|
+
return JSON.stringify([domain, ...parts]);
|
|
450
|
+
}
|
|
451
|
+
function hasCycle(topologies) {
|
|
452
|
+
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
453
|
+
const visiting = new Set;
|
|
454
|
+
const visited = new Set;
|
|
455
|
+
const cyclic = (stageId) => {
|
|
456
|
+
if (visiting.has(stageId))
|
|
457
|
+
return true;
|
|
458
|
+
if (visited.has(stageId))
|
|
459
|
+
return false;
|
|
460
|
+
visiting.add(stageId);
|
|
461
|
+
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
462
|
+
return true;
|
|
463
|
+
visiting.delete(stageId);
|
|
464
|
+
visited.add(stageId);
|
|
465
|
+
return false;
|
|
466
|
+
};
|
|
467
|
+
return topologies.some((topology) => cyclic(topology.stageId));
|
|
468
|
+
}
|
|
469
|
+
var init_stage_topology_validation = () => {};
|
|
470
|
+
|
|
471
|
+
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
472
|
+
function parseWorkflowChildResult(value2) {
|
|
473
|
+
if (!isOrdinaryObject(value2))
|
|
474
|
+
return;
|
|
475
|
+
const workflow = value2.workflow;
|
|
476
|
+
const runId = value2.runId;
|
|
477
|
+
const status = value2.status;
|
|
478
|
+
const exited = value2.exited;
|
|
479
|
+
const outputs = value2.outputs;
|
|
480
|
+
const exitReason = value2.exitReason;
|
|
481
|
+
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
482
|
+
return;
|
|
483
|
+
if (exited === false) {
|
|
484
|
+
if (status !== "completed" || exitReason !== undefined)
|
|
485
|
+
return;
|
|
486
|
+
return value2;
|
|
487
|
+
}
|
|
488
|
+
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
489
|
+
return;
|
|
490
|
+
return value2;
|
|
491
|
+
}
|
|
492
|
+
function isWorkflowChildResult(value2) {
|
|
493
|
+
return parseWorkflowChildResult(value2) !== undefined;
|
|
494
|
+
}
|
|
495
|
+
function isWorkflowExitStatus(value2) {
|
|
496
|
+
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
497
|
+
}
|
|
498
|
+
function isOrdinaryObject(value2) {
|
|
499
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
500
|
+
return false;
|
|
501
|
+
const prototype = Object.getPrototypeOf(value2);
|
|
502
|
+
return prototype === Object.prototype || prototype === null;
|
|
503
|
+
}
|
|
504
|
+
|
|
300
505
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
301
506
|
import { createHash } from "node:crypto";
|
|
302
507
|
function durableHash(value2) {
|
|
@@ -363,6 +568,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
|
|
|
363
568
|
function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
|
|
364
569
|
return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
|
|
365
570
|
}
|
|
571
|
+
|
|
572
|
+
class StageReceipts {
|
|
573
|
+
exact = new Map;
|
|
574
|
+
anySession = new Map;
|
|
575
|
+
has(logicalRunId, sessionId, admission) {
|
|
576
|
+
return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
|
|
577
|
+
}
|
|
578
|
+
hasAnySession(logicalRunId, admission) {
|
|
579
|
+
return this.anySession.get(logicalRunId)?.has(admission) ?? false;
|
|
580
|
+
}
|
|
581
|
+
add(logicalRunId, sessionId, admission) {
|
|
582
|
+
let bySession = this.exact.get(logicalRunId);
|
|
583
|
+
if (bySession === undefined) {
|
|
584
|
+
bySession = new Map;
|
|
585
|
+
this.exact.set(logicalRunId, bySession);
|
|
586
|
+
}
|
|
587
|
+
let admissions = bySession.get(sessionId);
|
|
588
|
+
if (admissions === undefined) {
|
|
589
|
+
admissions = new Set;
|
|
590
|
+
bySession.set(sessionId, admissions);
|
|
591
|
+
}
|
|
592
|
+
if (admissions.has(admission))
|
|
593
|
+
return false;
|
|
594
|
+
admissions.add(admission);
|
|
595
|
+
return true;
|
|
596
|
+
}
|
|
597
|
+
addAnySession(logicalRunId, admission) {
|
|
598
|
+
let admissions = this.anySession.get(logicalRunId);
|
|
599
|
+
if (admissions === undefined) {
|
|
600
|
+
admissions = new Set;
|
|
601
|
+
this.anySession.set(logicalRunId, admissions);
|
|
602
|
+
}
|
|
603
|
+
if (admissions.has(admission))
|
|
604
|
+
return false;
|
|
605
|
+
admissions.add(admission);
|
|
606
|
+
return true;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
class PendingStageReceiptIndex {
|
|
611
|
+
workflows = new Map;
|
|
612
|
+
add(workflowId, messageId, delivery, logicalRunId) {
|
|
613
|
+
let workflow = this.workflows.get(workflowId);
|
|
614
|
+
if (workflow === undefined) {
|
|
615
|
+
workflow = { byMessage: new Map, log: [] };
|
|
616
|
+
this.workflows.set(workflowId, workflow);
|
|
617
|
+
}
|
|
618
|
+
let byRun = workflow.byMessage.get(messageId);
|
|
619
|
+
if (byRun === undefined) {
|
|
620
|
+
byRun = new Map;
|
|
621
|
+
workflow.byMessage.set(messageId, byRun);
|
|
622
|
+
}
|
|
623
|
+
let byStage = byRun.get(delivery.runId);
|
|
624
|
+
if (byStage === undefined) {
|
|
625
|
+
byStage = new Map;
|
|
626
|
+
byRun.set(delivery.runId, byStage);
|
|
627
|
+
}
|
|
628
|
+
let receipts = byStage.get(delivery.stageId);
|
|
629
|
+
if (receipts === undefined) {
|
|
630
|
+
receipts = new StageReceipts;
|
|
631
|
+
byStage.set(delivery.stageId, receipts);
|
|
632
|
+
}
|
|
633
|
+
const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
|
|
634
|
+
let changed = added;
|
|
635
|
+
if (receipts.add(logicalRunId, undefined, undefined))
|
|
636
|
+
changed = true;
|
|
637
|
+
if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
|
|
638
|
+
changed = true;
|
|
639
|
+
if (changed)
|
|
640
|
+
workflow.log.push(messageId);
|
|
641
|
+
return added;
|
|
642
|
+
}
|
|
643
|
+
has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
|
|
644
|
+
const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
|
|
645
|
+
if (receipts === undefined)
|
|
646
|
+
return false;
|
|
647
|
+
if (anySession && delivery.admission === "context")
|
|
648
|
+
return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
|
|
649
|
+
return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
|
|
650
|
+
}
|
|
651
|
+
changesSince(workflowId, cursor) {
|
|
652
|
+
let workflow = this.workflows.get(workflowId);
|
|
653
|
+
if (workflow === undefined) {
|
|
654
|
+
workflow = { byMessage: new Map, log: [] };
|
|
655
|
+
this.workflows.set(workflowId, workflow);
|
|
656
|
+
}
|
|
657
|
+
const next = { log: workflow.log, length: workflow.log.length };
|
|
658
|
+
if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
|
|
659
|
+
return { cursor: next, messageIds: undefined };
|
|
660
|
+
return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
|
|
661
|
+
}
|
|
662
|
+
delete(workflowId) {
|
|
663
|
+
this.workflows.delete(workflowId);
|
|
664
|
+
}
|
|
665
|
+
clear() {
|
|
666
|
+
this.workflows.clear();
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
function nextVersion() {
|
|
670
|
+
lastVersion += 1;
|
|
671
|
+
return lastVersion;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
class NestedSnapshotVersion {
|
|
675
|
+
rootWorkflowId;
|
|
676
|
+
current = nextVersion();
|
|
677
|
+
latestCompletedAt = Number.NEGATIVE_INFINITY;
|
|
678
|
+
replayGroups = new Map;
|
|
679
|
+
boundaryReplayKeys = new Set;
|
|
680
|
+
constructor(rootWorkflowId) {
|
|
681
|
+
this.rootWorkflowId = rootWorkflowId;
|
|
682
|
+
}
|
|
683
|
+
get version() {
|
|
684
|
+
return this.current;
|
|
685
|
+
}
|
|
686
|
+
record(checkpoint) {
|
|
687
|
+
if (!this.isInert(checkpoint))
|
|
688
|
+
this.current = nextVersion();
|
|
689
|
+
this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
|
|
690
|
+
if (checkpoint.kind !== "stage")
|
|
691
|
+
return;
|
|
692
|
+
if (checkpoint.topology?.boundary !== undefined)
|
|
693
|
+
this.boundaryReplayKeys.add(checkpoint.replayKey);
|
|
694
|
+
if (checkpoint.topology?.occurrenceKey !== undefined)
|
|
695
|
+
return;
|
|
696
|
+
const identity = stageIdentity(checkpoint);
|
|
697
|
+
const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
|
|
698
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
699
|
+
if (summary === undefined) {
|
|
700
|
+
this.replayGroups.set(checkpoint.replayKey, {
|
|
701
|
+
name: checkpoint.name,
|
|
702
|
+
identity,
|
|
703
|
+
consistent: true,
|
|
704
|
+
hasOutput: checkpoint.output !== undefined,
|
|
705
|
+
hasExitedChildOutput: exited
|
|
706
|
+
});
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
|
|
710
|
+
summary.hasOutput ||= checkpoint.output !== undefined;
|
|
711
|
+
summary.hasExitedChildOutput ||= exited;
|
|
712
|
+
}
|
|
713
|
+
isInert(checkpoint) {
|
|
714
|
+
if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
|
|
715
|
+
return false;
|
|
716
|
+
}
|
|
717
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
718
|
+
if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
|
|
719
|
+
return false;
|
|
720
|
+
}
|
|
721
|
+
const id = checkpoint.checkpointId;
|
|
722
|
+
for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
|
|
723
|
+
if (this.boundaryReplayKeys.has(id.slice(0, colon)))
|
|
724
|
+
return false;
|
|
725
|
+
}
|
|
726
|
+
return true;
|
|
727
|
+
}
|
|
728
|
+
isRootOwned(checkpoint) {
|
|
729
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
730
|
+
return runId === undefined || runId === this.rootWorkflowId;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
function stageIdentity(checkpoint) {
|
|
734
|
+
const topology = checkpoint.topology;
|
|
735
|
+
return JSON.stringify([
|
|
736
|
+
durableStageGroupKey(checkpoint),
|
|
737
|
+
topology?.version,
|
|
738
|
+
topology?.stageId,
|
|
739
|
+
topology?.parentIds,
|
|
740
|
+
topology?.sourceOrder,
|
|
741
|
+
topology?.run === undefined ? undefined : [
|
|
742
|
+
topology.run.runId,
|
|
743
|
+
topology.run.runName,
|
|
744
|
+
topology.run.parentRunId,
|
|
745
|
+
topology.run.parentStageId,
|
|
746
|
+
topology.run.rootRunId
|
|
747
|
+
]
|
|
748
|
+
]);
|
|
749
|
+
}
|
|
366
750
|
function checkpointKey(c) {
|
|
367
751
|
return `${c.kind}:${c.checkpointId}`;
|
|
368
752
|
}
|
|
@@ -374,7 +758,7 @@ class JournalProjection {
|
|
|
374
758
|
workflows = new Map;
|
|
375
759
|
deletedWorkflowIds = new Set;
|
|
376
760
|
pendingReceipts = new Map;
|
|
377
|
-
stickyReceipts = new
|
|
761
|
+
stickyReceipts = new PendingStageReceiptIndex;
|
|
378
762
|
registerWorkflow(handle, authoritativeOwner = false) {
|
|
379
763
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
380
764
|
const existing = this.workflows.get(handle.workflowId);
|
|
@@ -422,7 +806,8 @@ class JournalProjection {
|
|
|
422
806
|
toolByHash: new Map,
|
|
423
807
|
uiByHash: new Map,
|
|
424
808
|
stageOutputByReplayKey: new Map,
|
|
425
|
-
stageSessionByReplayKey: new Map
|
|
809
|
+
stageSessionByReplayKey: new Map,
|
|
810
|
+
nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
|
|
426
811
|
});
|
|
427
812
|
}
|
|
428
813
|
applyWorkflowSnapshot(handle) {
|
|
@@ -448,39 +833,43 @@ class JournalProjection {
|
|
|
448
833
|
return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
|
|
449
834
|
}
|
|
450
835
|
stickyDeliveryCounts = new Map;
|
|
836
|
+
stickyDeliveryCountVersions = new Map;
|
|
451
837
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
452
|
-
return this.stickyDeliveryCounts.get(workflowId)?.get(
|
|
838
|
+
return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
|
|
839
|
+
}
|
|
840
|
+
pendingStageDeliveryCountVersion(workflowId) {
|
|
841
|
+
return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
|
|
842
|
+
}
|
|
843
|
+
bumpDeliveryCountVersion(workflowId) {
|
|
844
|
+
this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
|
|
453
845
|
}
|
|
454
846
|
async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
|
|
455
|
-
|
|
456
|
-
if (
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
|
|
464
|
-
if (delivery.admission === "context" && delivery.sessionId !== undefined)
|
|
465
|
-
receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
|
|
466
|
-
if (accounting !== undefined && !existing) {
|
|
467
|
-
let counts = this.stickyDeliveryCounts.get(workflowId);
|
|
847
|
+
const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
|
|
848
|
+
if (accounting !== undefined && added) {
|
|
849
|
+
let byRun = this.stickyDeliveryCounts.get(workflowId);
|
|
850
|
+
if (byRun === undefined) {
|
|
851
|
+
byRun = new Map;
|
|
852
|
+
this.stickyDeliveryCounts.set(workflowId, byRun);
|
|
853
|
+
}
|
|
854
|
+
let counts = byRun.get(accounting.messageRunId);
|
|
468
855
|
if (counts === undefined) {
|
|
469
856
|
counts = new Map;
|
|
470
|
-
|
|
857
|
+
byRun.set(accounting.messageRunId, counts);
|
|
471
858
|
}
|
|
472
|
-
const
|
|
473
|
-
counts.set(
|
|
859
|
+
const previous = counts.get(messageId) ?? 0;
|
|
860
|
+
counts.set(messageId, Math.max(previous, accounting.deliveryCount));
|
|
861
|
+
if (accounting.deliveryCount > previous)
|
|
862
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
474
863
|
}
|
|
475
864
|
}
|
|
476
865
|
async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
|
|
477
|
-
return
|
|
866
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
|
|
478
867
|
}
|
|
479
868
|
hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
|
|
480
|
-
return
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
869
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
|
|
870
|
+
}
|
|
871
|
+
pendingStageDeliveryReceiptChanges(workflowId, cursor) {
|
|
872
|
+
return this.stickyReceipts.changesSince(workflowId, cursor);
|
|
484
873
|
}
|
|
485
874
|
recordCheckpoint(recorded) {
|
|
486
875
|
const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
|
|
@@ -499,6 +888,7 @@ class JournalProjection {
|
|
|
499
888
|
if (rec.checkpoints.has(key))
|
|
500
889
|
return;
|
|
501
890
|
rec.checkpoints.set(key, currentCheckpoint);
|
|
891
|
+
rec.nestedSnapshotVersion.record(currentCheckpoint);
|
|
502
892
|
if (currentCheckpoint.kind === "tool") {
|
|
503
893
|
const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
|
|
504
894
|
if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
|
|
@@ -556,6 +946,9 @@ class JournalProjection {
|
|
|
556
946
|
getWorkflow(workflowId) {
|
|
557
947
|
return this.workflows.get(workflowId)?.handle;
|
|
558
948
|
}
|
|
949
|
+
nestedSnapshotVersion(workflowId) {
|
|
950
|
+
return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
|
|
951
|
+
}
|
|
559
952
|
getLoadableWorkflow(workflowId) {
|
|
560
953
|
return this.getWorkflow(workflowId);
|
|
561
954
|
}
|
|
@@ -633,6 +1026,7 @@ class JournalProjection {
|
|
|
633
1026
|
this.pendingReceipts.delete(workflowId);
|
|
634
1027
|
this.stickyReceipts.delete(workflowId);
|
|
635
1028
|
this.stickyDeliveryCounts.delete(workflowId);
|
|
1029
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
636
1030
|
this.deletedWorkflowIds.add(workflowId);
|
|
637
1031
|
}
|
|
638
1032
|
isWorkflowLoadable(workflowId) {
|
|
@@ -643,6 +1037,8 @@ class JournalProjection {
|
|
|
643
1037
|
this.pendingReceipts.clear();
|
|
644
1038
|
this.stickyReceipts.clear();
|
|
645
1039
|
this.stickyDeliveryCounts.clear();
|
|
1040
|
+
for (const workflowId of this.stickyDeliveryCountVersions.keys())
|
|
1041
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
646
1042
|
this.deletedWorkflowIds.clear();
|
|
647
1043
|
}
|
|
648
1044
|
}
|
|
@@ -696,216 +1092,32 @@ function toResumableEntry(handle) {
|
|
|
696
1092
|
updatedAt: handle.updatedAt
|
|
697
1093
|
};
|
|
698
1094
|
}
|
|
699
|
-
|
|
700
|
-
const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
|
|
701
|
-
return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
|
|
702
|
-
}
|
|
703
|
-
var SESSION_HEADER_READ_LIMIT_BYTES;
|
|
1095
|
+
var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
|
|
704
1096
|
var init_backend = __esm(() => {
|
|
705
1097
|
init_resume_eligibility();
|
|
1098
|
+
init_stage_topology_validation();
|
|
706
1099
|
init_durable_hash();
|
|
707
1100
|
SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
|
|
708
1101
|
});
|
|
709
1102
|
|
|
710
|
-
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
711
|
-
function immutableStageGroupError(stages) {
|
|
712
|
-
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
713
|
-
if (occurrenceError !== undefined)
|
|
714
|
-
return occurrenceError;
|
|
715
|
-
const byReplayKey = groupByDurableStageKey(stages);
|
|
716
|
-
for (const group of byReplayKey.values()) {
|
|
717
|
-
const firstName = group[0]?.name;
|
|
718
|
-
if (group.some((stage) => stage.name !== firstName))
|
|
719
|
-
return "stage name drift within one replay-key group";
|
|
720
|
-
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
721
|
-
const first = topologies[0];
|
|
722
|
-
if (first === undefined)
|
|
723
|
-
continue;
|
|
724
|
-
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
725
|
-
return "immutable stage identity drift within one replay-key group";
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
return;
|
|
729
|
-
}
|
|
730
|
-
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
731
|
-
const prefix = `${replayScope}:`;
|
|
732
|
-
const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
|
|
733
|
-
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
734
|
-
return "child run topology exists outside its durable invocation scope";
|
|
735
|
-
}
|
|
736
|
-
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
737
|
-
return "child checkpoint ownership is not reciprocal";
|
|
738
|
-
}
|
|
739
|
-
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
740
|
-
const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
741
|
-
const immutableError = immutableStageGroupError(direct);
|
|
742
|
-
if (immutableError !== undefined)
|
|
743
|
-
return immutableError;
|
|
744
|
-
const merged = [];
|
|
745
|
-
for (const group of groupByDurableStageKey(direct).values()) {
|
|
746
|
-
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
747
|
-
if (topology === undefined)
|
|
748
|
-
return "direct child stage group lacks owning-run topology";
|
|
749
|
-
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
750
|
-
return "direct child stage topology lacks current source identity";
|
|
751
|
-
}
|
|
752
|
-
if (topology.stageId.length === 0)
|
|
753
|
-
return "direct child stage id is empty";
|
|
754
|
-
merged.push(topology);
|
|
755
|
-
}
|
|
756
|
-
const ids = new Set(allowedNodeIds);
|
|
757
|
-
const stageIds = new Set;
|
|
758
|
-
const orders = new Set;
|
|
759
|
-
for (const topology of merged) {
|
|
760
|
-
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
761
|
-
return "direct child stage ids are not unique";
|
|
762
|
-
stageIds.add(topology.stageId);
|
|
763
|
-
ids.add(topology.stageId);
|
|
764
|
-
if (topology.sourceOrder !== undefined) {
|
|
765
|
-
if (orders.has(topology.sourceOrder))
|
|
766
|
-
return "direct child source orders are not unique";
|
|
767
|
-
orders.add(topology.sourceOrder);
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
771
|
-
return "direct child stage references an unknown parent";
|
|
772
|
-
}
|
|
773
|
-
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
774
|
-
}
|
|
775
|
-
function owningBoundaryIdError(stages, owningRunId) {
|
|
776
|
-
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
777
|
-
const seen = new Map;
|
|
778
|
-
for (const start of starts) {
|
|
779
|
-
const stageId = start.topology.stageId;
|
|
780
|
-
if (stageId.length === 0)
|
|
781
|
-
return "owning-run boundary stage id is empty";
|
|
782
|
-
const prior = seen.get(stageId);
|
|
783
|
-
if (prior !== undefined && prior !== start.replayKey)
|
|
784
|
-
return "owning-run boundary stage id is aliased";
|
|
785
|
-
seen.set(stageId, start.replayKey);
|
|
786
|
-
}
|
|
787
|
-
return;
|
|
788
|
-
}
|
|
789
|
-
function sameRun(actual, expected) {
|
|
790
|
-
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
791
|
-
}
|
|
792
|
-
function sameStrings(left, right) {
|
|
793
|
-
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
794
|
-
}
|
|
795
|
-
function sameOptionalRun(actual, expected) {
|
|
796
|
-
if (actual === undefined || expected === undefined)
|
|
797
|
-
return actual === expected;
|
|
798
|
-
return sameRun(actual, expected);
|
|
799
|
-
}
|
|
800
|
-
function promptOccurrenceIdentityError(stages) {
|
|
801
|
-
const replayByOccurrence = new Map;
|
|
802
|
-
const occurrenceByReplayStage = new Map;
|
|
803
|
-
const stageByReplayOccurrence = new Map;
|
|
804
|
-
for (const stage of stages) {
|
|
805
|
-
const topology = stage.topology;
|
|
806
|
-
const occurrence = topology?.occurrenceKey;
|
|
807
|
-
if (topology === undefined || occurrence === undefined)
|
|
808
|
-
continue;
|
|
809
|
-
const priorReplay = replayByOccurrence.get(occurrence);
|
|
810
|
-
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
811
|
-
return "prompt occurrence is bound to multiple replay keys";
|
|
812
|
-
}
|
|
813
|
-
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
814
|
-
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
815
|
-
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
816
|
-
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
817
|
-
return "prompt stage identity is bound to multiple occurrences";
|
|
818
|
-
}
|
|
819
|
-
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
820
|
-
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
821
|
-
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
822
|
-
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
823
|
-
return "prompt occurrence is bound to multiple stage identities";
|
|
824
|
-
}
|
|
825
|
-
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
826
|
-
}
|
|
827
|
-
return;
|
|
828
|
-
}
|
|
829
|
-
function durableStageGroupKey(stage) {
|
|
830
|
-
const occurrence = stage.topology?.occurrenceKey;
|
|
831
|
-
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
832
|
-
}
|
|
833
|
-
function groupByDurableStageKey(stages) {
|
|
834
|
-
const groups = new Map;
|
|
835
|
-
for (const stage of stages) {
|
|
836
|
-
const key = durableStageGroupKey(stage);
|
|
837
|
-
const group = groups.get(key) ?? [];
|
|
838
|
-
group.push(stage);
|
|
839
|
-
groups.set(key, group);
|
|
840
|
-
}
|
|
841
|
-
return groups;
|
|
842
|
-
}
|
|
843
|
-
function compositeKey(domain, ...parts) {
|
|
844
|
-
return JSON.stringify([domain, ...parts]);
|
|
845
|
-
}
|
|
846
|
-
function hasCycle(topologies) {
|
|
847
|
-
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
848
|
-
const visiting = new Set;
|
|
849
|
-
const visited = new Set;
|
|
850
|
-
const cyclic = (stageId) => {
|
|
851
|
-
if (visiting.has(stageId))
|
|
852
|
-
return true;
|
|
853
|
-
if (visited.has(stageId))
|
|
854
|
-
return false;
|
|
855
|
-
visiting.add(stageId);
|
|
856
|
-
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
857
|
-
return true;
|
|
858
|
-
visiting.delete(stageId);
|
|
859
|
-
visited.add(stageId);
|
|
860
|
-
return false;
|
|
861
|
-
};
|
|
862
|
-
return topologies.some((topology) => cyclic(topology.stageId));
|
|
863
|
-
}
|
|
864
|
-
var init_stage_topology_validation = () => {};
|
|
865
|
-
|
|
866
|
-
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
867
|
-
function parseWorkflowChildResult(value2) {
|
|
868
|
-
if (!isOrdinaryObject(value2))
|
|
869
|
-
return;
|
|
870
|
-
const workflow = value2.workflow;
|
|
871
|
-
const runId = value2.runId;
|
|
872
|
-
const status = value2.status;
|
|
873
|
-
const exited = value2.exited;
|
|
874
|
-
const outputs = value2.outputs;
|
|
875
|
-
const exitReason = value2.exitReason;
|
|
876
|
-
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
877
|
-
return;
|
|
878
|
-
if (exited === false) {
|
|
879
|
-
if (status !== "completed" || exitReason !== undefined)
|
|
880
|
-
return;
|
|
881
|
-
return value2;
|
|
882
|
-
}
|
|
883
|
-
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
884
|
-
return;
|
|
885
|
-
return value2;
|
|
886
|
-
}
|
|
887
|
-
function isWorkflowChildResult(value2) {
|
|
888
|
-
return parseWorkflowChildResult(value2) !== undefined;
|
|
889
|
-
}
|
|
890
|
-
function isWorkflowExitStatus(value2) {
|
|
891
|
-
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
892
|
-
}
|
|
893
|
-
function isOrdinaryObject(value2) {
|
|
894
|
-
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
895
|
-
return false;
|
|
896
|
-
const prototype = Object.getPrototypeOf(value2);
|
|
897
|
-
return prototype === Object.prototype || prototype === null;
|
|
898
|
-
}
|
|
899
|
-
|
|
900
1103
|
// dist/builtin/workflows/src/durable/boundary-lifecycle.ts
|
|
901
1104
|
function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
902
1105
|
const startTopology = start.topology;
|
|
903
1106
|
const startBoundary = startTopology?.boundary;
|
|
904
|
-
const startRun = startTopology?.run;
|
|
905
1107
|
const related = stages.filter((stage) => {
|
|
906
1108
|
const boundary = stage.topology?.boundary;
|
|
907
1109
|
return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
|
|
908
1110
|
});
|
|
1111
|
+
return resolveBoundaryLifecycleFromRecords(start, {
|
|
1112
|
+
related,
|
|
1113
|
+
outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
|
|
1114
|
+
}, requireTerminal);
|
|
1115
|
+
}
|
|
1116
|
+
function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
|
|
1117
|
+
const startTopology = start.topology;
|
|
1118
|
+
const startBoundary = startTopology?.boundary;
|
|
1119
|
+
const startRun = startTopology?.run;
|
|
1120
|
+
const related = records.related;
|
|
909
1121
|
const starts = related.filter((stage) => stage.topology.boundary.event === "start");
|
|
910
1122
|
if (starts.length !== 1 || starts[0] !== start)
|
|
911
1123
|
return failure("boundary has no unique start record");
|
|
@@ -942,7 +1154,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
|
942
1154
|
return failure("boundary has an illegal terminal transition");
|
|
943
1155
|
}
|
|
944
1156
|
const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
|
|
945
|
-
const outputRecords =
|
|
1157
|
+
const outputRecords = records.outputRecords();
|
|
946
1158
|
if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
|
|
947
1159
|
return failure("boundary replay key has conflicting output records");
|
|
948
1160
|
}
|
|
@@ -1683,9 +1895,19 @@ function* invoke(action) {
|
|
|
1683
1895
|
} catch (error) {
|
|
1684
1896
|
if (error instanceof JournalError || error instanceof StorageError)
|
|
1685
1897
|
throw error;
|
|
1686
|
-
throw
|
|
1898
|
+
throw customJournalUnavailable(error);
|
|
1687
1899
|
}
|
|
1688
1900
|
}
|
|
1901
|
+
function customJournalUnavailable(failure) {
|
|
1902
|
+
const cause = failure instanceof Error ? failure : new Error(String(failure));
|
|
1903
|
+
const code = hasErrorCode(cause) ? cause.code : undefined;
|
|
1904
|
+
const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
|
|
1905
|
+
return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
|
|
1906
|
+
}
|
|
1907
|
+
function hasErrorCode(error) {
|
|
1908
|
+
const code = error.code;
|
|
1909
|
+
return typeof code === "string" || typeof code === "number";
|
|
1910
|
+
}
|
|
1689
1911
|
async function execute(operation) {
|
|
1690
1912
|
let step = operation.next();
|
|
1691
1913
|
while (!step.done) {
|
|
@@ -1713,6 +1935,8 @@ class JournalDurableBackend {
|
|
|
1713
1935
|
executions = new Map;
|
|
1714
1936
|
executionContext = new AsyncLocalStorage;
|
|
1715
1937
|
pending = new Map;
|
|
1938
|
+
reconciledPendingStages = new Map;
|
|
1939
|
+
appendBatches = new Map;
|
|
1716
1940
|
catalogQueue = Symbol("catalog");
|
|
1717
1941
|
lost = new Map;
|
|
1718
1942
|
prompts = new Map;
|
|
@@ -1745,7 +1969,7 @@ class JournalDurableBackend {
|
|
|
1745
1969
|
await backend.close();
|
|
1746
1970
|
if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
|
|
1747
1971
|
throw error;
|
|
1748
|
-
throw
|
|
1972
|
+
throw customJournalUnavailable(error);
|
|
1749
1973
|
}
|
|
1750
1974
|
}
|
|
1751
1975
|
serialize(runId, action, requiresActiveExecution = true) {
|
|
@@ -1760,6 +1984,8 @@ class JournalDurableBackend {
|
|
|
1760
1984
|
return this.enqueue(runId, action, admitted);
|
|
1761
1985
|
}
|
|
1762
1986
|
enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
|
|
1987
|
+
if (typeof runId === "string")
|
|
1988
|
+
this.appendBatches.delete(runId);
|
|
1763
1989
|
const prior = this.pending.get(runId) ?? Promise.resolve();
|
|
1764
1990
|
const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
|
|
1765
1991
|
const settled = result.then(() => {}, () => {});
|
|
@@ -1774,6 +2000,7 @@ class JournalDurableBackend {
|
|
|
1774
2000
|
if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
|
|
1775
2001
|
return;
|
|
1776
2002
|
const event = entry.event;
|
|
2003
|
+
let raisedCount;
|
|
1777
2004
|
if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
|
|
1778
2005
|
const checkpoint = event.payload;
|
|
1779
2006
|
validateCheckpointInvocationFingerprint(checkpoint);
|
|
@@ -1782,7 +2009,10 @@ class JournalDurableBackend {
|
|
|
1782
2009
|
this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
|
|
1783
2010
|
} else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
|
|
1784
2011
|
const payload = event.payload;
|
|
2012
|
+
const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
|
|
1785
2013
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
2014
|
+
if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
|
|
2015
|
+
raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
|
|
1786
2016
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
1787
2017
|
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
1788
2018
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
@@ -1793,20 +2023,80 @@ class JournalDurableBackend {
|
|
|
1793
2023
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
1794
2024
|
this.prompts.delete(entry.runId);
|
|
1795
2025
|
}
|
|
1796
|
-
|
|
1797
|
-
if (handle?.pendingStageMessages !== undefined) {
|
|
1798
|
-
const messages = handle.pendingStageMessages.map((message) => {
|
|
1799
|
-
if (message.sticky !== true)
|
|
1800
|
-
return message;
|
|
1801
|
-
const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
|
|
1802
|
-
return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
|
|
1803
|
-
});
|
|
1804
|
-
if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
|
|
1805
|
-
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
|
|
1806
|
-
}
|
|
2026
|
+
this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
|
|
1807
2027
|
this.sequences.set(entry.runId, entry.seq);
|
|
1808
2028
|
this.foldedAt.set(entry.runId, entry.at);
|
|
1809
2029
|
}
|
|
2030
|
+
reconcileFoldedDeliveryCounts(runId, raisedCount) {
|
|
2031
|
+
const handle = this.projection.getWorkflow(runId);
|
|
2032
|
+
const list = handle?.pendingStageMessages;
|
|
2033
|
+
if (handle === undefined || list === undefined) {
|
|
2034
|
+
this.reconciledPendingStages.delete(runId);
|
|
2035
|
+
return;
|
|
2036
|
+
}
|
|
2037
|
+
let state = this.reconciledPendingStages.get(runId);
|
|
2038
|
+
let next;
|
|
2039
|
+
const raise = (index, count) => {
|
|
2040
|
+
const message = (next ?? list)[index];
|
|
2041
|
+
if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
|
|
2042
|
+
return;
|
|
2043
|
+
next ??= [...list];
|
|
2044
|
+
next[index] = { ...message, deliveryCount: count };
|
|
2045
|
+
};
|
|
2046
|
+
const reconcile = (index) => {
|
|
2047
|
+
const message = list[index];
|
|
2048
|
+
if (message.sticky !== true)
|
|
2049
|
+
return;
|
|
2050
|
+
raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
|
|
2051
|
+
};
|
|
2052
|
+
if (state?.list !== list) {
|
|
2053
|
+
const previous = state?.list ?? [];
|
|
2054
|
+
let aligned = list.length >= previous.length;
|
|
2055
|
+
const changed = [];
|
|
2056
|
+
for (let index = 0;index < list.length; index++) {
|
|
2057
|
+
const message = list[index];
|
|
2058
|
+
if (previous[index] === message)
|
|
2059
|
+
continue;
|
|
2060
|
+
if (index < previous.length && previous[index].id !== message.id)
|
|
2061
|
+
aligned = false;
|
|
2062
|
+
changed.push(index);
|
|
2063
|
+
}
|
|
2064
|
+
const reconciled = aligned ? undefined : new Set(previous);
|
|
2065
|
+
for (const index of changed)
|
|
2066
|
+
if (!reconciled?.has(list[index]))
|
|
2067
|
+
reconcile(index);
|
|
2068
|
+
const positions = aligned ? state?.positions : undefined;
|
|
2069
|
+
if (positions !== undefined) {
|
|
2070
|
+
for (let index = previous.length;index < list.length; index++) {
|
|
2071
|
+
const indexes = positions.get(list[index].id);
|
|
2072
|
+
if (indexes === undefined)
|
|
2073
|
+
positions.set(list[index].id, [index]);
|
|
2074
|
+
else
|
|
2075
|
+
indexes.push(index);
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
state = { list, positions };
|
|
2079
|
+
}
|
|
2080
|
+
if (raisedCount !== undefined) {
|
|
2081
|
+
const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
|
|
2082
|
+
state.positions ??= messagePositions(list);
|
|
2083
|
+
for (const index of state.positions.get(raisedCount.messageId) ?? []) {
|
|
2084
|
+
const message = list[index];
|
|
2085
|
+
if (message.sticky === true && message.runId === raisedCount.messageRunId)
|
|
2086
|
+
raise(index, count);
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
if (next === undefined) {
|
|
2090
|
+
this.reconciledPendingStages.set(runId, state);
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
|
|
2094
|
+
const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
|
|
2095
|
+
this.reconciledPendingStages.set(runId, {
|
|
2096
|
+
list: applied,
|
|
2097
|
+
positions: applied.length === list.length ? state.positions : undefined
|
|
2098
|
+
});
|
|
2099
|
+
}
|
|
1810
2100
|
summarize(runId, tail) {
|
|
1811
2101
|
const event = tail.snapshot.event;
|
|
1812
2102
|
const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
|
|
@@ -1830,6 +2120,7 @@ class JournalDurableBackend {
|
|
|
1830
2120
|
*rebuild(runId) {
|
|
1831
2121
|
const entries = yield* this.entries(runId);
|
|
1832
2122
|
this.projection.deleteWorkflow(runId);
|
|
2123
|
+
this.reconciledPendingStages.delete(runId);
|
|
1833
2124
|
this.sequences.delete(runId);
|
|
1834
2125
|
this.foldedAt.delete(runId);
|
|
1835
2126
|
this.summarized.delete(runId);
|
|
@@ -1925,9 +2216,9 @@ class JournalDurableBackend {
|
|
|
1925
2216
|
}
|
|
1926
2217
|
yield* this.write(runId, lease, event, options);
|
|
1927
2218
|
}
|
|
1928
|
-
*write(runId, lease, event, options) {
|
|
2219
|
+
*write(runId, lease, event, options, context = (action) => action()) {
|
|
1929
2220
|
try {
|
|
1930
|
-
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
2221
|
+
const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
|
|
1931
2222
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
1932
2223
|
if (outcome.entry.seq > nextSeq) {
|
|
1933
2224
|
for (const entry of yield* this.entries(runId, nextSeq))
|
|
@@ -2111,13 +2402,80 @@ class JournalDurableBackend {
|
|
|
2111
2402
|
return true;
|
|
2112
2403
|
}
|
|
2113
2404
|
async archivePendingStageMessage(runId, message) {
|
|
2114
|
-
await this.
|
|
2405
|
+
await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
|
|
2115
2406
|
}
|
|
2116
2407
|
async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
|
|
2117
|
-
await this.
|
|
2408
|
+
await this.queueAppend(runId, {
|
|
2118
2409
|
type: RUN_EVENT_TYPES.messageDelivered,
|
|
2119
2410
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
2120
|
-
})
|
|
2411
|
+
});
|
|
2412
|
+
}
|
|
2413
|
+
queueAppend(runId, event, key) {
|
|
2414
|
+
try {
|
|
2415
|
+
this.ensureOpen();
|
|
2416
|
+
this.assertExecutionActive();
|
|
2417
|
+
} catch (error) {
|
|
2418
|
+
return Promise.reject(error);
|
|
2419
|
+
}
|
|
2420
|
+
const open = this.appendBatches.get(runId);
|
|
2421
|
+
const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
|
|
2422
|
+
const queued = new Promise((resolve, reject) => {
|
|
2423
|
+
batch.push({
|
|
2424
|
+
event,
|
|
2425
|
+
...key === undefined ? {} : { key },
|
|
2426
|
+
context: AsyncLocalStorage.snapshot(),
|
|
2427
|
+
resolve,
|
|
2428
|
+
reject
|
|
2429
|
+
});
|
|
2430
|
+
});
|
|
2431
|
+
if (batch !== open) {
|
|
2432
|
+
this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
|
|
2433
|
+
for (const append of batch)
|
|
2434
|
+
append.reject(error);
|
|
2435
|
+
});
|
|
2436
|
+
this.appendBatches.set(runId, batch);
|
|
2437
|
+
}
|
|
2438
|
+
return queued;
|
|
2439
|
+
}
|
|
2440
|
+
*appendBatch(runId, batch) {
|
|
2441
|
+
if (this.appendBatches.get(runId) === batch)
|
|
2442
|
+
this.appendBatches.delete(runId);
|
|
2443
|
+
const appendMany = runJournalAppendMany(this.journal);
|
|
2444
|
+
if (appendMany === undefined || batch.length === 1) {
|
|
2445
|
+
for (const append of batch) {
|
|
2446
|
+
try {
|
|
2447
|
+
const lease = yield* this.ownership(runId);
|
|
2448
|
+
yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
|
|
2449
|
+
append.resolve();
|
|
2450
|
+
} catch (error) {
|
|
2451
|
+
append.reject(error);
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
return;
|
|
2455
|
+
}
|
|
2456
|
+
let settled = 0;
|
|
2457
|
+
try {
|
|
2458
|
+
const lease = yield* this.ownership(runId);
|
|
2459
|
+
const results = yield* invoke(() => appendMany(lease, batch));
|
|
2460
|
+
for (const result of results) {
|
|
2461
|
+
const append = batch[settled];
|
|
2462
|
+
if (result.status === "rejected") {
|
|
2463
|
+
append.reject(yield* this.mapOwnership(result.reason, runId));
|
|
2464
|
+
} else {
|
|
2465
|
+
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
2466
|
+
if (result.value.entry.seq > nextSeq)
|
|
2467
|
+
for (const entry of yield* this.entries(runId, nextSeq))
|
|
2468
|
+
this.fold(entry);
|
|
2469
|
+
this.fold(result.value.entry);
|
|
2470
|
+
append.resolve();
|
|
2471
|
+
}
|
|
2472
|
+
settled++;
|
|
2473
|
+
}
|
|
2474
|
+
} catch (error) {
|
|
2475
|
+
const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
|
|
2476
|
+
for (const append of batch.slice(settled))
|
|
2477
|
+
append.reject(failure);
|
|
2478
|
+
}
|
|
2121
2479
|
}
|
|
2122
2480
|
getToolOutput(...args) {
|
|
2123
2481
|
return this.projection.getToolOutput(...args);
|
|
@@ -2137,6 +2495,9 @@ class JournalDurableBackend {
|
|
|
2137
2495
|
listCheckpoints(runId) {
|
|
2138
2496
|
return this.projection.listCheckpoints(runId);
|
|
2139
2497
|
}
|
|
2498
|
+
nestedSnapshotVersion(runId) {
|
|
2499
|
+
return this.projection.nestedSnapshotVersion(runId);
|
|
2500
|
+
}
|
|
2140
2501
|
getWorkflow(runId) {
|
|
2141
2502
|
return this.projection.getWorkflow(runId);
|
|
2142
2503
|
}
|
|
@@ -2161,6 +2522,12 @@ class JournalDurableBackend {
|
|
|
2161
2522
|
getPendingStageMessageDeliveryCount(...args) {
|
|
2162
2523
|
return this.projection.getPendingStageMessageDeliveryCount(...args);
|
|
2163
2524
|
}
|
|
2525
|
+
pendingStageDeliveryReceiptChanges(...args) {
|
|
2526
|
+
return this.projection.pendingStageDeliveryReceiptChanges(...args);
|
|
2527
|
+
}
|
|
2528
|
+
pendingStageDeliveryCountVersion(runId) {
|
|
2529
|
+
return this.projection.pendingStageDeliveryCountVersion(runId);
|
|
2530
|
+
}
|
|
2164
2531
|
hasPendingStageDeliveryReceipt(...args) {
|
|
2165
2532
|
return this.projection.hasPendingStageDeliveryReceipt(...args);
|
|
2166
2533
|
}
|
|
@@ -2245,6 +2612,7 @@ class JournalDurableBackend {
|
|
|
2245
2612
|
this.foldedAt.delete(runId);
|
|
2246
2613
|
this.summarized.delete(runId);
|
|
2247
2614
|
this.prompts.delete(runId);
|
|
2615
|
+
this.reconciledPendingStages.delete(runId);
|
|
2248
2616
|
this.projection.deleteWorkflow(runId);
|
|
2249
2617
|
}
|
|
2250
2618
|
async deleteWorkflow(runId) {
|
|
@@ -2427,8 +2795,24 @@ class JournalDurableBackend {
|
|
|
2427
2795
|
}
|
|
2428
2796
|
}
|
|
2429
2797
|
}
|
|
2798
|
+
function messagePositions(list) {
|
|
2799
|
+
const positions = new Map;
|
|
2800
|
+
list.forEach((message, index) => {
|
|
2801
|
+
const indexes = positions.get(message.id);
|
|
2802
|
+
if (indexes === undefined)
|
|
2803
|
+
positions.set(message.id, [index]);
|
|
2804
|
+
else
|
|
2805
|
+
indexes.push(index);
|
|
2806
|
+
});
|
|
2807
|
+
return positions;
|
|
2808
|
+
}
|
|
2430
2809
|
function identity(message) {
|
|
2431
|
-
|
|
2810
|
+
let key = entryIdentity.get(message);
|
|
2811
|
+
if (key === undefined) {
|
|
2812
|
+
key = JSON.stringify([message.runId, message.id]);
|
|
2813
|
+
entryIdentity.set(message, key);
|
|
2814
|
+
}
|
|
2815
|
+
return key;
|
|
2432
2816
|
}
|
|
2433
2817
|
function uniqueByIdentity(messages) {
|
|
2434
2818
|
const result = new Map;
|
|
@@ -2602,7 +2986,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
2602
2986
|
function hasPendingStageDeliveryDelta(messages) {
|
|
2603
2987
|
return messages?.some(isDeltaEncoded) === true;
|
|
2604
2988
|
}
|
|
2605
|
-
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
2989
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
|
|
2606
2990
|
var init_journal_backend = __esm(() => {
|
|
2607
2991
|
init_workflow_instance_ownership_error();
|
|
2608
2992
|
init_backend();
|
|
@@ -2614,6 +2998,7 @@ var init_journal_backend = __esm(() => {
|
|
|
2614
2998
|
init_types();
|
|
2615
2999
|
PROCESS_HOLDER_ID = randomUUID();
|
|
2616
3000
|
processExitCleanups = new Set;
|
|
3001
|
+
CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
|
|
2617
3002
|
JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
|
|
2618
3003
|
runId;
|
|
2619
3004
|
holderId;
|
|
@@ -2660,6 +3045,7 @@ var init_journal_backend = __esm(() => {
|
|
|
2660
3045
|
this.name = "JournalMessageScopeError";
|
|
2661
3046
|
}
|
|
2662
3047
|
};
|
|
3048
|
+
entryIdentity = new WeakMap;
|
|
2663
3049
|
entryJson = new WeakMap;
|
|
2664
3050
|
});
|
|
2665
3051
|
|
|
@@ -2787,6 +3173,7 @@ class SqlRunJournal {
|
|
|
2787
3173
|
runVersions: () => this.runVersions(),
|
|
2788
3174
|
snapshotTail: (runId) => this.snapshotTail(runId)
|
|
2789
3175
|
});
|
|
3176
|
+
registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
|
|
2790
3177
|
registerSqlPoolStorage(this, {
|
|
2791
3178
|
transaction: (body) => this.operation(() => this.database.transaction(body)),
|
|
2792
3179
|
withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
|
|
@@ -3051,38 +3438,76 @@ class SqlRunJournal {
|
|
|
3051
3438
|
this.assertOpen();
|
|
3052
3439
|
validateText("runId", lease.runId);
|
|
3053
3440
|
validateText("holderId", lease.holderId);
|
|
3054
|
-
if (options?.key !== undefined)
|
|
3055
|
-
validateText("key", options.key);
|
|
3056
3441
|
const owned = { ...lease };
|
|
3057
|
-
const
|
|
3442
|
+
const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
|
|
3058
3443
|
const expected = options?.expectSeq;
|
|
3444
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
3445
|
+
await this.checkLease(tx, owned);
|
|
3446
|
+
return this.insert(tx, owned, prepared, expected);
|
|
3447
|
+
}));
|
|
3448
|
+
}
|
|
3449
|
+
async appendMany(lease, appends) {
|
|
3450
|
+
this.assertOpen();
|
|
3451
|
+
validateText("runId", lease.runId);
|
|
3452
|
+
validateText("holderId", lease.holderId);
|
|
3453
|
+
const owned = { ...lease };
|
|
3454
|
+
const prepared = appends.map((append) => {
|
|
3455
|
+
try {
|
|
3456
|
+
return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
|
|
3457
|
+
} catch (error) {
|
|
3458
|
+
return error;
|
|
3459
|
+
}
|
|
3460
|
+
});
|
|
3461
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
3462
|
+
await this.checkLease(tx, owned);
|
|
3463
|
+
const results = [];
|
|
3464
|
+
for (const append of prepared) {
|
|
3465
|
+
if (append instanceof Error) {
|
|
3466
|
+
results.push({ status: "rejected", reason: append });
|
|
3467
|
+
continue;
|
|
3468
|
+
}
|
|
3469
|
+
await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
|
|
3470
|
+
try {
|
|
3471
|
+
results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
|
|
3472
|
+
} catch (error) {
|
|
3473
|
+
await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
|
|
3474
|
+
results.push({ status: "rejected", reason: error });
|
|
3475
|
+
}
|
|
3476
|
+
await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
|
|
3477
|
+
}
|
|
3478
|
+
return results;
|
|
3479
|
+
}));
|
|
3480
|
+
}
|
|
3481
|
+
prepare(event, key, appendGuard) {
|
|
3482
|
+
if (key !== undefined)
|
|
3483
|
+
validateText("key", key);
|
|
3059
3484
|
const type = event.type;
|
|
3060
3485
|
const payload = this.snapshotJson(event.payload);
|
|
3061
|
-
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ?
|
|
3486
|
+
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
|
|
3062
3487
|
const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
|
|
3063
3488
|
if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
|
|
3064
3489
|
validateText("status", status);
|
|
3065
|
-
return
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
}
|
|
3490
|
+
return { key, type, payload, guard, status };
|
|
3491
|
+
}
|
|
3492
|
+
async insert(tx, owned, { key, type, payload, guard, status }, expected) {
|
|
3493
|
+
await guard?.(tx, owned.runId);
|
|
3494
|
+
if (key !== undefined) {
|
|
3495
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
|
|
3496
|
+
if (row)
|
|
3497
|
+
return { status: "exists", entry: entryFromRow(row) };
|
|
3498
|
+
}
|
|
3499
|
+
const actual = await this.sequence(tx, owned.runId);
|
|
3500
|
+
if (expected !== undefined && expected !== actual)
|
|
3501
|
+
throw new JournalCasConflictError(owned.runId, expected, actual);
|
|
3502
|
+
const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
|
|
3503
|
+
if (payload instanceof StorageUnavailableError2)
|
|
3504
|
+
throw payload;
|
|
3505
|
+
await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
|
|
3506
|
+
if (status !== undefined)
|
|
3507
|
+
await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
|
|
3508
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
|
|
3509
|
+
await guard?.(tx, owned.runId);
|
|
3510
|
+
return { status: "appended", entry: entryFromRow(row) };
|
|
3086
3511
|
}
|
|
3087
3512
|
async* read(runId, fromSeq = 1) {
|
|
3088
3513
|
this.assertOpen();
|
|
@@ -3150,7 +3575,7 @@ class SqlRunJournal {
|
|
|
3150
3575
|
return this.closePromise;
|
|
3151
3576
|
}
|
|
3152
3577
|
}
|
|
3153
|
-
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
|
|
3578
|
+
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
|
|
3154
3579
|
var init_sql_run_journal = __esm(() => {
|
|
3155
3580
|
init_internal();
|
|
3156
3581
|
init_legacy_journal_tables();
|
|
@@ -3707,8 +4132,59 @@ function firstError(schema, value2) {
|
|
|
3707
4132
|
}
|
|
3708
4133
|
return;
|
|
3709
4134
|
}
|
|
4135
|
+
function certainlySerializable(root, rootMustBeObject) {
|
|
4136
|
+
if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
|
|
4137
|
+
return false;
|
|
4138
|
+
const pending = [root];
|
|
4139
|
+
const ancestors = new Set;
|
|
4140
|
+
const exits = [undefined];
|
|
4141
|
+
while (pending.length > 0) {
|
|
4142
|
+
const exit = exits.pop();
|
|
4143
|
+
const value2 = pending.pop();
|
|
4144
|
+
if (exit !== undefined) {
|
|
4145
|
+
ancestors.delete(exit);
|
|
4146
|
+
continue;
|
|
4147
|
+
}
|
|
4148
|
+
if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
|
|
4149
|
+
continue;
|
|
4150
|
+
if (typeof value2 === "number") {
|
|
4151
|
+
if (!Number.isFinite(value2))
|
|
4152
|
+
return false;
|
|
4153
|
+
continue;
|
|
4154
|
+
}
|
|
4155
|
+
if (typeof value2 !== "object" || ancestors.has(value2))
|
|
4156
|
+
return false;
|
|
4157
|
+
if (Object.getOwnPropertySymbols(value2).length > 0)
|
|
4158
|
+
return false;
|
|
4159
|
+
const names = Object.getOwnPropertyNames(value2);
|
|
4160
|
+
let children;
|
|
4161
|
+
if (Array.isArray(value2)) {
|
|
4162
|
+
if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
|
|
4163
|
+
return false;
|
|
4164
|
+
children = [];
|
|
4165
|
+
for (let index = 0;index < value2.length; index++)
|
|
4166
|
+
children.push(value2[index]);
|
|
4167
|
+
} else {
|
|
4168
|
+
const proto = Object.getPrototypeOf(value2);
|
|
4169
|
+
if (proto !== Object.prototype && proto !== null)
|
|
4170
|
+
return false;
|
|
4171
|
+
const keys = Object.keys(value2);
|
|
4172
|
+
if (keys.length !== names.length)
|
|
4173
|
+
return false;
|
|
4174
|
+
children = keys.map((key) => value2[key]);
|
|
4175
|
+
}
|
|
4176
|
+
ancestors.add(value2);
|
|
4177
|
+
pending.push(value2);
|
|
4178
|
+
exits.push(value2);
|
|
4179
|
+
for (const child of children) {
|
|
4180
|
+
pending.push(child);
|
|
4181
|
+
exits.push(undefined);
|
|
4182
|
+
}
|
|
4183
|
+
}
|
|
4184
|
+
return true;
|
|
4185
|
+
}
|
|
3710
4186
|
function workflowSerializableValidationError(value2, label) {
|
|
3711
|
-
if (Value.Check(workflowSerializableValueSchema, value2))
|
|
4187
|
+
if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
|
|
3712
4188
|
return;
|
|
3713
4189
|
const issue = firstError(workflowSerializableValueSchema, value2);
|
|
3714
4190
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -3717,7 +4193,7 @@ function workflowSerializableValidationError(value2, label) {
|
|
|
3717
4193
|
return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
|
|
3718
4194
|
}
|
|
3719
4195
|
function workflowSerializableObjectValidationError(value2, label) {
|
|
3720
|
-
if (Value.Check(workflowSerializableObjectSchema, value2))
|
|
4196
|
+
if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
|
|
3721
4197
|
return;
|
|
3722
4198
|
const issue = firstError(workflowSerializableObjectSchema, value2);
|
|
3723
4199
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -6089,6 +6565,12 @@ class ScopedDurableBackend {
|
|
|
6089
6565
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
6090
6566
|
return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
|
|
6091
6567
|
}
|
|
6568
|
+
pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
|
|
6569
|
+
return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
|
|
6570
|
+
}
|
|
6571
|
+
pendingStageDeliveryCountVersion(_workflowId) {
|
|
6572
|
+
return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
|
|
6573
|
+
}
|
|
6092
6574
|
async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
6093
6575
|
return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
|
|
6094
6576
|
}
|
|
@@ -8354,8 +8836,11 @@ var init_store_internal = __esm(() => {
|
|
|
8354
8836
|
});
|
|
8355
8837
|
|
|
8356
8838
|
// dist/builtin/workflows/src/shared/pending-stage-retention.ts
|
|
8839
|
+
function isSettledPendingStageMessage(entry) {
|
|
8840
|
+
return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
|
|
8841
|
+
}
|
|
8357
8842
|
async function retainPendingStageMessages(messages, archive) {
|
|
8358
|
-
const settled = messages.filter(
|
|
8843
|
+
const settled = messages.filter(isSettledPendingStageMessage);
|
|
8359
8844
|
const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
|
|
8360
8845
|
for (const entry of expired)
|
|
8361
8846
|
await archive(entry);
|
|
@@ -8363,53 +8848,210 @@ async function retainPendingStageMessages(messages, archive) {
|
|
|
8363
8848
|
}
|
|
8364
8849
|
var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
|
|
8365
8850
|
|
|
8851
|
+
// dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
|
|
8852
|
+
class PendingStageTransitionLedger {
|
|
8853
|
+
compacted = [];
|
|
8854
|
+
settled = 0;
|
|
8855
|
+
stages = new Map;
|
|
8856
|
+
fixed = new WeakSet;
|
|
8857
|
+
reconciled = [];
|
|
8858
|
+
reconciledVersion;
|
|
8859
|
+
reconcile(backend, runId, messages) {
|
|
8860
|
+
if (backend.getPendingStageMessageDeliveryCount === undefined)
|
|
8861
|
+
return messages;
|
|
8862
|
+
const version = backend.pendingStageDeliveryCountVersion?.(runId);
|
|
8863
|
+
const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
|
|
8864
|
+
let recovered;
|
|
8865
|
+
for (let index = 0;index < messages.length; index++) {
|
|
8866
|
+
const entry = messages[index];
|
|
8867
|
+
if (entry.sticky !== true || base !== undefined && base[index] === entry)
|
|
8868
|
+
continue;
|
|
8869
|
+
const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
|
|
8870
|
+
if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
|
|
8871
|
+
continue;
|
|
8872
|
+
recovered ??= [...messages];
|
|
8873
|
+
recovered[index] = { ...entry, deliveryCount: count };
|
|
8874
|
+
}
|
|
8875
|
+
const result = recovered ?? messages;
|
|
8876
|
+
this.reconciled = result;
|
|
8877
|
+
this.reconciledVersion = version;
|
|
8878
|
+
return result;
|
|
8879
|
+
}
|
|
8880
|
+
async compact(backend, runId, messages, runs, archiveSettled) {
|
|
8881
|
+
const stageStatus = createStageStatusLookup(runs);
|
|
8882
|
+
const endedStages = [];
|
|
8883
|
+
const dirty = new Set;
|
|
8884
|
+
for (const [deliveryRunId, byStage] of this.stages) {
|
|
8885
|
+
for (const [stageId, entries] of byStage) {
|
|
8886
|
+
if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
|
|
8887
|
+
continue;
|
|
8888
|
+
endedStages.push([deliveryRunId, stageId]);
|
|
8889
|
+
for (const entry of entries)
|
|
8890
|
+
dirty.add(entry);
|
|
8891
|
+
}
|
|
8892
|
+
}
|
|
8893
|
+
const base = this.compacted;
|
|
8894
|
+
const output = [];
|
|
8895
|
+
const removed = [];
|
|
8896
|
+
const added = [];
|
|
8897
|
+
let settled = this.settled;
|
|
8898
|
+
const length = Math.max(base.length, messages.length);
|
|
8899
|
+
for (let index = 0;index < length; index++) {
|
|
8900
|
+
const previous = base[index];
|
|
8901
|
+
const entry = messages[index];
|
|
8902
|
+
if (previous === entry && !dirty.has(entry)) {
|
|
8903
|
+
output.push(entry);
|
|
8904
|
+
continue;
|
|
8905
|
+
}
|
|
8906
|
+
if (previous !== undefined) {
|
|
8907
|
+
removed.push(previous);
|
|
8908
|
+
if (isSettledPendingStageMessage(previous))
|
|
8909
|
+
settled--;
|
|
8910
|
+
}
|
|
8911
|
+
if (entry === undefined)
|
|
8912
|
+
continue;
|
|
8913
|
+
if (isSettledPendingStageMessage(entry))
|
|
8914
|
+
settled++;
|
|
8915
|
+
const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
|
|
8916
|
+
output.push(next);
|
|
8917
|
+
added.push(next);
|
|
8918
|
+
}
|
|
8919
|
+
let retained = output;
|
|
8920
|
+
if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
|
|
8921
|
+
retained = await retainPendingStageMessages(output, archiveSettled);
|
|
8922
|
+
settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
|
|
8923
|
+
}
|
|
8924
|
+
return {
|
|
8925
|
+
messages: retained,
|
|
8926
|
+
commit: () => {
|
|
8927
|
+
for (const [deliveryRunId, stageId] of endedStages) {
|
|
8928
|
+
const byStage = this.stages.get(deliveryRunId);
|
|
8929
|
+
byStage?.delete(stageId);
|
|
8930
|
+
if (byStage?.size === 0)
|
|
8931
|
+
this.stages.delete(deliveryRunId);
|
|
8932
|
+
}
|
|
8933
|
+
for (const entry of dirty)
|
|
8934
|
+
this.fixed.delete(entry);
|
|
8935
|
+
for (const entry of removed)
|
|
8936
|
+
this.unindex(entry);
|
|
8937
|
+
for (const entry of added)
|
|
8938
|
+
this.index(entry);
|
|
8939
|
+
this.compacted = retained;
|
|
8940
|
+
this.settled = settled;
|
|
8941
|
+
}
|
|
8942
|
+
};
|
|
8943
|
+
}
|
|
8944
|
+
index(entry) {
|
|
8945
|
+
this.fixed.add(entry);
|
|
8946
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
8947
|
+
let byStage = this.stages.get(delivery.runId);
|
|
8948
|
+
if (byStage === undefined) {
|
|
8949
|
+
byStage = new Map;
|
|
8950
|
+
this.stages.set(delivery.runId, byStage);
|
|
8951
|
+
}
|
|
8952
|
+
let entries = byStage.get(delivery.stageId);
|
|
8953
|
+
if (entries === undefined) {
|
|
8954
|
+
entries = new Set;
|
|
8955
|
+
byStage.set(delivery.stageId, entries);
|
|
8956
|
+
}
|
|
8957
|
+
entries.add(entry);
|
|
8958
|
+
}
|
|
8959
|
+
}
|
|
8960
|
+
unindex(entry) {
|
|
8961
|
+
this.fixed.delete(entry);
|
|
8962
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
8963
|
+
const byStage = this.stages.get(delivery.runId);
|
|
8964
|
+
const entries = byStage?.get(delivery.stageId);
|
|
8965
|
+
if (entries === undefined)
|
|
8966
|
+
continue;
|
|
8967
|
+
entries.delete(entry);
|
|
8968
|
+
if (entries.size > 0)
|
|
8969
|
+
continue;
|
|
8970
|
+
byStage.delete(delivery.stageId);
|
|
8971
|
+
if (byStage.size === 0)
|
|
8972
|
+
this.stages.delete(delivery.runId);
|
|
8973
|
+
}
|
|
8974
|
+
}
|
|
8975
|
+
}
|
|
8976
|
+
function isCompactingBackend(backend) {
|
|
8977
|
+
return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
|
|
8978
|
+
}
|
|
8979
|
+
function createStageStatusLookup(runs) {
|
|
8980
|
+
let runById;
|
|
8981
|
+
return (runId, stageId) => {
|
|
8982
|
+
if (runById === undefined) {
|
|
8983
|
+
runById = new Map;
|
|
8984
|
+
for (const run of runs)
|
|
8985
|
+
if (!runById.has(run.id))
|
|
8986
|
+
runById.set(run.id, run);
|
|
8987
|
+
}
|
|
8988
|
+
return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
|
|
8989
|
+
};
|
|
8990
|
+
}
|
|
8991
|
+
function isEndedStageStatus(status) {
|
|
8992
|
+
return status === "completed" || status === "skipped" || status === "failed";
|
|
8993
|
+
}
|
|
8994
|
+
async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
|
|
8995
|
+
if (entry.sticky !== true || entry.deliveries === undefined)
|
|
8996
|
+
return entry;
|
|
8997
|
+
const latest = new Map;
|
|
8998
|
+
entry.deliveries.forEach((delivery, index) => {
|
|
8999
|
+
latest.set(deliverySlot(delivery), index);
|
|
9000
|
+
});
|
|
9001
|
+
const deliveries = [];
|
|
9002
|
+
let archivedThrough = -1;
|
|
9003
|
+
for (let index = 0;index < entry.deliveries.length; index++) {
|
|
9004
|
+
const delivery = entry.deliveries[index];
|
|
9005
|
+
if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
|
|
9006
|
+
while (archivedThrough < index) {
|
|
9007
|
+
const prefixIndex = archivedThrough + 1;
|
|
9008
|
+
const receipt = entry.deliveries[prefixIndex];
|
|
9009
|
+
if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
|
|
9010
|
+
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
|
|
9011
|
+
messageRunId: entry.runId,
|
|
9012
|
+
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
9013
|
+
});
|
|
9014
|
+
archivedThrough = prefixIndex;
|
|
9015
|
+
}
|
|
9016
|
+
} else {
|
|
9017
|
+
deliveries.push(delivery);
|
|
9018
|
+
}
|
|
9019
|
+
}
|
|
9020
|
+
return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
|
|
9021
|
+
}
|
|
9022
|
+
function deliverySlot(delivery) {
|
|
9023
|
+
return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
|
|
9024
|
+
}
|
|
9025
|
+
var init_pending_stage_transition_ledger = () => {};
|
|
9026
|
+
|
|
8366
9027
|
// dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
|
|
8367
9028
|
function createPendingStageDeliveryStoreMethods(context) {
|
|
8368
|
-
const
|
|
8369
|
-
const
|
|
8370
|
-
const previous = transitions.get(runId) ?? Promise.resolve();
|
|
8371
|
-
const result = previous.catch(() => {
|
|
8372
|
-
return;
|
|
8373
|
-
}).then(transition);
|
|
8374
|
-
const settled = result.then(() => {
|
|
8375
|
-
return;
|
|
8376
|
-
}, () => {
|
|
8377
|
-
return;
|
|
8378
|
-
});
|
|
8379
|
-
transitions.set(runId, settled);
|
|
8380
|
-
settled.finally(() => {
|
|
8381
|
-
if (transitions.get(runId) === settled)
|
|
8382
|
-
transitions.delete(runId);
|
|
8383
|
-
});
|
|
8384
|
-
return await result;
|
|
8385
|
-
};
|
|
9029
|
+
const ledgers = new PendingStageLedgers;
|
|
9030
|
+
const batches = new PendingStageTransitionBatches(context, ledgers);
|
|
8386
9031
|
return {
|
|
8387
9032
|
async queueStageMessage(input, senderGroup, runGroup, backend) {
|
|
8388
|
-
return await
|
|
9033
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
8389
9034
|
const run = context.findRun(input.runId);
|
|
8390
9035
|
if (run === undefined)
|
|
8391
9036
|
return;
|
|
8392
9037
|
const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
|
|
8393
9038
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
8394
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
9039
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
|
|
8395
9040
|
if (result.ok && !result.deduplicated) {
|
|
8396
|
-
|
|
8397
|
-
const live = context.findRun(input.runId) ?? run;
|
|
8398
|
-
live.pendingStageMessages = [...messages];
|
|
8399
|
-
context.bumpAndNotify();
|
|
9041
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
8400
9042
|
}
|
|
8401
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
9043
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
8402
9044
|
});
|
|
8403
9045
|
},
|
|
8404
9046
|
async validateLiveStageMessage(input) {
|
|
8405
|
-
return await
|
|
9047
|
+
return await batches.run(input.runId, undefined, async (batch) => {
|
|
8406
9048
|
const run = context.findRun(input.runId);
|
|
8407
9049
|
if (run === undefined)
|
|
8408
9050
|
return;
|
|
8409
9051
|
const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
|
|
8410
9052
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
|
|
8411
9053
|
const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
8412
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
9054
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
|
|
8413
9055
|
if (!result.ok) {
|
|
8414
9056
|
return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
|
|
8415
9057
|
}
|
|
@@ -8431,77 +9073,65 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
8431
9073
|
return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
|
|
8432
9074
|
},
|
|
8433
9075
|
async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
|
|
8434
|
-
return await
|
|
9076
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8435
9077
|
const run = context.findRun(runId);
|
|
8436
9078
|
if (run === undefined)
|
|
8437
9079
|
return false;
|
|
8438
|
-
const current = run
|
|
9080
|
+
const current = batch.messages(run);
|
|
8439
9081
|
const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
|
|
8440
9082
|
if (next === current)
|
|
8441
9083
|
return false;
|
|
8442
|
-
|
|
8443
|
-
const live = context.findRun(runId) ?? run;
|
|
8444
|
-
live.pendingStageMessages = [...messages];
|
|
8445
|
-
context.bumpAndNotify();
|
|
9084
|
+
await batch.apply(backend, runId, next);
|
|
8446
9085
|
return true;
|
|
8447
9086
|
});
|
|
8448
9087
|
},
|
|
8449
9088
|
async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
|
|
8450
|
-
return await
|
|
9089
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8451
9090
|
const run = context.findRun(runId);
|
|
8452
9091
|
if (run === undefined)
|
|
8453
9092
|
return false;
|
|
8454
|
-
const current = run
|
|
9093
|
+
const current = batch.messages(run);
|
|
8455
9094
|
const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
|
|
8456
9095
|
if (next === current)
|
|
8457
9096
|
return false;
|
|
8458
|
-
|
|
8459
|
-
const live = context.findRun(runId) ?? run;
|
|
8460
|
-
live.pendingStageMessages = [...messages];
|
|
8461
|
-
context.bumpAndNotify();
|
|
9097
|
+
await batch.apply(backend, runId, next);
|
|
8462
9098
|
return true;
|
|
8463
9099
|
});
|
|
8464
9100
|
},
|
|
8465
9101
|
async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
|
|
8466
|
-
return await
|
|
9102
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8467
9103
|
const run = context.findRun(runId);
|
|
8468
9104
|
if (run === undefined)
|
|
8469
9105
|
return false;
|
|
8470
|
-
const current = run
|
|
9106
|
+
const current = batch.messages(run);
|
|
8471
9107
|
const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
|
|
8472
9108
|
if (next === current)
|
|
8473
9109
|
return false;
|
|
8474
|
-
|
|
8475
|
-
const live = context.findRun(runId) ?? run;
|
|
8476
|
-
live.pendingStageMessages = [...messages];
|
|
8477
|
-
context.bumpAndNotify();
|
|
9110
|
+
await batch.apply(backend, runId, next);
|
|
8478
9111
|
return true;
|
|
8479
9112
|
});
|
|
8480
9113
|
},
|
|
8481
9114
|
async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
|
|
8482
|
-
return await
|
|
9115
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
8483
9116
|
const run = context.findRun(input.runId);
|
|
8484
9117
|
if (run === undefined)
|
|
8485
9118
|
return;
|
|
8486
9119
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
8487
|
-
const result = queueStickyStageMessage(receipt === undefined ? run
|
|
9120
|
+
const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
|
|
8488
9121
|
if (result.ok && !result.deduplicated) {
|
|
8489
|
-
|
|
8490
|
-
const live = context.findRun(input.runId) ?? run;
|
|
8491
|
-
live.pendingStageMessages = [...messages];
|
|
8492
|
-
context.bumpAndNotify();
|
|
9122
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
8493
9123
|
}
|
|
8494
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
9124
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
8495
9125
|
});
|
|
8496
9126
|
},
|
|
8497
9127
|
async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
|
|
8498
|
-
return await
|
|
9128
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8499
9129
|
const run = context.findRun(runId);
|
|
8500
9130
|
if (run === undefined)
|
|
8501
9131
|
return false;
|
|
8502
9132
|
if (records.length === 0)
|
|
8503
9133
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
8504
|
-
let current =
|
|
9134
|
+
let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
8505
9135
|
const unconfirmedRecords = [];
|
|
8506
9136
|
for (const record of records) {
|
|
8507
9137
|
const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
|
|
@@ -8509,35 +9139,26 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
8509
9139
|
unconfirmedRecords.push(record);
|
|
8510
9140
|
}
|
|
8511
9141
|
}
|
|
8512
|
-
current =
|
|
8513
|
-
|
|
8514
|
-
run.pendingStageMessages = [...current];
|
|
8515
|
-
context.bumpAndNotify();
|
|
8516
|
-
}
|
|
9142
|
+
current = ledgers.for(run).reconcile(backend, runId, current);
|
|
9143
|
+
batch.reconciled(run, current);
|
|
8517
9144
|
const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
|
|
8518
9145
|
if (next === current)
|
|
8519
9146
|
return false;
|
|
8520
|
-
|
|
8521
|
-
const live = context.findRun(runId) ?? run;
|
|
8522
|
-
live.pendingStageMessages = [...messages];
|
|
8523
|
-
context.bumpAndNotify();
|
|
9147
|
+
await batch.apply(backend, runId, next);
|
|
8524
9148
|
return true;
|
|
8525
9149
|
});
|
|
8526
9150
|
},
|
|
8527
9151
|
async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
|
|
8528
|
-
return await
|
|
9152
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8529
9153
|
const run = context.findRun(runId);
|
|
8530
9154
|
if (run === undefined)
|
|
8531
9155
|
return false;
|
|
8532
9156
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
8533
|
-
const current =
|
|
9157
|
+
const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
8534
9158
|
const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
|
|
8535
9159
|
if (next === current)
|
|
8536
9160
|
return false;
|
|
8537
|
-
|
|
8538
|
-
const live = context.findRun(runId) ?? run;
|
|
8539
|
-
live.pendingStageMessages = [...messages];
|
|
8540
|
-
context.bumpAndNotify();
|
|
9161
|
+
await batch.apply(backend, runId, next);
|
|
8541
9162
|
return true;
|
|
8542
9163
|
});
|
|
8543
9164
|
}
|
|
@@ -8555,78 +9176,186 @@ function resolvePendingStageIdentity(run, stageKey) {
|
|
|
8555
9176
|
aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
|
|
8556
9177
|
};
|
|
8557
9178
|
}
|
|
8558
|
-
|
|
8559
|
-
|
|
8560
|
-
|
|
8561
|
-
|
|
8562
|
-
|
|
8563
|
-
|
|
8564
|
-
|
|
8565
|
-
|
|
8566
|
-
|
|
8567
|
-
|
|
8568
|
-
|
|
8569
|
-
|
|
8570
|
-
|
|
8571
|
-
|
|
8572
|
-
|
|
8573
|
-
|
|
8574
|
-
|
|
8575
|
-
|
|
8576
|
-
|
|
8577
|
-
|
|
8578
|
-
|
|
8579
|
-
|
|
8580
|
-
|
|
8581
|
-
|
|
8582
|
-
|
|
8583
|
-
|
|
8584
|
-
|
|
8585
|
-
|
|
8586
|
-
|
|
8587
|
-
|
|
8588
|
-
|
|
8589
|
-
|
|
8590
|
-
|
|
9179
|
+
|
|
9180
|
+
class PendingStageTransitionBatches {
|
|
9181
|
+
queues = new Map;
|
|
9182
|
+
context;
|
|
9183
|
+
ledgers;
|
|
9184
|
+
constructor(context, ledgers) {
|
|
9185
|
+
this.context = context;
|
|
9186
|
+
this.ledgers = ledgers;
|
|
9187
|
+
}
|
|
9188
|
+
run(runId, backend, step) {
|
|
9189
|
+
return new Promise((resolve, reject) => {
|
|
9190
|
+
const transition = {
|
|
9191
|
+
backend,
|
|
9192
|
+
step,
|
|
9193
|
+
resolve: (value2) => resolve(value2),
|
|
9194
|
+
reject
|
|
9195
|
+
};
|
|
9196
|
+
const queue = this.queues.get(runId);
|
|
9197
|
+
if (queue !== undefined) {
|
|
9198
|
+
queue.push(transition);
|
|
9199
|
+
return;
|
|
9200
|
+
}
|
|
9201
|
+
const created = [transition];
|
|
9202
|
+
this.queues.set(runId, created);
|
|
9203
|
+
Promise.resolve().then(() => this.drain(runId, created));
|
|
9204
|
+
});
|
|
9205
|
+
}
|
|
9206
|
+
async drain(runId, queue) {
|
|
9207
|
+
while (queue.length > 0)
|
|
9208
|
+
await this.runBatch(runId, takeBatch(queue));
|
|
9209
|
+
this.queues.delete(runId);
|
|
9210
|
+
}
|
|
9211
|
+
async runBatch(runId, transitions) {
|
|
9212
|
+
const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
|
|
9213
|
+
const outcomes = [];
|
|
9214
|
+
for (const transition of transitions) {
|
|
9215
|
+
const changedBefore = batch.changed;
|
|
9216
|
+
try {
|
|
9217
|
+
const value2 = await transition.step(batch);
|
|
9218
|
+
outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
|
|
9219
|
+
} catch (error) {
|
|
9220
|
+
outcomes.push({ ok: false, error });
|
|
8591
9221
|
}
|
|
8592
|
-
retained = compacted;
|
|
8593
9222
|
}
|
|
8594
|
-
|
|
8595
|
-
|
|
8596
|
-
|
|
9223
|
+
let failure;
|
|
9224
|
+
try {
|
|
9225
|
+
await batch.persist();
|
|
9226
|
+
} catch (error) {
|
|
9227
|
+
failure = { error };
|
|
9228
|
+
}
|
|
9229
|
+
transitions.forEach((transition, index) => {
|
|
9230
|
+
const outcome = outcomes[index];
|
|
9231
|
+
if (!outcome.ok)
|
|
9232
|
+
transition.reject(outcome.error);
|
|
9233
|
+
else if (failure !== undefined && outcome.uncommitted)
|
|
9234
|
+
transition.reject(failure.error);
|
|
9235
|
+
else
|
|
9236
|
+
transition.resolve(outcome.value);
|
|
9237
|
+
});
|
|
9238
|
+
}
|
|
9239
|
+
}
|
|
9240
|
+
function takeBatch(queue) {
|
|
9241
|
+
let count = 1;
|
|
9242
|
+
let backend = queue[0].backend;
|
|
9243
|
+
while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
|
|
9244
|
+
const next = queue[count].backend;
|
|
9245
|
+
if (next !== undefined && backend !== undefined && next !== backend)
|
|
9246
|
+
break;
|
|
9247
|
+
backend ??= next;
|
|
9248
|
+
count++;
|
|
9249
|
+
}
|
|
9250
|
+
return queue.splice(0, count);
|
|
9251
|
+
}
|
|
9252
|
+
|
|
9253
|
+
class PendingStageTransitionBatch {
|
|
9254
|
+
changed = false;
|
|
9255
|
+
list;
|
|
9256
|
+
pending;
|
|
9257
|
+
context;
|
|
9258
|
+
ledgers;
|
|
9259
|
+
runId;
|
|
9260
|
+
constructor(context, ledgers, runId) {
|
|
9261
|
+
this.context = context;
|
|
9262
|
+
this.ledgers = ledgers;
|
|
9263
|
+
this.runId = runId;
|
|
9264
|
+
}
|
|
9265
|
+
messages(run) {
|
|
9266
|
+
return this.list ?? run.pendingStageMessages ?? [];
|
|
9267
|
+
}
|
|
9268
|
+
reconciled(run, messages) {
|
|
9269
|
+
if (this.changed) {
|
|
9270
|
+
this.list = messages;
|
|
9271
|
+
return;
|
|
8597
9272
|
}
|
|
8598
|
-
if (
|
|
8599
|
-
|
|
9273
|
+
if (messages !== run.pendingStageMessages) {
|
|
9274
|
+
run.pendingStageMessages = [...messages];
|
|
9275
|
+
this.context.bumpAndNotify();
|
|
8600
9276
|
}
|
|
8601
|
-
|
|
8602
|
-
|
|
8603
|
-
const run = context.findRun(runId);
|
|
8604
|
-
|
|
8605
|
-
const
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
8609
|
-
|
|
8610
|
-
|
|
9277
|
+
}
|
|
9278
|
+
async apply(backend, runId, messages) {
|
|
9279
|
+
const run = this.context.findRun(runId);
|
|
9280
|
+
try {
|
|
9281
|
+
const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
|
|
9282
|
+
this.list = compacted.messages;
|
|
9283
|
+
this.changed = true;
|
|
9284
|
+
this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
|
|
9285
|
+
return compacted.messages;
|
|
9286
|
+
} catch (error) {
|
|
9287
|
+
if (this.changed && this.list !== undefined) {
|
|
9288
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
|
|
9289
|
+
this.list = ledger.reconcile(backend, runId, this.list);
|
|
9290
|
+
} else
|
|
9291
|
+
reconcileLive(backend, runId, this.context, this.ledgers);
|
|
9292
|
+
throw error;
|
|
8611
9293
|
}
|
|
8612
9294
|
}
|
|
9295
|
+
async persist() {
|
|
9296
|
+
if (this.pending === undefined)
|
|
9297
|
+
return;
|
|
9298
|
+
const { backend, messages, commit, run } = this.pending;
|
|
9299
|
+
const published = this.list ?? messages;
|
|
9300
|
+
try {
|
|
9301
|
+
if (!await backend.persistPendingStageMessages(this.runId, messages)) {
|
|
9302
|
+
throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
|
|
9303
|
+
}
|
|
9304
|
+
commit?.();
|
|
9305
|
+
} finally {
|
|
9306
|
+
reconcileLive(backend, this.runId, this.context, this.ledgers);
|
|
9307
|
+
}
|
|
9308
|
+
const live = this.context.findRun(this.runId) ?? run;
|
|
9309
|
+
if (live !== undefined)
|
|
9310
|
+
live.pendingStageMessages = [...published];
|
|
9311
|
+
this.context.bumpAndNotify();
|
|
9312
|
+
}
|
|
9313
|
+
}
|
|
9314
|
+
async function compactTransition(backend, runId, messages, context, ledgers, run) {
|
|
9315
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
|
|
9316
|
+
const reconciled = ledger.reconcile(backend, runId, messages);
|
|
9317
|
+
const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
|
|
9318
|
+
if (isCompactingBackend(backend)) {
|
|
9319
|
+
return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
|
|
9320
|
+
}
|
|
9321
|
+
let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
|
|
9322
|
+
if (archiveSettled !== undefined)
|
|
9323
|
+
retained = await retainPendingStageMessages(retained, archiveSettled);
|
|
9324
|
+
if (run !== undefined)
|
|
9325
|
+
ledgers.reset(run);
|
|
9326
|
+
return { messages: retained };
|
|
9327
|
+
}
|
|
9328
|
+
function reconcileLive(backend, runId, context, ledgers) {
|
|
9329
|
+
const current = context.findRun(runId);
|
|
9330
|
+
if (current === undefined)
|
|
9331
|
+
return;
|
|
9332
|
+
const messages = current.pendingStageMessages ?? [];
|
|
9333
|
+
const recovered = ledgers.for(current).reconcile(backend, runId, messages);
|
|
9334
|
+
if (recovered !== messages) {
|
|
9335
|
+
current.pendingStageMessages = [...recovered];
|
|
9336
|
+
context.bumpAndNotify();
|
|
9337
|
+
}
|
|
8613
9338
|
}
|
|
8614
|
-
|
|
8615
|
-
|
|
8616
|
-
|
|
8617
|
-
|
|
8618
|
-
|
|
8619
|
-
|
|
8620
|
-
|
|
8621
|
-
|
|
8622
|
-
|
|
8623
|
-
return
|
|
8624
|
-
}
|
|
8625
|
-
|
|
9339
|
+
|
|
9340
|
+
class PendingStageLedgers {
|
|
9341
|
+
ledgers = new WeakMap;
|
|
9342
|
+
for(run) {
|
|
9343
|
+
let ledger = this.ledgers.get(run);
|
|
9344
|
+
if (ledger === undefined) {
|
|
9345
|
+
ledger = new PendingStageTransitionLedger;
|
|
9346
|
+
this.ledgers.set(run, ledger);
|
|
9347
|
+
}
|
|
9348
|
+
return ledger;
|
|
9349
|
+
}
|
|
9350
|
+
reset(run) {
|
|
9351
|
+
this.ledgers.delete(run);
|
|
9352
|
+
}
|
|
8626
9353
|
}
|
|
9354
|
+
var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
|
|
8627
9355
|
var init_store_pending_stage_delivery_methods = __esm(() => {
|
|
8628
9356
|
init_run_owner_backend();
|
|
8629
9357
|
init_pending_stage_delivery();
|
|
9358
|
+
init_pending_stage_transition_ledger();
|
|
8630
9359
|
});
|
|
8631
9360
|
|
|
8632
9361
|
// dist/builtin/workflows/src/shared/store-prompt-methods.ts
|
|
@@ -57401,6 +58130,27 @@ var init_executor = __esm(() => {
|
|
|
57401
58130
|
init_executor_run();
|
|
57402
58131
|
});
|
|
57403
58132
|
|
|
58133
|
+
// dist/builtin/workflows/src/engine/session-closed-error.ts
|
|
58134
|
+
import { isStaleExtensionContextError as isStaleExtensionContextError2 } from "@bastani/atomic";
|
|
58135
|
+
function reportsClosedSession(error) {
|
|
58136
|
+
return "code" in error && error.code === "SessionClosed" || isStaleExtensionContextError2(error);
|
|
58137
|
+
}
|
|
58138
|
+
function isSessionClosedError(error) {
|
|
58139
|
+
let current = error;
|
|
58140
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined; depth++) {
|
|
58141
|
+
try {
|
|
58142
|
+
if (reportsClosedSession(current))
|
|
58143
|
+
return true;
|
|
58144
|
+
current = current.cause instanceof Error ? current.cause : undefined;
|
|
58145
|
+
} catch {
|
|
58146
|
+
return false;
|
|
58147
|
+
}
|
|
58148
|
+
}
|
|
58149
|
+
return false;
|
|
58150
|
+
}
|
|
58151
|
+
var MAX_CAUSE_DEPTH = 8;
|
|
58152
|
+
var init_session_closed_error = () => {};
|
|
58153
|
+
|
|
57404
58154
|
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
57405
58155
|
function claimExternalRunRoute(store, runId, owner) {
|
|
57406
58156
|
let owners = registry.owners.get(store);
|
|
@@ -57726,39 +58476,399 @@ var init_pending_stage_status = __esm(() => {
|
|
|
57726
58476
|
init_workflow_stage_target();
|
|
57727
58477
|
});
|
|
57728
58478
|
|
|
57729
|
-
// dist/builtin/workflows/src/shared/sticky-
|
|
57730
|
-
function
|
|
57731
|
-
|
|
57732
|
-
|
|
57733
|
-
const
|
|
57734
|
-
|
|
57735
|
-
const
|
|
57736
|
-
|
|
57737
|
-
const
|
|
57738
|
-
|
|
58479
|
+
// dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
|
|
58480
|
+
function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
|
|
58481
|
+
if (states.length === 0)
|
|
58482
|
+
return { deliveryStates: states };
|
|
58483
|
+
const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
|
|
58484
|
+
counts.set("delivered", 0);
|
|
58485
|
+
for (const delivery of states)
|
|
58486
|
+
counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
|
|
58487
|
+
const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
|
|
58488
|
+
if (states.length <= limit)
|
|
58489
|
+
return { deliveryStates: states, deliveryStateCounts };
|
|
58490
|
+
const exemplars = [];
|
|
58491
|
+
for (const state of UNRESOLVED_STATES) {
|
|
58492
|
+
for (const delivery of states) {
|
|
58493
|
+
if (exemplars.length === limit)
|
|
58494
|
+
break;
|
|
58495
|
+
if (delivery.state === state)
|
|
58496
|
+
exemplars.push(delivery);
|
|
58497
|
+
}
|
|
58498
|
+
}
|
|
58499
|
+
for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
|
|
58500
|
+
const delivery = states[index];
|
|
58501
|
+
if (delivery.state === "delivered")
|
|
58502
|
+
exemplars.push(delivery);
|
|
58503
|
+
}
|
|
58504
|
+
return {
|
|
58505
|
+
deliveryStates: exemplars,
|
|
58506
|
+
deliveryStateCounts,
|
|
58507
|
+
omittedDeliveryStates: states.length - exemplars.length
|
|
58508
|
+
};
|
|
58509
|
+
}
|
|
58510
|
+
var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
|
|
58511
|
+
var init_sticky_delivery_state_bound = __esm(() => {
|
|
58512
|
+
UNRESOLVED_STATES = [
|
|
58513
|
+
"queued",
|
|
58514
|
+
"receipt-unverified",
|
|
58515
|
+
"session-replaced",
|
|
58516
|
+
"delivery-unavailable",
|
|
58517
|
+
"root-unresolved",
|
|
58518
|
+
"stage-terminal"
|
|
58519
|
+
];
|
|
58520
|
+
});
|
|
58521
|
+
|
|
58522
|
+
// dist/builtin/workflows/src/shared/possible-stage-rows.ts
|
|
58523
|
+
function boundPossibleStageDeliveryStates(states) {
|
|
58524
|
+
const bounded = boundStickyDeliveryStates(states);
|
|
58525
|
+
return {
|
|
58526
|
+
...bounded,
|
|
58527
|
+
deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
|
|
58528
|
+
};
|
|
58529
|
+
}
|
|
58530
|
+
|
|
58531
|
+
class WorkflowTopologyIndex {
|
|
58532
|
+
lineage = [];
|
|
58533
|
+
runById = new Map;
|
|
58534
|
+
roots = new Map;
|
|
58535
|
+
hops = new Map;
|
|
58536
|
+
refresh(runs) {
|
|
58537
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58538
|
+
const lineage = [];
|
|
58539
|
+
for (const run of runs) {
|
|
58540
|
+
lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
|
|
58541
|
+
if (run.parentRunId === undefined)
|
|
58542
|
+
continue;
|
|
58543
|
+
const parent = runById.get(run.parentRunId);
|
|
58544
|
+
const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
|
|
58545
|
+
lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
|
|
58546
|
+
}
|
|
58547
|
+
if (sameValues(lineage, this.lineage))
|
|
58548
|
+
return;
|
|
58549
|
+
this.lineage = lineage;
|
|
58550
|
+
this.runById = runById;
|
|
58551
|
+
this.roots.clear();
|
|
58552
|
+
this.hops.clear();
|
|
58553
|
+
}
|
|
58554
|
+
rootOf(runId) {
|
|
58555
|
+
if (this.roots.has(runId))
|
|
58556
|
+
return this.roots.get(runId);
|
|
58557
|
+
const root = reciprocalWorkflowRootRunId(this.runById, runId);
|
|
58558
|
+
this.roots.set(runId, root);
|
|
58559
|
+
return root;
|
|
58560
|
+
}
|
|
58561
|
+
hopsOf(runId) {
|
|
58562
|
+
if (this.hops.has(runId))
|
|
58563
|
+
return this.hops.get(runId);
|
|
58564
|
+
const hops = boundaryHops(this.runById, runId);
|
|
58565
|
+
this.hops.set(runId, hops);
|
|
58566
|
+
return hops;
|
|
58567
|
+
}
|
|
58568
|
+
}
|
|
58569
|
+
function boundaryHops(runById, runId) {
|
|
58570
|
+
const hops = [];
|
|
58571
|
+
let current = runById.get(runId);
|
|
58572
|
+
while (current !== undefined && current.parentRunId !== undefined) {
|
|
58573
|
+
const parent = runById.get(current.parentRunId);
|
|
58574
|
+
const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
|
|
58575
|
+
if (parent === undefined || boundary === undefined)
|
|
58576
|
+
return;
|
|
58577
|
+
hops.unshift({ name: boundary.name, runId: current.id });
|
|
58578
|
+
current = parent;
|
|
58579
|
+
}
|
|
58580
|
+
return current === undefined ? undefined : hops;
|
|
58581
|
+
}
|
|
58582
|
+
function sameValues(left, right) {
|
|
58583
|
+
if (left.length !== right.length)
|
|
58584
|
+
return false;
|
|
58585
|
+
for (let index = 0;index < left.length; index++)
|
|
58586
|
+
if (left[index] !== right[index])
|
|
58587
|
+
return false;
|
|
58588
|
+
return true;
|
|
58589
|
+
}
|
|
58590
|
+
|
|
58591
|
+
class PossibleStageRowsIndex {
|
|
58592
|
+
rootRunId;
|
|
58593
|
+
topology;
|
|
58594
|
+
targets = new Map;
|
|
58595
|
+
patterns = [];
|
|
58596
|
+
targetOf = new Map;
|
|
58597
|
+
targetsByMessage = new Map;
|
|
58598
|
+
list = [];
|
|
58599
|
+
stageFields = new WeakMap;
|
|
58600
|
+
backend;
|
|
58601
|
+
receiptCursor;
|
|
58602
|
+
scanMatches = new Map;
|
|
58603
|
+
rowCache = new Map;
|
|
58604
|
+
boundedStates = new WeakMap;
|
|
58605
|
+
deliveriesByEntry = new WeakMap;
|
|
58606
|
+
statusCache = new WeakMap;
|
|
58607
|
+
receiptGeneration = 0;
|
|
58608
|
+
receiptEpochs = new Map;
|
|
58609
|
+
constructor(rootRunId, topology) {
|
|
58610
|
+
this.rootRunId = rootRunId;
|
|
58611
|
+
this.topology = topology;
|
|
58612
|
+
}
|
|
58613
|
+
rows(runs, rootRun, availableBackend) {
|
|
58614
|
+
if (isTerminalRunStatus(rootRun.status))
|
|
58615
|
+
return [];
|
|
58616
|
+
this.refreshEntries(rootRun.pendingStageMessages ?? []);
|
|
58617
|
+
const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
|
|
58618
|
+
const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
|
|
58619
|
+
this.refreshReceipts(backend);
|
|
58620
|
+
this.refreshStages(runs);
|
|
58621
|
+
for (const state of this.patterns)
|
|
58622
|
+
if (state.dirty)
|
|
58623
|
+
this.recompute(state, backend);
|
|
58624
|
+
return this.assemble(runs, rootRun);
|
|
58625
|
+
}
|
|
58626
|
+
targetFor(entry) {
|
|
57739
58627
|
const target = entry.targetPath ?? entry.stageKey;
|
|
57740
|
-
let
|
|
57741
|
-
if (
|
|
58628
|
+
let state = this.targets.get(target);
|
|
58629
|
+
if (state === undefined) {
|
|
57742
58630
|
const parsed = parseWorkflowStageTarget(target);
|
|
57743
|
-
|
|
57744
|
-
|
|
57745
|
-
|
|
57746
|
-
|
|
58631
|
+
const owned = parsed?.rootRunId === this.rootRunId;
|
|
58632
|
+
state = {
|
|
58633
|
+
target,
|
|
58634
|
+
segments: owned ? parsed.segments : undefined,
|
|
58635
|
+
pattern: owned ? this.patterns.length : -1,
|
|
58636
|
+
entries: [],
|
|
58637
|
+
positions: [],
|
|
58638
|
+
queued: 0,
|
|
58639
|
+
stages: [],
|
|
58640
|
+
statuses: [],
|
|
58641
|
+
dirty: true
|
|
58642
|
+
};
|
|
58643
|
+
this.targets.set(target, state);
|
|
58644
|
+
if (owned)
|
|
58645
|
+
this.patterns.push(state);
|
|
57747
58646
|
}
|
|
57748
|
-
|
|
58647
|
+
return state;
|
|
58648
|
+
}
|
|
58649
|
+
refreshEntries(list) {
|
|
58650
|
+
const previous = this.list;
|
|
58651
|
+
if (list === previous)
|
|
57749
58652
|
return;
|
|
57750
|
-
|
|
57751
|
-
|
|
57752
|
-
|
|
57753
|
-
|
|
57754
|
-
|
|
57755
|
-
|
|
57756
|
-
|
|
57757
|
-
|
|
57758
|
-
|
|
58653
|
+
this.list = list;
|
|
58654
|
+
let incremental = list.length >= previous.length;
|
|
58655
|
+
const replaced = [];
|
|
58656
|
+
for (let index = 0;incremental && index < previous.length; index++) {
|
|
58657
|
+
if (list[index] === previous[index])
|
|
58658
|
+
continue;
|
|
58659
|
+
const before = previous[index];
|
|
58660
|
+
const after = list[index];
|
|
58661
|
+
if (before.sticky !== true && after.sticky !== true)
|
|
58662
|
+
continue;
|
|
58663
|
+
if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
|
|
58664
|
+
incremental = false;
|
|
58665
|
+
break;
|
|
58666
|
+
}
|
|
58667
|
+
replaced.push(index);
|
|
58668
|
+
}
|
|
58669
|
+
if (!incremental) {
|
|
58670
|
+
this.rebuildEntries(list);
|
|
58671
|
+
return;
|
|
58672
|
+
}
|
|
58673
|
+
for (const index of replaced) {
|
|
58674
|
+
const before = previous[index];
|
|
58675
|
+
const after = list[index];
|
|
58676
|
+
const state = this.targetOf.get(before);
|
|
58677
|
+
const slot = state.entries.indexOf(before);
|
|
58678
|
+
state.entries[slot] = after;
|
|
58679
|
+
this.targetOf.delete(before);
|
|
58680
|
+
this.targetOf.set(after, state);
|
|
58681
|
+
state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
|
|
58682
|
+
if (before.id !== after.id)
|
|
58683
|
+
this.rebuildMessageIndex();
|
|
58684
|
+
state.dirty = true;
|
|
58685
|
+
}
|
|
58686
|
+
for (let index = previous.length;index < list.length; index++)
|
|
58687
|
+
this.addEntry(list[index], index);
|
|
58688
|
+
}
|
|
58689
|
+
addEntry(entry, position) {
|
|
58690
|
+
if (entry.sticky !== true)
|
|
58691
|
+
return;
|
|
58692
|
+
const state = this.targetFor(entry);
|
|
58693
|
+
state.entries.push(entry);
|
|
58694
|
+
state.positions.push(position);
|
|
58695
|
+
if (entry.status === "queued")
|
|
58696
|
+
state.queued++;
|
|
58697
|
+
state.dirty = true;
|
|
58698
|
+
this.targetOf.set(entry, state);
|
|
58699
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
58700
|
+
if (targets === undefined) {
|
|
58701
|
+
targets = new Set;
|
|
58702
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
58703
|
+
}
|
|
58704
|
+
targets.add(state);
|
|
58705
|
+
}
|
|
58706
|
+
rebuildEntries(list) {
|
|
58707
|
+
const before = new Map;
|
|
58708
|
+
for (const state of this.targets.values()) {
|
|
58709
|
+
before.set(state, state.entries);
|
|
58710
|
+
state.entries = [];
|
|
58711
|
+
state.positions = [];
|
|
58712
|
+
state.queued = 0;
|
|
58713
|
+
}
|
|
58714
|
+
this.targetOf.clear();
|
|
58715
|
+
this.targetsByMessage.clear();
|
|
58716
|
+
list.forEach((entry, position) => {
|
|
58717
|
+
this.addEntry(entry, position);
|
|
58718
|
+
});
|
|
58719
|
+
for (const state of this.targets.values()) {
|
|
58720
|
+
const previous = before.get(state);
|
|
58721
|
+
state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
|
|
58722
|
+
}
|
|
58723
|
+
}
|
|
58724
|
+
rebuildMessageIndex() {
|
|
58725
|
+
this.targetsByMessage.clear();
|
|
58726
|
+
for (const [entry, state] of this.targetOf) {
|
|
58727
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
58728
|
+
if (targets === undefined) {
|
|
58729
|
+
targets = new Set;
|
|
58730
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
58731
|
+
}
|
|
58732
|
+
targets.add(state);
|
|
58733
|
+
}
|
|
58734
|
+
}
|
|
58735
|
+
refreshReceipts(backend) {
|
|
58736
|
+
if (backend !== this.backend) {
|
|
58737
|
+
this.backend = backend;
|
|
58738
|
+
this.receiptCursor = undefined;
|
|
58739
|
+
this.invalidateReceipts();
|
|
58740
|
+
}
|
|
58741
|
+
if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
|
|
58742
|
+
return;
|
|
58743
|
+
const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
|
|
58744
|
+
if (changes === undefined) {
|
|
58745
|
+
this.invalidateReceipts();
|
|
58746
|
+
return;
|
|
58747
|
+
}
|
|
58748
|
+
const firstRead = this.receiptCursor === undefined;
|
|
58749
|
+
this.receiptCursor = changes.cursor;
|
|
58750
|
+
if (changes.messageIds === undefined) {
|
|
58751
|
+
if (!firstRead)
|
|
58752
|
+
this.invalidateReceipts();
|
|
58753
|
+
return;
|
|
58754
|
+
}
|
|
58755
|
+
for (const messageId of changes.messageIds) {
|
|
58756
|
+
this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
|
|
58757
|
+
for (const state of this.targetsByMessage.get(messageId) ?? [])
|
|
58758
|
+
state.dirty = true;
|
|
58759
|
+
}
|
|
58760
|
+
}
|
|
58761
|
+
invalidateReceipts() {
|
|
58762
|
+
this.receiptGeneration++;
|
|
58763
|
+
this.receiptEpochs.clear();
|
|
58764
|
+
for (const state of this.patterns)
|
|
58765
|
+
state.dirty = true;
|
|
58766
|
+
}
|
|
58767
|
+
refreshStages(runs) {
|
|
58768
|
+
const current = new Map;
|
|
58769
|
+
let ordinal = 0;
|
|
58770
|
+
for (const run of runs) {
|
|
58771
|
+
const root = this.topology.rootOf(run.id);
|
|
58772
|
+
if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
|
|
58773
|
+
continue;
|
|
58774
|
+
const resolved = root === this.rootRunId;
|
|
58775
|
+
const hops = this.topology.hopsOf(run.id);
|
|
58776
|
+
for (const stage of run.stages) {
|
|
58777
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
58778
|
+
continue;
|
|
58779
|
+
const fields = this.fieldsOf(run, stage, resolved, hops);
|
|
58780
|
+
const status = { run, stage, resolved, hops, ordinal: ordinal++ };
|
|
58781
|
+
for (const state of fields.matched) {
|
|
58782
|
+
const stages = current.get(state);
|
|
58783
|
+
if (stages === undefined)
|
|
58784
|
+
current.set(state, [status]);
|
|
58785
|
+
else
|
|
58786
|
+
stages.push(status);
|
|
58787
|
+
}
|
|
58788
|
+
}
|
|
58789
|
+
}
|
|
58790
|
+
for (const state of this.patterns) {
|
|
58791
|
+
const stages = current.get(state) ?? [];
|
|
58792
|
+
if (stages.length !== state.stages.length || stages.some((status, index) => {
|
|
58793
|
+
const previous = state.stages[index];
|
|
58794
|
+
return previous.stage !== status.stage || previous.run !== status.run;
|
|
58795
|
+
}))
|
|
58796
|
+
state.dirty = true;
|
|
58797
|
+
state.stages = stages;
|
|
58798
|
+
}
|
|
58799
|
+
}
|
|
58800
|
+
fieldsOf(run, stage, resolved, hops) {
|
|
58801
|
+
let fields = this.stageFields.get(stage);
|
|
58802
|
+
const matchable = resolved && hops !== undefined;
|
|
58803
|
+
let changed = false;
|
|
58804
|
+
if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
|
|
58805
|
+
const previous = fields;
|
|
58806
|
+
const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
|
|
58807
|
+
fields = {
|
|
58808
|
+
run,
|
|
58809
|
+
status: stage.status,
|
|
58810
|
+
error: stage.error,
|
|
58811
|
+
sessionId: stage.sessionId,
|
|
58812
|
+
pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
|
|
58813
|
+
resolved,
|
|
58814
|
+
hops,
|
|
58815
|
+
id: stage.id,
|
|
58816
|
+
name: stage.name,
|
|
58817
|
+
tested: sameMatching ? previous.tested : 0,
|
|
58818
|
+
matched: sameMatching ? previous.matched : []
|
|
58819
|
+
};
|
|
58820
|
+
this.stageFields.set(stage, fields);
|
|
58821
|
+
changed = true;
|
|
58822
|
+
}
|
|
58823
|
+
if (fields.tested < this.patterns.length) {
|
|
58824
|
+
const added = this.patterns.slice(fields.tested);
|
|
58825
|
+
if (!matchable)
|
|
58826
|
+
fields.matched.push(...added);
|
|
58827
|
+
else {
|
|
58828
|
+
const index = createStagePathPatternIndex(added.map((state) => state.segments));
|
|
58829
|
+
for (const position of index.matching(hops, [stage.id, stage.name]))
|
|
58830
|
+
fields.matched.push(added[position]);
|
|
58831
|
+
}
|
|
58832
|
+
fields.tested = this.patterns.length;
|
|
58833
|
+
}
|
|
58834
|
+
if (changed)
|
|
58835
|
+
for (const state of fields.matched)
|
|
58836
|
+
state.dirty = true;
|
|
58837
|
+
return fields;
|
|
58838
|
+
}
|
|
58839
|
+
recompute(state, backend) {
|
|
58840
|
+
state.dirty = false;
|
|
58841
|
+
const statuses = [];
|
|
58842
|
+
for (const { run, stage, resolved, hops } of state.stages) {
|
|
58843
|
+
const fields = this.stageFields.get(stage);
|
|
58844
|
+
for (const entry of state.entries) {
|
|
58845
|
+
let byStage = this.statusCache.get(entry);
|
|
58846
|
+
if (byStage === undefined) {
|
|
58847
|
+
byStage = new Map;
|
|
58848
|
+
this.statusCache.set(entry, byStage);
|
|
58849
|
+
}
|
|
58850
|
+
const epoch = this.receiptEpochs.get(entry.id) ?? 0;
|
|
58851
|
+
let cached = byStage.get(stage);
|
|
58852
|
+
if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
|
|
58853
|
+
cached = {
|
|
58854
|
+
fields,
|
|
58855
|
+
generation: this.receiptGeneration,
|
|
58856
|
+
epoch,
|
|
58857
|
+
status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
|
|
58858
|
+
};
|
|
58859
|
+
byStage.set(stage, cached);
|
|
58860
|
+
}
|
|
58861
|
+
statuses.push(cached.status);
|
|
58862
|
+
}
|
|
58863
|
+
}
|
|
58864
|
+
if (!sameStatuses(statuses, state.statuses))
|
|
58865
|
+
state.statuses = statuses;
|
|
58866
|
+
}
|
|
58867
|
+
receiptsOf(entry, runId, stageId) {
|
|
58868
|
+
let byStage = this.deliveriesByEntry.get(entry);
|
|
57759
58869
|
if (byStage === undefined) {
|
|
57760
58870
|
byStage = new Map;
|
|
57761
|
-
for (const delivery of
|
|
58871
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
57762
58872
|
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
57763
58873
|
const list = byStage.get(key);
|
|
57764
58874
|
if (list === undefined)
|
|
@@ -57766,75 +58876,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
57766
58876
|
else
|
|
57767
58877
|
list.push(delivery);
|
|
57768
58878
|
}
|
|
57769
|
-
deliveriesByEntry.set(
|
|
58879
|
+
this.deliveriesByEntry.set(entry, byStage);
|
|
57770
58880
|
}
|
|
57771
58881
|
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
57772
|
-
}
|
|
57773
|
-
|
|
57774
|
-
|
|
57775
|
-
|
|
57776
|
-
|
|
57777
|
-
|
|
57778
|
-
|
|
57779
|
-
if (
|
|
57780
|
-
|
|
57781
|
-
|
|
57782
|
-
|
|
57783
|
-
|
|
57784
|
-
|
|
58882
|
+
}
|
|
58883
|
+
status(entry, run, stage, matchable, backend) {
|
|
58884
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
58885
|
+
const receipts = this.receiptsOf(entry, run.id, stage.id);
|
|
58886
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
58887
|
+
let state;
|
|
58888
|
+
let reason;
|
|
58889
|
+
if (!matchable) {
|
|
58890
|
+
state = "root-unresolved";
|
|
58891
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
58892
|
+
} else if (delivered) {
|
|
58893
|
+
state = "delivered";
|
|
58894
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
58895
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
58896
|
+
state = "stage-terminal";
|
|
58897
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
58898
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
58899
|
+
state = "receipt-unverified";
|
|
58900
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
58901
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
58902
|
+
state = "delivery-unavailable";
|
|
58903
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
58904
|
+
} else if (receipts.length > 0) {
|
|
58905
|
+
state = "session-replaced";
|
|
58906
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
58907
|
+
} else {
|
|
58908
|
+
state = "queued";
|
|
58909
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
58910
|
+
}
|
|
58911
|
+
return {
|
|
58912
|
+
messageId: entry.message.id,
|
|
58913
|
+
target,
|
|
58914
|
+
runId: run.id,
|
|
58915
|
+
stageId: stage.id,
|
|
58916
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
58917
|
+
state,
|
|
58918
|
+
reason
|
|
58919
|
+
};
|
|
58920
|
+
}
|
|
58921
|
+
assemble(runs, rootRun) {
|
|
58922
|
+
const rows = [];
|
|
58923
|
+
const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
|
|
58924
|
+
let liveStageMatches;
|
|
58925
|
+
const rowTargets = new Set;
|
|
58926
|
+
for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
|
|
58927
|
+
const rowSegments = splitStagePathSegments(scanEntry);
|
|
58928
|
+
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
57785
58929
|
continue;
|
|
57786
|
-
|
|
57787
|
-
|
|
57788
|
-
|
|
57789
|
-
|
|
57790
|
-
const receipts = entryReceipts(index, run.id, stage.id);
|
|
57791
|
-
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
57792
|
-
runId: run.id,
|
|
57793
|
-
stageId: stage.id,
|
|
57794
|
-
sessionId: stage.sessionId,
|
|
57795
|
-
admission: "context"
|
|
57796
|
-
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
57797
|
-
let state;
|
|
57798
|
-
let reason;
|
|
57799
|
-
if (!resolved || !hops) {
|
|
57800
|
-
state = "root-unresolved";
|
|
57801
|
-
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
57802
|
-
} else if (delivered) {
|
|
57803
|
-
state = "delivered";
|
|
57804
|
-
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
57805
|
-
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
57806
|
-
state = "stage-terminal";
|
|
57807
|
-
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
57808
|
-
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
57809
|
-
state = "receipt-unverified";
|
|
57810
|
-
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
57811
|
-
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
57812
|
-
state = "delivery-unavailable";
|
|
57813
|
-
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
57814
|
-
} else if (receipts.length > 0) {
|
|
57815
|
-
state = "session-replaced";
|
|
57816
|
-
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
57817
|
-
} else {
|
|
57818
|
-
state = "queued";
|
|
57819
|
-
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
57820
|
-
}
|
|
57821
|
-
result.push({
|
|
57822
|
-
messageId: entry.message.id,
|
|
57823
|
-
target,
|
|
57824
|
-
runId: run.id,
|
|
57825
|
-
stageId: stage.id,
|
|
57826
|
-
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
57827
|
-
state,
|
|
57828
|
-
reason
|
|
57829
|
-
});
|
|
58930
|
+
if (!rowSegments.some((segment) => segment.includes("*"))) {
|
|
58931
|
+
liveStageMatches ??= this.materializedStageMatcher(runs);
|
|
58932
|
+
if (liveStageMatches(rowSegments))
|
|
58933
|
+
continue;
|
|
57830
58934
|
}
|
|
58935
|
+
let matches = this.scanMatches.get(scanEntry);
|
|
58936
|
+
if (matches === undefined) {
|
|
58937
|
+
matches = new Map;
|
|
58938
|
+
this.scanMatches.set(scanEntry, matches);
|
|
58939
|
+
}
|
|
58940
|
+
let queuedCount = 0;
|
|
58941
|
+
for (const state of scanTargets) {
|
|
58942
|
+
let match = matches.get(state);
|
|
58943
|
+
if (match === undefined) {
|
|
58944
|
+
const segments = state.segments;
|
|
58945
|
+
match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
58946
|
+
matches.set(state, match);
|
|
58947
|
+
}
|
|
58948
|
+
if (match)
|
|
58949
|
+
queuedCount += state.queued;
|
|
58950
|
+
}
|
|
58951
|
+
const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
|
|
58952
|
+
rows.push({ slot: `s${position}`, target, queuedCount });
|
|
58953
|
+
rowTargets.add(target);
|
|
58954
|
+
}
|
|
58955
|
+
let broadcastCount = 0;
|
|
58956
|
+
for (const state of this.targets.values()) {
|
|
58957
|
+
if (state.segments?.length === 1 && state.segments[0] === "**")
|
|
58958
|
+
broadcastCount += state.queued;
|
|
58959
|
+
}
|
|
58960
|
+
const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
|
|
58961
|
+
rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
|
|
58962
|
+
rowTargets.add(broadcast);
|
|
58963
|
+
const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
|
|
58964
|
+
const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
|
|
58965
|
+
for (const state of statusTargets) {
|
|
58966
|
+
if (rowTargets.has(state.target))
|
|
58967
|
+
continue;
|
|
58968
|
+
rowTargets.add(state.target);
|
|
58969
|
+
rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
|
|
57831
58970
|
}
|
|
58971
|
+
const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
|
|
58972
|
+
this.pruneRows(result);
|
|
58973
|
+
return result;
|
|
58974
|
+
}
|
|
58975
|
+
row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
|
|
58976
|
+
const cached = this.rowCache.get(slot);
|
|
58977
|
+
const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
|
|
58978
|
+
if (same && cached.statuses === deliveryStates)
|
|
58979
|
+
return cached.row;
|
|
58980
|
+
const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
|
|
58981
|
+
if (same && sameBoundedRow(cached.row, bounded)) {
|
|
58982
|
+
this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
|
|
58983
|
+
return cached.row;
|
|
58984
|
+
}
|
|
58985
|
+
const row = {
|
|
58986
|
+
target,
|
|
58987
|
+
queuedCount,
|
|
58988
|
+
...deliveryOnly === true ? { deliveryOnly } : {},
|
|
58989
|
+
...bounded
|
|
58990
|
+
};
|
|
58991
|
+
this.rowCache.set(slot, { row, statuses: deliveryStates });
|
|
58992
|
+
return row;
|
|
58993
|
+
}
|
|
58994
|
+
bounded(states) {
|
|
58995
|
+
let bounded = this.boundedStates.get(states);
|
|
58996
|
+
if (bounded === undefined) {
|
|
58997
|
+
bounded = boundPossibleStageDeliveryStates(states);
|
|
58998
|
+
this.boundedStates.set(states, bounded);
|
|
58999
|
+
}
|
|
59000
|
+
return bounded;
|
|
59001
|
+
}
|
|
59002
|
+
pruneRows(rows) {
|
|
59003
|
+
if (this.rowCache.size <= rows.length * 4 + 16)
|
|
59004
|
+
return;
|
|
59005
|
+
const kept = new Set(rows);
|
|
59006
|
+
for (const [key, cached] of this.rowCache)
|
|
59007
|
+
if (!kept.has(cached.row))
|
|
59008
|
+
this.rowCache.delete(key);
|
|
59009
|
+
}
|
|
59010
|
+
materializedStageMatcher(runs) {
|
|
59011
|
+
const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
|
|
59012
|
+
return (entrySegments) => {
|
|
59013
|
+
for (const run of invocationRuns) {
|
|
59014
|
+
const hops = this.topology.hopsOf(run.id);
|
|
59015
|
+
if (hops === undefined)
|
|
59016
|
+
continue;
|
|
59017
|
+
for (const stage of run.stages) {
|
|
59018
|
+
if (stage.pendingStageDeliveryAvailable !== true)
|
|
59019
|
+
continue;
|
|
59020
|
+
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
59021
|
+
continue;
|
|
59022
|
+
}
|
|
59023
|
+
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
59024
|
+
return true;
|
|
59025
|
+
}
|
|
59026
|
+
}
|
|
59027
|
+
return false;
|
|
59028
|
+
};
|
|
57832
59029
|
}
|
|
57833
|
-
return result;
|
|
57834
59030
|
}
|
|
57835
|
-
|
|
59031
|
+
function sameBoundedRow(row, bounded) {
|
|
59032
|
+
if (bounded === undefined)
|
|
59033
|
+
return row.deliveryStates === undefined;
|
|
59034
|
+
if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
|
|
59035
|
+
return false;
|
|
59036
|
+
const counts = Object.entries(row.deliveryStateCounts ?? {});
|
|
59037
|
+
const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
|
|
59038
|
+
if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
|
|
59039
|
+
return false;
|
|
59040
|
+
return sameStatuses(row.deliveryStates, bounded.deliveryStates);
|
|
59041
|
+
}
|
|
59042
|
+
function sameStatuses(left, right) {
|
|
59043
|
+
if (left.length !== right.length)
|
|
59044
|
+
return false;
|
|
59045
|
+
for (let index = 0;index < left.length; index++) {
|
|
59046
|
+
const a = left[index];
|
|
59047
|
+
const b = right[index];
|
|
59048
|
+
if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
|
|
59049
|
+
return false;
|
|
59050
|
+
}
|
|
59051
|
+
return true;
|
|
59052
|
+
}
|
|
59053
|
+
var init_possible_stage_rows = __esm(() => {
|
|
57836
59054
|
init_run_owner_backend();
|
|
57837
59055
|
init_pending_stage_status();
|
|
59056
|
+
init_sticky_delivery_state_bound();
|
|
59057
|
+
init_store_internal();
|
|
57838
59058
|
init_workflow_stage_target();
|
|
57839
59059
|
});
|
|
57840
59060
|
|
|
@@ -57895,12 +59115,36 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57895
59115
|
};
|
|
57896
59116
|
const announcedRoutes = new Map;
|
|
57897
59117
|
const routeCompletions = new Map;
|
|
57898
|
-
const disposeReadiness = registerWorkflowPendingStageRouteReadiness(activeStore, (runId) =>
|
|
59118
|
+
const disposeReadiness = registerWorkflowPendingStageRouteReadiness(activeStore, (runId) => {
|
|
59119
|
+
announceIfPending();
|
|
59120
|
+
return routeCompletions.get(runId);
|
|
59121
|
+
});
|
|
59122
|
+
const topology = new WorkflowTopologyIndex;
|
|
59123
|
+
const possibleStageRowIndexes = new Map;
|
|
59124
|
+
const rowIdentities = new WeakMap;
|
|
59125
|
+
let nextRowIdentity = 0;
|
|
59126
|
+
const rowIdentity = (row) => {
|
|
59127
|
+
let identity = rowIdentities.get(row);
|
|
59128
|
+
if (identity === undefined) {
|
|
59129
|
+
identity = nextRowIdentity++;
|
|
59130
|
+
rowIdentities.set(row, identity);
|
|
59131
|
+
}
|
|
59132
|
+
return identity;
|
|
59133
|
+
};
|
|
59134
|
+
const possibleStageRowsIndex = (rootRunId) => {
|
|
59135
|
+
let index = possibleStageRowIndexes.get(rootRunId);
|
|
59136
|
+
if (index === undefined) {
|
|
59137
|
+
index = new PossibleStageRowsIndex(rootRunId, topology);
|
|
59138
|
+
possibleStageRowIndexes.set(rootRunId, index);
|
|
59139
|
+
}
|
|
59140
|
+
return index;
|
|
59141
|
+
};
|
|
57899
59142
|
const announceRoutes = () => {
|
|
57900
59143
|
if (disposed)
|
|
57901
59144
|
return;
|
|
57902
59145
|
const ownedRunIds = new Set;
|
|
57903
59146
|
const runs = activeStore.runs();
|
|
59147
|
+
topology.refresh(runs);
|
|
57904
59148
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
57905
59149
|
const childrenByBoundary = new Map;
|
|
57906
59150
|
for (const run of runs) {
|
|
@@ -57915,7 +59159,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57915
59159
|
for (const run of runs) {
|
|
57916
59160
|
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
57917
59161
|
continue;
|
|
57918
|
-
const rootRunId =
|
|
59162
|
+
const rootRunId = topology.rootOf(run.id);
|
|
57919
59163
|
if (rootRunId === undefined)
|
|
57920
59164
|
continue;
|
|
57921
59165
|
ownedRunIds.add(run.id);
|
|
@@ -57925,6 +59169,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57925
59169
|
runId: parentRun.id,
|
|
57926
59170
|
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
57927
59171
|
};
|
|
59172
|
+
let possibleStages;
|
|
59173
|
+
if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
|
|
59174
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
59175
|
+
possibleStages = [];
|
|
59176
|
+
} else if (run.id === rootRunId) {
|
|
59177
|
+
possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
|
|
59178
|
+
}
|
|
57928
59179
|
const announcement = {
|
|
57929
59180
|
runId: run.id,
|
|
57930
59181
|
group: workflowInvocationIntercomGroup(rootRunId),
|
|
@@ -57949,11 +59200,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57949
59200
|
group: workflowInvocationIntercomGroup(rootRunId)
|
|
57950
59201
|
}))
|
|
57951
59202
|
],
|
|
57952
|
-
...
|
|
57953
|
-
possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
|
|
57954
|
-
} : {}
|
|
59203
|
+
...possibleStages === undefined ? {} : { possibleStages }
|
|
57955
59204
|
};
|
|
57956
|
-
const signature = JSON.stringify(announcement)
|
|
59205
|
+
const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
|
|
57957
59206
|
if (announcedRoutes.get(run.id)?.signature === signature)
|
|
57958
59207
|
continue;
|
|
57959
59208
|
const entry = { signature };
|
|
@@ -57988,15 +59237,61 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57988
59237
|
if (!ownedRunIds.has(runId))
|
|
57989
59238
|
routeCompletions.delete(runId);
|
|
57990
59239
|
}
|
|
59240
|
+
for (const rootRunId of possibleStageRowIndexes.keys()) {
|
|
59241
|
+
if (!ownedRunIds.has(rootRunId))
|
|
59242
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
59243
|
+
}
|
|
57991
59244
|
sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
|
|
57992
59245
|
return;
|
|
57993
59246
|
}).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
|
|
57994
59247
|
};
|
|
57995
|
-
|
|
57996
|
-
|
|
59248
|
+
let announcementPending = false;
|
|
59249
|
+
let announcementTimer;
|
|
59250
|
+
let announcedAt = Number.NEGATIVE_INFINITY;
|
|
59251
|
+
let announcementCost = 0;
|
|
59252
|
+
const announceNow = () => {
|
|
59253
|
+
if (announcementTimer !== undefined)
|
|
59254
|
+
clearTimeout(announcementTimer);
|
|
59255
|
+
announcementTimer = undefined;
|
|
59256
|
+
announcementPending = false;
|
|
59257
|
+
const started = performance.now();
|
|
59258
|
+
announceRoutes();
|
|
59259
|
+
announcedAt = performance.now();
|
|
59260
|
+
announcementCost = announcedAt - started;
|
|
59261
|
+
};
|
|
59262
|
+
function announceIfPending() {
|
|
59263
|
+
if (announcementPending)
|
|
59264
|
+
announceNow();
|
|
59265
|
+
}
|
|
59266
|
+
const announceBatch = () => {
|
|
59267
|
+
try {
|
|
59268
|
+
announceNow();
|
|
59269
|
+
} catch (error) {
|
|
59270
|
+
if (error instanceof Error && isSessionClosedError(error)) {
|
|
59271
|
+
dispose();
|
|
59272
|
+
return;
|
|
59273
|
+
}
|
|
59274
|
+
try {
|
|
59275
|
+
reportWarning(`atomic-workflows: pending stage route announcement failed: ${String(error)}`);
|
|
59276
|
+
} catch {
|
|
59277
|
+
dispose();
|
|
59278
|
+
}
|
|
59279
|
+
}
|
|
59280
|
+
};
|
|
59281
|
+
const scheduleAnnouncement = () => {
|
|
59282
|
+
if (disposed || announcementTimer !== undefined)
|
|
59283
|
+
return;
|
|
59284
|
+
const idle = announcementCost / ROUTE_ANNOUNCEMENT_EVENT_LOOP_SHARE - announcementCost;
|
|
59285
|
+
const wait = Math.max(ROUTE_ANNOUNCEMENT_BATCH_MS, announcedAt + idle - performance.now());
|
|
59286
|
+
announcementPending = true;
|
|
59287
|
+
announcementTimer = setTimeout(announceBatch, wait);
|
|
59288
|
+
announcementTimer.unref?.();
|
|
59289
|
+
};
|
|
59290
|
+
const unsubscribeStore = activeStore.subscribeInvalidation(scheduleAnnouncement);
|
|
59291
|
+
announceNow();
|
|
57997
59292
|
pi.on?.("session_start", () => {
|
|
57998
59293
|
announcedRoutes.clear();
|
|
57999
|
-
|
|
59294
|
+
announceNow();
|
|
58000
59295
|
});
|
|
58001
59296
|
const routePendingStageMessage = (payload) => {
|
|
58002
59297
|
if (disposed || !isPendingStageMessageEvent(payload) || payload.handled)
|
|
@@ -58078,6 +59373,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58078
59373
|
});
|
|
58079
59374
|
const dispose = () => {
|
|
58080
59375
|
disposed = true;
|
|
59376
|
+
if (announcementTimer !== undefined)
|
|
59377
|
+
clearTimeout(announcementTimer);
|
|
59378
|
+
announcementTimer = undefined;
|
|
59379
|
+
announcementPending = false;
|
|
58081
59380
|
disposeReadiness();
|
|
58082
59381
|
disposeRouter();
|
|
58083
59382
|
routeCompletions.clear();
|
|
@@ -58091,112 +59390,17 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58091
59390
|
pi.on?.("session_shutdown", dispose);
|
|
58092
59391
|
return Object.assign(dispose, {
|
|
58093
59392
|
async flush() {
|
|
59393
|
+
announceIfPending();
|
|
58094
59394
|
await Promise.allSettled(routeCompletions.values());
|
|
58095
59395
|
await sweepPromise;
|
|
58096
59396
|
},
|
|
58097
|
-
refresh:
|
|
59397
|
+
refresh: announceNow
|
|
58098
59398
|
});
|
|
58099
59399
|
}
|
|
58100
59400
|
function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegments = (id) => workflowBoundarySegments(runs, id)) {
|
|
58101
59401
|
const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
|
|
58102
59402
|
return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
|
|
58103
59403
|
}
|
|
58104
|
-
function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
58105
|
-
if (isTerminalRunStatus(rootRun.status))
|
|
58106
|
-
return [];
|
|
58107
|
-
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
58108
|
-
const segmentsByTarget = new Map;
|
|
58109
|
-
const stickySegments = (entry) => {
|
|
58110
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58111
|
-
if (segmentsByTarget.has(target))
|
|
58112
|
-
return segmentsByTarget.get(target);
|
|
58113
|
-
const parsed = parseWorkflowStageTarget(target);
|
|
58114
|
-
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
58115
|
-
segmentsByTarget.set(target, segments);
|
|
58116
|
-
return segments;
|
|
58117
|
-
};
|
|
58118
|
-
const scanEntries = stickyQueued.filter((entry) => {
|
|
58119
|
-
const segments = stickySegments(entry);
|
|
58120
|
-
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
58121
|
-
});
|
|
58122
|
-
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
58123
|
-
const rows = [];
|
|
58124
|
-
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
58125
|
-
const rowSegments = splitStagePathSegments(scanEntry);
|
|
58126
|
-
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
58127
|
-
continue;
|
|
58128
|
-
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
58129
|
-
continue;
|
|
58130
|
-
}
|
|
58131
|
-
rows.push({
|
|
58132
|
-
target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
|
|
58133
|
-
queuedCount: scanEntries.filter((entry) => {
|
|
58134
|
-
const segments = stickySegments(entry);
|
|
58135
|
-
if (segments === undefined)
|
|
58136
|
-
return false;
|
|
58137
|
-
return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
58138
|
-
}).length
|
|
58139
|
-
});
|
|
58140
|
-
}
|
|
58141
|
-
rows.push({
|
|
58142
|
-
target: formatWorkflowStageTarget(rootRunId, "**"),
|
|
58143
|
-
queuedCount: stickyQueued.filter((entry) => {
|
|
58144
|
-
const segments = stickySegments(entry);
|
|
58145
|
-
return segments !== undefined && segments.length === 1 && segments[0] === "**";
|
|
58146
|
-
}).length
|
|
58147
|
-
});
|
|
58148
|
-
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
58149
|
-
const statusesByTarget = new Map;
|
|
58150
|
-
for (const status of statuses) {
|
|
58151
|
-
const forTarget = statusesByTarget.get(status.target);
|
|
58152
|
-
if (forTarget === undefined)
|
|
58153
|
-
statusesByTarget.set(status.target, [status]);
|
|
58154
|
-
else
|
|
58155
|
-
forTarget.push(status);
|
|
58156
|
-
}
|
|
58157
|
-
const queuedByTarget = new Map;
|
|
58158
|
-
for (const entry of stickyQueued) {
|
|
58159
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58160
|
-
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
58161
|
-
}
|
|
58162
|
-
const rowTargets = new Set(rows.map((row) => row.target));
|
|
58163
|
-
for (const target of statusesByTarget.keys()) {
|
|
58164
|
-
if (rowTargets.has(target))
|
|
58165
|
-
continue;
|
|
58166
|
-
rowTargets.add(target);
|
|
58167
|
-
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
58168
|
-
}
|
|
58169
|
-
for (let index = 0;index < rows.length; index++) {
|
|
58170
|
-
const row = rows[index];
|
|
58171
|
-
const deliveryStates = statusesByTarget.get(row.target);
|
|
58172
|
-
if (deliveryStates !== undefined)
|
|
58173
|
-
rows[index] = { ...row, deliveryStates };
|
|
58174
|
-
}
|
|
58175
|
-
return rows;
|
|
58176
|
-
}
|
|
58177
|
-
function materializedStageMatcher(runs, rootRunId) {
|
|
58178
|
-
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58179
|
-
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
58180
|
-
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
58181
|
-
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
58182
|
-
}
|
|
58183
|
-
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
58184
|
-
for (const run of invocationRuns) {
|
|
58185
|
-
const hops = hopsFor(run.id);
|
|
58186
|
-
if (hops === undefined)
|
|
58187
|
-
continue;
|
|
58188
|
-
for (const stage of run.stages) {
|
|
58189
|
-
if (stage.pendingStageDeliveryAvailable !== true)
|
|
58190
|
-
continue;
|
|
58191
|
-
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
58192
|
-
continue;
|
|
58193
|
-
}
|
|
58194
|
-
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
58195
|
-
return true;
|
|
58196
|
-
}
|
|
58197
|
-
}
|
|
58198
|
-
return false;
|
|
58199
|
-
}
|
|
58200
59404
|
function isPendingStageMessageEvent(value2) {
|
|
58201
59405
|
if (typeof value2 !== "object" || value2 === null)
|
|
58202
59406
|
return false;
|
|
@@ -58569,17 +59773,18 @@ async function settleUndeliverablePendingStageMessages(activeStore, notify, owns
|
|
|
58569
59773
|
}
|
|
58570
59774
|
return settled;
|
|
58571
59775
|
}
|
|
58572
|
-
var PENDING_STAGE_ROUTE_EVENT = "atomic:workflow-pending-stage-route", PENDING_STAGE_MESSAGE_EVENT = "atomic:workflow-pending-stage-message", PENDING_STAGE_UNDELIVERABLE_EVENT = "atomic:workflow-pending-stage-undeliverable", STICKY_LIVE_DELIVERED_EVENT = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_ASK_REFUSAL = "Cannot ask a workflow stage whose session has not initialized. Use send; Atomic will queue the message until the stage session initializes.";
|
|
59776
|
+
var PENDING_STAGE_ROUTE_EVENT = "atomic:workflow-pending-stage-route", PENDING_STAGE_MESSAGE_EVENT = "atomic:workflow-pending-stage-message", PENDING_STAGE_UNDELIVERABLE_EVENT = "atomic:workflow-pending-stage-undeliverable", STICKY_LIVE_DELIVERED_EVENT = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_ASK_REFUSAL = "Cannot ask a workflow stage whose session has not initialized. Use send; Atomic will queue the message until the stage session initializes.", ROUTE_ANNOUNCEMENT_BATCH_MS = 25, ROUTE_ANNOUNCEMENT_EVENT_LOOP_SHARE = 0.2;
|
|
58573
59777
|
var init_pending_stage_intercom = __esm(() => {
|
|
58574
59778
|
init_execution_context_boundary();
|
|
58575
59779
|
init_run_owner_backend();
|
|
58576
59780
|
init_store_backend();
|
|
59781
|
+
init_session_closed_error();
|
|
58577
59782
|
init_external_run_routes();
|
|
58578
59783
|
init_intercom_group();
|
|
58579
59784
|
init_pending_stage_route_capability();
|
|
58580
59785
|
init_pending_stage_route_readiness();
|
|
58581
59786
|
init_pending_stage_status();
|
|
58582
|
-
|
|
59787
|
+
init_possible_stage_rows();
|
|
58583
59788
|
init_store_internal();
|
|
58584
59789
|
init_workflow_stage_target();
|
|
58585
59790
|
});
|
|
@@ -61130,7 +62335,7 @@ function isInfrastructureError(value2) {
|
|
|
61130
62335
|
}
|
|
61131
62336
|
function isRetryableWorkflowError(error) {
|
|
61132
62337
|
let current = error;
|
|
61133
|
-
for (let depth = 0;depth <
|
|
62338
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined && current !== null; depth++) {
|
|
61134
62339
|
if (isInfrastructureError(current))
|
|
61135
62340
|
return true;
|
|
61136
62341
|
current = current instanceof Error ? current.cause : undefined;
|
|
@@ -61148,7 +62353,7 @@ function asWorkflowInfrastructureError(error) {
|
|
|
61148
62353
|
const code = workflowInfrastructureFault(error.message);
|
|
61149
62354
|
return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
|
|
61150
62355
|
}
|
|
61151
|
-
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError",
|
|
62356
|
+
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH2 = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
|
|
61152
62357
|
var init_workflow_infrastructure_error = __esm(() => {
|
|
61153
62358
|
FAULT_PATTERNS = [
|
|
61154
62359
|
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
@@ -61409,9 +62614,146 @@ var init_definitions = __esm(() => {
|
|
|
61409
62614
|
workerDefinitionSources = new WeakMap;
|
|
61410
62615
|
});
|
|
61411
62616
|
|
|
62617
|
+
// dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
|
|
62618
|
+
function push(map, key, value2) {
|
|
62619
|
+
if (key === undefined)
|
|
62620
|
+
return;
|
|
62621
|
+
const list = map.get(key);
|
|
62622
|
+
if (list)
|
|
62623
|
+
list.push(value2);
|
|
62624
|
+
else
|
|
62625
|
+
map.set(key, [value2]);
|
|
62626
|
+
}
|
|
62627
|
+
|
|
62628
|
+
class Partitions {
|
|
62629
|
+
byRun = new Map;
|
|
62630
|
+
startsByChild = new Map;
|
|
62631
|
+
stagesByGroup = new Map;
|
|
62632
|
+
stagesByReplayKey = new Map;
|
|
62633
|
+
boundariesByStageId = new Map;
|
|
62634
|
+
boundariesByChild = new Map;
|
|
62635
|
+
add(entry) {
|
|
62636
|
+
const checkpoint = entry.checkpoint;
|
|
62637
|
+
push(this.byRun, checkpoint.topology?.run?.runId, entry);
|
|
62638
|
+
if (checkpoint.kind !== "stage")
|
|
62639
|
+
return;
|
|
62640
|
+
const stage = entry;
|
|
62641
|
+
push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
|
|
62642
|
+
push(this.stagesByReplayKey, checkpoint.replayKey, stage);
|
|
62643
|
+
const boundary = checkpoint.topology?.boundary;
|
|
62644
|
+
if (!boundary)
|
|
62645
|
+
return;
|
|
62646
|
+
push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
|
|
62647
|
+
push(this.boundariesByChild, boundary.child?.runId, stage);
|
|
62648
|
+
if (boundary.event === "start")
|
|
62649
|
+
push(this.startsByChild, boundary.child?.runId, stage);
|
|
62650
|
+
}
|
|
62651
|
+
}
|
|
62652
|
+
function inKeyOrder(entries) {
|
|
62653
|
+
return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
|
|
62654
|
+
}
|
|
62655
|
+
|
|
62656
|
+
class PoolCheckpointView {
|
|
62657
|
+
base;
|
|
62658
|
+
from;
|
|
62659
|
+
delta;
|
|
62660
|
+
constructor(base, from, delta) {
|
|
62661
|
+
this.base = base;
|
|
62662
|
+
this.from = from;
|
|
62663
|
+
this.delta = delta;
|
|
62664
|
+
}
|
|
62665
|
+
entries(partition, key) {
|
|
62666
|
+
const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
|
|
62667
|
+
return [...committed, ...this.delta[partition].get(key) ?? []];
|
|
62668
|
+
}
|
|
62669
|
+
}
|
|
62670
|
+
|
|
62671
|
+
class PoolCheckpointRead {
|
|
62672
|
+
index;
|
|
62673
|
+
root;
|
|
62674
|
+
pending;
|
|
62675
|
+
constructor(index, root) {
|
|
62676
|
+
this.index = index;
|
|
62677
|
+
this.root = root;
|
|
62678
|
+
}
|
|
62679
|
+
read(tx, events) {
|
|
62680
|
+
return this.index.read(tx, events, this.root, (apply) => {
|
|
62681
|
+
this.pending = apply;
|
|
62682
|
+
});
|
|
62683
|
+
}
|
|
62684
|
+
commit() {
|
|
62685
|
+
this.pending?.();
|
|
62686
|
+
this.pending = undefined;
|
|
62687
|
+
}
|
|
62688
|
+
}
|
|
62689
|
+
|
|
62690
|
+
class PoolCheckpointIndex {
|
|
62691
|
+
roots = new Map;
|
|
62692
|
+
begin(root) {
|
|
62693
|
+
return new PoolCheckpointRead(this, root);
|
|
62694
|
+
}
|
|
62695
|
+
async read(tx, events, root, onCommit) {
|
|
62696
|
+
const current = this.roots.get(root);
|
|
62697
|
+
let base = current;
|
|
62698
|
+
const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
|
|
62699
|
+
if (base && base.seq > 0) {
|
|
62700
|
+
const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
|
|
62701
|
+
if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
|
|
62702
|
+
base = undefined;
|
|
62703
|
+
}
|
|
62704
|
+
const from = base?.seq ?? 0;
|
|
62705
|
+
const to = head ? Number(head.seq) : 0;
|
|
62706
|
+
const delta = new Partitions;
|
|
62707
|
+
const added = [];
|
|
62708
|
+
if (to > from) {
|
|
62709
|
+
const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
|
|
62710
|
+
for (const row of rows) {
|
|
62711
|
+
const checkpoint = JSON.parse(String(row.payload));
|
|
62712
|
+
if (checkpoint.kind === "ui")
|
|
62713
|
+
continue;
|
|
62714
|
+
const entry = {
|
|
62715
|
+
seq: Number(row.seq),
|
|
62716
|
+
order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
|
|
62717
|
+
checkpoint
|
|
62718
|
+
};
|
|
62719
|
+
delta.add(entry);
|
|
62720
|
+
added.push(entry);
|
|
62721
|
+
}
|
|
62722
|
+
}
|
|
62723
|
+
onCommit(() => {
|
|
62724
|
+
const latest = this.roots.get(root);
|
|
62725
|
+
if (!base) {
|
|
62726
|
+
if (latest !== current)
|
|
62727
|
+
return;
|
|
62728
|
+
const fresh = {
|
|
62729
|
+
seq: to,
|
|
62730
|
+
key: head?.key ?? null,
|
|
62731
|
+
at: Number(head?.at ?? 0),
|
|
62732
|
+
partitions: delta
|
|
62733
|
+
};
|
|
62734
|
+
this.roots.set(root, fresh);
|
|
62735
|
+
return;
|
|
62736
|
+
}
|
|
62737
|
+
if (latest !== base || to <= base.seq)
|
|
62738
|
+
return;
|
|
62739
|
+
for (const entry of added)
|
|
62740
|
+
if (entry.seq > base.seq)
|
|
62741
|
+
base.partitions.add(entry);
|
|
62742
|
+
base.seq = to;
|
|
62743
|
+
base.key = head?.key ?? null;
|
|
62744
|
+
base.at = Number(head?.at ?? 0);
|
|
62745
|
+
});
|
|
62746
|
+
return new PoolCheckpointView(base, from, delta);
|
|
62747
|
+
}
|
|
62748
|
+
}
|
|
62749
|
+
var init_pool_checkpoint_index = __esm(() => {
|
|
62750
|
+
init_stage_topology_validation();
|
|
62751
|
+
init_types();
|
|
62752
|
+
});
|
|
62753
|
+
|
|
61412
62754
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
61413
62755
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
61414
|
-
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
62756
|
+
async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
|
|
61415
62757
|
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
61416
62758
|
throw new Error("Invalid pool mutation");
|
|
61417
62759
|
const checkpoint = mutation.checkpoint;
|
|
@@ -61431,42 +62773,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
|
61431
62773
|
}
|
|
61432
62774
|
if (checkpoint.kind === "ui")
|
|
61433
62775
|
return;
|
|
61434
|
-
const
|
|
61435
|
-
const records = rows.map((row) => JSON.parse(String(row.payload)));
|
|
61436
|
-
const stages = records.filter((entry) => entry.kind === "stage");
|
|
62776
|
+
const view = await checkpoints.read(tx, journal.events);
|
|
61437
62777
|
const topology = checkpoint.topology;
|
|
61438
62778
|
const run = topology?.run;
|
|
61439
62779
|
if (!topology || !run || run.rootRunId !== root || run.runId === root)
|
|
61440
62780
|
throw new Error("Pool topology scope mismatch");
|
|
61441
|
-
const owner =
|
|
62781
|
+
const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
|
|
61442
62782
|
if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
|
|
61443
62783
|
throw new Error("Pool topology scope mismatch");
|
|
61444
62784
|
if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
|
|
61445
62785
|
throw new Error("Pool session scope mismatch");
|
|
62786
|
+
const runEntries = view.entries("byRun", run.runId);
|
|
61446
62787
|
if (checkpoint.kind === "tool") {
|
|
61447
62788
|
const tool = checkpoint.topology;
|
|
61448
|
-
const peers =
|
|
62789
|
+
const peers = runEntries.map((entry) => entry.checkpoint);
|
|
61449
62790
|
const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
|
|
61450
62791
|
if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
|
|
61451
62792
|
throw new Error("Pool tool topology scope mismatch");
|
|
61452
62793
|
return;
|
|
61453
62794
|
}
|
|
61454
|
-
const
|
|
61455
|
-
const
|
|
62795
|
+
const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
|
|
62796
|
+
const relevant = [...runStages];
|
|
62797
|
+
const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
|
|
62798
|
+
for (const group of groups)
|
|
62799
|
+
relevant.push(...view.entries("stagesByGroup", group));
|
|
62800
|
+
const boundary = checkpoint.topology?.boundary;
|
|
62801
|
+
let start;
|
|
62802
|
+
if (boundary) {
|
|
62803
|
+
const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
|
|
62804
|
+
start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
62805
|
+
relevant.push(...replayed);
|
|
62806
|
+
if (start) {
|
|
62807
|
+
const startTopology = start.topology;
|
|
62808
|
+
relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
|
|
62809
|
+
relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
|
|
62810
|
+
}
|
|
62811
|
+
}
|
|
62812
|
+
const next = [...stagesExcept(relevant, checkpoint), checkpoint];
|
|
62813
|
+
const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
|
|
61456
62814
|
const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
|
|
61457
62815
|
if (stageError)
|
|
61458
62816
|
throw new Error(`Pool topology scope: ${stageError}`);
|
|
61459
|
-
const boundary = checkpoint.topology?.boundary;
|
|
61460
62817
|
if (!boundary) {
|
|
61461
62818
|
if (parseWorkflowChildResult(checkpoint.output))
|
|
61462
62819
|
throw new Error("Pool boundary output without ownership");
|
|
61463
62820
|
return;
|
|
61464
62821
|
}
|
|
61465
|
-
const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
61466
62822
|
const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
|
|
61467
62823
|
if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
|
|
61468
62824
|
throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
|
|
61469
62825
|
}
|
|
62826
|
+
function stagesExcept(entries, checkpoint) {
|
|
62827
|
+
return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
|
|
62828
|
+
}
|
|
61470
62829
|
async function sessionStore(tx, prefix) {
|
|
61471
62830
|
const database = {
|
|
61472
62831
|
descriptor: { adapter: "pool-transaction", location: prefix },
|
|
@@ -61515,20 +62874,52 @@ var init_pool_mutations = __esm(() => {
|
|
|
61515
62874
|
init_boundary_lifecycle();
|
|
61516
62875
|
init_boundary_topology();
|
|
61517
62876
|
init_stage_topology_validation();
|
|
61518
|
-
|
|
62877
|
+
init_pool_checkpoint_index();
|
|
61519
62878
|
});
|
|
61520
62879
|
|
|
61521
62880
|
// dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
|
|
61522
62881
|
function poolUsageTable(runs) {
|
|
61523
62882
|
return `${runs}_pool_usage`;
|
|
61524
62883
|
}
|
|
62884
|
+
function poolUsageTotalsTable(runs) {
|
|
62885
|
+
return `${runs}_pool_usage_totals`;
|
|
62886
|
+
}
|
|
62887
|
+
function addToTotals(totals, runs, work, sign, row) {
|
|
62888
|
+
const assignments = POOL_USAGE_FIELDS.map((field) => {
|
|
62889
|
+
const total = field.toLowerCase();
|
|
62890
|
+
const value2 = `(${sign}${row}.${total})`;
|
|
62891
|
+
return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
|
|
62892
|
+
});
|
|
62893
|
+
return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
|
|
62894
|
+
}
|
|
61525
62895
|
function poolUsageSchema(runs) {
|
|
61526
|
-
|
|
62896
|
+
const usage = poolUsageTable(runs);
|
|
62897
|
+
const totals = poolUsageTotalsTable(runs);
|
|
62898
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
62899
|
+
return `CREATE TABLE IF NOT EXISTS ${usage}(
|
|
61527
62900
|
work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
|
|
61528
62901
|
generation INTEGER NOT NULL, key TEXT NOT NULL,
|
|
61529
62902
|
input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
|
|
61530
62903
|
PRIMARY KEY(work, generation, key)
|
|
61531
|
-
)
|
|
62904
|
+
);
|
|
62905
|
+
CREATE TABLE IF NOT EXISTS ${totals}(
|
|
62906
|
+
root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
|
|
62907
|
+
${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
|
|
62908
|
+
);
|
|
62909
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
|
|
62910
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
62911
|
+
END;
|
|
62912
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
|
|
62913
|
+
WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
|
|
62914
|
+
${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
|
|
62915
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
62916
|
+
END;
|
|
62917
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
|
|
62918
|
+
DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
|
|
62919
|
+
END;
|
|
62920
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
|
|
62921
|
+
DELETE FROM ${totals} WHERE root = OLD.root;
|
|
62922
|
+
END;`;
|
|
61532
62923
|
}
|
|
61533
62924
|
function poolStageUsageKey(runId, stageId) {
|
|
61534
62925
|
return `stage:${runId}:${stageId}`;
|
|
@@ -61555,7 +62946,7 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
61555
62946
|
|
|
61556
62947
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
61557
62948
|
import { createHash as createHash4, randomUUID as randomUUID15 } from "node:crypto";
|
|
61558
|
-
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
62949
|
+
import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
|
|
61559
62950
|
function requestIdentity(request) {
|
|
61560
62951
|
return createHash4("sha256").update(request).digest("hex");
|
|
61561
62952
|
}
|
|
@@ -61594,6 +62985,7 @@ class SqlPoolQueue {
|
|
|
61594
62985
|
operations;
|
|
61595
62986
|
transaction;
|
|
61596
62987
|
journal;
|
|
62988
|
+
checkpoints = new PoolCheckpointIndex;
|
|
61597
62989
|
constructor(storage) {
|
|
61598
62990
|
this.storage = storage;
|
|
61599
62991
|
const { transaction, tables: journal } = storage;
|
|
@@ -61705,17 +63097,23 @@ class SqlPoolQueue {
|
|
|
61705
63097
|
lease = snapshotLease(lease);
|
|
61706
63098
|
token = { ...token };
|
|
61707
63099
|
return this.storage.withAppendGuard(async (tx, runId) => {
|
|
61708
|
-
|
|
61709
|
-
|
|
61710
|
-
|
|
61711
|
-
|
|
61712
|
-
|
|
61713
|
-
|
|
61714
|
-
|
|
61715
|
-
|
|
61716
|
-
|
|
61717
|
-
|
|
61718
|
-
|
|
63100
|
+
try {
|
|
63101
|
+
if (runId !== lease.runId)
|
|
63102
|
+
throw new Error("Pool delivery cannot write another root");
|
|
63103
|
+
await this.authority(tx, lease).catch((error) => {
|
|
63104
|
+
if (error instanceof StaleLeaseError)
|
|
63105
|
+
throw new Error("Stale pool coordinator authority", { cause: error });
|
|
63106
|
+
throw error;
|
|
63107
|
+
});
|
|
63108
|
+
const row = await this.current(tx, lease, token);
|
|
63109
|
+
if (row.epoch !== lease.epoch)
|
|
63110
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63111
|
+
assertSession();
|
|
63112
|
+
} catch (error) {
|
|
63113
|
+
if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
|
|
63114
|
+
throw error;
|
|
63115
|
+
throw new PoolAppendRefusedError(error.message, { cause: error });
|
|
63116
|
+
}
|
|
61719
63117
|
}, action);
|
|
61720
63118
|
}
|
|
61721
63119
|
async assertCurrent(lease, token) {
|
|
@@ -61828,7 +63226,8 @@ class SqlPoolQueue {
|
|
|
61828
63226
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
61829
63227
|
const identity = requestIdentity(request);
|
|
61830
63228
|
mutation = JSON.parse(request).mutation;
|
|
61831
|
-
|
|
63229
|
+
const checkpoints = this.checkpoints.begin(lease.runId);
|
|
63230
|
+
const receipt = await this.transaction(async (tx) => {
|
|
61832
63231
|
const { now, status } = await this.authority(tx, lease);
|
|
61833
63232
|
const row = await this.current(tx, lease, token);
|
|
61834
63233
|
if (row.epoch !== lease.epoch)
|
|
@@ -61843,24 +63242,26 @@ class SqlPoolQueue {
|
|
|
61843
63242
|
throw new Error(STALE_WORK_LEASE);
|
|
61844
63243
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
61845
63244
|
throw new Error("Pool root is not running");
|
|
61846
|
-
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
63245
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
|
|
61847
63246
|
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
61848
63247
|
return receipt;
|
|
61849
63248
|
});
|
|
63249
|
+
checkpoints.commit();
|
|
63250
|
+
return receipt;
|
|
61850
63251
|
}
|
|
61851
63252
|
async appendSession(tx, row, mutation, now) {
|
|
61852
63253
|
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
61853
63254
|
}
|
|
61854
|
-
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
63255
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
|
|
61855
63256
|
const checkpoint = mutation.checkpoint;
|
|
61856
|
-
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
63257
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
|
|
61857
63258
|
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
61858
63259
|
throw new Error("Pool checkpoint timestamp is in the future");
|
|
61859
63260
|
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
61860
63261
|
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
61861
63262
|
const payload = serializeJson(checkpoint);
|
|
61862
63263
|
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
61863
|
-
if (existing && existing.payload
|
|
63264
|
+
if (existing && !isRecordedCheckpoint(existing.payload, payload))
|
|
61864
63265
|
throw new Error("Pool checkpoint identity conflict");
|
|
61865
63266
|
if (existing && mutation.kind !== "checkpoint")
|
|
61866
63267
|
throw new Error("Pool session checkpoint identity conflict");
|
|
@@ -61958,22 +63359,37 @@ class SqlPoolQueue {
|
|
|
61958
63359
|
return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
|
|
61959
63360
|
}
|
|
61960
63361
|
}
|
|
63362
|
+
function isRecordedCheckpoint(stored, payload) {
|
|
63363
|
+
if (stored === payload)
|
|
63364
|
+
return true;
|
|
63365
|
+
const { completedAt } = JSON.parse(stored);
|
|
63366
|
+
if (typeof completedAt !== "number")
|
|
63367
|
+
return false;
|
|
63368
|
+
return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
|
|
63369
|
+
}
|
|
61961
63370
|
function openSqlPoolQueue(journal) {
|
|
61962
63371
|
const storage = sqlPoolStorage(journal);
|
|
61963
63372
|
if (!storage)
|
|
61964
63373
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
61965
63374
|
return SqlPoolQueue.open(storage);
|
|
61966
63375
|
}
|
|
61967
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
63376
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
|
|
61968
63377
|
var init_sql_pool_queue = __esm(() => {
|
|
61969
63378
|
init_serializable();
|
|
61970
63379
|
init_boundary_topology();
|
|
61971
63380
|
init_durable_hash();
|
|
61972
63381
|
init_internal();
|
|
63382
|
+
init_pool_checkpoint_index();
|
|
61973
63383
|
init_pool_mutations();
|
|
61974
63384
|
init_pool_stage_usage();
|
|
61975
63385
|
init_types();
|
|
61976
63386
|
init_values();
|
|
63387
|
+
PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
|
|
63388
|
+
constructor(message, options) {
|
|
63389
|
+
super(message, options);
|
|
63390
|
+
this.name = "PoolAppendRefusedError";
|
|
63391
|
+
}
|
|
63392
|
+
};
|
|
61977
63393
|
});
|
|
61978
63394
|
|
|
61979
63395
|
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
@@ -61996,15 +63412,22 @@ function validate(report) {
|
|
|
61996
63412
|
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
61997
63413
|
}
|
|
61998
63414
|
}
|
|
63415
|
+
function addCompensated(total, value2) {
|
|
63416
|
+
const sum = total.sum + value2;
|
|
63417
|
+
total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
|
|
63418
|
+
total.sum = sum;
|
|
63419
|
+
}
|
|
61999
63420
|
|
|
62000
63421
|
class SqlPoolUsage {
|
|
62001
63422
|
transaction;
|
|
62002
63423
|
journal;
|
|
62003
63424
|
table;
|
|
63425
|
+
totals;
|
|
62004
63426
|
constructor(transaction, journal) {
|
|
62005
63427
|
this.transaction = transaction;
|
|
62006
63428
|
this.journal = journal;
|
|
62007
63429
|
this.table = poolUsageTable(journal.runs);
|
|
63430
|
+
this.totals = poolUsageTotalsTable(journal.runs);
|
|
62008
63431
|
}
|
|
62009
63432
|
static async open(transaction, journal) {
|
|
62010
63433
|
const usage = new SqlPoolUsage(transaction, journal);
|
|
@@ -62012,9 +63435,27 @@ class SqlPoolUsage {
|
|
|
62012
63435
|
return usage;
|
|
62013
63436
|
}
|
|
62014
63437
|
async totalIn(tx, root) {
|
|
63438
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
63439
|
+
const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
|
|
63440
|
+
if (kept)
|
|
63441
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
|
|
63442
|
+
field,
|
|
63443
|
+
Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
|
|
63444
|
+
]));
|
|
62015
63445
|
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
62016
63446
|
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
62017
|
-
|
|
63447
|
+
const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
63448
|
+
if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
|
|
63449
|
+
return total;
|
|
63450
|
+
const sums = columns.map(() => ({ sum: 0, error: 0 }));
|
|
63451
|
+
const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
|
|
63452
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
63453
|
+
for (const usage of rows)
|
|
63454
|
+
for (const [index, column] of columns.entries())
|
|
63455
|
+
addCompensated(sums[index], Number(usage[column]));
|
|
63456
|
+
await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
|
|
63457
|
+
VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
|
|
63458
|
+
return total;
|
|
62018
63459
|
}
|
|
62019
63460
|
async report(lease, token, reports) {
|
|
62020
63461
|
validateText("runId", lease.runId);
|
|
@@ -65169,7 +66610,7 @@ function describeErrorWithCauses(error) {
|
|
|
65169
66610
|
const messages = [];
|
|
65170
66611
|
const visited = new Set;
|
|
65171
66612
|
let current = error;
|
|
65172
|
-
for (let depth = 0;current !== undefined && depth <
|
|
66613
|
+
for (let depth = 0;current !== undefined && depth < MAX_CAUSE_DEPTH3 && !visited.has(current); depth++) {
|
|
65173
66614
|
visited.add(current);
|
|
65174
66615
|
const message = current instanceof Error ? current.message : String(current);
|
|
65175
66616
|
if (!messages.includes(message))
|
|
@@ -65178,7 +66619,7 @@ function describeErrorWithCauses(error) {
|
|
|
65178
66619
|
}
|
|
65179
66620
|
return messages.join(": ");
|
|
65180
66621
|
}
|
|
65181
|
-
var
|
|
66622
|
+
var MAX_CAUSE_DEPTH3 = 5;
|
|
65182
66623
|
|
|
65183
66624
|
// dist/builtin/workflows/src/shared/returned-run-status.ts
|
|
65184
66625
|
function normalizeReturnedWorkflowStatus(status) {
|
|
@@ -65558,10 +66999,120 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
|
65558
66999
|
}
|
|
65559
67000
|
var init_stage_session_store = () => {};
|
|
65560
67001
|
|
|
67002
|
+
// dist/builtin/workflows/src/durable/completed-catalog-index.ts
|
|
67003
|
+
class CheckpointCatalogIndex {
|
|
67004
|
+
checkpoints;
|
|
67005
|
+
stages;
|
|
67006
|
+
positions = new Map;
|
|
67007
|
+
groupKeys = new Map;
|
|
67008
|
+
groups = new Map;
|
|
67009
|
+
stagesByRunId = new Map;
|
|
67010
|
+
boundariesByReplayKey = new Map;
|
|
67011
|
+
boundariesByStageId = new Map;
|
|
67012
|
+
boundariesByChildRunId = new Map;
|
|
67013
|
+
outputsByReplayKey = new Map;
|
|
67014
|
+
evidence;
|
|
67015
|
+
constructor(checkpoints) {
|
|
67016
|
+
this.checkpoints = checkpoints;
|
|
67017
|
+
const stages = [];
|
|
67018
|
+
checkpoints.forEach((checkpoint, index) => {
|
|
67019
|
+
if (!this.positions.has(checkpoint))
|
|
67020
|
+
this.positions.set(checkpoint, index);
|
|
67021
|
+
if (checkpoint.kind !== "stage")
|
|
67022
|
+
return;
|
|
67023
|
+
stages.push(checkpoint);
|
|
67024
|
+
const key = durableStageGroupKey(checkpoint);
|
|
67025
|
+
this.groupKeys.set(checkpoint, key);
|
|
67026
|
+
append(this.groups, key, checkpoint);
|
|
67027
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
67028
|
+
if (runId !== undefined)
|
|
67029
|
+
append(this.stagesByRunId, runId, checkpoint);
|
|
67030
|
+
if (checkpoint.output !== undefined)
|
|
67031
|
+
append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
|
|
67032
|
+
const boundary = checkpoint.topology?.boundary;
|
|
67033
|
+
if (boundary === undefined)
|
|
67034
|
+
return;
|
|
67035
|
+
append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
|
|
67036
|
+
append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
|
|
67037
|
+
append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
|
|
67038
|
+
});
|
|
67039
|
+
this.stages = stages;
|
|
67040
|
+
}
|
|
67041
|
+
sequence(checkpoint) {
|
|
67042
|
+
return (this.positions.get(checkpoint) ?? -1) + 1;
|
|
67043
|
+
}
|
|
67044
|
+
stageGroups() {
|
|
67045
|
+
return this.groups.values();
|
|
67046
|
+
}
|
|
67047
|
+
boundaryLifecycle(start, requireTerminal) {
|
|
67048
|
+
const topology = start.topology;
|
|
67049
|
+
const childRunId = topology?.boundary?.child.runId;
|
|
67050
|
+
const related = this.inCheckpointOrder([
|
|
67051
|
+
this.boundariesByReplayKey.get(start.replayKey),
|
|
67052
|
+
topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
|
|
67053
|
+
childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
|
|
67054
|
+
]);
|
|
67055
|
+
return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
|
|
67056
|
+
}
|
|
67057
|
+
directChildRecords(childRunId) {
|
|
67058
|
+
return {
|
|
67059
|
+
claiming: this.stagesByRunId.get(childRunId) ?? [],
|
|
67060
|
+
direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
|
|
67061
|
+
};
|
|
67062
|
+
}
|
|
67063
|
+
childScopedEvidenceExists(boundaryReplayKey, childRunId) {
|
|
67064
|
+
this.evidence ??= scopedEvidenceIndex(this.checkpoints);
|
|
67065
|
+
return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
|
|
67066
|
+
}
|
|
67067
|
+
inCheckpointOrder(lists) {
|
|
67068
|
+
const present = lists.filter((list) => list !== undefined);
|
|
67069
|
+
if (present.length === 1)
|
|
67070
|
+
return present[0];
|
|
67071
|
+
return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
|
|
67072
|
+
}
|
|
67073
|
+
}
|
|
67074
|
+
function scopedEvidenceIndex(checkpoints) {
|
|
67075
|
+
const runIds = new Set;
|
|
67076
|
+
const identities = [];
|
|
67077
|
+
for (const checkpoint of checkpoints) {
|
|
67078
|
+
identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
|
|
67079
|
+
if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
|
|
67080
|
+
runIds.add(checkpoint.topology.run.runId);
|
|
67081
|
+
}
|
|
67082
|
+
}
|
|
67083
|
+
identities.sort(compareCodeUnits);
|
|
67084
|
+
return { runIds, identities };
|
|
67085
|
+
}
|
|
67086
|
+
function hasPrefix(sorted, prefix) {
|
|
67087
|
+
let low = 0;
|
|
67088
|
+
let high = sorted.length;
|
|
67089
|
+
while (low < high) {
|
|
67090
|
+
const middle = low + high >>> 1;
|
|
67091
|
+
if (compareCodeUnits(sorted[middle], prefix) < 0)
|
|
67092
|
+
low = middle + 1;
|
|
67093
|
+
else
|
|
67094
|
+
high = middle;
|
|
67095
|
+
}
|
|
67096
|
+
return low < sorted.length && sorted[low].startsWith(prefix);
|
|
67097
|
+
}
|
|
67098
|
+
function compareCodeUnits(left, right) {
|
|
67099
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
67100
|
+
}
|
|
67101
|
+
function append(map, key, value2) {
|
|
67102
|
+
const list = map.get(key);
|
|
67103
|
+
if (list === undefined)
|
|
67104
|
+
map.set(key, [value2]);
|
|
67105
|
+
else
|
|
67106
|
+
list.push(value2);
|
|
67107
|
+
}
|
|
67108
|
+
var init_completed_catalog_index = __esm(() => {
|
|
67109
|
+
init_boundary_lifecycle();
|
|
67110
|
+
init_stage_topology_validation();
|
|
67111
|
+
});
|
|
67112
|
+
|
|
65561
67113
|
// dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
|
|
65562
|
-
function validBoundaryRecordSet(
|
|
65563
|
-
const
|
|
65564
|
-
const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
|
|
67114
|
+
function validBoundaryRecordSet(index, strict) {
|
|
67115
|
+
const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
|
|
65565
67116
|
const startsByReplay = new Map;
|
|
65566
67117
|
for (const checkpoint of boundaries) {
|
|
65567
67118
|
if (checkpoint.topology.boundary.event !== "start")
|
|
@@ -65576,7 +67127,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
|
|
|
65576
67127
|
return false;
|
|
65577
67128
|
return [...startsByReplay.values()].every((starts) => {
|
|
65578
67129
|
const start = starts[0];
|
|
65579
|
-
if (
|
|
67130
|
+
if ("error" in index.boundaryLifecycle(start, strict))
|
|
65580
67131
|
return false;
|
|
65581
67132
|
return start.topology.boundary.invocationFingerprint !== undefined;
|
|
65582
67133
|
});
|
|
@@ -65640,8 +67191,8 @@ function compareDraftSourceOrder(left, right) {
|
|
|
65640
67191
|
function runTopologyFor(drafts, tools) {
|
|
65641
67192
|
return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
|
|
65642
67193
|
}
|
|
65643
|
-
function validateRunGroups(grouped, rootRunId,
|
|
65644
|
-
const toolsFor = (runId) =>
|
|
67194
|
+
function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
|
|
67195
|
+
const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
|
|
65645
67196
|
const owners = new Map;
|
|
65646
67197
|
const syntheticChildIds = new Set;
|
|
65647
67198
|
for (const [parentRunId, drafts] of grouped) {
|
|
@@ -65652,12 +67203,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
|
|
|
65652
67203
|
if (childRunId === parentRunId || owners.has(childRunId))
|
|
65653
67204
|
return false;
|
|
65654
67205
|
if (!grouped.has(childRunId)) {
|
|
65655
|
-
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(
|
|
67206
|
+
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
|
|
65656
67207
|
syntheticChildIds.add(childRunId);
|
|
65657
67208
|
owners.set(childRunId, draft);
|
|
65658
67209
|
continue;
|
|
65659
67210
|
}
|
|
65660
|
-
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(
|
|
67211
|
+
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
|
|
65661
67212
|
return false;
|
|
65662
67213
|
syntheticChildIds.add(childRunId);
|
|
65663
67214
|
owners.set(childRunId, draft);
|
|
@@ -65698,13 +67249,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
|
|
|
65698
67249
|
const boundary = draft.topology?.boundary;
|
|
65699
67250
|
return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
|
|
65700
67251
|
}
|
|
65701
|
-
function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
65702
|
-
const prefix = `${boundaryReplayKey}:`;
|
|
65703
|
-
return checkpoints.some((checkpoint) => {
|
|
65704
|
-
const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
|
|
65705
|
-
return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
|
|
65706
|
-
});
|
|
65707
|
-
}
|
|
65708
67252
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
65709
67253
|
const reachable = new Set([rootRunId]);
|
|
65710
67254
|
const settled = new Set;
|
|
@@ -65745,14 +67289,6 @@ function childRunIdFromDraft(draft) {
|
|
|
65745
67289
|
return;
|
|
65746
67290
|
return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
|
|
65747
67291
|
}
|
|
65748
|
-
function boundaryOwner(grouped, childRunId) {
|
|
65749
|
-
for (const drafts of grouped.values()) {
|
|
65750
|
-
const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
|
|
65751
|
-
if (owner !== undefined)
|
|
65752
|
-
return owner;
|
|
65753
|
-
}
|
|
65754
|
-
return;
|
|
65755
|
-
}
|
|
65756
67292
|
function childRunStatus(owner) {
|
|
65757
67293
|
const child = workflowChildFromDraft(owner);
|
|
65758
67294
|
if (child !== undefined)
|
|
@@ -65760,15 +67296,15 @@ function childRunStatus(owner) {
|
|
|
65760
67296
|
const status = owner.topology?.boundary?.status;
|
|
65761
67297
|
return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
|
|
65762
67298
|
}
|
|
65763
|
-
function mergeStageGroup(records,
|
|
67299
|
+
function mergeStageGroup(records, index) {
|
|
65764
67300
|
let merged;
|
|
65765
67301
|
for (const record of records)
|
|
65766
|
-
merged = mergeStageDraft(merged, record,
|
|
67302
|
+
merged = mergeStageDraft(merged, record, index.sequence(record));
|
|
65767
67303
|
const draft = merged;
|
|
65768
67304
|
const start = records.find((record) => record.topology?.boundary?.event === "start");
|
|
65769
67305
|
if (start === undefined)
|
|
65770
67306
|
return draft;
|
|
65771
|
-
const resolved =
|
|
67307
|
+
const resolved = index.boundaryLifecycle(start, false);
|
|
65772
67308
|
if ("error" in resolved)
|
|
65773
67309
|
return draft;
|
|
65774
67310
|
const terminal = resolved.lifecycle.latestTerminal;
|
|
@@ -65831,9 +67367,7 @@ function workflowChildFromDraft(draft) {
|
|
|
65831
67367
|
...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
|
|
65832
67368
|
};
|
|
65833
67369
|
}
|
|
65834
|
-
var init_completed_catalog_stage_groups =
|
|
65835
|
-
init_boundary_lifecycle();
|
|
65836
|
-
});
|
|
67370
|
+
var init_completed_catalog_stage_groups = () => {};
|
|
65837
67371
|
|
|
65838
67372
|
// dist/builtin/workflows/src/engine/run-tool-control-registry.ts
|
|
65839
67373
|
function createToolControlRegistry() {
|
|
@@ -66038,11 +67572,68 @@ var init_resume_runtime = __esm(() => {
|
|
|
66038
67572
|
function isReservedRunCheckpointHash(argsHash) {
|
|
66039
67573
|
return [RUN_TIMING_CHECKPOINT_NAME, RUN_USAGE_CHECKPOINT_NAME, RUN_RESULT_CHECKPOINT_NAME].includes(argsHash) || argsHash === WORKFLOW_HEARTBEAT_ANCHOR_CHECKPOINT_NAME;
|
|
66040
67574
|
}
|
|
66041
|
-
function
|
|
67575
|
+
function durableNestedRunCatalog(backend, rootWorkflowId) {
|
|
66042
67576
|
const handle = backend.getWorkflow(rootWorkflowId);
|
|
66043
67577
|
if (handle === undefined)
|
|
66044
|
-
return
|
|
66045
|
-
|
|
67578
|
+
return EMPTY_NESTED_RUN_CATALOG;
|
|
67579
|
+
const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
|
|
67580
|
+
const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
|
|
67581
|
+
if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
|
|
67582
|
+
return cached.catalog;
|
|
67583
|
+
}
|
|
67584
|
+
const checkpoints = backend.listCheckpoints(rootWorkflowId);
|
|
67585
|
+
const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
|
|
67586
|
+
const catalog = nestedRunCatalog(runs, checkpoints);
|
|
67587
|
+
if (version !== undefined) {
|
|
67588
|
+
const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
|
|
67589
|
+
nestedRunCatalogs.set(backend, byRoot);
|
|
67590
|
+
byRoot.delete(rootWorkflowId);
|
|
67591
|
+
byRoot.set(rootWorkflowId, {
|
|
67592
|
+
version,
|
|
67593
|
+
name: handle.name,
|
|
67594
|
+
updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
|
|
67595
|
+
catalog
|
|
67596
|
+
});
|
|
67597
|
+
if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
|
|
67598
|
+
byRoot.delete(byRoot.keys().next().value);
|
|
67599
|
+
}
|
|
67600
|
+
return catalog;
|
|
67601
|
+
}
|
|
67602
|
+
function nestedRunCatalog(runs, checkpoints) {
|
|
67603
|
+
const positionsById = new Map;
|
|
67604
|
+
const childIdsByParent = new Map;
|
|
67605
|
+
runs.forEach((run, position) => {
|
|
67606
|
+
const positions = positionsById.get(run.id);
|
|
67607
|
+
if (positions === undefined)
|
|
67608
|
+
positionsById.set(run.id, [position]);
|
|
67609
|
+
else
|
|
67610
|
+
positions.push(position);
|
|
67611
|
+
if (run.parentRunId === undefined)
|
|
67612
|
+
return;
|
|
67613
|
+
const children = childIdsByParent.get(run.parentRunId);
|
|
67614
|
+
if (children === undefined)
|
|
67615
|
+
childIdsByParent.set(run.parentRunId, [run.id]);
|
|
67616
|
+
else
|
|
67617
|
+
children.push(run.id);
|
|
67618
|
+
});
|
|
67619
|
+
const intentionalExitStatuses = new Map;
|
|
67620
|
+
for (const checkpoint of checkpoints) {
|
|
67621
|
+
if (checkpoint.kind !== "stage")
|
|
67622
|
+
continue;
|
|
67623
|
+
const child = parseWorkflowChildResult(checkpoint.output);
|
|
67624
|
+
if (child?.exited === true)
|
|
67625
|
+
intentionalExitStatuses.set(child.runId, child.status);
|
|
67626
|
+
}
|
|
67627
|
+
return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
|
|
67628
|
+
}
|
|
67629
|
+
function readsFallbackCompletedAt(run) {
|
|
67630
|
+
return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
|
|
67631
|
+
}
|
|
67632
|
+
function cloneRunSnapshot(run) {
|
|
67633
|
+
const copy = structuredClone(run);
|
|
67634
|
+
for (const tool of copy.toolNodes)
|
|
67635
|
+
Object.freeze(tool.parentIds);
|
|
67636
|
+
return copy;
|
|
66046
67637
|
}
|
|
66047
67638
|
function checkpointDrafts(checkpoints) {
|
|
66048
67639
|
const stageByReplayKey = new Map;
|
|
@@ -66066,11 +67657,8 @@ function checkpointDrafts(checkpoints) {
|
|
|
66066
67657
|
tools: [...toolByHash.values()]
|
|
66067
67658
|
};
|
|
66068
67659
|
}
|
|
66069
|
-
function completedToolNodes(
|
|
66070
|
-
return
|
|
66071
|
-
const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
66072
|
-
if (topologyRunId !== runId)
|
|
66073
|
-
return [];
|
|
67660
|
+
function completedToolNodes(ownedCheckpoints, sourceIds) {
|
|
67661
|
+
return ownedCheckpoints.flatMap((checkpoint) => {
|
|
66074
67662
|
const topology = checkpoint.topology;
|
|
66075
67663
|
if (topology === undefined)
|
|
66076
67664
|
return [];
|
|
@@ -66107,18 +67695,34 @@ function summarizeCompletedToolResult(value2) {
|
|
|
66107
67695
|
return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
|
|
66108
67696
|
}
|
|
66109
67697
|
function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
|
|
66110
|
-
|
|
67698
|
+
const index = new CheckpointCatalogIndex(checkpoints);
|
|
67699
|
+
if (!validBoundaryRecordSet(index, policy.requireTerminal))
|
|
66111
67700
|
return [];
|
|
66112
|
-
const stageRecords =
|
|
67701
|
+
const stageRecords = index.stages;
|
|
66113
67702
|
const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
|
|
66114
67703
|
if (immutableStageGroupError(identityRecords) !== undefined)
|
|
66115
67704
|
return [];
|
|
66116
|
-
const
|
|
66117
|
-
const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
67705
|
+
const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
66118
67706
|
const drafts = checkpointDrafts(checkpoints);
|
|
66119
67707
|
const toolCheckpoints = drafts.tools;
|
|
66120
67708
|
if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
|
|
66121
67709
|
return [];
|
|
67710
|
+
const toolNodeIdsByClaimedRun = new Map;
|
|
67711
|
+
const toolsByRun = new Map;
|
|
67712
|
+
for (const checkpoint of toolCheckpoints) {
|
|
67713
|
+
const claimedRunId = checkpoint.topology?.run?.runId;
|
|
67714
|
+
if (claimedRunId !== undefined) {
|
|
67715
|
+
const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
|
|
67716
|
+
ids.add(checkpoint.topology.nodeId);
|
|
67717
|
+
toolNodeIdsByClaimedRun.set(claimedRunId, ids);
|
|
67718
|
+
}
|
|
67719
|
+
const ownerRunId = claimedRunId ?? rootRunId;
|
|
67720
|
+
const owned = toolsByRun.get(ownerRunId);
|
|
67721
|
+
if (owned === undefined)
|
|
67722
|
+
toolsByRun.set(ownerRunId, [checkpoint]);
|
|
67723
|
+
else
|
|
67724
|
+
owned.push(checkpoint);
|
|
67725
|
+
}
|
|
66122
67726
|
for (const draft of candidates) {
|
|
66123
67727
|
const boundary = draft.topology?.boundary;
|
|
66124
67728
|
if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
|
|
@@ -66132,8 +67736,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66132
67736
|
parentStageId: boundary.child.parentStageId,
|
|
66133
67737
|
rootRunId: boundary.child.rootRunId
|
|
66134
67738
|
};
|
|
66135
|
-
const childToolIds =
|
|
66136
|
-
if (
|
|
67739
|
+
const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
|
|
67740
|
+
if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
|
|
66137
67741
|
return [];
|
|
66138
67742
|
}
|
|
66139
67743
|
if (candidates.length === 0 && toolCheckpoints.length === 0) {
|
|
@@ -66161,7 +67765,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66161
67765
|
group.sort(compareDraftSourceOrder);
|
|
66162
67766
|
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
66163
67767
|
return [];
|
|
66164
|
-
if (!validateRunGroups(grouped, rootRunId,
|
|
67768
|
+
if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
|
|
66165
67769
|
return [];
|
|
66166
67770
|
const syntheticChildren = [];
|
|
66167
67771
|
const pendingChildren = [];
|
|
@@ -66175,7 +67779,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66175
67779
|
const child = workflowChildFromDraft(draft);
|
|
66176
67780
|
if (child?.exited !== true || grouped.has(child.runId))
|
|
66177
67781
|
continue;
|
|
66178
|
-
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(
|
|
67782
|
+
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
|
|
66179
67783
|
return [];
|
|
66180
67784
|
}
|
|
66181
67785
|
syntheticChildren.push({ parentRunId, owner: draft, child });
|
|
@@ -66183,7 +67787,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66183
67787
|
}
|
|
66184
67788
|
const idMaps = new Map;
|
|
66185
67789
|
for (const [runId, runDrafts] of grouped) {
|
|
66186
|
-
const ownedTools =
|
|
67790
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
66187
67791
|
if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
|
|
66188
67792
|
return [];
|
|
66189
67793
|
const ids = new Map;
|
|
@@ -66201,32 +67805,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66201
67805
|
});
|
|
66202
67806
|
if (hasIdentityConflict)
|
|
66203
67807
|
return [];
|
|
66204
|
-
for (const checkpoint of
|
|
66205
|
-
const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
67808
|
+
for (const checkpoint of ownedTools) {
|
|
66206
67809
|
const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
|
|
66207
|
-
|
|
66208
|
-
ids.set(sourceId, sourceId);
|
|
67810
|
+
ids.set(sourceId, sourceId);
|
|
66209
67811
|
}
|
|
66210
67812
|
idMaps.set(runId, ids);
|
|
66211
67813
|
}
|
|
67814
|
+
const boundaryOwners = new Map;
|
|
67815
|
+
const boundarySourcesByParent = new Map;
|
|
67816
|
+
for (const [parentRunId, runDrafts] of grouped) {
|
|
67817
|
+
const sources = new Map;
|
|
67818
|
+
for (const draft of runDrafts) {
|
|
67819
|
+
const childRunId = childRunIdFromDraft(draft);
|
|
67820
|
+
if (childRunId === undefined)
|
|
67821
|
+
continue;
|
|
67822
|
+
if (!boundaryOwners.has(childRunId))
|
|
67823
|
+
boundaryOwners.set(childRunId, draft);
|
|
67824
|
+
if (!sources.has(childRunId))
|
|
67825
|
+
sources.set(childRunId, draft);
|
|
67826
|
+
}
|
|
67827
|
+
boundarySourcesByParent.set(parentRunId, sources);
|
|
67828
|
+
}
|
|
66212
67829
|
const runs = [];
|
|
66213
67830
|
const emittedRunIds = new Set;
|
|
66214
67831
|
for (const [runId, runDrafts] of grouped) {
|
|
66215
67832
|
const ids = idMaps.get(runId);
|
|
66216
|
-
const ownedTools =
|
|
67833
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
66217
67834
|
const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
|
|
66218
67835
|
if (hasUnknownParents)
|
|
66219
67836
|
return [];
|
|
66220
67837
|
const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
|
|
66221
|
-
const toolNodes = completedToolNodes(
|
|
67838
|
+
const toolNodes = completedToolNodes(ownedTools, ids);
|
|
66222
67839
|
const run = runTopologyFor(runDrafts, ownedTools);
|
|
66223
67840
|
const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
|
|
66224
67841
|
const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
|
|
66225
|
-
const owner = runId === rootRunId ? undefined :
|
|
67842
|
+
const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
|
|
66226
67843
|
const status = owner === undefined ? "completed" : childRunStatus(owner);
|
|
66227
67844
|
const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
|
|
66228
67845
|
const parentRunId = run?.parentRunId ?? rootRunId;
|
|
66229
|
-
const boundarySourceId =
|
|
67846
|
+
const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
|
|
66230
67847
|
const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
|
|
66231
67848
|
const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
|
|
66232
67849
|
const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
|
|
@@ -66362,10 +67979,11 @@ function emptyGraphRun(runId, runName, completedAt) {
|
|
|
66362
67979
|
resumable: false
|
|
66363
67980
|
};
|
|
66364
67981
|
}
|
|
66365
|
-
var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
67982
|
+
var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
66366
67983
|
var init_completed_catalog = __esm(() => {
|
|
66367
67984
|
init_stage_session_store();
|
|
66368
67985
|
init_tool_payload_bounds();
|
|
67986
|
+
init_completed_catalog_index();
|
|
66369
67987
|
init_completed_catalog_stage_groups();
|
|
66370
67988
|
init_resume_eligibility();
|
|
66371
67989
|
init_resume_runtime();
|
|
@@ -66374,6 +67992,13 @@ var init_completed_catalog = __esm(() => {
|
|
|
66374
67992
|
init_workflow_heartbeat_anchor();
|
|
66375
67993
|
init_stage_topology_validation();
|
|
66376
67994
|
init_tool_outcome();
|
|
67995
|
+
nestedRunCatalogs = new WeakMap;
|
|
67996
|
+
EMPTY_NESTED_RUN_CATALOG = {
|
|
67997
|
+
runs: [],
|
|
67998
|
+
positionsById: new Map,
|
|
67999
|
+
childIdsByParent: new Map,
|
|
68000
|
+
intentionalExitStatuses: new Map
|
|
68001
|
+
};
|
|
66377
68002
|
COMPLETED_RECONSTRUCTION = {
|
|
66378
68003
|
requireTerminal: true,
|
|
66379
68004
|
retainActive: false,
|
|
@@ -66388,29 +68013,23 @@ var init_completed_catalog = __esm(() => {
|
|
|
66388
68013
|
|
|
66389
68014
|
// dist/builtin/workflows/src/durable/completed-subtree.ts
|
|
66390
68015
|
function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
|
|
66391
|
-
const
|
|
66392
|
-
if (!
|
|
68016
|
+
const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
|
|
68017
|
+
if (!catalog.positionsById.has(childRunId))
|
|
66393
68018
|
return;
|
|
66394
|
-
const intentionalExitStatuses = new Map;
|
|
66395
|
-
for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
|
|
66396
|
-
if (checkpoint.kind !== "stage")
|
|
66397
|
-
continue;
|
|
66398
|
-
const child = parseWorkflowChildResult(checkpoint.output);
|
|
66399
|
-
if (child?.exited === true)
|
|
66400
|
-
intentionalExitStatuses.set(child.runId, child.status);
|
|
66401
|
-
}
|
|
66402
68019
|
const included = new Set([childRunId]);
|
|
66403
|
-
|
|
66404
|
-
|
|
66405
|
-
|
|
66406
|
-
|
|
66407
|
-
|
|
66408
|
-
|
|
68020
|
+
const pending = [childRunId];
|
|
68021
|
+
for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
|
|
68022
|
+
for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
|
|
68023
|
+
if (included.has(childId))
|
|
68024
|
+
continue;
|
|
68025
|
+
included.add(childId);
|
|
68026
|
+
pending.push(childId);
|
|
66409
68027
|
}
|
|
66410
68028
|
}
|
|
66411
|
-
const
|
|
66412
|
-
const
|
|
66413
|
-
|
|
68029
|
+
const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
|
|
68030
|
+
const subtree = positions.map((position) => catalog.runs[position]);
|
|
68031
|
+
const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
|
|
68032
|
+
return complete ? subtree.map(cloneRunSnapshot) : undefined;
|
|
66414
68033
|
}
|
|
66415
68034
|
function isCompleteCachedRun(run, intentionalExitStatuses) {
|
|
66416
68035
|
if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
|
|
@@ -66531,7 +68150,7 @@ function createDurableCachedStageRecorder(input) {
|
|
|
66531
68150
|
const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
|
|
66532
68151
|
kind: "stage",
|
|
66533
68152
|
workflowId: input.run.id,
|
|
66534
|
-
checkpointId:
|
|
68153
|
+
checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
|
|
66535
68154
|
name,
|
|
66536
68155
|
replayKey,
|
|
66537
68156
|
completedAt: Date.now(),
|
|
@@ -66703,7 +68322,7 @@ function recordedOwners(boundary, reason) {
|
|
|
66703
68322
|
const pending = [{ value: reason, depth: 0 }];
|
|
66704
68323
|
while (pending.length > 0) {
|
|
66705
68324
|
const { value: value2, depth } = pending.pop();
|
|
66706
|
-
if (depth >
|
|
68325
|
+
if (depth > MAX_CAUSE_DEPTH4)
|
|
66707
68326
|
continue;
|
|
66708
68327
|
if (isPrimitive(value2)) {
|
|
66709
68328
|
if (depth === 0 || value2 !== undefined && value2 !== null) {
|
|
@@ -67025,7 +68644,7 @@ function containUiRejections(ui, scope) {
|
|
|
67025
68644
|
custom: (factory, options) => scope.watch(ui.custom(factory, options))
|
|
67026
68645
|
};
|
|
67027
68646
|
}
|
|
67028
|
-
var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections",
|
|
68647
|
+
var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections", MAX_CAUSE_DEPTH4 = 8, MAX_PRIMITIVES_PER_RUN = 64, MAX_STOP_STRINGS = 64, ENDED_RUN_GRACE_MS = 30000, NATIVE_REPORTING_MODES, WATCHED_STAGE_METHODS;
|
|
67029
68648
|
var init_run_rejection_boundary = __esm(() => {
|
|
67030
68649
|
init_prompt_callsite();
|
|
67031
68650
|
init_workflow_source();
|
|
@@ -72916,27 +74535,6 @@ var init_runtime = __esm(() => {
|
|
|
72916
74535
|
init_intercom_group();
|
|
72917
74536
|
});
|
|
72918
74537
|
|
|
72919
|
-
// dist/builtin/workflows/src/engine/session-closed-error.ts
|
|
72920
|
-
import { isStaleExtensionContextError as isStaleExtensionContextError2 } from "@bastani/atomic";
|
|
72921
|
-
function reportsClosedSession(error) {
|
|
72922
|
-
return "code" in error && error.code === "SessionClosed" || isStaleExtensionContextError2(error);
|
|
72923
|
-
}
|
|
72924
|
-
function isSessionClosedError(error) {
|
|
72925
|
-
let current = error;
|
|
72926
|
-
for (let depth = 0;depth < MAX_CAUSE_DEPTH4 && current !== undefined; depth++) {
|
|
72927
|
-
try {
|
|
72928
|
-
if (reportsClosedSession(current))
|
|
72929
|
-
return true;
|
|
72930
|
-
current = current.cause instanceof Error ? current.cause : undefined;
|
|
72931
|
-
} catch {
|
|
72932
|
-
return false;
|
|
72933
|
-
}
|
|
72934
|
-
}
|
|
72935
|
-
return false;
|
|
72936
|
-
}
|
|
72937
|
-
var MAX_CAUSE_DEPTH4 = 8;
|
|
72938
|
-
var init_session_closed_error = () => {};
|
|
72939
|
-
|
|
72940
74538
|
// dist/builtin/workflows/src/engine/workflow-activity.ts
|
|
72941
74539
|
import { runCallback as runCallback4 } from "@bastani/atomic";
|
|
72942
74540
|
function nextEventLoopTurn() {
|
|
@@ -74178,6 +75776,7 @@ async function runWorker(start) {
|
|
|
74178
75776
|
"getStageOutput",
|
|
74179
75777
|
"getStageSession",
|
|
74180
75778
|
"listCheckpoints",
|
|
75779
|
+
"nestedSnapshotVersion",
|
|
74181
75780
|
"getWorkflow",
|
|
74182
75781
|
"getLoadableWorkflow",
|
|
74183
75782
|
"toMetadata",
|