@bastani/atomic 0.9.44-alpha.5 → 0.9.44-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/builtin/intercom/CHANGELOG.md +6 -0
- package/dist/builtin/intercom/broker/broker.ts +13 -1
- package/dist/builtin/intercom/index.bundle.mjs +41 -5
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/types.ts +7 -0
- package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +20 -0
- package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
- package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
- package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +2514 -741
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2384 -727
- package/dist/builtin/workflows/src/index.js +2383 -727
- package/dist/builtin/workflows/src/local-worker-entry.mjs +2384 -727
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +16 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/storage/node-sqlite.d.ts.map +1 -1
- package/dist/core/storage/node-sqlite.js +130 -13
- package/dist/core/storage/node-sqlite.js.map +1 -1
- package/docs/intercom/reference.md +2 -0
- package/docs/workflows/operations.md +1 -1
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -172,13 +172,19 @@ function registerRunJournalVersions(journal, versions) {
|
|
|
172
172
|
function runJournalVersions(journal) {
|
|
173
173
|
return versionReaders.get(journal);
|
|
174
174
|
}
|
|
175
|
+
function registerRunJournalAppendMany(journal, appendMany) {
|
|
176
|
+
appendManyWriters.set(journal, appendMany);
|
|
177
|
+
}
|
|
178
|
+
function runJournalAppendMany(journal) {
|
|
179
|
+
return appendManyWriters.get(journal);
|
|
180
|
+
}
|
|
175
181
|
function registerSqlPoolStorage(journal, storage) {
|
|
176
182
|
poolStorage.set(journal, storage);
|
|
177
183
|
}
|
|
178
184
|
function sqlPoolStorage(journal) {
|
|
179
185
|
return poolStorage.get(journal);
|
|
180
186
|
}
|
|
181
|
-
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
|
|
187
|
+
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
|
|
182
188
|
var init_internal = __esm(() => {
|
|
183
189
|
init_types();
|
|
184
190
|
NON_SNAPSHOT_EVENT_TYPES = [
|
|
@@ -187,6 +193,7 @@ var init_internal = __esm(() => {
|
|
|
187
193
|
RUN_EVENT_TYPES.messageDelivered
|
|
188
194
|
];
|
|
189
195
|
versionReaders = new WeakMap;
|
|
196
|
+
appendManyWriters = new WeakMap;
|
|
190
197
|
poolStorage = new WeakMap;
|
|
191
198
|
});
|
|
192
199
|
|
|
@@ -297,6 +304,204 @@ var init_resume_eligibility = () => {};
|
|
|
297
304
|
// dist/builtin/workflows/src/durable/types.ts
|
|
298
305
|
var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
|
|
299
306
|
|
|
307
|
+
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
308
|
+
function immutableStageGroupError(stages) {
|
|
309
|
+
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
310
|
+
if (occurrenceError !== undefined)
|
|
311
|
+
return occurrenceError;
|
|
312
|
+
const byReplayKey = groupByDurableStageKey(stages);
|
|
313
|
+
for (const group of byReplayKey.values()) {
|
|
314
|
+
const firstName = group[0]?.name;
|
|
315
|
+
if (group.some((stage) => stage.name !== firstName))
|
|
316
|
+
return "stage name drift within one replay-key group";
|
|
317
|
+
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
318
|
+
const first = topologies[0];
|
|
319
|
+
if (first === undefined)
|
|
320
|
+
continue;
|
|
321
|
+
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
322
|
+
return "immutable stage identity drift within one replay-key group";
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
328
|
+
return directChildTopologyErrorFromRecords({
|
|
329
|
+
claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
|
|
330
|
+
direct: (claiming) => {
|
|
331
|
+
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
332
|
+
return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
333
|
+
}
|
|
334
|
+
}, replayScope, expectedRun, allowedNodeIds);
|
|
335
|
+
}
|
|
336
|
+
function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
337
|
+
const prefix = `${replayScope}:`;
|
|
338
|
+
const claiming = records.claiming;
|
|
339
|
+
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
340
|
+
return "child run topology exists outside its durable invocation scope";
|
|
341
|
+
}
|
|
342
|
+
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
343
|
+
return "child checkpoint ownership is not reciprocal";
|
|
344
|
+
}
|
|
345
|
+
const direct = records.direct(claiming);
|
|
346
|
+
const immutableError = immutableStageGroupError(direct);
|
|
347
|
+
if (immutableError !== undefined)
|
|
348
|
+
return immutableError;
|
|
349
|
+
const merged = [];
|
|
350
|
+
for (const group of groupByDurableStageKey(direct).values()) {
|
|
351
|
+
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
352
|
+
if (topology === undefined)
|
|
353
|
+
return "direct child stage group lacks owning-run topology";
|
|
354
|
+
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
355
|
+
return "direct child stage topology lacks current source identity";
|
|
356
|
+
}
|
|
357
|
+
if (topology.stageId.length === 0)
|
|
358
|
+
return "direct child stage id is empty";
|
|
359
|
+
merged.push(topology);
|
|
360
|
+
}
|
|
361
|
+
const ids = new Set(allowedNodeIds);
|
|
362
|
+
const stageIds = new Set;
|
|
363
|
+
const orders = new Set;
|
|
364
|
+
for (const topology of merged) {
|
|
365
|
+
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
366
|
+
return "direct child stage ids are not unique";
|
|
367
|
+
stageIds.add(topology.stageId);
|
|
368
|
+
ids.add(topology.stageId);
|
|
369
|
+
if (topology.sourceOrder !== undefined) {
|
|
370
|
+
if (orders.has(topology.sourceOrder))
|
|
371
|
+
return "direct child source orders are not unique";
|
|
372
|
+
orders.add(topology.sourceOrder);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
376
|
+
return "direct child stage references an unknown parent";
|
|
377
|
+
}
|
|
378
|
+
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
379
|
+
}
|
|
380
|
+
function owningBoundaryIdError(stages, owningRunId) {
|
|
381
|
+
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
382
|
+
const seen = new Map;
|
|
383
|
+
for (const start of starts) {
|
|
384
|
+
const stageId = start.topology.stageId;
|
|
385
|
+
if (stageId.length === 0)
|
|
386
|
+
return "owning-run boundary stage id is empty";
|
|
387
|
+
const prior = seen.get(stageId);
|
|
388
|
+
if (prior !== undefined && prior !== start.replayKey)
|
|
389
|
+
return "owning-run boundary stage id is aliased";
|
|
390
|
+
seen.set(stageId, start.replayKey);
|
|
391
|
+
}
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
function sameRun(actual, expected) {
|
|
395
|
+
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
396
|
+
}
|
|
397
|
+
function sameStrings(left, right) {
|
|
398
|
+
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
399
|
+
}
|
|
400
|
+
function sameOptionalRun(actual, expected) {
|
|
401
|
+
if (actual === undefined || expected === undefined)
|
|
402
|
+
return actual === expected;
|
|
403
|
+
return sameRun(actual, expected);
|
|
404
|
+
}
|
|
405
|
+
function promptOccurrenceIdentityError(stages) {
|
|
406
|
+
const replayByOccurrence = new Map;
|
|
407
|
+
const occurrenceByReplayStage = new Map;
|
|
408
|
+
const stageByReplayOccurrence = new Map;
|
|
409
|
+
for (const stage of stages) {
|
|
410
|
+
const topology = stage.topology;
|
|
411
|
+
const occurrence = topology?.occurrenceKey;
|
|
412
|
+
if (topology === undefined || occurrence === undefined)
|
|
413
|
+
continue;
|
|
414
|
+
const priorReplay = replayByOccurrence.get(occurrence);
|
|
415
|
+
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
416
|
+
return "prompt occurrence is bound to multiple replay keys";
|
|
417
|
+
}
|
|
418
|
+
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
419
|
+
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
420
|
+
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
421
|
+
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
422
|
+
return "prompt stage identity is bound to multiple occurrences";
|
|
423
|
+
}
|
|
424
|
+
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
425
|
+
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
426
|
+
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
427
|
+
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
428
|
+
return "prompt occurrence is bound to multiple stage identities";
|
|
429
|
+
}
|
|
430
|
+
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
431
|
+
}
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
function durableStageGroupKey(stage) {
|
|
435
|
+
const occurrence = stage.topology?.occurrenceKey;
|
|
436
|
+
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
437
|
+
}
|
|
438
|
+
function groupByDurableStageKey(stages) {
|
|
439
|
+
const groups = new Map;
|
|
440
|
+
for (const stage of stages) {
|
|
441
|
+
const key = durableStageGroupKey(stage);
|
|
442
|
+
const group = groups.get(key) ?? [];
|
|
443
|
+
group.push(stage);
|
|
444
|
+
groups.set(key, group);
|
|
445
|
+
}
|
|
446
|
+
return groups;
|
|
447
|
+
}
|
|
448
|
+
function compositeKey(domain, ...parts) {
|
|
449
|
+
return JSON.stringify([domain, ...parts]);
|
|
450
|
+
}
|
|
451
|
+
function hasCycle(topologies) {
|
|
452
|
+
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
453
|
+
const visiting = new Set;
|
|
454
|
+
const visited = new Set;
|
|
455
|
+
const cyclic = (stageId) => {
|
|
456
|
+
if (visiting.has(stageId))
|
|
457
|
+
return true;
|
|
458
|
+
if (visited.has(stageId))
|
|
459
|
+
return false;
|
|
460
|
+
visiting.add(stageId);
|
|
461
|
+
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
462
|
+
return true;
|
|
463
|
+
visiting.delete(stageId);
|
|
464
|
+
visited.add(stageId);
|
|
465
|
+
return false;
|
|
466
|
+
};
|
|
467
|
+
return topologies.some((topology) => cyclic(topology.stageId));
|
|
468
|
+
}
|
|
469
|
+
var init_stage_topology_validation = () => {};
|
|
470
|
+
|
|
471
|
+
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
472
|
+
function parseWorkflowChildResult(value2) {
|
|
473
|
+
if (!isOrdinaryObject(value2))
|
|
474
|
+
return;
|
|
475
|
+
const workflow = value2.workflow;
|
|
476
|
+
const runId = value2.runId;
|
|
477
|
+
const status = value2.status;
|
|
478
|
+
const exited = value2.exited;
|
|
479
|
+
const outputs = value2.outputs;
|
|
480
|
+
const exitReason = value2.exitReason;
|
|
481
|
+
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
482
|
+
return;
|
|
483
|
+
if (exited === false) {
|
|
484
|
+
if (status !== "completed" || exitReason !== undefined)
|
|
485
|
+
return;
|
|
486
|
+
return value2;
|
|
487
|
+
}
|
|
488
|
+
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
489
|
+
return;
|
|
490
|
+
return value2;
|
|
491
|
+
}
|
|
492
|
+
function isWorkflowChildResult(value2) {
|
|
493
|
+
return parseWorkflowChildResult(value2) !== undefined;
|
|
494
|
+
}
|
|
495
|
+
function isWorkflowExitStatus(value2) {
|
|
496
|
+
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
497
|
+
}
|
|
498
|
+
function isOrdinaryObject(value2) {
|
|
499
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
500
|
+
return false;
|
|
501
|
+
const prototype = Object.getPrototypeOf(value2);
|
|
502
|
+
return prototype === Object.prototype || prototype === null;
|
|
503
|
+
}
|
|
504
|
+
|
|
300
505
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
301
506
|
import { createHash } from "node:crypto";
|
|
302
507
|
function durableHash(value2) {
|
|
@@ -363,6 +568,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
|
|
|
363
568
|
function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
|
|
364
569
|
return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
|
|
365
570
|
}
|
|
571
|
+
|
|
572
|
+
class StageReceipts {
|
|
573
|
+
exact = new Map;
|
|
574
|
+
anySession = new Map;
|
|
575
|
+
has(logicalRunId, sessionId, admission) {
|
|
576
|
+
return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
|
|
577
|
+
}
|
|
578
|
+
hasAnySession(logicalRunId, admission) {
|
|
579
|
+
return this.anySession.get(logicalRunId)?.has(admission) ?? false;
|
|
580
|
+
}
|
|
581
|
+
add(logicalRunId, sessionId, admission) {
|
|
582
|
+
let bySession = this.exact.get(logicalRunId);
|
|
583
|
+
if (bySession === undefined) {
|
|
584
|
+
bySession = new Map;
|
|
585
|
+
this.exact.set(logicalRunId, bySession);
|
|
586
|
+
}
|
|
587
|
+
let admissions = bySession.get(sessionId);
|
|
588
|
+
if (admissions === undefined) {
|
|
589
|
+
admissions = new Set;
|
|
590
|
+
bySession.set(sessionId, admissions);
|
|
591
|
+
}
|
|
592
|
+
if (admissions.has(admission))
|
|
593
|
+
return false;
|
|
594
|
+
admissions.add(admission);
|
|
595
|
+
return true;
|
|
596
|
+
}
|
|
597
|
+
addAnySession(logicalRunId, admission) {
|
|
598
|
+
let admissions = this.anySession.get(logicalRunId);
|
|
599
|
+
if (admissions === undefined) {
|
|
600
|
+
admissions = new Set;
|
|
601
|
+
this.anySession.set(logicalRunId, admissions);
|
|
602
|
+
}
|
|
603
|
+
if (admissions.has(admission))
|
|
604
|
+
return false;
|
|
605
|
+
admissions.add(admission);
|
|
606
|
+
return true;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
class PendingStageReceiptIndex {
|
|
611
|
+
workflows = new Map;
|
|
612
|
+
add(workflowId, messageId, delivery, logicalRunId) {
|
|
613
|
+
let workflow = this.workflows.get(workflowId);
|
|
614
|
+
if (workflow === undefined) {
|
|
615
|
+
workflow = { byMessage: new Map, log: [] };
|
|
616
|
+
this.workflows.set(workflowId, workflow);
|
|
617
|
+
}
|
|
618
|
+
let byRun = workflow.byMessage.get(messageId);
|
|
619
|
+
if (byRun === undefined) {
|
|
620
|
+
byRun = new Map;
|
|
621
|
+
workflow.byMessage.set(messageId, byRun);
|
|
622
|
+
}
|
|
623
|
+
let byStage = byRun.get(delivery.runId);
|
|
624
|
+
if (byStage === undefined) {
|
|
625
|
+
byStage = new Map;
|
|
626
|
+
byRun.set(delivery.runId, byStage);
|
|
627
|
+
}
|
|
628
|
+
let receipts = byStage.get(delivery.stageId);
|
|
629
|
+
if (receipts === undefined) {
|
|
630
|
+
receipts = new StageReceipts;
|
|
631
|
+
byStage.set(delivery.stageId, receipts);
|
|
632
|
+
}
|
|
633
|
+
const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
|
|
634
|
+
let changed = added;
|
|
635
|
+
if (receipts.add(logicalRunId, undefined, undefined))
|
|
636
|
+
changed = true;
|
|
637
|
+
if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
|
|
638
|
+
changed = true;
|
|
639
|
+
if (changed)
|
|
640
|
+
workflow.log.push(messageId);
|
|
641
|
+
return added;
|
|
642
|
+
}
|
|
643
|
+
has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
|
|
644
|
+
const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
|
|
645
|
+
if (receipts === undefined)
|
|
646
|
+
return false;
|
|
647
|
+
if (anySession && delivery.admission === "context")
|
|
648
|
+
return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
|
|
649
|
+
return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
|
|
650
|
+
}
|
|
651
|
+
changesSince(workflowId, cursor) {
|
|
652
|
+
let workflow = this.workflows.get(workflowId);
|
|
653
|
+
if (workflow === undefined) {
|
|
654
|
+
workflow = { byMessage: new Map, log: [] };
|
|
655
|
+
this.workflows.set(workflowId, workflow);
|
|
656
|
+
}
|
|
657
|
+
const next = { log: workflow.log, length: workflow.log.length };
|
|
658
|
+
if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
|
|
659
|
+
return { cursor: next, messageIds: undefined };
|
|
660
|
+
return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
|
|
661
|
+
}
|
|
662
|
+
delete(workflowId) {
|
|
663
|
+
this.workflows.delete(workflowId);
|
|
664
|
+
}
|
|
665
|
+
clear() {
|
|
666
|
+
this.workflows.clear();
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
function nextVersion() {
|
|
670
|
+
lastVersion += 1;
|
|
671
|
+
return lastVersion;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
class NestedSnapshotVersion {
|
|
675
|
+
rootWorkflowId;
|
|
676
|
+
current = nextVersion();
|
|
677
|
+
latestCompletedAt = Number.NEGATIVE_INFINITY;
|
|
678
|
+
replayGroups = new Map;
|
|
679
|
+
boundaryReplayKeys = new Set;
|
|
680
|
+
constructor(rootWorkflowId) {
|
|
681
|
+
this.rootWorkflowId = rootWorkflowId;
|
|
682
|
+
}
|
|
683
|
+
get version() {
|
|
684
|
+
return this.current;
|
|
685
|
+
}
|
|
686
|
+
record(checkpoint) {
|
|
687
|
+
if (!this.isInert(checkpoint))
|
|
688
|
+
this.current = nextVersion();
|
|
689
|
+
this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
|
|
690
|
+
if (checkpoint.kind !== "stage")
|
|
691
|
+
return;
|
|
692
|
+
if (checkpoint.topology?.boundary !== undefined)
|
|
693
|
+
this.boundaryReplayKeys.add(checkpoint.replayKey);
|
|
694
|
+
if (checkpoint.topology?.occurrenceKey !== undefined)
|
|
695
|
+
return;
|
|
696
|
+
const identity = stageIdentity(checkpoint);
|
|
697
|
+
const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
|
|
698
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
699
|
+
if (summary === undefined) {
|
|
700
|
+
this.replayGroups.set(checkpoint.replayKey, {
|
|
701
|
+
name: checkpoint.name,
|
|
702
|
+
identity,
|
|
703
|
+
consistent: true,
|
|
704
|
+
hasOutput: checkpoint.output !== undefined,
|
|
705
|
+
hasExitedChildOutput: exited
|
|
706
|
+
});
|
|
707
|
+
return;
|
|
708
|
+
}
|
|
709
|
+
summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
|
|
710
|
+
summary.hasOutput ||= checkpoint.output !== undefined;
|
|
711
|
+
summary.hasExitedChildOutput ||= exited;
|
|
712
|
+
}
|
|
713
|
+
isInert(checkpoint) {
|
|
714
|
+
if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
|
|
715
|
+
return false;
|
|
716
|
+
}
|
|
717
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
718
|
+
if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
|
|
719
|
+
return false;
|
|
720
|
+
}
|
|
721
|
+
const id = checkpoint.checkpointId;
|
|
722
|
+
for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
|
|
723
|
+
if (this.boundaryReplayKeys.has(id.slice(0, colon)))
|
|
724
|
+
return false;
|
|
725
|
+
}
|
|
726
|
+
return true;
|
|
727
|
+
}
|
|
728
|
+
isRootOwned(checkpoint) {
|
|
729
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
730
|
+
return runId === undefined || runId === this.rootWorkflowId;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
function stageIdentity(checkpoint) {
|
|
734
|
+
const topology = checkpoint.topology;
|
|
735
|
+
return JSON.stringify([
|
|
736
|
+
durableStageGroupKey(checkpoint),
|
|
737
|
+
topology?.version,
|
|
738
|
+
topology?.stageId,
|
|
739
|
+
topology?.parentIds,
|
|
740
|
+
topology?.sourceOrder,
|
|
741
|
+
topology?.run === undefined ? undefined : [
|
|
742
|
+
topology.run.runId,
|
|
743
|
+
topology.run.runName,
|
|
744
|
+
topology.run.parentRunId,
|
|
745
|
+
topology.run.parentStageId,
|
|
746
|
+
topology.run.rootRunId
|
|
747
|
+
]
|
|
748
|
+
]);
|
|
749
|
+
}
|
|
366
750
|
function checkpointKey(c) {
|
|
367
751
|
return `${c.kind}:${c.checkpointId}`;
|
|
368
752
|
}
|
|
@@ -374,7 +758,7 @@ class JournalProjection {
|
|
|
374
758
|
workflows = new Map;
|
|
375
759
|
deletedWorkflowIds = new Set;
|
|
376
760
|
pendingReceipts = new Map;
|
|
377
|
-
stickyReceipts = new
|
|
761
|
+
stickyReceipts = new PendingStageReceiptIndex;
|
|
378
762
|
registerWorkflow(handle, authoritativeOwner = false) {
|
|
379
763
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
380
764
|
const existing = this.workflows.get(handle.workflowId);
|
|
@@ -422,7 +806,8 @@ class JournalProjection {
|
|
|
422
806
|
toolByHash: new Map,
|
|
423
807
|
uiByHash: new Map,
|
|
424
808
|
stageOutputByReplayKey: new Map,
|
|
425
|
-
stageSessionByReplayKey: new Map
|
|
809
|
+
stageSessionByReplayKey: new Map,
|
|
810
|
+
nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
|
|
426
811
|
});
|
|
427
812
|
}
|
|
428
813
|
applyWorkflowSnapshot(handle) {
|
|
@@ -448,39 +833,43 @@ class JournalProjection {
|
|
|
448
833
|
return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
|
|
449
834
|
}
|
|
450
835
|
stickyDeliveryCounts = new Map;
|
|
836
|
+
stickyDeliveryCountVersions = new Map;
|
|
451
837
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
452
|
-
return this.stickyDeliveryCounts.get(workflowId)?.get(
|
|
838
|
+
return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
|
|
839
|
+
}
|
|
840
|
+
pendingStageDeliveryCountVersion(workflowId) {
|
|
841
|
+
return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
|
|
842
|
+
}
|
|
843
|
+
bumpDeliveryCountVersion(workflowId) {
|
|
844
|
+
this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
|
|
453
845
|
}
|
|
454
846
|
async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
|
|
455
|
-
|
|
456
|
-
if (
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
|
|
464
|
-
if (delivery.admission === "context" && delivery.sessionId !== undefined)
|
|
465
|
-
receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
|
|
466
|
-
if (accounting !== undefined && !existing) {
|
|
467
|
-
let counts = this.stickyDeliveryCounts.get(workflowId);
|
|
847
|
+
const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
|
|
848
|
+
if (accounting !== undefined && added) {
|
|
849
|
+
let byRun = this.stickyDeliveryCounts.get(workflowId);
|
|
850
|
+
if (byRun === undefined) {
|
|
851
|
+
byRun = new Map;
|
|
852
|
+
this.stickyDeliveryCounts.set(workflowId, byRun);
|
|
853
|
+
}
|
|
854
|
+
let counts = byRun.get(accounting.messageRunId);
|
|
468
855
|
if (counts === undefined) {
|
|
469
856
|
counts = new Map;
|
|
470
|
-
|
|
857
|
+
byRun.set(accounting.messageRunId, counts);
|
|
471
858
|
}
|
|
472
|
-
const
|
|
473
|
-
counts.set(
|
|
859
|
+
const previous = counts.get(messageId) ?? 0;
|
|
860
|
+
counts.set(messageId, Math.max(previous, accounting.deliveryCount));
|
|
861
|
+
if (accounting.deliveryCount > previous)
|
|
862
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
474
863
|
}
|
|
475
864
|
}
|
|
476
865
|
async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
|
|
477
|
-
return
|
|
866
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
|
|
478
867
|
}
|
|
479
868
|
hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
|
|
480
|
-
return
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
869
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
|
|
870
|
+
}
|
|
871
|
+
pendingStageDeliveryReceiptChanges(workflowId, cursor) {
|
|
872
|
+
return this.stickyReceipts.changesSince(workflowId, cursor);
|
|
484
873
|
}
|
|
485
874
|
recordCheckpoint(recorded) {
|
|
486
875
|
const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
|
|
@@ -499,6 +888,7 @@ class JournalProjection {
|
|
|
499
888
|
if (rec.checkpoints.has(key))
|
|
500
889
|
return;
|
|
501
890
|
rec.checkpoints.set(key, currentCheckpoint);
|
|
891
|
+
rec.nestedSnapshotVersion.record(currentCheckpoint);
|
|
502
892
|
if (currentCheckpoint.kind === "tool") {
|
|
503
893
|
const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
|
|
504
894
|
if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
|
|
@@ -556,6 +946,9 @@ class JournalProjection {
|
|
|
556
946
|
getWorkflow(workflowId) {
|
|
557
947
|
return this.workflows.get(workflowId)?.handle;
|
|
558
948
|
}
|
|
949
|
+
nestedSnapshotVersion(workflowId) {
|
|
950
|
+
return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
|
|
951
|
+
}
|
|
559
952
|
getLoadableWorkflow(workflowId) {
|
|
560
953
|
return this.getWorkflow(workflowId);
|
|
561
954
|
}
|
|
@@ -633,6 +1026,7 @@ class JournalProjection {
|
|
|
633
1026
|
this.pendingReceipts.delete(workflowId);
|
|
634
1027
|
this.stickyReceipts.delete(workflowId);
|
|
635
1028
|
this.stickyDeliveryCounts.delete(workflowId);
|
|
1029
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
636
1030
|
this.deletedWorkflowIds.add(workflowId);
|
|
637
1031
|
}
|
|
638
1032
|
isWorkflowLoadable(workflowId) {
|
|
@@ -643,6 +1037,8 @@ class JournalProjection {
|
|
|
643
1037
|
this.pendingReceipts.clear();
|
|
644
1038
|
this.stickyReceipts.clear();
|
|
645
1039
|
this.stickyDeliveryCounts.clear();
|
|
1040
|
+
for (const workflowId of this.stickyDeliveryCountVersions.keys())
|
|
1041
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
646
1042
|
this.deletedWorkflowIds.clear();
|
|
647
1043
|
}
|
|
648
1044
|
}
|
|
@@ -696,216 +1092,32 @@ function toResumableEntry(handle) {
|
|
|
696
1092
|
updatedAt: handle.updatedAt
|
|
697
1093
|
};
|
|
698
1094
|
}
|
|
699
|
-
|
|
700
|
-
const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
|
|
701
|
-
return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
|
|
702
|
-
}
|
|
703
|
-
var SESSION_HEADER_READ_LIMIT_BYTES;
|
|
1095
|
+
var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
|
|
704
1096
|
var init_backend = __esm(() => {
|
|
705
1097
|
init_resume_eligibility();
|
|
1098
|
+
init_stage_topology_validation();
|
|
706
1099
|
init_durable_hash();
|
|
707
1100
|
SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
|
|
708
1101
|
});
|
|
709
1102
|
|
|
710
|
-
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
711
|
-
function immutableStageGroupError(stages) {
|
|
712
|
-
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
713
|
-
if (occurrenceError !== undefined)
|
|
714
|
-
return occurrenceError;
|
|
715
|
-
const byReplayKey = groupByDurableStageKey(stages);
|
|
716
|
-
for (const group of byReplayKey.values()) {
|
|
717
|
-
const firstName = group[0]?.name;
|
|
718
|
-
if (group.some((stage) => stage.name !== firstName))
|
|
719
|
-
return "stage name drift within one replay-key group";
|
|
720
|
-
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
721
|
-
const first = topologies[0];
|
|
722
|
-
if (first === undefined)
|
|
723
|
-
continue;
|
|
724
|
-
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
725
|
-
return "immutable stage identity drift within one replay-key group";
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
return;
|
|
729
|
-
}
|
|
730
|
-
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
731
|
-
const prefix = `${replayScope}:`;
|
|
732
|
-
const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
|
|
733
|
-
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
734
|
-
return "child run topology exists outside its durable invocation scope";
|
|
735
|
-
}
|
|
736
|
-
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
737
|
-
return "child checkpoint ownership is not reciprocal";
|
|
738
|
-
}
|
|
739
|
-
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
740
|
-
const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
741
|
-
const immutableError = immutableStageGroupError(direct);
|
|
742
|
-
if (immutableError !== undefined)
|
|
743
|
-
return immutableError;
|
|
744
|
-
const merged = [];
|
|
745
|
-
for (const group of groupByDurableStageKey(direct).values()) {
|
|
746
|
-
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
747
|
-
if (topology === undefined)
|
|
748
|
-
return "direct child stage group lacks owning-run topology";
|
|
749
|
-
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
750
|
-
return "direct child stage topology lacks current source identity";
|
|
751
|
-
}
|
|
752
|
-
if (topology.stageId.length === 0)
|
|
753
|
-
return "direct child stage id is empty";
|
|
754
|
-
merged.push(topology);
|
|
755
|
-
}
|
|
756
|
-
const ids = new Set(allowedNodeIds);
|
|
757
|
-
const stageIds = new Set;
|
|
758
|
-
const orders = new Set;
|
|
759
|
-
for (const topology of merged) {
|
|
760
|
-
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
761
|
-
return "direct child stage ids are not unique";
|
|
762
|
-
stageIds.add(topology.stageId);
|
|
763
|
-
ids.add(topology.stageId);
|
|
764
|
-
if (topology.sourceOrder !== undefined) {
|
|
765
|
-
if (orders.has(topology.sourceOrder))
|
|
766
|
-
return "direct child source orders are not unique";
|
|
767
|
-
orders.add(topology.sourceOrder);
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
771
|
-
return "direct child stage references an unknown parent";
|
|
772
|
-
}
|
|
773
|
-
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
774
|
-
}
|
|
775
|
-
function owningBoundaryIdError(stages, owningRunId) {
|
|
776
|
-
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
777
|
-
const seen = new Map;
|
|
778
|
-
for (const start of starts) {
|
|
779
|
-
const stageId = start.topology.stageId;
|
|
780
|
-
if (stageId.length === 0)
|
|
781
|
-
return "owning-run boundary stage id is empty";
|
|
782
|
-
const prior = seen.get(stageId);
|
|
783
|
-
if (prior !== undefined && prior !== start.replayKey)
|
|
784
|
-
return "owning-run boundary stage id is aliased";
|
|
785
|
-
seen.set(stageId, start.replayKey);
|
|
786
|
-
}
|
|
787
|
-
return;
|
|
788
|
-
}
|
|
789
|
-
function sameRun(actual, expected) {
|
|
790
|
-
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
791
|
-
}
|
|
792
|
-
function sameStrings(left, right) {
|
|
793
|
-
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
794
|
-
}
|
|
795
|
-
function sameOptionalRun(actual, expected) {
|
|
796
|
-
if (actual === undefined || expected === undefined)
|
|
797
|
-
return actual === expected;
|
|
798
|
-
return sameRun(actual, expected);
|
|
799
|
-
}
|
|
800
|
-
function promptOccurrenceIdentityError(stages) {
|
|
801
|
-
const replayByOccurrence = new Map;
|
|
802
|
-
const occurrenceByReplayStage = new Map;
|
|
803
|
-
const stageByReplayOccurrence = new Map;
|
|
804
|
-
for (const stage of stages) {
|
|
805
|
-
const topology = stage.topology;
|
|
806
|
-
const occurrence = topology?.occurrenceKey;
|
|
807
|
-
if (topology === undefined || occurrence === undefined)
|
|
808
|
-
continue;
|
|
809
|
-
const priorReplay = replayByOccurrence.get(occurrence);
|
|
810
|
-
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
811
|
-
return "prompt occurrence is bound to multiple replay keys";
|
|
812
|
-
}
|
|
813
|
-
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
814
|
-
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
815
|
-
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
816
|
-
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
817
|
-
return "prompt stage identity is bound to multiple occurrences";
|
|
818
|
-
}
|
|
819
|
-
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
820
|
-
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
821
|
-
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
822
|
-
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
823
|
-
return "prompt occurrence is bound to multiple stage identities";
|
|
824
|
-
}
|
|
825
|
-
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
826
|
-
}
|
|
827
|
-
return;
|
|
828
|
-
}
|
|
829
|
-
function durableStageGroupKey(stage) {
|
|
830
|
-
const occurrence = stage.topology?.occurrenceKey;
|
|
831
|
-
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
832
|
-
}
|
|
833
|
-
function groupByDurableStageKey(stages) {
|
|
834
|
-
const groups = new Map;
|
|
835
|
-
for (const stage of stages) {
|
|
836
|
-
const key = durableStageGroupKey(stage);
|
|
837
|
-
const group = groups.get(key) ?? [];
|
|
838
|
-
group.push(stage);
|
|
839
|
-
groups.set(key, group);
|
|
840
|
-
}
|
|
841
|
-
return groups;
|
|
842
|
-
}
|
|
843
|
-
function compositeKey(domain, ...parts) {
|
|
844
|
-
return JSON.stringify([domain, ...parts]);
|
|
845
|
-
}
|
|
846
|
-
function hasCycle(topologies) {
|
|
847
|
-
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
848
|
-
const visiting = new Set;
|
|
849
|
-
const visited = new Set;
|
|
850
|
-
const cyclic = (stageId) => {
|
|
851
|
-
if (visiting.has(stageId))
|
|
852
|
-
return true;
|
|
853
|
-
if (visited.has(stageId))
|
|
854
|
-
return false;
|
|
855
|
-
visiting.add(stageId);
|
|
856
|
-
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
857
|
-
return true;
|
|
858
|
-
visiting.delete(stageId);
|
|
859
|
-
visited.add(stageId);
|
|
860
|
-
return false;
|
|
861
|
-
};
|
|
862
|
-
return topologies.some((topology) => cyclic(topology.stageId));
|
|
863
|
-
}
|
|
864
|
-
var init_stage_topology_validation = () => {};
|
|
865
|
-
|
|
866
|
-
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
867
|
-
function parseWorkflowChildResult(value2) {
|
|
868
|
-
if (!isOrdinaryObject(value2))
|
|
869
|
-
return;
|
|
870
|
-
const workflow = value2.workflow;
|
|
871
|
-
const runId = value2.runId;
|
|
872
|
-
const status = value2.status;
|
|
873
|
-
const exited = value2.exited;
|
|
874
|
-
const outputs = value2.outputs;
|
|
875
|
-
const exitReason = value2.exitReason;
|
|
876
|
-
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
877
|
-
return;
|
|
878
|
-
if (exited === false) {
|
|
879
|
-
if (status !== "completed" || exitReason !== undefined)
|
|
880
|
-
return;
|
|
881
|
-
return value2;
|
|
882
|
-
}
|
|
883
|
-
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
884
|
-
return;
|
|
885
|
-
return value2;
|
|
886
|
-
}
|
|
887
|
-
function isWorkflowChildResult(value2) {
|
|
888
|
-
return parseWorkflowChildResult(value2) !== undefined;
|
|
889
|
-
}
|
|
890
|
-
function isWorkflowExitStatus(value2) {
|
|
891
|
-
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
892
|
-
}
|
|
893
|
-
function isOrdinaryObject(value2) {
|
|
894
|
-
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
895
|
-
return false;
|
|
896
|
-
const prototype = Object.getPrototypeOf(value2);
|
|
897
|
-
return prototype === Object.prototype || prototype === null;
|
|
898
|
-
}
|
|
899
|
-
|
|
900
1103
|
// dist/builtin/workflows/src/durable/boundary-lifecycle.ts
|
|
901
1104
|
function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
902
1105
|
const startTopology = start.topology;
|
|
903
1106
|
const startBoundary = startTopology?.boundary;
|
|
904
|
-
const startRun = startTopology?.run;
|
|
905
1107
|
const related = stages.filter((stage) => {
|
|
906
1108
|
const boundary = stage.topology?.boundary;
|
|
907
1109
|
return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
|
|
908
1110
|
});
|
|
1111
|
+
return resolveBoundaryLifecycleFromRecords(start, {
|
|
1112
|
+
related,
|
|
1113
|
+
outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
|
|
1114
|
+
}, requireTerminal);
|
|
1115
|
+
}
|
|
1116
|
+
function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
|
|
1117
|
+
const startTopology = start.topology;
|
|
1118
|
+
const startBoundary = startTopology?.boundary;
|
|
1119
|
+
const startRun = startTopology?.run;
|
|
1120
|
+
const related = records.related;
|
|
909
1121
|
const starts = related.filter((stage) => stage.topology.boundary.event === "start");
|
|
910
1122
|
if (starts.length !== 1 || starts[0] !== start)
|
|
911
1123
|
return failure("boundary has no unique start record");
|
|
@@ -942,7 +1154,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
|
942
1154
|
return failure("boundary has an illegal terminal transition");
|
|
943
1155
|
}
|
|
944
1156
|
const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
|
|
945
|
-
const outputRecords =
|
|
1157
|
+
const outputRecords = records.outputRecords();
|
|
946
1158
|
if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
|
|
947
1159
|
return failure("boundary replay key has conflicting output records");
|
|
948
1160
|
}
|
|
@@ -1683,9 +1895,19 @@ function* invoke(action) {
|
|
|
1683
1895
|
} catch (error) {
|
|
1684
1896
|
if (error instanceof JournalError || error instanceof StorageError)
|
|
1685
1897
|
throw error;
|
|
1686
|
-
throw
|
|
1898
|
+
throw customJournalUnavailable(error);
|
|
1687
1899
|
}
|
|
1688
1900
|
}
|
|
1901
|
+
function customJournalUnavailable(failure) {
|
|
1902
|
+
const cause = failure instanceof Error ? failure : new Error(String(failure));
|
|
1903
|
+
const code = hasErrorCode(cause) ? cause.code : undefined;
|
|
1904
|
+
const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
|
|
1905
|
+
return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
|
|
1906
|
+
}
|
|
1907
|
+
function hasErrorCode(error) {
|
|
1908
|
+
const code = error.code;
|
|
1909
|
+
return typeof code === "string" || typeof code === "number";
|
|
1910
|
+
}
|
|
1689
1911
|
async function execute(operation) {
|
|
1690
1912
|
let step = operation.next();
|
|
1691
1913
|
while (!step.done) {
|
|
@@ -1713,6 +1935,8 @@ class JournalDurableBackend {
|
|
|
1713
1935
|
executions = new Map;
|
|
1714
1936
|
executionContext = new AsyncLocalStorage;
|
|
1715
1937
|
pending = new Map;
|
|
1938
|
+
reconciledPendingStages = new Map;
|
|
1939
|
+
appendBatches = new Map;
|
|
1716
1940
|
catalogQueue = Symbol("catalog");
|
|
1717
1941
|
lost = new Map;
|
|
1718
1942
|
prompts = new Map;
|
|
@@ -1745,7 +1969,7 @@ class JournalDurableBackend {
|
|
|
1745
1969
|
await backend.close();
|
|
1746
1970
|
if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
|
|
1747
1971
|
throw error;
|
|
1748
|
-
throw
|
|
1972
|
+
throw customJournalUnavailable(error);
|
|
1749
1973
|
}
|
|
1750
1974
|
}
|
|
1751
1975
|
serialize(runId, action, requiresActiveExecution = true) {
|
|
@@ -1760,6 +1984,8 @@ class JournalDurableBackend {
|
|
|
1760
1984
|
return this.enqueue(runId, action, admitted);
|
|
1761
1985
|
}
|
|
1762
1986
|
enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
|
|
1987
|
+
if (typeof runId === "string")
|
|
1988
|
+
this.appendBatches.delete(runId);
|
|
1763
1989
|
const prior = this.pending.get(runId) ?? Promise.resolve();
|
|
1764
1990
|
const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
|
|
1765
1991
|
const settled = result.then(() => {}, () => {});
|
|
@@ -1774,6 +2000,7 @@ class JournalDurableBackend {
|
|
|
1774
2000
|
if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
|
|
1775
2001
|
return;
|
|
1776
2002
|
const event = entry.event;
|
|
2003
|
+
let raisedCount;
|
|
1777
2004
|
if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
|
|
1778
2005
|
const checkpoint = event.payload;
|
|
1779
2006
|
validateCheckpointInvocationFingerprint(checkpoint);
|
|
@@ -1782,7 +2009,10 @@ class JournalDurableBackend {
|
|
|
1782
2009
|
this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
|
|
1783
2010
|
} else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
|
|
1784
2011
|
const payload = event.payload;
|
|
2012
|
+
const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
|
|
1785
2013
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
2014
|
+
if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
|
|
2015
|
+
raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
|
|
1786
2016
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
1787
2017
|
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
1788
2018
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
@@ -1793,20 +2023,80 @@ class JournalDurableBackend {
|
|
|
1793
2023
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
1794
2024
|
this.prompts.delete(entry.runId);
|
|
1795
2025
|
}
|
|
1796
|
-
|
|
1797
|
-
if (handle?.pendingStageMessages !== undefined) {
|
|
1798
|
-
const messages = handle.pendingStageMessages.map((message) => {
|
|
1799
|
-
if (message.sticky !== true)
|
|
1800
|
-
return message;
|
|
1801
|
-
const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
|
|
1802
|
-
return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
|
|
1803
|
-
});
|
|
1804
|
-
if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
|
|
1805
|
-
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
|
|
1806
|
-
}
|
|
2026
|
+
this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
|
|
1807
2027
|
this.sequences.set(entry.runId, entry.seq);
|
|
1808
2028
|
this.foldedAt.set(entry.runId, entry.at);
|
|
1809
2029
|
}
|
|
2030
|
+
reconcileFoldedDeliveryCounts(runId, raisedCount) {
|
|
2031
|
+
const handle = this.projection.getWorkflow(runId);
|
|
2032
|
+
const list = handle?.pendingStageMessages;
|
|
2033
|
+
if (handle === undefined || list === undefined) {
|
|
2034
|
+
this.reconciledPendingStages.delete(runId);
|
|
2035
|
+
return;
|
|
2036
|
+
}
|
|
2037
|
+
let state = this.reconciledPendingStages.get(runId);
|
|
2038
|
+
let next;
|
|
2039
|
+
const raise = (index, count) => {
|
|
2040
|
+
const message = (next ?? list)[index];
|
|
2041
|
+
if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
|
|
2042
|
+
return;
|
|
2043
|
+
next ??= [...list];
|
|
2044
|
+
next[index] = { ...message, deliveryCount: count };
|
|
2045
|
+
};
|
|
2046
|
+
const reconcile = (index) => {
|
|
2047
|
+
const message = list[index];
|
|
2048
|
+
if (message.sticky !== true)
|
|
2049
|
+
return;
|
|
2050
|
+
raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
|
|
2051
|
+
};
|
|
2052
|
+
if (state?.list !== list) {
|
|
2053
|
+
const previous = state?.list ?? [];
|
|
2054
|
+
let aligned = list.length >= previous.length;
|
|
2055
|
+
const changed = [];
|
|
2056
|
+
for (let index = 0;index < list.length; index++) {
|
|
2057
|
+
const message = list[index];
|
|
2058
|
+
if (previous[index] === message)
|
|
2059
|
+
continue;
|
|
2060
|
+
if (index < previous.length && previous[index].id !== message.id)
|
|
2061
|
+
aligned = false;
|
|
2062
|
+
changed.push(index);
|
|
2063
|
+
}
|
|
2064
|
+
const reconciled = aligned ? undefined : new Set(previous);
|
|
2065
|
+
for (const index of changed)
|
|
2066
|
+
if (!reconciled?.has(list[index]))
|
|
2067
|
+
reconcile(index);
|
|
2068
|
+
const positions = aligned ? state?.positions : undefined;
|
|
2069
|
+
if (positions !== undefined) {
|
|
2070
|
+
for (let index = previous.length;index < list.length; index++) {
|
|
2071
|
+
const indexes = positions.get(list[index].id);
|
|
2072
|
+
if (indexes === undefined)
|
|
2073
|
+
positions.set(list[index].id, [index]);
|
|
2074
|
+
else
|
|
2075
|
+
indexes.push(index);
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
state = { list, positions };
|
|
2079
|
+
}
|
|
2080
|
+
if (raisedCount !== undefined) {
|
|
2081
|
+
const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
|
|
2082
|
+
state.positions ??= messagePositions(list);
|
|
2083
|
+
for (const index of state.positions.get(raisedCount.messageId) ?? []) {
|
|
2084
|
+
const message = list[index];
|
|
2085
|
+
if (message.sticky === true && message.runId === raisedCount.messageRunId)
|
|
2086
|
+
raise(index, count);
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
if (next === undefined) {
|
|
2090
|
+
this.reconciledPendingStages.set(runId, state);
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
|
|
2094
|
+
const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
|
|
2095
|
+
this.reconciledPendingStages.set(runId, {
|
|
2096
|
+
list: applied,
|
|
2097
|
+
positions: applied.length === list.length ? state.positions : undefined
|
|
2098
|
+
});
|
|
2099
|
+
}
|
|
1810
2100
|
summarize(runId, tail) {
|
|
1811
2101
|
const event = tail.snapshot.event;
|
|
1812
2102
|
const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
|
|
@@ -1830,6 +2120,7 @@ class JournalDurableBackend {
|
|
|
1830
2120
|
*rebuild(runId) {
|
|
1831
2121
|
const entries = yield* this.entries(runId);
|
|
1832
2122
|
this.projection.deleteWorkflow(runId);
|
|
2123
|
+
this.reconciledPendingStages.delete(runId);
|
|
1833
2124
|
this.sequences.delete(runId);
|
|
1834
2125
|
this.foldedAt.delete(runId);
|
|
1835
2126
|
this.summarized.delete(runId);
|
|
@@ -1925,9 +2216,9 @@ class JournalDurableBackend {
|
|
|
1925
2216
|
}
|
|
1926
2217
|
yield* this.write(runId, lease, event, options);
|
|
1927
2218
|
}
|
|
1928
|
-
*write(runId, lease, event, options) {
|
|
2219
|
+
*write(runId, lease, event, options, context = (action) => action()) {
|
|
1929
2220
|
try {
|
|
1930
|
-
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
2221
|
+
const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
|
|
1931
2222
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
1932
2223
|
if (outcome.entry.seq > nextSeq) {
|
|
1933
2224
|
for (const entry of yield* this.entries(runId, nextSeq))
|
|
@@ -2111,13 +2402,80 @@ class JournalDurableBackend {
|
|
|
2111
2402
|
return true;
|
|
2112
2403
|
}
|
|
2113
2404
|
async archivePendingStageMessage(runId, message) {
|
|
2114
|
-
await this.
|
|
2405
|
+
await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
|
|
2115
2406
|
}
|
|
2116
2407
|
async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
|
|
2117
|
-
await this.
|
|
2408
|
+
await this.queueAppend(runId, {
|
|
2118
2409
|
type: RUN_EVENT_TYPES.messageDelivered,
|
|
2119
2410
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
2120
|
-
})
|
|
2411
|
+
});
|
|
2412
|
+
}
|
|
2413
|
+
queueAppend(runId, event, key) {
|
|
2414
|
+
try {
|
|
2415
|
+
this.ensureOpen();
|
|
2416
|
+
this.assertExecutionActive();
|
|
2417
|
+
} catch (error) {
|
|
2418
|
+
return Promise.reject(error);
|
|
2419
|
+
}
|
|
2420
|
+
const open = this.appendBatches.get(runId);
|
|
2421
|
+
const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
|
|
2422
|
+
const queued = new Promise((resolve, reject) => {
|
|
2423
|
+
batch.push({
|
|
2424
|
+
event,
|
|
2425
|
+
...key === undefined ? {} : { key },
|
|
2426
|
+
context: AsyncLocalStorage.snapshot(),
|
|
2427
|
+
resolve,
|
|
2428
|
+
reject
|
|
2429
|
+
});
|
|
2430
|
+
});
|
|
2431
|
+
if (batch !== open) {
|
|
2432
|
+
this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
|
|
2433
|
+
for (const append of batch)
|
|
2434
|
+
append.reject(error);
|
|
2435
|
+
});
|
|
2436
|
+
this.appendBatches.set(runId, batch);
|
|
2437
|
+
}
|
|
2438
|
+
return queued;
|
|
2439
|
+
}
|
|
2440
|
+
*appendBatch(runId, batch) {
|
|
2441
|
+
if (this.appendBatches.get(runId) === batch)
|
|
2442
|
+
this.appendBatches.delete(runId);
|
|
2443
|
+
const appendMany = runJournalAppendMany(this.journal);
|
|
2444
|
+
if (appendMany === undefined || batch.length === 1) {
|
|
2445
|
+
for (const append of batch) {
|
|
2446
|
+
try {
|
|
2447
|
+
const lease = yield* this.ownership(runId);
|
|
2448
|
+
yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
|
|
2449
|
+
append.resolve();
|
|
2450
|
+
} catch (error) {
|
|
2451
|
+
append.reject(error);
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
return;
|
|
2455
|
+
}
|
|
2456
|
+
let settled = 0;
|
|
2457
|
+
try {
|
|
2458
|
+
const lease = yield* this.ownership(runId);
|
|
2459
|
+
const results = yield* invoke(() => appendMany(lease, batch));
|
|
2460
|
+
for (const result of results) {
|
|
2461
|
+
const append = batch[settled];
|
|
2462
|
+
if (result.status === "rejected") {
|
|
2463
|
+
append.reject(yield* this.mapOwnership(result.reason, runId));
|
|
2464
|
+
} else {
|
|
2465
|
+
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
2466
|
+
if (result.value.entry.seq > nextSeq)
|
|
2467
|
+
for (const entry of yield* this.entries(runId, nextSeq))
|
|
2468
|
+
this.fold(entry);
|
|
2469
|
+
this.fold(result.value.entry);
|
|
2470
|
+
append.resolve();
|
|
2471
|
+
}
|
|
2472
|
+
settled++;
|
|
2473
|
+
}
|
|
2474
|
+
} catch (error) {
|
|
2475
|
+
const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
|
|
2476
|
+
for (const append of batch.slice(settled))
|
|
2477
|
+
append.reject(failure);
|
|
2478
|
+
}
|
|
2121
2479
|
}
|
|
2122
2480
|
getToolOutput(...args) {
|
|
2123
2481
|
return this.projection.getToolOutput(...args);
|
|
@@ -2137,6 +2495,9 @@ class JournalDurableBackend {
|
|
|
2137
2495
|
listCheckpoints(runId) {
|
|
2138
2496
|
return this.projection.listCheckpoints(runId);
|
|
2139
2497
|
}
|
|
2498
|
+
nestedSnapshotVersion(runId) {
|
|
2499
|
+
return this.projection.nestedSnapshotVersion(runId);
|
|
2500
|
+
}
|
|
2140
2501
|
getWorkflow(runId) {
|
|
2141
2502
|
return this.projection.getWorkflow(runId);
|
|
2142
2503
|
}
|
|
@@ -2161,6 +2522,12 @@ class JournalDurableBackend {
|
|
|
2161
2522
|
getPendingStageMessageDeliveryCount(...args) {
|
|
2162
2523
|
return this.projection.getPendingStageMessageDeliveryCount(...args);
|
|
2163
2524
|
}
|
|
2525
|
+
pendingStageDeliveryReceiptChanges(...args) {
|
|
2526
|
+
return this.projection.pendingStageDeliveryReceiptChanges(...args);
|
|
2527
|
+
}
|
|
2528
|
+
pendingStageDeliveryCountVersion(runId) {
|
|
2529
|
+
return this.projection.pendingStageDeliveryCountVersion(runId);
|
|
2530
|
+
}
|
|
2164
2531
|
hasPendingStageDeliveryReceipt(...args) {
|
|
2165
2532
|
return this.projection.hasPendingStageDeliveryReceipt(...args);
|
|
2166
2533
|
}
|
|
@@ -2245,6 +2612,7 @@ class JournalDurableBackend {
|
|
|
2245
2612
|
this.foldedAt.delete(runId);
|
|
2246
2613
|
this.summarized.delete(runId);
|
|
2247
2614
|
this.prompts.delete(runId);
|
|
2615
|
+
this.reconciledPendingStages.delete(runId);
|
|
2248
2616
|
this.projection.deleteWorkflow(runId);
|
|
2249
2617
|
}
|
|
2250
2618
|
async deleteWorkflow(runId) {
|
|
@@ -2427,8 +2795,24 @@ class JournalDurableBackend {
|
|
|
2427
2795
|
}
|
|
2428
2796
|
}
|
|
2429
2797
|
}
|
|
2798
|
+
function messagePositions(list) {
|
|
2799
|
+
const positions = new Map;
|
|
2800
|
+
list.forEach((message, index) => {
|
|
2801
|
+
const indexes = positions.get(message.id);
|
|
2802
|
+
if (indexes === undefined)
|
|
2803
|
+
positions.set(message.id, [index]);
|
|
2804
|
+
else
|
|
2805
|
+
indexes.push(index);
|
|
2806
|
+
});
|
|
2807
|
+
return positions;
|
|
2808
|
+
}
|
|
2430
2809
|
function identity(message) {
|
|
2431
|
-
|
|
2810
|
+
let key = entryIdentity.get(message);
|
|
2811
|
+
if (key === undefined) {
|
|
2812
|
+
key = JSON.stringify([message.runId, message.id]);
|
|
2813
|
+
entryIdentity.set(message, key);
|
|
2814
|
+
}
|
|
2815
|
+
return key;
|
|
2432
2816
|
}
|
|
2433
2817
|
function uniqueByIdentity(messages) {
|
|
2434
2818
|
const result = new Map;
|
|
@@ -2602,7 +2986,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
2602
2986
|
function hasPendingStageDeliveryDelta(messages) {
|
|
2603
2987
|
return messages?.some(isDeltaEncoded) === true;
|
|
2604
2988
|
}
|
|
2605
|
-
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
2989
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
|
|
2606
2990
|
var init_journal_backend = __esm(() => {
|
|
2607
2991
|
init_workflow_instance_ownership_error();
|
|
2608
2992
|
init_backend();
|
|
@@ -2614,6 +2998,7 @@ var init_journal_backend = __esm(() => {
|
|
|
2614
2998
|
init_types();
|
|
2615
2999
|
PROCESS_HOLDER_ID = randomUUID();
|
|
2616
3000
|
processExitCleanups = new Set;
|
|
3001
|
+
CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
|
|
2617
3002
|
JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
|
|
2618
3003
|
runId;
|
|
2619
3004
|
holderId;
|
|
@@ -2660,6 +3045,7 @@ var init_journal_backend = __esm(() => {
|
|
|
2660
3045
|
this.name = "JournalMessageScopeError";
|
|
2661
3046
|
}
|
|
2662
3047
|
};
|
|
3048
|
+
entryIdentity = new WeakMap;
|
|
2663
3049
|
entryJson = new WeakMap;
|
|
2664
3050
|
});
|
|
2665
3051
|
|
|
@@ -2787,6 +3173,7 @@ class SqlRunJournal {
|
|
|
2787
3173
|
runVersions: () => this.runVersions(),
|
|
2788
3174
|
snapshotTail: (runId) => this.snapshotTail(runId)
|
|
2789
3175
|
});
|
|
3176
|
+
registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
|
|
2790
3177
|
registerSqlPoolStorage(this, {
|
|
2791
3178
|
transaction: (body) => this.operation(() => this.database.transaction(body)),
|
|
2792
3179
|
withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
|
|
@@ -3051,38 +3438,76 @@ class SqlRunJournal {
|
|
|
3051
3438
|
this.assertOpen();
|
|
3052
3439
|
validateText("runId", lease.runId);
|
|
3053
3440
|
validateText("holderId", lease.holderId);
|
|
3054
|
-
if (options?.key !== undefined)
|
|
3055
|
-
validateText("key", options.key);
|
|
3056
3441
|
const owned = { ...lease };
|
|
3057
|
-
const
|
|
3442
|
+
const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
|
|
3058
3443
|
const expected = options?.expectSeq;
|
|
3444
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
3445
|
+
await this.checkLease(tx, owned);
|
|
3446
|
+
return this.insert(tx, owned, prepared, expected);
|
|
3447
|
+
}));
|
|
3448
|
+
}
|
|
3449
|
+
async appendMany(lease, appends) {
|
|
3450
|
+
this.assertOpen();
|
|
3451
|
+
validateText("runId", lease.runId);
|
|
3452
|
+
validateText("holderId", lease.holderId);
|
|
3453
|
+
const owned = { ...lease };
|
|
3454
|
+
const prepared = appends.map((append) => {
|
|
3455
|
+
try {
|
|
3456
|
+
return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
|
|
3457
|
+
} catch (error) {
|
|
3458
|
+
return error;
|
|
3459
|
+
}
|
|
3460
|
+
});
|
|
3461
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
3462
|
+
await this.checkLease(tx, owned);
|
|
3463
|
+
const results = [];
|
|
3464
|
+
for (const append of prepared) {
|
|
3465
|
+
if (append instanceof Error) {
|
|
3466
|
+
results.push({ status: "rejected", reason: append });
|
|
3467
|
+
continue;
|
|
3468
|
+
}
|
|
3469
|
+
await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
|
|
3470
|
+
try {
|
|
3471
|
+
results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
|
|
3472
|
+
} catch (error) {
|
|
3473
|
+
await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
|
|
3474
|
+
results.push({ status: "rejected", reason: error });
|
|
3475
|
+
}
|
|
3476
|
+
await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
|
|
3477
|
+
}
|
|
3478
|
+
return results;
|
|
3479
|
+
}));
|
|
3480
|
+
}
|
|
3481
|
+
prepare(event, key, appendGuard) {
|
|
3482
|
+
if (key !== undefined)
|
|
3483
|
+
validateText("key", key);
|
|
3059
3484
|
const type = event.type;
|
|
3060
3485
|
const payload = this.snapshotJson(event.payload);
|
|
3061
|
-
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ?
|
|
3486
|
+
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
|
|
3062
3487
|
const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
|
|
3063
3488
|
if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
|
|
3064
3489
|
validateText("status", status);
|
|
3065
|
-
return
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
}
|
|
3490
|
+
return { key, type, payload, guard, status };
|
|
3491
|
+
}
|
|
3492
|
+
async insert(tx, owned, { key, type, payload, guard, status }, expected) {
|
|
3493
|
+
await guard?.(tx, owned.runId);
|
|
3494
|
+
if (key !== undefined) {
|
|
3495
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
|
|
3496
|
+
if (row)
|
|
3497
|
+
return { status: "exists", entry: entryFromRow(row) };
|
|
3498
|
+
}
|
|
3499
|
+
const actual = await this.sequence(tx, owned.runId);
|
|
3500
|
+
if (expected !== undefined && expected !== actual)
|
|
3501
|
+
throw new JournalCasConflictError(owned.runId, expected, actual);
|
|
3502
|
+
const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
|
|
3503
|
+
if (payload instanceof StorageUnavailableError2)
|
|
3504
|
+
throw payload;
|
|
3505
|
+
await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
|
|
3506
|
+
if (status !== undefined)
|
|
3507
|
+
await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
|
|
3508
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
|
|
3509
|
+
await guard?.(tx, owned.runId);
|
|
3510
|
+
return { status: "appended", entry: entryFromRow(row) };
|
|
3086
3511
|
}
|
|
3087
3512
|
async* read(runId, fromSeq = 1) {
|
|
3088
3513
|
this.assertOpen();
|
|
@@ -3150,7 +3575,7 @@ class SqlRunJournal {
|
|
|
3150
3575
|
return this.closePromise;
|
|
3151
3576
|
}
|
|
3152
3577
|
}
|
|
3153
|
-
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
|
|
3578
|
+
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
|
|
3154
3579
|
var init_sql_run_journal = __esm(() => {
|
|
3155
3580
|
init_internal();
|
|
3156
3581
|
init_legacy_journal_tables();
|
|
@@ -3707,8 +4132,59 @@ function firstError(schema, value2) {
|
|
|
3707
4132
|
}
|
|
3708
4133
|
return;
|
|
3709
4134
|
}
|
|
4135
|
+
function certainlySerializable(root, rootMustBeObject) {
|
|
4136
|
+
if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
|
|
4137
|
+
return false;
|
|
4138
|
+
const pending = [root];
|
|
4139
|
+
const ancestors = new Set;
|
|
4140
|
+
const exits = [undefined];
|
|
4141
|
+
while (pending.length > 0) {
|
|
4142
|
+
const exit = exits.pop();
|
|
4143
|
+
const value2 = pending.pop();
|
|
4144
|
+
if (exit !== undefined) {
|
|
4145
|
+
ancestors.delete(exit);
|
|
4146
|
+
continue;
|
|
4147
|
+
}
|
|
4148
|
+
if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
|
|
4149
|
+
continue;
|
|
4150
|
+
if (typeof value2 === "number") {
|
|
4151
|
+
if (!Number.isFinite(value2))
|
|
4152
|
+
return false;
|
|
4153
|
+
continue;
|
|
4154
|
+
}
|
|
4155
|
+
if (typeof value2 !== "object" || ancestors.has(value2))
|
|
4156
|
+
return false;
|
|
4157
|
+
if (Object.getOwnPropertySymbols(value2).length > 0)
|
|
4158
|
+
return false;
|
|
4159
|
+
const names = Object.getOwnPropertyNames(value2);
|
|
4160
|
+
let children;
|
|
4161
|
+
if (Array.isArray(value2)) {
|
|
4162
|
+
if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
|
|
4163
|
+
return false;
|
|
4164
|
+
children = [];
|
|
4165
|
+
for (let index = 0;index < value2.length; index++)
|
|
4166
|
+
children.push(value2[index]);
|
|
4167
|
+
} else {
|
|
4168
|
+
const proto = Object.getPrototypeOf(value2);
|
|
4169
|
+
if (proto !== Object.prototype && proto !== null)
|
|
4170
|
+
return false;
|
|
4171
|
+
const keys = Object.keys(value2);
|
|
4172
|
+
if (keys.length !== names.length)
|
|
4173
|
+
return false;
|
|
4174
|
+
children = keys.map((key) => value2[key]);
|
|
4175
|
+
}
|
|
4176
|
+
ancestors.add(value2);
|
|
4177
|
+
pending.push(value2);
|
|
4178
|
+
exits.push(value2);
|
|
4179
|
+
for (const child of children) {
|
|
4180
|
+
pending.push(child);
|
|
4181
|
+
exits.push(undefined);
|
|
4182
|
+
}
|
|
4183
|
+
}
|
|
4184
|
+
return true;
|
|
4185
|
+
}
|
|
3710
4186
|
function workflowSerializableValidationError(value2, label) {
|
|
3711
|
-
if (Value.Check(workflowSerializableValueSchema, value2))
|
|
4187
|
+
if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
|
|
3712
4188
|
return;
|
|
3713
4189
|
const issue = firstError(workflowSerializableValueSchema, value2);
|
|
3714
4190
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -3717,7 +4193,7 @@ function workflowSerializableValidationError(value2, label) {
|
|
|
3717
4193
|
return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
|
|
3718
4194
|
}
|
|
3719
4195
|
function workflowSerializableObjectValidationError(value2, label) {
|
|
3720
|
-
if (Value.Check(workflowSerializableObjectSchema, value2))
|
|
4196
|
+
if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
|
|
3721
4197
|
return;
|
|
3722
4198
|
const issue = firstError(workflowSerializableObjectSchema, value2);
|
|
3723
4199
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -6089,6 +6565,12 @@ class ScopedDurableBackend {
|
|
|
6089
6565
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
6090
6566
|
return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
|
|
6091
6567
|
}
|
|
6568
|
+
pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
|
|
6569
|
+
return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
|
|
6570
|
+
}
|
|
6571
|
+
pendingStageDeliveryCountVersion(_workflowId) {
|
|
6572
|
+
return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
|
|
6573
|
+
}
|
|
6092
6574
|
async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
6093
6575
|
return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
|
|
6094
6576
|
}
|
|
@@ -8354,8 +8836,11 @@ var init_store_internal = __esm(() => {
|
|
|
8354
8836
|
});
|
|
8355
8837
|
|
|
8356
8838
|
// dist/builtin/workflows/src/shared/pending-stage-retention.ts
|
|
8839
|
+
function isSettledPendingStageMessage(entry) {
|
|
8840
|
+
return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
|
|
8841
|
+
}
|
|
8357
8842
|
async function retainPendingStageMessages(messages, archive) {
|
|
8358
|
-
const settled = messages.filter(
|
|
8843
|
+
const settled = messages.filter(isSettledPendingStageMessage);
|
|
8359
8844
|
const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
|
|
8360
8845
|
for (const entry of expired)
|
|
8361
8846
|
await archive(entry);
|
|
@@ -8363,53 +8848,210 @@ async function retainPendingStageMessages(messages, archive) {
|
|
|
8363
8848
|
}
|
|
8364
8849
|
var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
|
|
8365
8850
|
|
|
8851
|
+
// dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
|
|
8852
|
+
class PendingStageTransitionLedger {
|
|
8853
|
+
compacted = [];
|
|
8854
|
+
settled = 0;
|
|
8855
|
+
stages = new Map;
|
|
8856
|
+
fixed = new WeakSet;
|
|
8857
|
+
reconciled = [];
|
|
8858
|
+
reconciledVersion;
|
|
8859
|
+
reconcile(backend, runId, messages) {
|
|
8860
|
+
if (backend.getPendingStageMessageDeliveryCount === undefined)
|
|
8861
|
+
return messages;
|
|
8862
|
+
const version = backend.pendingStageDeliveryCountVersion?.(runId);
|
|
8863
|
+
const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
|
|
8864
|
+
let recovered;
|
|
8865
|
+
for (let index = 0;index < messages.length; index++) {
|
|
8866
|
+
const entry = messages[index];
|
|
8867
|
+
if (entry.sticky !== true || base !== undefined && base[index] === entry)
|
|
8868
|
+
continue;
|
|
8869
|
+
const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
|
|
8870
|
+
if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
|
|
8871
|
+
continue;
|
|
8872
|
+
recovered ??= [...messages];
|
|
8873
|
+
recovered[index] = { ...entry, deliveryCount: count };
|
|
8874
|
+
}
|
|
8875
|
+
const result = recovered ?? messages;
|
|
8876
|
+
this.reconciled = result;
|
|
8877
|
+
this.reconciledVersion = version;
|
|
8878
|
+
return result;
|
|
8879
|
+
}
|
|
8880
|
+
async compact(backend, runId, messages, runs, archiveSettled) {
|
|
8881
|
+
const stageStatus = createStageStatusLookup(runs);
|
|
8882
|
+
const endedStages = [];
|
|
8883
|
+
const dirty = new Set;
|
|
8884
|
+
for (const [deliveryRunId, byStage] of this.stages) {
|
|
8885
|
+
for (const [stageId, entries] of byStage) {
|
|
8886
|
+
if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
|
|
8887
|
+
continue;
|
|
8888
|
+
endedStages.push([deliveryRunId, stageId]);
|
|
8889
|
+
for (const entry of entries)
|
|
8890
|
+
dirty.add(entry);
|
|
8891
|
+
}
|
|
8892
|
+
}
|
|
8893
|
+
const base = this.compacted;
|
|
8894
|
+
const output = [];
|
|
8895
|
+
const removed = [];
|
|
8896
|
+
const added = [];
|
|
8897
|
+
let settled = this.settled;
|
|
8898
|
+
const length = Math.max(base.length, messages.length);
|
|
8899
|
+
for (let index = 0;index < length; index++) {
|
|
8900
|
+
const previous = base[index];
|
|
8901
|
+
const entry = messages[index];
|
|
8902
|
+
if (previous === entry && !dirty.has(entry)) {
|
|
8903
|
+
output.push(entry);
|
|
8904
|
+
continue;
|
|
8905
|
+
}
|
|
8906
|
+
if (previous !== undefined) {
|
|
8907
|
+
removed.push(previous);
|
|
8908
|
+
if (isSettledPendingStageMessage(previous))
|
|
8909
|
+
settled--;
|
|
8910
|
+
}
|
|
8911
|
+
if (entry === undefined)
|
|
8912
|
+
continue;
|
|
8913
|
+
if (isSettledPendingStageMessage(entry))
|
|
8914
|
+
settled++;
|
|
8915
|
+
const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
|
|
8916
|
+
output.push(next);
|
|
8917
|
+
added.push(next);
|
|
8918
|
+
}
|
|
8919
|
+
let retained = output;
|
|
8920
|
+
if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
|
|
8921
|
+
retained = await retainPendingStageMessages(output, archiveSettled);
|
|
8922
|
+
settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
|
|
8923
|
+
}
|
|
8924
|
+
return {
|
|
8925
|
+
messages: retained,
|
|
8926
|
+
commit: () => {
|
|
8927
|
+
for (const [deliveryRunId, stageId] of endedStages) {
|
|
8928
|
+
const byStage = this.stages.get(deliveryRunId);
|
|
8929
|
+
byStage?.delete(stageId);
|
|
8930
|
+
if (byStage?.size === 0)
|
|
8931
|
+
this.stages.delete(deliveryRunId);
|
|
8932
|
+
}
|
|
8933
|
+
for (const entry of dirty)
|
|
8934
|
+
this.fixed.delete(entry);
|
|
8935
|
+
for (const entry of removed)
|
|
8936
|
+
this.unindex(entry);
|
|
8937
|
+
for (const entry of added)
|
|
8938
|
+
this.index(entry);
|
|
8939
|
+
this.compacted = retained;
|
|
8940
|
+
this.settled = settled;
|
|
8941
|
+
}
|
|
8942
|
+
};
|
|
8943
|
+
}
|
|
8944
|
+
index(entry) {
|
|
8945
|
+
this.fixed.add(entry);
|
|
8946
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
8947
|
+
let byStage = this.stages.get(delivery.runId);
|
|
8948
|
+
if (byStage === undefined) {
|
|
8949
|
+
byStage = new Map;
|
|
8950
|
+
this.stages.set(delivery.runId, byStage);
|
|
8951
|
+
}
|
|
8952
|
+
let entries = byStage.get(delivery.stageId);
|
|
8953
|
+
if (entries === undefined) {
|
|
8954
|
+
entries = new Set;
|
|
8955
|
+
byStage.set(delivery.stageId, entries);
|
|
8956
|
+
}
|
|
8957
|
+
entries.add(entry);
|
|
8958
|
+
}
|
|
8959
|
+
}
|
|
8960
|
+
unindex(entry) {
|
|
8961
|
+
this.fixed.delete(entry);
|
|
8962
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
8963
|
+
const byStage = this.stages.get(delivery.runId);
|
|
8964
|
+
const entries = byStage?.get(delivery.stageId);
|
|
8965
|
+
if (entries === undefined)
|
|
8966
|
+
continue;
|
|
8967
|
+
entries.delete(entry);
|
|
8968
|
+
if (entries.size > 0)
|
|
8969
|
+
continue;
|
|
8970
|
+
byStage.delete(delivery.stageId);
|
|
8971
|
+
if (byStage.size === 0)
|
|
8972
|
+
this.stages.delete(delivery.runId);
|
|
8973
|
+
}
|
|
8974
|
+
}
|
|
8975
|
+
}
|
|
8976
|
+
function isCompactingBackend(backend) {
|
|
8977
|
+
return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
|
|
8978
|
+
}
|
|
8979
|
+
function createStageStatusLookup(runs) {
|
|
8980
|
+
let runById;
|
|
8981
|
+
return (runId, stageId) => {
|
|
8982
|
+
if (runById === undefined) {
|
|
8983
|
+
runById = new Map;
|
|
8984
|
+
for (const run of runs)
|
|
8985
|
+
if (!runById.has(run.id))
|
|
8986
|
+
runById.set(run.id, run);
|
|
8987
|
+
}
|
|
8988
|
+
return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
|
|
8989
|
+
};
|
|
8990
|
+
}
|
|
8991
|
+
function isEndedStageStatus(status) {
|
|
8992
|
+
return status === "completed" || status === "skipped" || status === "failed";
|
|
8993
|
+
}
|
|
8994
|
+
async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
|
|
8995
|
+
if (entry.sticky !== true || entry.deliveries === undefined)
|
|
8996
|
+
return entry;
|
|
8997
|
+
const latest = new Map;
|
|
8998
|
+
entry.deliveries.forEach((delivery, index) => {
|
|
8999
|
+
latest.set(deliverySlot(delivery), index);
|
|
9000
|
+
});
|
|
9001
|
+
const deliveries = [];
|
|
9002
|
+
let archivedThrough = -1;
|
|
9003
|
+
for (let index = 0;index < entry.deliveries.length; index++) {
|
|
9004
|
+
const delivery = entry.deliveries[index];
|
|
9005
|
+
if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
|
|
9006
|
+
while (archivedThrough < index) {
|
|
9007
|
+
const prefixIndex = archivedThrough + 1;
|
|
9008
|
+
const receipt = entry.deliveries[prefixIndex];
|
|
9009
|
+
if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
|
|
9010
|
+
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
|
|
9011
|
+
messageRunId: entry.runId,
|
|
9012
|
+
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
9013
|
+
});
|
|
9014
|
+
archivedThrough = prefixIndex;
|
|
9015
|
+
}
|
|
9016
|
+
} else {
|
|
9017
|
+
deliveries.push(delivery);
|
|
9018
|
+
}
|
|
9019
|
+
}
|
|
9020
|
+
return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
|
|
9021
|
+
}
|
|
9022
|
+
function deliverySlot(delivery) {
|
|
9023
|
+
return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
|
|
9024
|
+
}
|
|
9025
|
+
var init_pending_stage_transition_ledger = () => {};
|
|
9026
|
+
|
|
8366
9027
|
// dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
|
|
8367
9028
|
function createPendingStageDeliveryStoreMethods(context) {
|
|
8368
|
-
const
|
|
8369
|
-
const
|
|
8370
|
-
const previous = transitions.get(runId) ?? Promise.resolve();
|
|
8371
|
-
const result = previous.catch(() => {
|
|
8372
|
-
return;
|
|
8373
|
-
}).then(transition);
|
|
8374
|
-
const settled = result.then(() => {
|
|
8375
|
-
return;
|
|
8376
|
-
}, () => {
|
|
8377
|
-
return;
|
|
8378
|
-
});
|
|
8379
|
-
transitions.set(runId, settled);
|
|
8380
|
-
settled.finally(() => {
|
|
8381
|
-
if (transitions.get(runId) === settled)
|
|
8382
|
-
transitions.delete(runId);
|
|
8383
|
-
});
|
|
8384
|
-
return await result;
|
|
8385
|
-
};
|
|
9029
|
+
const ledgers = new PendingStageLedgers;
|
|
9030
|
+
const batches = new PendingStageTransitionBatches(context, ledgers);
|
|
8386
9031
|
return {
|
|
8387
9032
|
async queueStageMessage(input, senderGroup, runGroup, backend) {
|
|
8388
|
-
return await
|
|
9033
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
8389
9034
|
const run = context.findRun(input.runId);
|
|
8390
9035
|
if (run === undefined)
|
|
8391
9036
|
return;
|
|
8392
9037
|
const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
|
|
8393
9038
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
8394
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
9039
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
|
|
8395
9040
|
if (result.ok && !result.deduplicated) {
|
|
8396
|
-
|
|
8397
|
-
const live = context.findRun(input.runId) ?? run;
|
|
8398
|
-
live.pendingStageMessages = [...messages];
|
|
8399
|
-
context.bumpAndNotify();
|
|
9041
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
8400
9042
|
}
|
|
8401
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
9043
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
8402
9044
|
});
|
|
8403
9045
|
},
|
|
8404
9046
|
async validateLiveStageMessage(input) {
|
|
8405
|
-
return await
|
|
9047
|
+
return await batches.run(input.runId, undefined, async (batch) => {
|
|
8406
9048
|
const run = context.findRun(input.runId);
|
|
8407
9049
|
if (run === undefined)
|
|
8408
9050
|
return;
|
|
8409
9051
|
const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
|
|
8410
9052
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
|
|
8411
9053
|
const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
8412
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
9054
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
|
|
8413
9055
|
if (!result.ok) {
|
|
8414
9056
|
return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
|
|
8415
9057
|
}
|
|
@@ -8431,77 +9073,65 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
8431
9073
|
return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
|
|
8432
9074
|
},
|
|
8433
9075
|
async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
|
|
8434
|
-
return await
|
|
9076
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8435
9077
|
const run = context.findRun(runId);
|
|
8436
9078
|
if (run === undefined)
|
|
8437
9079
|
return false;
|
|
8438
|
-
const current = run
|
|
9080
|
+
const current = batch.messages(run);
|
|
8439
9081
|
const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
|
|
8440
9082
|
if (next === current)
|
|
8441
9083
|
return false;
|
|
8442
|
-
|
|
8443
|
-
const live = context.findRun(runId) ?? run;
|
|
8444
|
-
live.pendingStageMessages = [...messages];
|
|
8445
|
-
context.bumpAndNotify();
|
|
9084
|
+
await batch.apply(backend, runId, next);
|
|
8446
9085
|
return true;
|
|
8447
9086
|
});
|
|
8448
9087
|
},
|
|
8449
9088
|
async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
|
|
8450
|
-
return await
|
|
9089
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8451
9090
|
const run = context.findRun(runId);
|
|
8452
9091
|
if (run === undefined)
|
|
8453
9092
|
return false;
|
|
8454
|
-
const current = run
|
|
9093
|
+
const current = batch.messages(run);
|
|
8455
9094
|
const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
|
|
8456
9095
|
if (next === current)
|
|
8457
9096
|
return false;
|
|
8458
|
-
|
|
8459
|
-
const live = context.findRun(runId) ?? run;
|
|
8460
|
-
live.pendingStageMessages = [...messages];
|
|
8461
|
-
context.bumpAndNotify();
|
|
9097
|
+
await batch.apply(backend, runId, next);
|
|
8462
9098
|
return true;
|
|
8463
9099
|
});
|
|
8464
9100
|
},
|
|
8465
9101
|
async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
|
|
8466
|
-
return await
|
|
9102
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8467
9103
|
const run = context.findRun(runId);
|
|
8468
9104
|
if (run === undefined)
|
|
8469
9105
|
return false;
|
|
8470
|
-
const current = run
|
|
9106
|
+
const current = batch.messages(run);
|
|
8471
9107
|
const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
|
|
8472
9108
|
if (next === current)
|
|
8473
9109
|
return false;
|
|
8474
|
-
|
|
8475
|
-
const live = context.findRun(runId) ?? run;
|
|
8476
|
-
live.pendingStageMessages = [...messages];
|
|
8477
|
-
context.bumpAndNotify();
|
|
9110
|
+
await batch.apply(backend, runId, next);
|
|
8478
9111
|
return true;
|
|
8479
9112
|
});
|
|
8480
9113
|
},
|
|
8481
9114
|
async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
|
|
8482
|
-
return await
|
|
9115
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
8483
9116
|
const run = context.findRun(input.runId);
|
|
8484
9117
|
if (run === undefined)
|
|
8485
9118
|
return;
|
|
8486
9119
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
8487
|
-
const result = queueStickyStageMessage(receipt === undefined ? run
|
|
9120
|
+
const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
|
|
8488
9121
|
if (result.ok && !result.deduplicated) {
|
|
8489
|
-
|
|
8490
|
-
const live = context.findRun(input.runId) ?? run;
|
|
8491
|
-
live.pendingStageMessages = [...messages];
|
|
8492
|
-
context.bumpAndNotify();
|
|
9122
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
8493
9123
|
}
|
|
8494
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
9124
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
8495
9125
|
});
|
|
8496
9126
|
},
|
|
8497
9127
|
async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
|
|
8498
|
-
return await
|
|
9128
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8499
9129
|
const run = context.findRun(runId);
|
|
8500
9130
|
if (run === undefined)
|
|
8501
9131
|
return false;
|
|
8502
9132
|
if (records.length === 0)
|
|
8503
9133
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
8504
|
-
let current =
|
|
9134
|
+
let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
8505
9135
|
const unconfirmedRecords = [];
|
|
8506
9136
|
for (const record of records) {
|
|
8507
9137
|
const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
|
|
@@ -8509,35 +9139,26 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
8509
9139
|
unconfirmedRecords.push(record);
|
|
8510
9140
|
}
|
|
8511
9141
|
}
|
|
8512
|
-
current =
|
|
8513
|
-
|
|
8514
|
-
run.pendingStageMessages = [...current];
|
|
8515
|
-
context.bumpAndNotify();
|
|
8516
|
-
}
|
|
9142
|
+
current = ledgers.for(run).reconcile(backend, runId, current);
|
|
9143
|
+
batch.reconciled(run, current);
|
|
8517
9144
|
const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
|
|
8518
9145
|
if (next === current)
|
|
8519
9146
|
return false;
|
|
8520
|
-
|
|
8521
|
-
const live = context.findRun(runId) ?? run;
|
|
8522
|
-
live.pendingStageMessages = [...messages];
|
|
8523
|
-
context.bumpAndNotify();
|
|
9147
|
+
await batch.apply(backend, runId, next);
|
|
8524
9148
|
return true;
|
|
8525
9149
|
});
|
|
8526
9150
|
},
|
|
8527
9151
|
async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
|
|
8528
|
-
return await
|
|
9152
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
8529
9153
|
const run = context.findRun(runId);
|
|
8530
9154
|
if (run === undefined)
|
|
8531
9155
|
return false;
|
|
8532
9156
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
8533
|
-
const current =
|
|
9157
|
+
const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
8534
9158
|
const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
|
|
8535
9159
|
if (next === current)
|
|
8536
9160
|
return false;
|
|
8537
|
-
|
|
8538
|
-
const live = context.findRun(runId) ?? run;
|
|
8539
|
-
live.pendingStageMessages = [...messages];
|
|
8540
|
-
context.bumpAndNotify();
|
|
9161
|
+
await batch.apply(backend, runId, next);
|
|
8541
9162
|
return true;
|
|
8542
9163
|
});
|
|
8543
9164
|
}
|
|
@@ -8555,78 +9176,186 @@ function resolvePendingStageIdentity(run, stageKey) {
|
|
|
8555
9176
|
aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
|
|
8556
9177
|
};
|
|
8557
9178
|
}
|
|
8558
|
-
|
|
8559
|
-
|
|
8560
|
-
|
|
8561
|
-
|
|
8562
|
-
|
|
8563
|
-
|
|
8564
|
-
|
|
8565
|
-
|
|
8566
|
-
|
|
8567
|
-
|
|
8568
|
-
|
|
8569
|
-
|
|
8570
|
-
|
|
8571
|
-
|
|
8572
|
-
|
|
8573
|
-
|
|
8574
|
-
|
|
8575
|
-
|
|
8576
|
-
|
|
8577
|
-
|
|
8578
|
-
|
|
8579
|
-
|
|
8580
|
-
|
|
8581
|
-
|
|
8582
|
-
|
|
8583
|
-
|
|
8584
|
-
|
|
8585
|
-
|
|
8586
|
-
|
|
8587
|
-
|
|
8588
|
-
|
|
8589
|
-
|
|
8590
|
-
|
|
9179
|
+
|
|
9180
|
+
class PendingStageTransitionBatches {
|
|
9181
|
+
queues = new Map;
|
|
9182
|
+
context;
|
|
9183
|
+
ledgers;
|
|
9184
|
+
constructor(context, ledgers) {
|
|
9185
|
+
this.context = context;
|
|
9186
|
+
this.ledgers = ledgers;
|
|
9187
|
+
}
|
|
9188
|
+
run(runId, backend, step) {
|
|
9189
|
+
return new Promise((resolve, reject) => {
|
|
9190
|
+
const transition = {
|
|
9191
|
+
backend,
|
|
9192
|
+
step,
|
|
9193
|
+
resolve: (value2) => resolve(value2),
|
|
9194
|
+
reject
|
|
9195
|
+
};
|
|
9196
|
+
const queue = this.queues.get(runId);
|
|
9197
|
+
if (queue !== undefined) {
|
|
9198
|
+
queue.push(transition);
|
|
9199
|
+
return;
|
|
9200
|
+
}
|
|
9201
|
+
const created = [transition];
|
|
9202
|
+
this.queues.set(runId, created);
|
|
9203
|
+
Promise.resolve().then(() => this.drain(runId, created));
|
|
9204
|
+
});
|
|
9205
|
+
}
|
|
9206
|
+
async drain(runId, queue) {
|
|
9207
|
+
while (queue.length > 0)
|
|
9208
|
+
await this.runBatch(runId, takeBatch(queue));
|
|
9209
|
+
this.queues.delete(runId);
|
|
9210
|
+
}
|
|
9211
|
+
async runBatch(runId, transitions) {
|
|
9212
|
+
const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
|
|
9213
|
+
const outcomes = [];
|
|
9214
|
+
for (const transition of transitions) {
|
|
9215
|
+
const changedBefore = batch.changed;
|
|
9216
|
+
try {
|
|
9217
|
+
const value2 = await transition.step(batch);
|
|
9218
|
+
outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
|
|
9219
|
+
} catch (error) {
|
|
9220
|
+
outcomes.push({ ok: false, error });
|
|
8591
9221
|
}
|
|
8592
|
-
retained = compacted;
|
|
8593
9222
|
}
|
|
8594
|
-
|
|
8595
|
-
|
|
8596
|
-
|
|
9223
|
+
let failure;
|
|
9224
|
+
try {
|
|
9225
|
+
await batch.persist();
|
|
9226
|
+
} catch (error) {
|
|
9227
|
+
failure = { error };
|
|
9228
|
+
}
|
|
9229
|
+
transitions.forEach((transition, index) => {
|
|
9230
|
+
const outcome = outcomes[index];
|
|
9231
|
+
if (!outcome.ok)
|
|
9232
|
+
transition.reject(outcome.error);
|
|
9233
|
+
else if (failure !== undefined && outcome.uncommitted)
|
|
9234
|
+
transition.reject(failure.error);
|
|
9235
|
+
else
|
|
9236
|
+
transition.resolve(outcome.value);
|
|
9237
|
+
});
|
|
9238
|
+
}
|
|
9239
|
+
}
|
|
9240
|
+
function takeBatch(queue) {
|
|
9241
|
+
let count = 1;
|
|
9242
|
+
let backend = queue[0].backend;
|
|
9243
|
+
while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
|
|
9244
|
+
const next = queue[count].backend;
|
|
9245
|
+
if (next !== undefined && backend !== undefined && next !== backend)
|
|
9246
|
+
break;
|
|
9247
|
+
backend ??= next;
|
|
9248
|
+
count++;
|
|
9249
|
+
}
|
|
9250
|
+
return queue.splice(0, count);
|
|
9251
|
+
}
|
|
9252
|
+
|
|
9253
|
+
class PendingStageTransitionBatch {
|
|
9254
|
+
changed = false;
|
|
9255
|
+
list;
|
|
9256
|
+
pending;
|
|
9257
|
+
context;
|
|
9258
|
+
ledgers;
|
|
9259
|
+
runId;
|
|
9260
|
+
constructor(context, ledgers, runId) {
|
|
9261
|
+
this.context = context;
|
|
9262
|
+
this.ledgers = ledgers;
|
|
9263
|
+
this.runId = runId;
|
|
9264
|
+
}
|
|
9265
|
+
messages(run) {
|
|
9266
|
+
return this.list ?? run.pendingStageMessages ?? [];
|
|
9267
|
+
}
|
|
9268
|
+
reconciled(run, messages) {
|
|
9269
|
+
if (this.changed) {
|
|
9270
|
+
this.list = messages;
|
|
9271
|
+
return;
|
|
8597
9272
|
}
|
|
8598
|
-
if (
|
|
8599
|
-
|
|
9273
|
+
if (messages !== run.pendingStageMessages) {
|
|
9274
|
+
run.pendingStageMessages = [...messages];
|
|
9275
|
+
this.context.bumpAndNotify();
|
|
8600
9276
|
}
|
|
8601
|
-
|
|
8602
|
-
|
|
8603
|
-
const run = context.findRun(runId);
|
|
8604
|
-
|
|
8605
|
-
const
|
|
8606
|
-
|
|
8607
|
-
|
|
8608
|
-
|
|
8609
|
-
|
|
8610
|
-
|
|
9277
|
+
}
|
|
9278
|
+
async apply(backend, runId, messages) {
|
|
9279
|
+
const run = this.context.findRun(runId);
|
|
9280
|
+
try {
|
|
9281
|
+
const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
|
|
9282
|
+
this.list = compacted.messages;
|
|
9283
|
+
this.changed = true;
|
|
9284
|
+
this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
|
|
9285
|
+
return compacted.messages;
|
|
9286
|
+
} catch (error) {
|
|
9287
|
+
if (this.changed && this.list !== undefined) {
|
|
9288
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
|
|
9289
|
+
this.list = ledger.reconcile(backend, runId, this.list);
|
|
9290
|
+
} else
|
|
9291
|
+
reconcileLive(backend, runId, this.context, this.ledgers);
|
|
9292
|
+
throw error;
|
|
8611
9293
|
}
|
|
8612
9294
|
}
|
|
9295
|
+
async persist() {
|
|
9296
|
+
if (this.pending === undefined)
|
|
9297
|
+
return;
|
|
9298
|
+
const { backend, messages, commit, run } = this.pending;
|
|
9299
|
+
const published = this.list ?? messages;
|
|
9300
|
+
try {
|
|
9301
|
+
if (!await backend.persistPendingStageMessages(this.runId, messages)) {
|
|
9302
|
+
throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
|
|
9303
|
+
}
|
|
9304
|
+
commit?.();
|
|
9305
|
+
} finally {
|
|
9306
|
+
reconcileLive(backend, this.runId, this.context, this.ledgers);
|
|
9307
|
+
}
|
|
9308
|
+
const live = this.context.findRun(this.runId) ?? run;
|
|
9309
|
+
if (live !== undefined)
|
|
9310
|
+
live.pendingStageMessages = [...published];
|
|
9311
|
+
this.context.bumpAndNotify();
|
|
9312
|
+
}
|
|
9313
|
+
}
|
|
9314
|
+
async function compactTransition(backend, runId, messages, context, ledgers, run) {
|
|
9315
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
|
|
9316
|
+
const reconciled = ledger.reconcile(backend, runId, messages);
|
|
9317
|
+
const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
|
|
9318
|
+
if (isCompactingBackend(backend)) {
|
|
9319
|
+
return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
|
|
9320
|
+
}
|
|
9321
|
+
let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
|
|
9322
|
+
if (archiveSettled !== undefined)
|
|
9323
|
+
retained = await retainPendingStageMessages(retained, archiveSettled);
|
|
9324
|
+
if (run !== undefined)
|
|
9325
|
+
ledgers.reset(run);
|
|
9326
|
+
return { messages: retained };
|
|
9327
|
+
}
|
|
9328
|
+
function reconcileLive(backend, runId, context, ledgers) {
|
|
9329
|
+
const current = context.findRun(runId);
|
|
9330
|
+
if (current === undefined)
|
|
9331
|
+
return;
|
|
9332
|
+
const messages = current.pendingStageMessages ?? [];
|
|
9333
|
+
const recovered = ledgers.for(current).reconcile(backend, runId, messages);
|
|
9334
|
+
if (recovered !== messages) {
|
|
9335
|
+
current.pendingStageMessages = [...recovered];
|
|
9336
|
+
context.bumpAndNotify();
|
|
9337
|
+
}
|
|
8613
9338
|
}
|
|
8614
|
-
|
|
8615
|
-
|
|
8616
|
-
|
|
8617
|
-
|
|
8618
|
-
|
|
8619
|
-
|
|
8620
|
-
|
|
8621
|
-
|
|
8622
|
-
|
|
8623
|
-
return
|
|
8624
|
-
}
|
|
8625
|
-
|
|
9339
|
+
|
|
9340
|
+
class PendingStageLedgers {
|
|
9341
|
+
ledgers = new WeakMap;
|
|
9342
|
+
for(run) {
|
|
9343
|
+
let ledger = this.ledgers.get(run);
|
|
9344
|
+
if (ledger === undefined) {
|
|
9345
|
+
ledger = new PendingStageTransitionLedger;
|
|
9346
|
+
this.ledgers.set(run, ledger);
|
|
9347
|
+
}
|
|
9348
|
+
return ledger;
|
|
9349
|
+
}
|
|
9350
|
+
reset(run) {
|
|
9351
|
+
this.ledgers.delete(run);
|
|
9352
|
+
}
|
|
8626
9353
|
}
|
|
9354
|
+
var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
|
|
8627
9355
|
var init_store_pending_stage_delivery_methods = __esm(() => {
|
|
8628
9356
|
init_run_owner_backend();
|
|
8629
9357
|
init_pending_stage_delivery();
|
|
9358
|
+
init_pending_stage_transition_ledger();
|
|
8630
9359
|
});
|
|
8631
9360
|
|
|
8632
9361
|
// dist/builtin/workflows/src/shared/store-prompt-methods.ts
|
|
@@ -57620,6 +58349,9 @@ var init_workflow_stage_target = __esm(() => {
|
|
|
57620
58349
|
});
|
|
57621
58350
|
|
|
57622
58351
|
// dist/builtin/workflows/src/shared/pending-stage-status.ts
|
|
58352
|
+
function isAddressableStageStatus(status) {
|
|
58353
|
+
return ADDRESSABLE_STAGE_STATUSES.has(status);
|
|
58354
|
+
}
|
|
57623
58355
|
function workflowBoundarySegments(runs, runId) {
|
|
57624
58356
|
return createWorkflowBoundarySegmentsResolver(runs)(runId);
|
|
57625
58357
|
}
|
|
@@ -57742,44 +58474,412 @@ function createStagePathPatternIndex(patterns) {
|
|
|
57742
58474
|
}
|
|
57743
58475
|
};
|
|
57744
58476
|
}
|
|
58477
|
+
var ADDRESSABLE_STAGE_STATUSES;
|
|
57745
58478
|
var init_pending_stage_status = __esm(() => {
|
|
57746
58479
|
init_store_internal();
|
|
57747
58480
|
init_workflow_stage_target();
|
|
58481
|
+
ADDRESSABLE_STAGE_STATUSES = new Set([
|
|
58482
|
+
"pending",
|
|
58483
|
+
"running",
|
|
58484
|
+
"awaiting_input",
|
|
58485
|
+
"paused",
|
|
58486
|
+
"blocked"
|
|
58487
|
+
]);
|
|
57748
58488
|
});
|
|
57749
58489
|
|
|
57750
|
-
// dist/builtin/workflows/src/shared/sticky-
|
|
57751
|
-
function
|
|
57752
|
-
|
|
57753
|
-
|
|
57754
|
-
const
|
|
57755
|
-
|
|
57756
|
-
const
|
|
57757
|
-
|
|
57758
|
-
const
|
|
57759
|
-
|
|
58490
|
+
// dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
|
|
58491
|
+
function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
|
|
58492
|
+
if (states.length === 0)
|
|
58493
|
+
return { deliveryStates: states };
|
|
58494
|
+
const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
|
|
58495
|
+
counts.set("delivered", 0);
|
|
58496
|
+
for (const delivery of states)
|
|
58497
|
+
counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
|
|
58498
|
+
const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
|
|
58499
|
+
if (states.length <= limit)
|
|
58500
|
+
return { deliveryStates: states, deliveryStateCounts };
|
|
58501
|
+
const exemplars = [];
|
|
58502
|
+
for (const state of UNRESOLVED_STATES) {
|
|
58503
|
+
for (const delivery of states) {
|
|
58504
|
+
if (exemplars.length === limit)
|
|
58505
|
+
break;
|
|
58506
|
+
if (delivery.state === state)
|
|
58507
|
+
exemplars.push(delivery);
|
|
58508
|
+
}
|
|
58509
|
+
}
|
|
58510
|
+
for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
|
|
58511
|
+
const delivery = states[index];
|
|
58512
|
+
if (delivery.state === "delivered")
|
|
58513
|
+
exemplars.push(delivery);
|
|
58514
|
+
}
|
|
58515
|
+
return {
|
|
58516
|
+
deliveryStates: exemplars,
|
|
58517
|
+
deliveryStateCounts,
|
|
58518
|
+
omittedDeliveryStates: states.length - exemplars.length
|
|
58519
|
+
};
|
|
58520
|
+
}
|
|
58521
|
+
var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
|
|
58522
|
+
var init_sticky_delivery_state_bound = __esm(() => {
|
|
58523
|
+
UNRESOLVED_STATES = [
|
|
58524
|
+
"queued",
|
|
58525
|
+
"receipt-unverified",
|
|
58526
|
+
"session-replaced",
|
|
58527
|
+
"delivery-unavailable",
|
|
58528
|
+
"root-unresolved",
|
|
58529
|
+
"stage-terminal"
|
|
58530
|
+
];
|
|
58531
|
+
});
|
|
58532
|
+
|
|
58533
|
+
// dist/builtin/workflows/src/shared/possible-stage-rows.ts
|
|
58534
|
+
function boundPossibleStageDeliveryStates(states) {
|
|
58535
|
+
const bounded = boundStickyDeliveryStates(states);
|
|
58536
|
+
return {
|
|
58537
|
+
...bounded,
|
|
58538
|
+
deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
|
|
58539
|
+
};
|
|
58540
|
+
}
|
|
58541
|
+
|
|
58542
|
+
class WorkflowTopologyIndex {
|
|
58543
|
+
lineage = [];
|
|
58544
|
+
runById = new Map;
|
|
58545
|
+
roots = new Map;
|
|
58546
|
+
hops = new Map;
|
|
58547
|
+
refresh(runs) {
|
|
58548
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58549
|
+
const lineage = [];
|
|
58550
|
+
for (const run of runs) {
|
|
58551
|
+
lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
|
|
58552
|
+
if (run.parentRunId === undefined)
|
|
58553
|
+
continue;
|
|
58554
|
+
const parent = runById.get(run.parentRunId);
|
|
58555
|
+
const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
|
|
58556
|
+
lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
|
|
58557
|
+
}
|
|
58558
|
+
if (sameValues(lineage, this.lineage))
|
|
58559
|
+
return;
|
|
58560
|
+
this.lineage = lineage;
|
|
58561
|
+
this.runById = runById;
|
|
58562
|
+
this.roots.clear();
|
|
58563
|
+
this.hops.clear();
|
|
58564
|
+
}
|
|
58565
|
+
rootOf(runId) {
|
|
58566
|
+
if (this.roots.has(runId))
|
|
58567
|
+
return this.roots.get(runId);
|
|
58568
|
+
const root = reciprocalWorkflowRootRunId(this.runById, runId);
|
|
58569
|
+
this.roots.set(runId, root);
|
|
58570
|
+
return root;
|
|
58571
|
+
}
|
|
58572
|
+
hopsOf(runId) {
|
|
58573
|
+
if (this.hops.has(runId))
|
|
58574
|
+
return this.hops.get(runId);
|
|
58575
|
+
const hops = boundaryHops(this.runById, runId);
|
|
58576
|
+
this.hops.set(runId, hops);
|
|
58577
|
+
return hops;
|
|
58578
|
+
}
|
|
58579
|
+
}
|
|
58580
|
+
function boundaryHops(runById, runId) {
|
|
58581
|
+
const hops = [];
|
|
58582
|
+
let current = runById.get(runId);
|
|
58583
|
+
while (current !== undefined && current.parentRunId !== undefined) {
|
|
58584
|
+
const parent = runById.get(current.parentRunId);
|
|
58585
|
+
const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
|
|
58586
|
+
if (parent === undefined || boundary === undefined)
|
|
58587
|
+
return;
|
|
58588
|
+
hops.unshift({ name: boundary.name, runId: current.id });
|
|
58589
|
+
current = parent;
|
|
58590
|
+
}
|
|
58591
|
+
return current === undefined ? undefined : hops;
|
|
58592
|
+
}
|
|
58593
|
+
function sameValues(left, right) {
|
|
58594
|
+
if (left.length !== right.length)
|
|
58595
|
+
return false;
|
|
58596
|
+
for (let index = 0;index < left.length; index++)
|
|
58597
|
+
if (left[index] !== right[index])
|
|
58598
|
+
return false;
|
|
58599
|
+
return true;
|
|
58600
|
+
}
|
|
58601
|
+
|
|
58602
|
+
class PossibleStageRowsIndex {
|
|
58603
|
+
rootRunId;
|
|
58604
|
+
topology;
|
|
58605
|
+
targets = new Map;
|
|
58606
|
+
patterns = [];
|
|
58607
|
+
targetOf = new Map;
|
|
58608
|
+
targetsByMessage = new Map;
|
|
58609
|
+
list = [];
|
|
58610
|
+
stageFields = new WeakMap;
|
|
58611
|
+
backend;
|
|
58612
|
+
receiptCursor;
|
|
58613
|
+
scanMatches = new Map;
|
|
58614
|
+
rowCache = new Map;
|
|
58615
|
+
boundedStates = new WeakMap;
|
|
58616
|
+
deliveriesByEntry = new WeakMap;
|
|
58617
|
+
statusCache = new WeakMap;
|
|
58618
|
+
receiptGeneration = 0;
|
|
58619
|
+
receiptEpochs = new Map;
|
|
58620
|
+
constructor(rootRunId, topology) {
|
|
58621
|
+
this.rootRunId = rootRunId;
|
|
58622
|
+
this.topology = topology;
|
|
58623
|
+
}
|
|
58624
|
+
rows(runs, rootRun, availableBackend) {
|
|
58625
|
+
if (isTerminalRunStatus(rootRun.status))
|
|
58626
|
+
return [];
|
|
58627
|
+
this.refreshEntries(rootRun.pendingStageMessages ?? []);
|
|
58628
|
+
const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
|
|
58629
|
+
const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
|
|
58630
|
+
this.refreshReceipts(backend);
|
|
58631
|
+
this.refreshStages(runs);
|
|
58632
|
+
for (const state of this.patterns)
|
|
58633
|
+
if (state.dirty)
|
|
58634
|
+
this.recompute(state, backend);
|
|
58635
|
+
return this.assemble(runs, rootRun);
|
|
58636
|
+
}
|
|
58637
|
+
targetFor(entry) {
|
|
57760
58638
|
const target = entry.targetPath ?? entry.stageKey;
|
|
57761
|
-
let
|
|
57762
|
-
if (
|
|
58639
|
+
let state = this.targets.get(target);
|
|
58640
|
+
if (state === undefined) {
|
|
57763
58641
|
const parsed = parseWorkflowStageTarget(target);
|
|
57764
|
-
|
|
57765
|
-
|
|
57766
|
-
|
|
57767
|
-
|
|
58642
|
+
const owned = parsed?.rootRunId === this.rootRunId;
|
|
58643
|
+
state = {
|
|
58644
|
+
target,
|
|
58645
|
+
segments: owned ? parsed.segments : undefined,
|
|
58646
|
+
pattern: owned ? this.patterns.length : -1,
|
|
58647
|
+
entries: [],
|
|
58648
|
+
positions: [],
|
|
58649
|
+
queued: 0,
|
|
58650
|
+
stages: [],
|
|
58651
|
+
statuses: [],
|
|
58652
|
+
dirty: true
|
|
58653
|
+
};
|
|
58654
|
+
this.targets.set(target, state);
|
|
58655
|
+
if (owned)
|
|
58656
|
+
this.patterns.push(state);
|
|
57768
58657
|
}
|
|
57769
|
-
|
|
58658
|
+
return state;
|
|
58659
|
+
}
|
|
58660
|
+
refreshEntries(list) {
|
|
58661
|
+
const previous = this.list;
|
|
58662
|
+
if (list === previous)
|
|
57770
58663
|
return;
|
|
57771
|
-
|
|
57772
|
-
|
|
57773
|
-
|
|
57774
|
-
|
|
57775
|
-
|
|
57776
|
-
|
|
57777
|
-
|
|
57778
|
-
|
|
57779
|
-
|
|
58664
|
+
this.list = list;
|
|
58665
|
+
let incremental = list.length >= previous.length;
|
|
58666
|
+
const replaced = [];
|
|
58667
|
+
for (let index = 0;incremental && index < previous.length; index++) {
|
|
58668
|
+
if (list[index] === previous[index])
|
|
58669
|
+
continue;
|
|
58670
|
+
const before = previous[index];
|
|
58671
|
+
const after = list[index];
|
|
58672
|
+
if (before.sticky !== true && after.sticky !== true)
|
|
58673
|
+
continue;
|
|
58674
|
+
if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
|
|
58675
|
+
incremental = false;
|
|
58676
|
+
break;
|
|
58677
|
+
}
|
|
58678
|
+
replaced.push(index);
|
|
58679
|
+
}
|
|
58680
|
+
if (!incremental) {
|
|
58681
|
+
this.rebuildEntries(list);
|
|
58682
|
+
return;
|
|
58683
|
+
}
|
|
58684
|
+
for (const index of replaced) {
|
|
58685
|
+
const before = previous[index];
|
|
58686
|
+
const after = list[index];
|
|
58687
|
+
const state = this.targetOf.get(before);
|
|
58688
|
+
const slot = state.entries.indexOf(before);
|
|
58689
|
+
state.entries[slot] = after;
|
|
58690
|
+
this.targetOf.delete(before);
|
|
58691
|
+
this.targetOf.set(after, state);
|
|
58692
|
+
state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
|
|
58693
|
+
if (before.id !== after.id)
|
|
58694
|
+
this.rebuildMessageIndex();
|
|
58695
|
+
state.dirty = true;
|
|
58696
|
+
}
|
|
58697
|
+
for (let index = previous.length;index < list.length; index++)
|
|
58698
|
+
this.addEntry(list[index], index);
|
|
58699
|
+
}
|
|
58700
|
+
addEntry(entry, position) {
|
|
58701
|
+
if (entry.sticky !== true)
|
|
58702
|
+
return;
|
|
58703
|
+
const state = this.targetFor(entry);
|
|
58704
|
+
state.entries.push(entry);
|
|
58705
|
+
state.positions.push(position);
|
|
58706
|
+
if (entry.status === "queued")
|
|
58707
|
+
state.queued++;
|
|
58708
|
+
state.dirty = true;
|
|
58709
|
+
this.targetOf.set(entry, state);
|
|
58710
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
58711
|
+
if (targets === undefined) {
|
|
58712
|
+
targets = new Set;
|
|
58713
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
58714
|
+
}
|
|
58715
|
+
targets.add(state);
|
|
58716
|
+
}
|
|
58717
|
+
rebuildEntries(list) {
|
|
58718
|
+
const before = new Map;
|
|
58719
|
+
for (const state of this.targets.values()) {
|
|
58720
|
+
before.set(state, state.entries);
|
|
58721
|
+
state.entries = [];
|
|
58722
|
+
state.positions = [];
|
|
58723
|
+
state.queued = 0;
|
|
58724
|
+
}
|
|
58725
|
+
this.targetOf.clear();
|
|
58726
|
+
this.targetsByMessage.clear();
|
|
58727
|
+
list.forEach((entry, position) => {
|
|
58728
|
+
this.addEntry(entry, position);
|
|
58729
|
+
});
|
|
58730
|
+
for (const state of this.targets.values()) {
|
|
58731
|
+
const previous = before.get(state);
|
|
58732
|
+
state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
|
|
58733
|
+
}
|
|
58734
|
+
}
|
|
58735
|
+
rebuildMessageIndex() {
|
|
58736
|
+
this.targetsByMessage.clear();
|
|
58737
|
+
for (const [entry, state] of this.targetOf) {
|
|
58738
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
58739
|
+
if (targets === undefined) {
|
|
58740
|
+
targets = new Set;
|
|
58741
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
58742
|
+
}
|
|
58743
|
+
targets.add(state);
|
|
58744
|
+
}
|
|
58745
|
+
}
|
|
58746
|
+
refreshReceipts(backend) {
|
|
58747
|
+
if (backend !== this.backend) {
|
|
58748
|
+
this.backend = backend;
|
|
58749
|
+
this.receiptCursor = undefined;
|
|
58750
|
+
this.invalidateReceipts();
|
|
58751
|
+
}
|
|
58752
|
+
if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
|
|
58753
|
+
return;
|
|
58754
|
+
const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
|
|
58755
|
+
if (changes === undefined) {
|
|
58756
|
+
this.invalidateReceipts();
|
|
58757
|
+
return;
|
|
58758
|
+
}
|
|
58759
|
+
const firstRead = this.receiptCursor === undefined;
|
|
58760
|
+
this.receiptCursor = changes.cursor;
|
|
58761
|
+
if (changes.messageIds === undefined) {
|
|
58762
|
+
if (!firstRead)
|
|
58763
|
+
this.invalidateReceipts();
|
|
58764
|
+
return;
|
|
58765
|
+
}
|
|
58766
|
+
for (const messageId of changes.messageIds) {
|
|
58767
|
+
this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
|
|
58768
|
+
for (const state of this.targetsByMessage.get(messageId) ?? [])
|
|
58769
|
+
state.dirty = true;
|
|
58770
|
+
}
|
|
58771
|
+
}
|
|
58772
|
+
invalidateReceipts() {
|
|
58773
|
+
this.receiptGeneration++;
|
|
58774
|
+
this.receiptEpochs.clear();
|
|
58775
|
+
for (const state of this.patterns)
|
|
58776
|
+
state.dirty = true;
|
|
58777
|
+
}
|
|
58778
|
+
refreshStages(runs) {
|
|
58779
|
+
const current = new Map;
|
|
58780
|
+
let ordinal = 0;
|
|
58781
|
+
for (const run of runs) {
|
|
58782
|
+
const root = this.topology.rootOf(run.id);
|
|
58783
|
+
if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
|
|
58784
|
+
continue;
|
|
58785
|
+
const resolved = root === this.rootRunId;
|
|
58786
|
+
const hops = this.topology.hopsOf(run.id);
|
|
58787
|
+
for (const stage of run.stages) {
|
|
58788
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
58789
|
+
continue;
|
|
58790
|
+
const fields = this.fieldsOf(run, stage, resolved, hops);
|
|
58791
|
+
const status = { run, stage, resolved, hops, ordinal: ordinal++ };
|
|
58792
|
+
for (const state of fields.matched) {
|
|
58793
|
+
const stages = current.get(state);
|
|
58794
|
+
if (stages === undefined)
|
|
58795
|
+
current.set(state, [status]);
|
|
58796
|
+
else
|
|
58797
|
+
stages.push(status);
|
|
58798
|
+
}
|
|
58799
|
+
}
|
|
58800
|
+
}
|
|
58801
|
+
for (const state of this.patterns) {
|
|
58802
|
+
const stages = current.get(state) ?? [];
|
|
58803
|
+
if (stages.length !== state.stages.length || stages.some((status, index) => {
|
|
58804
|
+
const previous = state.stages[index];
|
|
58805
|
+
return previous.stage !== status.stage || previous.run !== status.run;
|
|
58806
|
+
}))
|
|
58807
|
+
state.dirty = true;
|
|
58808
|
+
state.stages = stages;
|
|
58809
|
+
}
|
|
58810
|
+
}
|
|
58811
|
+
fieldsOf(run, stage, resolved, hops) {
|
|
58812
|
+
let fields = this.stageFields.get(stage);
|
|
58813
|
+
const matchable = resolved && hops !== undefined;
|
|
58814
|
+
let changed = false;
|
|
58815
|
+
if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
|
|
58816
|
+
const previous = fields;
|
|
58817
|
+
const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
|
|
58818
|
+
fields = {
|
|
58819
|
+
run,
|
|
58820
|
+
status: stage.status,
|
|
58821
|
+
error: stage.error,
|
|
58822
|
+
sessionId: stage.sessionId,
|
|
58823
|
+
pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
|
|
58824
|
+
resolved,
|
|
58825
|
+
hops,
|
|
58826
|
+
id: stage.id,
|
|
58827
|
+
name: stage.name,
|
|
58828
|
+
tested: sameMatching ? previous.tested : 0,
|
|
58829
|
+
matched: sameMatching ? previous.matched : []
|
|
58830
|
+
};
|
|
58831
|
+
this.stageFields.set(stage, fields);
|
|
58832
|
+
changed = true;
|
|
58833
|
+
}
|
|
58834
|
+
if (fields.tested < this.patterns.length) {
|
|
58835
|
+
const added = this.patterns.slice(fields.tested);
|
|
58836
|
+
if (!matchable)
|
|
58837
|
+
fields.matched.push(...added);
|
|
58838
|
+
else {
|
|
58839
|
+
const index = createStagePathPatternIndex(added.map((state) => state.segments));
|
|
58840
|
+
for (const position of index.matching(hops, [stage.id, stage.name]))
|
|
58841
|
+
fields.matched.push(added[position]);
|
|
58842
|
+
}
|
|
58843
|
+
fields.tested = this.patterns.length;
|
|
58844
|
+
}
|
|
58845
|
+
if (changed)
|
|
58846
|
+
for (const state of fields.matched)
|
|
58847
|
+
state.dirty = true;
|
|
58848
|
+
return fields;
|
|
58849
|
+
}
|
|
58850
|
+
recompute(state, backend) {
|
|
58851
|
+
state.dirty = false;
|
|
58852
|
+
const statuses = [];
|
|
58853
|
+
for (const { run, stage, resolved, hops } of state.stages) {
|
|
58854
|
+
const fields = this.stageFields.get(stage);
|
|
58855
|
+
for (const entry of state.entries) {
|
|
58856
|
+
let byStage = this.statusCache.get(entry);
|
|
58857
|
+
if (byStage === undefined) {
|
|
58858
|
+
byStage = new Map;
|
|
58859
|
+
this.statusCache.set(entry, byStage);
|
|
58860
|
+
}
|
|
58861
|
+
const epoch = this.receiptEpochs.get(entry.id) ?? 0;
|
|
58862
|
+
let cached = byStage.get(stage);
|
|
58863
|
+
if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
|
|
58864
|
+
cached = {
|
|
58865
|
+
fields,
|
|
58866
|
+
generation: this.receiptGeneration,
|
|
58867
|
+
epoch,
|
|
58868
|
+
status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
|
|
58869
|
+
};
|
|
58870
|
+
byStage.set(stage, cached);
|
|
58871
|
+
}
|
|
58872
|
+
statuses.push(cached.status);
|
|
58873
|
+
}
|
|
58874
|
+
}
|
|
58875
|
+
if (!sameStatuses(statuses, state.statuses))
|
|
58876
|
+
state.statuses = statuses;
|
|
58877
|
+
}
|
|
58878
|
+
receiptsOf(entry, runId, stageId) {
|
|
58879
|
+
let byStage = this.deliveriesByEntry.get(entry);
|
|
57780
58880
|
if (byStage === undefined) {
|
|
57781
58881
|
byStage = new Map;
|
|
57782
|
-
for (const delivery of
|
|
58882
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
57783
58883
|
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
57784
58884
|
const list = byStage.get(key);
|
|
57785
58885
|
if (list === undefined)
|
|
@@ -57787,75 +58887,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
57787
58887
|
else
|
|
57788
58888
|
list.push(delivery);
|
|
57789
58889
|
}
|
|
57790
|
-
deliveriesByEntry.set(
|
|
58890
|
+
this.deliveriesByEntry.set(entry, byStage);
|
|
57791
58891
|
}
|
|
57792
58892
|
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
57793
|
-
}
|
|
57794
|
-
|
|
57795
|
-
|
|
57796
|
-
|
|
57797
|
-
|
|
57798
|
-
|
|
57799
|
-
|
|
57800
|
-
if (
|
|
57801
|
-
|
|
57802
|
-
|
|
57803
|
-
|
|
57804
|
-
|
|
57805
|
-
|
|
58893
|
+
}
|
|
58894
|
+
status(entry, run, stage, matchable, backend) {
|
|
58895
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
58896
|
+
const receipts = this.receiptsOf(entry, run.id, stage.id);
|
|
58897
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
58898
|
+
let state;
|
|
58899
|
+
let reason;
|
|
58900
|
+
if (!matchable) {
|
|
58901
|
+
state = "root-unresolved";
|
|
58902
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
58903
|
+
} else if (delivered) {
|
|
58904
|
+
state = "delivered";
|
|
58905
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
58906
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
58907
|
+
state = "stage-terminal";
|
|
58908
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
58909
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
58910
|
+
state = "receipt-unverified";
|
|
58911
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
58912
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
58913
|
+
state = "delivery-unavailable";
|
|
58914
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
58915
|
+
} else if (receipts.length > 0) {
|
|
58916
|
+
state = "session-replaced";
|
|
58917
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
58918
|
+
} else {
|
|
58919
|
+
state = "queued";
|
|
58920
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
58921
|
+
}
|
|
58922
|
+
return {
|
|
58923
|
+
messageId: entry.message.id,
|
|
58924
|
+
target,
|
|
58925
|
+
runId: run.id,
|
|
58926
|
+
stageId: stage.id,
|
|
58927
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
58928
|
+
state,
|
|
58929
|
+
reason
|
|
58930
|
+
};
|
|
58931
|
+
}
|
|
58932
|
+
assemble(runs, rootRun) {
|
|
58933
|
+
const rows = [];
|
|
58934
|
+
const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
|
|
58935
|
+
let liveStageMatches;
|
|
58936
|
+
const rowTargets = new Set;
|
|
58937
|
+
for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
|
|
58938
|
+
const rowSegments = splitStagePathSegments(scanEntry);
|
|
58939
|
+
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
57806
58940
|
continue;
|
|
57807
|
-
|
|
57808
|
-
|
|
57809
|
-
|
|
57810
|
-
|
|
57811
|
-
const receipts = entryReceipts(index, run.id, stage.id);
|
|
57812
|
-
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
57813
|
-
runId: run.id,
|
|
57814
|
-
stageId: stage.id,
|
|
57815
|
-
sessionId: stage.sessionId,
|
|
57816
|
-
admission: "context"
|
|
57817
|
-
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
57818
|
-
let state;
|
|
57819
|
-
let reason;
|
|
57820
|
-
if (!resolved || !hops) {
|
|
57821
|
-
state = "root-unresolved";
|
|
57822
|
-
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
57823
|
-
} else if (delivered) {
|
|
57824
|
-
state = "delivered";
|
|
57825
|
-
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
57826
|
-
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
57827
|
-
state = "stage-terminal";
|
|
57828
|
-
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
57829
|
-
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
57830
|
-
state = "receipt-unverified";
|
|
57831
|
-
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
57832
|
-
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
57833
|
-
state = "delivery-unavailable";
|
|
57834
|
-
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
57835
|
-
} else if (receipts.length > 0) {
|
|
57836
|
-
state = "session-replaced";
|
|
57837
|
-
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
57838
|
-
} else {
|
|
57839
|
-
state = "queued";
|
|
57840
|
-
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
57841
|
-
}
|
|
57842
|
-
result.push({
|
|
57843
|
-
messageId: entry.message.id,
|
|
57844
|
-
target,
|
|
57845
|
-
runId: run.id,
|
|
57846
|
-
stageId: stage.id,
|
|
57847
|
-
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
57848
|
-
state,
|
|
57849
|
-
reason
|
|
57850
|
-
});
|
|
58941
|
+
if (!rowSegments.some((segment) => segment.includes("*"))) {
|
|
58942
|
+
liveStageMatches ??= this.materializedStageMatcher(runs);
|
|
58943
|
+
if (liveStageMatches(rowSegments))
|
|
58944
|
+
continue;
|
|
57851
58945
|
}
|
|
58946
|
+
let matches = this.scanMatches.get(scanEntry);
|
|
58947
|
+
if (matches === undefined) {
|
|
58948
|
+
matches = new Map;
|
|
58949
|
+
this.scanMatches.set(scanEntry, matches);
|
|
58950
|
+
}
|
|
58951
|
+
let queuedCount = 0;
|
|
58952
|
+
for (const state of scanTargets) {
|
|
58953
|
+
let match = matches.get(state);
|
|
58954
|
+
if (match === undefined) {
|
|
58955
|
+
const segments = state.segments;
|
|
58956
|
+
match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
58957
|
+
matches.set(state, match);
|
|
58958
|
+
}
|
|
58959
|
+
if (match)
|
|
58960
|
+
queuedCount += state.queued;
|
|
58961
|
+
}
|
|
58962
|
+
const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
|
|
58963
|
+
rows.push({ slot: `s${position}`, target, queuedCount });
|
|
58964
|
+
rowTargets.add(target);
|
|
58965
|
+
}
|
|
58966
|
+
let broadcastCount = 0;
|
|
58967
|
+
for (const state of this.targets.values()) {
|
|
58968
|
+
if (state.segments?.length === 1 && state.segments[0] === "**")
|
|
58969
|
+
broadcastCount += state.queued;
|
|
58970
|
+
}
|
|
58971
|
+
const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
|
|
58972
|
+
rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
|
|
58973
|
+
rowTargets.add(broadcast);
|
|
58974
|
+
const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
|
|
58975
|
+
const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
|
|
58976
|
+
for (const state of statusTargets) {
|
|
58977
|
+
if (rowTargets.has(state.target))
|
|
58978
|
+
continue;
|
|
58979
|
+
rowTargets.add(state.target);
|
|
58980
|
+
rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
|
|
57852
58981
|
}
|
|
58982
|
+
const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
|
|
58983
|
+
this.pruneRows(result);
|
|
58984
|
+
return result;
|
|
57853
58985
|
}
|
|
57854
|
-
|
|
58986
|
+
row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
|
|
58987
|
+
const cached = this.rowCache.get(slot);
|
|
58988
|
+
const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
|
|
58989
|
+
if (same && cached.statuses === deliveryStates)
|
|
58990
|
+
return cached.row;
|
|
58991
|
+
const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
|
|
58992
|
+
if (same && sameBoundedRow(cached.row, bounded)) {
|
|
58993
|
+
this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
|
|
58994
|
+
return cached.row;
|
|
58995
|
+
}
|
|
58996
|
+
const row = {
|
|
58997
|
+
target,
|
|
58998
|
+
queuedCount,
|
|
58999
|
+
...deliveryOnly === true ? { deliveryOnly } : {},
|
|
59000
|
+
...bounded
|
|
59001
|
+
};
|
|
59002
|
+
this.rowCache.set(slot, { row, statuses: deliveryStates });
|
|
59003
|
+
return row;
|
|
59004
|
+
}
|
|
59005
|
+
bounded(states) {
|
|
59006
|
+
let bounded = this.boundedStates.get(states);
|
|
59007
|
+
if (bounded === undefined) {
|
|
59008
|
+
bounded = boundPossibleStageDeliveryStates(states);
|
|
59009
|
+
this.boundedStates.set(states, bounded);
|
|
59010
|
+
}
|
|
59011
|
+
return bounded;
|
|
59012
|
+
}
|
|
59013
|
+
pruneRows(rows) {
|
|
59014
|
+
if (this.rowCache.size <= rows.length * 4 + 16)
|
|
59015
|
+
return;
|
|
59016
|
+
const kept = new Set(rows);
|
|
59017
|
+
for (const [key, cached] of this.rowCache)
|
|
59018
|
+
if (!kept.has(cached.row))
|
|
59019
|
+
this.rowCache.delete(key);
|
|
59020
|
+
}
|
|
59021
|
+
materializedStageMatcher(runs) {
|
|
59022
|
+
const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
|
|
59023
|
+
return (entrySegments) => {
|
|
59024
|
+
for (const run of invocationRuns) {
|
|
59025
|
+
const hops = this.topology.hopsOf(run.id);
|
|
59026
|
+
if (hops === undefined)
|
|
59027
|
+
continue;
|
|
59028
|
+
for (const stage of run.stages) {
|
|
59029
|
+
if (stage.pendingStageDeliveryAvailable !== true)
|
|
59030
|
+
continue;
|
|
59031
|
+
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
59032
|
+
continue;
|
|
59033
|
+
}
|
|
59034
|
+
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
59035
|
+
return true;
|
|
59036
|
+
}
|
|
59037
|
+
}
|
|
59038
|
+
return false;
|
|
59039
|
+
};
|
|
59040
|
+
}
|
|
59041
|
+
}
|
|
59042
|
+
function sameBoundedRow(row, bounded) {
|
|
59043
|
+
if (bounded === undefined)
|
|
59044
|
+
return row.deliveryStates === undefined;
|
|
59045
|
+
if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
|
|
59046
|
+
return false;
|
|
59047
|
+
const counts = Object.entries(row.deliveryStateCounts ?? {});
|
|
59048
|
+
const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
|
|
59049
|
+
if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
|
|
59050
|
+
return false;
|
|
59051
|
+
return sameStatuses(row.deliveryStates, bounded.deliveryStates);
|
|
59052
|
+
}
|
|
59053
|
+
function sameStatuses(left, right) {
|
|
59054
|
+
if (left.length !== right.length)
|
|
59055
|
+
return false;
|
|
59056
|
+
for (let index = 0;index < left.length; index++) {
|
|
59057
|
+
const a = left[index];
|
|
59058
|
+
const b = right[index];
|
|
59059
|
+
if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
|
|
59060
|
+
return false;
|
|
59061
|
+
}
|
|
59062
|
+
return true;
|
|
57855
59063
|
}
|
|
57856
|
-
var
|
|
59064
|
+
var init_possible_stage_rows = __esm(() => {
|
|
57857
59065
|
init_run_owner_backend();
|
|
57858
59066
|
init_pending_stage_status();
|
|
59067
|
+
init_sticky_delivery_state_bound();
|
|
59068
|
+
init_store_internal();
|
|
57859
59069
|
init_workflow_stage_target();
|
|
57860
59070
|
});
|
|
57861
59071
|
|
|
@@ -57920,11 +59130,32 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57920
59130
|
announceIfPending();
|
|
57921
59131
|
return routeCompletions.get(runId);
|
|
57922
59132
|
});
|
|
59133
|
+
const topology = new WorkflowTopologyIndex;
|
|
59134
|
+
const possibleStageRowIndexes = new Map;
|
|
59135
|
+
const rowIdentities = new WeakMap;
|
|
59136
|
+
let nextRowIdentity = 0;
|
|
59137
|
+
const rowIdentity = (row) => {
|
|
59138
|
+
let identity = rowIdentities.get(row);
|
|
59139
|
+
if (identity === undefined) {
|
|
59140
|
+
identity = nextRowIdentity++;
|
|
59141
|
+
rowIdentities.set(row, identity);
|
|
59142
|
+
}
|
|
59143
|
+
return identity;
|
|
59144
|
+
};
|
|
59145
|
+
const possibleStageRowsIndex = (rootRunId) => {
|
|
59146
|
+
let index = possibleStageRowIndexes.get(rootRunId);
|
|
59147
|
+
if (index === undefined) {
|
|
59148
|
+
index = new PossibleStageRowsIndex(rootRunId, topology);
|
|
59149
|
+
possibleStageRowIndexes.set(rootRunId, index);
|
|
59150
|
+
}
|
|
59151
|
+
return index;
|
|
59152
|
+
};
|
|
57923
59153
|
const announceRoutes = () => {
|
|
57924
59154
|
if (disposed)
|
|
57925
59155
|
return;
|
|
57926
59156
|
const ownedRunIds = new Set;
|
|
57927
59157
|
const runs = activeStore.runs();
|
|
59158
|
+
topology.refresh(runs);
|
|
57928
59159
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
57929
59160
|
const childrenByBoundary = new Map;
|
|
57930
59161
|
for (const run of runs) {
|
|
@@ -57939,7 +59170,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57939
59170
|
for (const run of runs) {
|
|
57940
59171
|
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
57941
59172
|
continue;
|
|
57942
|
-
const rootRunId =
|
|
59173
|
+
const rootRunId = topology.rootOf(run.id);
|
|
57943
59174
|
if (rootRunId === undefined)
|
|
57944
59175
|
continue;
|
|
57945
59176
|
ownedRunIds.add(run.id);
|
|
@@ -57949,6 +59180,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57949
59180
|
runId: parentRun.id,
|
|
57950
59181
|
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
57951
59182
|
};
|
|
59183
|
+
let possibleStages;
|
|
59184
|
+
if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
|
|
59185
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
59186
|
+
possibleStages = [];
|
|
59187
|
+
} else if (run.id === rootRunId) {
|
|
59188
|
+
possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
|
|
59189
|
+
}
|
|
57952
59190
|
const announcement = {
|
|
57953
59191
|
runId: run.id,
|
|
57954
59192
|
group: workflowInvocationIntercomGroup(rootRunId),
|
|
@@ -57960,7 +59198,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57960
59198
|
stageName: stage.name,
|
|
57961
59199
|
target: stageRouteTarget(runs, rootRunId, run.id, stage.id, resolveBoundarySegments),
|
|
57962
59200
|
lifecycle: stage.sessionId === undefined && stage.sessionFile === undefined ? "pending" : "running",
|
|
57963
|
-
routeEligible: stage.pendingStageDeliveryAvailable === true && (stage.status
|
|
59201
|
+
routeEligible: stage.pendingStageDeliveryAvailable === true && isAddressableStageStatus(stage.status),
|
|
57964
59202
|
group: stage.intercomGroup ?? workflowInvocationIntercomGroup(rootRunId)
|
|
57965
59203
|
})),
|
|
57966
59204
|
...nonAgentNodes(run).map((node) => ({
|
|
@@ -57973,11 +59211,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
57973
59211
|
group: workflowInvocationIntercomGroup(rootRunId)
|
|
57974
59212
|
}))
|
|
57975
59213
|
],
|
|
57976
|
-
...
|
|
57977
|
-
possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
|
|
57978
|
-
} : {}
|
|
59214
|
+
...possibleStages === undefined ? {} : { possibleStages }
|
|
57979
59215
|
};
|
|
57980
|
-
const signature = JSON.stringify(announcement)
|
|
59216
|
+
const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
|
|
57981
59217
|
if (announcedRoutes.get(run.id)?.signature === signature)
|
|
57982
59218
|
continue;
|
|
57983
59219
|
const entry = { signature };
|
|
@@ -58012,6 +59248,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58012
59248
|
if (!ownedRunIds.has(runId))
|
|
58013
59249
|
routeCompletions.delete(runId);
|
|
58014
59250
|
}
|
|
59251
|
+
for (const rootRunId of possibleStageRowIndexes.keys()) {
|
|
59252
|
+
if (!ownedRunIds.has(rootRunId))
|
|
59253
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
59254
|
+
}
|
|
58015
59255
|
sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
|
|
58016
59256
|
return;
|
|
58017
59257
|
}).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
|
|
@@ -58172,102 +59412,6 @@ function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegmen
|
|
|
58172
59412
|
const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
|
|
58173
59413
|
return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
|
|
58174
59414
|
}
|
|
58175
|
-
function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
58176
|
-
if (isTerminalRunStatus(rootRun.status))
|
|
58177
|
-
return [];
|
|
58178
|
-
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
58179
|
-
const segmentsByTarget = new Map;
|
|
58180
|
-
const stickySegments = (entry) => {
|
|
58181
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58182
|
-
if (segmentsByTarget.has(target))
|
|
58183
|
-
return segmentsByTarget.get(target);
|
|
58184
|
-
const parsed = parseWorkflowStageTarget(target);
|
|
58185
|
-
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
58186
|
-
segmentsByTarget.set(target, segments);
|
|
58187
|
-
return segments;
|
|
58188
|
-
};
|
|
58189
|
-
const scanEntries = stickyQueued.filter((entry) => {
|
|
58190
|
-
const segments = stickySegments(entry);
|
|
58191
|
-
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
58192
|
-
});
|
|
58193
|
-
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
58194
|
-
const rows = [];
|
|
58195
|
-
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
58196
|
-
const rowSegments = splitStagePathSegments(scanEntry);
|
|
58197
|
-
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
58198
|
-
continue;
|
|
58199
|
-
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
58200
|
-
continue;
|
|
58201
|
-
}
|
|
58202
|
-
rows.push({
|
|
58203
|
-
target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
|
|
58204
|
-
queuedCount: scanEntries.filter((entry) => {
|
|
58205
|
-
const segments = stickySegments(entry);
|
|
58206
|
-
if (segments === undefined)
|
|
58207
|
-
return false;
|
|
58208
|
-
return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
58209
|
-
}).length
|
|
58210
|
-
});
|
|
58211
|
-
}
|
|
58212
|
-
rows.push({
|
|
58213
|
-
target: formatWorkflowStageTarget(rootRunId, "**"),
|
|
58214
|
-
queuedCount: stickyQueued.filter((entry) => {
|
|
58215
|
-
const segments = stickySegments(entry);
|
|
58216
|
-
return segments !== undefined && segments.length === 1 && segments[0] === "**";
|
|
58217
|
-
}).length
|
|
58218
|
-
});
|
|
58219
|
-
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
58220
|
-
const statusesByTarget = new Map;
|
|
58221
|
-
for (const status of statuses) {
|
|
58222
|
-
const forTarget = statusesByTarget.get(status.target);
|
|
58223
|
-
if (forTarget === undefined)
|
|
58224
|
-
statusesByTarget.set(status.target, [status]);
|
|
58225
|
-
else
|
|
58226
|
-
forTarget.push(status);
|
|
58227
|
-
}
|
|
58228
|
-
const queuedByTarget = new Map;
|
|
58229
|
-
for (const entry of stickyQueued) {
|
|
58230
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58231
|
-
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
58232
|
-
}
|
|
58233
|
-
const rowTargets = new Set(rows.map((row) => row.target));
|
|
58234
|
-
for (const target of statusesByTarget.keys()) {
|
|
58235
|
-
if (rowTargets.has(target))
|
|
58236
|
-
continue;
|
|
58237
|
-
rowTargets.add(target);
|
|
58238
|
-
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
58239
|
-
}
|
|
58240
|
-
for (let index = 0;index < rows.length; index++) {
|
|
58241
|
-
const row = rows[index];
|
|
58242
|
-
const deliveryStates = statusesByTarget.get(row.target);
|
|
58243
|
-
if (deliveryStates !== undefined)
|
|
58244
|
-
rows[index] = { ...row, deliveryStates };
|
|
58245
|
-
}
|
|
58246
|
-
return rows;
|
|
58247
|
-
}
|
|
58248
|
-
function materializedStageMatcher(runs, rootRunId) {
|
|
58249
|
-
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58250
|
-
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
58251
|
-
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
58252
|
-
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
58253
|
-
}
|
|
58254
|
-
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
58255
|
-
for (const run of invocationRuns) {
|
|
58256
|
-
const hops = hopsFor(run.id);
|
|
58257
|
-
if (hops === undefined)
|
|
58258
|
-
continue;
|
|
58259
|
-
for (const stage of run.stages) {
|
|
58260
|
-
if (stage.pendingStageDeliveryAvailable !== true)
|
|
58261
|
-
continue;
|
|
58262
|
-
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
58263
|
-
continue;
|
|
58264
|
-
}
|
|
58265
|
-
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
58266
|
-
return true;
|
|
58267
|
-
}
|
|
58268
|
-
}
|
|
58269
|
-
return false;
|
|
58270
|
-
}
|
|
58271
59415
|
function isPendingStageMessageEvent(value2) {
|
|
58272
59416
|
if (typeof value2 !== "object" || value2 === null)
|
|
58273
59417
|
return false;
|
|
@@ -58557,7 +59701,7 @@ function knownUninitializedStage(run, stageKey) {
|
|
|
58557
59701
|
if (candidates.length !== 1)
|
|
58558
59702
|
return;
|
|
58559
59703
|
const stage = candidates[0];
|
|
58560
|
-
return (stage.status
|
|
59704
|
+
return isAddressableStageStatus(stage.status) && stage.sessionId === undefined && stage.sessionFile === undefined ? stage : undefined;
|
|
58561
59705
|
}
|
|
58562
59706
|
function pendingStageDestination(run, entry) {
|
|
58563
59707
|
if (entry.stageId !== undefined) {
|
|
@@ -58651,7 +59795,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
58651
59795
|
init_pending_stage_route_capability();
|
|
58652
59796
|
init_pending_stage_route_readiness();
|
|
58653
59797
|
init_pending_stage_status();
|
|
58654
|
-
|
|
59798
|
+
init_possible_stage_rows();
|
|
58655
59799
|
init_store_internal();
|
|
58656
59800
|
init_workflow_stage_target();
|
|
58657
59801
|
});
|
|
@@ -61481,9 +62625,146 @@ var init_definitions = __esm(() => {
|
|
|
61481
62625
|
workerDefinitionSources = new WeakMap;
|
|
61482
62626
|
});
|
|
61483
62627
|
|
|
62628
|
+
// dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
|
|
62629
|
+
function push(map, key, value2) {
|
|
62630
|
+
if (key === undefined)
|
|
62631
|
+
return;
|
|
62632
|
+
const list = map.get(key);
|
|
62633
|
+
if (list)
|
|
62634
|
+
list.push(value2);
|
|
62635
|
+
else
|
|
62636
|
+
map.set(key, [value2]);
|
|
62637
|
+
}
|
|
62638
|
+
|
|
62639
|
+
class Partitions {
|
|
62640
|
+
byRun = new Map;
|
|
62641
|
+
startsByChild = new Map;
|
|
62642
|
+
stagesByGroup = new Map;
|
|
62643
|
+
stagesByReplayKey = new Map;
|
|
62644
|
+
boundariesByStageId = new Map;
|
|
62645
|
+
boundariesByChild = new Map;
|
|
62646
|
+
add(entry) {
|
|
62647
|
+
const checkpoint = entry.checkpoint;
|
|
62648
|
+
push(this.byRun, checkpoint.topology?.run?.runId, entry);
|
|
62649
|
+
if (checkpoint.kind !== "stage")
|
|
62650
|
+
return;
|
|
62651
|
+
const stage = entry;
|
|
62652
|
+
push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
|
|
62653
|
+
push(this.stagesByReplayKey, checkpoint.replayKey, stage);
|
|
62654
|
+
const boundary = checkpoint.topology?.boundary;
|
|
62655
|
+
if (!boundary)
|
|
62656
|
+
return;
|
|
62657
|
+
push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
|
|
62658
|
+
push(this.boundariesByChild, boundary.child?.runId, stage);
|
|
62659
|
+
if (boundary.event === "start")
|
|
62660
|
+
push(this.startsByChild, boundary.child?.runId, stage);
|
|
62661
|
+
}
|
|
62662
|
+
}
|
|
62663
|
+
function inKeyOrder(entries) {
|
|
62664
|
+
return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
|
|
62665
|
+
}
|
|
62666
|
+
|
|
62667
|
+
class PoolCheckpointView {
|
|
62668
|
+
base;
|
|
62669
|
+
from;
|
|
62670
|
+
delta;
|
|
62671
|
+
constructor(base, from, delta) {
|
|
62672
|
+
this.base = base;
|
|
62673
|
+
this.from = from;
|
|
62674
|
+
this.delta = delta;
|
|
62675
|
+
}
|
|
62676
|
+
entries(partition, key) {
|
|
62677
|
+
const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
|
|
62678
|
+
return [...committed, ...this.delta[partition].get(key) ?? []];
|
|
62679
|
+
}
|
|
62680
|
+
}
|
|
62681
|
+
|
|
62682
|
+
class PoolCheckpointRead {
|
|
62683
|
+
index;
|
|
62684
|
+
root;
|
|
62685
|
+
pending;
|
|
62686
|
+
constructor(index, root) {
|
|
62687
|
+
this.index = index;
|
|
62688
|
+
this.root = root;
|
|
62689
|
+
}
|
|
62690
|
+
read(tx, events) {
|
|
62691
|
+
return this.index.read(tx, events, this.root, (apply) => {
|
|
62692
|
+
this.pending = apply;
|
|
62693
|
+
});
|
|
62694
|
+
}
|
|
62695
|
+
commit() {
|
|
62696
|
+
this.pending?.();
|
|
62697
|
+
this.pending = undefined;
|
|
62698
|
+
}
|
|
62699
|
+
}
|
|
62700
|
+
|
|
62701
|
+
class PoolCheckpointIndex {
|
|
62702
|
+
roots = new Map;
|
|
62703
|
+
begin(root) {
|
|
62704
|
+
return new PoolCheckpointRead(this, root);
|
|
62705
|
+
}
|
|
62706
|
+
async read(tx, events, root, onCommit) {
|
|
62707
|
+
const current = this.roots.get(root);
|
|
62708
|
+
let base = current;
|
|
62709
|
+
const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
|
|
62710
|
+
if (base && base.seq > 0) {
|
|
62711
|
+
const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
|
|
62712
|
+
if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
|
|
62713
|
+
base = undefined;
|
|
62714
|
+
}
|
|
62715
|
+
const from = base?.seq ?? 0;
|
|
62716
|
+
const to = head ? Number(head.seq) : 0;
|
|
62717
|
+
const delta = new Partitions;
|
|
62718
|
+
const added = [];
|
|
62719
|
+
if (to > from) {
|
|
62720
|
+
const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
|
|
62721
|
+
for (const row of rows) {
|
|
62722
|
+
const checkpoint = JSON.parse(String(row.payload));
|
|
62723
|
+
if (checkpoint.kind === "ui")
|
|
62724
|
+
continue;
|
|
62725
|
+
const entry = {
|
|
62726
|
+
seq: Number(row.seq),
|
|
62727
|
+
order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
|
|
62728
|
+
checkpoint
|
|
62729
|
+
};
|
|
62730
|
+
delta.add(entry);
|
|
62731
|
+
added.push(entry);
|
|
62732
|
+
}
|
|
62733
|
+
}
|
|
62734
|
+
onCommit(() => {
|
|
62735
|
+
const latest = this.roots.get(root);
|
|
62736
|
+
if (!base) {
|
|
62737
|
+
if (latest !== current)
|
|
62738
|
+
return;
|
|
62739
|
+
const fresh = {
|
|
62740
|
+
seq: to,
|
|
62741
|
+
key: head?.key ?? null,
|
|
62742
|
+
at: Number(head?.at ?? 0),
|
|
62743
|
+
partitions: delta
|
|
62744
|
+
};
|
|
62745
|
+
this.roots.set(root, fresh);
|
|
62746
|
+
return;
|
|
62747
|
+
}
|
|
62748
|
+
if (latest !== base || to <= base.seq)
|
|
62749
|
+
return;
|
|
62750
|
+
for (const entry of added)
|
|
62751
|
+
if (entry.seq > base.seq)
|
|
62752
|
+
base.partitions.add(entry);
|
|
62753
|
+
base.seq = to;
|
|
62754
|
+
base.key = head?.key ?? null;
|
|
62755
|
+
base.at = Number(head?.at ?? 0);
|
|
62756
|
+
});
|
|
62757
|
+
return new PoolCheckpointView(base, from, delta);
|
|
62758
|
+
}
|
|
62759
|
+
}
|
|
62760
|
+
var init_pool_checkpoint_index = __esm(() => {
|
|
62761
|
+
init_stage_topology_validation();
|
|
62762
|
+
init_types();
|
|
62763
|
+
});
|
|
62764
|
+
|
|
61484
62765
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
61485
62766
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
61486
|
-
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
62767
|
+
async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
|
|
61487
62768
|
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
61488
62769
|
throw new Error("Invalid pool mutation");
|
|
61489
62770
|
const checkpoint = mutation.checkpoint;
|
|
@@ -61503,42 +62784,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
|
61503
62784
|
}
|
|
61504
62785
|
if (checkpoint.kind === "ui")
|
|
61505
62786
|
return;
|
|
61506
|
-
const
|
|
61507
|
-
const records = rows.map((row) => JSON.parse(String(row.payload)));
|
|
61508
|
-
const stages = records.filter((entry) => entry.kind === "stage");
|
|
62787
|
+
const view = await checkpoints.read(tx, journal.events);
|
|
61509
62788
|
const topology = checkpoint.topology;
|
|
61510
62789
|
const run = topology?.run;
|
|
61511
62790
|
if (!topology || !run || run.rootRunId !== root || run.runId === root)
|
|
61512
62791
|
throw new Error("Pool topology scope mismatch");
|
|
61513
|
-
const owner =
|
|
62792
|
+
const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
|
|
61514
62793
|
if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
|
|
61515
62794
|
throw new Error("Pool topology scope mismatch");
|
|
61516
62795
|
if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
|
|
61517
62796
|
throw new Error("Pool session scope mismatch");
|
|
62797
|
+
const runEntries = view.entries("byRun", run.runId);
|
|
61518
62798
|
if (checkpoint.kind === "tool") {
|
|
61519
62799
|
const tool = checkpoint.topology;
|
|
61520
|
-
const peers =
|
|
62800
|
+
const peers = runEntries.map((entry) => entry.checkpoint);
|
|
61521
62801
|
const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
|
|
61522
62802
|
if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
|
|
61523
62803
|
throw new Error("Pool tool topology scope mismatch");
|
|
61524
62804
|
return;
|
|
61525
62805
|
}
|
|
61526
|
-
const
|
|
61527
|
-
const
|
|
62806
|
+
const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
|
|
62807
|
+
const relevant = [...runStages];
|
|
62808
|
+
const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
|
|
62809
|
+
for (const group of groups)
|
|
62810
|
+
relevant.push(...view.entries("stagesByGroup", group));
|
|
62811
|
+
const boundary = checkpoint.topology?.boundary;
|
|
62812
|
+
let start;
|
|
62813
|
+
if (boundary) {
|
|
62814
|
+
const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
|
|
62815
|
+
start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
62816
|
+
relevant.push(...replayed);
|
|
62817
|
+
if (start) {
|
|
62818
|
+
const startTopology = start.topology;
|
|
62819
|
+
relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
|
|
62820
|
+
relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
|
|
62821
|
+
}
|
|
62822
|
+
}
|
|
62823
|
+
const next = [...stagesExcept(relevant, checkpoint), checkpoint];
|
|
62824
|
+
const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
|
|
61528
62825
|
const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
|
|
61529
62826
|
if (stageError)
|
|
61530
62827
|
throw new Error(`Pool topology scope: ${stageError}`);
|
|
61531
|
-
const boundary = checkpoint.topology?.boundary;
|
|
61532
62828
|
if (!boundary) {
|
|
61533
62829
|
if (parseWorkflowChildResult(checkpoint.output))
|
|
61534
62830
|
throw new Error("Pool boundary output without ownership");
|
|
61535
62831
|
return;
|
|
61536
62832
|
}
|
|
61537
|
-
const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
61538
62833
|
const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
|
|
61539
62834
|
if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
|
|
61540
62835
|
throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
|
|
61541
62836
|
}
|
|
62837
|
+
function stagesExcept(entries, checkpoint) {
|
|
62838
|
+
return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
|
|
62839
|
+
}
|
|
61542
62840
|
async function sessionStore(tx, prefix) {
|
|
61543
62841
|
const database = {
|
|
61544
62842
|
descriptor: { adapter: "pool-transaction", location: prefix },
|
|
@@ -61587,20 +62885,52 @@ var init_pool_mutations = __esm(() => {
|
|
|
61587
62885
|
init_boundary_lifecycle();
|
|
61588
62886
|
init_boundary_topology();
|
|
61589
62887
|
init_stage_topology_validation();
|
|
61590
|
-
|
|
62888
|
+
init_pool_checkpoint_index();
|
|
61591
62889
|
});
|
|
61592
62890
|
|
|
61593
62891
|
// dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
|
|
61594
62892
|
function poolUsageTable(runs) {
|
|
61595
62893
|
return `${runs}_pool_usage`;
|
|
61596
62894
|
}
|
|
62895
|
+
function poolUsageTotalsTable(runs) {
|
|
62896
|
+
return `${runs}_pool_usage_totals`;
|
|
62897
|
+
}
|
|
62898
|
+
function addToTotals(totals, runs, work, sign, row) {
|
|
62899
|
+
const assignments = POOL_USAGE_FIELDS.map((field) => {
|
|
62900
|
+
const total = field.toLowerCase();
|
|
62901
|
+
const value2 = `(${sign}${row}.${total})`;
|
|
62902
|
+
return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
|
|
62903
|
+
});
|
|
62904
|
+
return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
|
|
62905
|
+
}
|
|
61597
62906
|
function poolUsageSchema(runs) {
|
|
61598
|
-
|
|
62907
|
+
const usage = poolUsageTable(runs);
|
|
62908
|
+
const totals = poolUsageTotalsTable(runs);
|
|
62909
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
62910
|
+
return `CREATE TABLE IF NOT EXISTS ${usage}(
|
|
61599
62911
|
work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
|
|
61600
62912
|
generation INTEGER NOT NULL, key TEXT NOT NULL,
|
|
61601
62913
|
input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
|
|
61602
62914
|
PRIMARY KEY(work, generation, key)
|
|
61603
|
-
)
|
|
62915
|
+
);
|
|
62916
|
+
CREATE TABLE IF NOT EXISTS ${totals}(
|
|
62917
|
+
root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
|
|
62918
|
+
${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
|
|
62919
|
+
);
|
|
62920
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
|
|
62921
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
62922
|
+
END;
|
|
62923
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
|
|
62924
|
+
WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
|
|
62925
|
+
${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
|
|
62926
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
62927
|
+
END;
|
|
62928
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
|
|
62929
|
+
DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
|
|
62930
|
+
END;
|
|
62931
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
|
|
62932
|
+
DELETE FROM ${totals} WHERE root = OLD.root;
|
|
62933
|
+
END;`;
|
|
61604
62934
|
}
|
|
61605
62935
|
function poolStageUsageKey(runId, stageId) {
|
|
61606
62936
|
return `stage:${runId}:${stageId}`;
|
|
@@ -61627,7 +62957,7 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
61627
62957
|
|
|
61628
62958
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
61629
62959
|
import { createHash as createHash4, randomUUID as randomUUID15 } from "node:crypto";
|
|
61630
|
-
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
62960
|
+
import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
|
|
61631
62961
|
function requestIdentity(request) {
|
|
61632
62962
|
return createHash4("sha256").update(request).digest("hex");
|
|
61633
62963
|
}
|
|
@@ -61666,6 +62996,7 @@ class SqlPoolQueue {
|
|
|
61666
62996
|
operations;
|
|
61667
62997
|
transaction;
|
|
61668
62998
|
journal;
|
|
62999
|
+
checkpoints = new PoolCheckpointIndex;
|
|
61669
63000
|
constructor(storage) {
|
|
61670
63001
|
this.storage = storage;
|
|
61671
63002
|
const { transaction, tables: journal } = storage;
|
|
@@ -61777,17 +63108,23 @@ class SqlPoolQueue {
|
|
|
61777
63108
|
lease = snapshotLease(lease);
|
|
61778
63109
|
token = { ...token };
|
|
61779
63110
|
return this.storage.withAppendGuard(async (tx, runId) => {
|
|
61780
|
-
|
|
61781
|
-
|
|
61782
|
-
|
|
61783
|
-
|
|
61784
|
-
|
|
61785
|
-
|
|
61786
|
-
|
|
61787
|
-
|
|
61788
|
-
|
|
61789
|
-
|
|
61790
|
-
|
|
63111
|
+
try {
|
|
63112
|
+
if (runId !== lease.runId)
|
|
63113
|
+
throw new Error("Pool delivery cannot write another root");
|
|
63114
|
+
await this.authority(tx, lease).catch((error) => {
|
|
63115
|
+
if (error instanceof StaleLeaseError)
|
|
63116
|
+
throw new Error("Stale pool coordinator authority", { cause: error });
|
|
63117
|
+
throw error;
|
|
63118
|
+
});
|
|
63119
|
+
const row = await this.current(tx, lease, token);
|
|
63120
|
+
if (row.epoch !== lease.epoch)
|
|
63121
|
+
throw new Error(STALE_WORK_LEASE);
|
|
63122
|
+
assertSession();
|
|
63123
|
+
} catch (error) {
|
|
63124
|
+
if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
|
|
63125
|
+
throw error;
|
|
63126
|
+
throw new PoolAppendRefusedError(error.message, { cause: error });
|
|
63127
|
+
}
|
|
61791
63128
|
}, action);
|
|
61792
63129
|
}
|
|
61793
63130
|
async assertCurrent(lease, token) {
|
|
@@ -61900,7 +63237,8 @@ class SqlPoolQueue {
|
|
|
61900
63237
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
61901
63238
|
const identity = requestIdentity(request);
|
|
61902
63239
|
mutation = JSON.parse(request).mutation;
|
|
61903
|
-
|
|
63240
|
+
const checkpoints = this.checkpoints.begin(lease.runId);
|
|
63241
|
+
const receipt = await this.transaction(async (tx) => {
|
|
61904
63242
|
const { now, status } = await this.authority(tx, lease);
|
|
61905
63243
|
const row = await this.current(tx, lease, token);
|
|
61906
63244
|
if (row.epoch !== lease.epoch)
|
|
@@ -61915,24 +63253,26 @@ class SqlPoolQueue {
|
|
|
61915
63253
|
throw new Error(STALE_WORK_LEASE);
|
|
61916
63254
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
61917
63255
|
throw new Error("Pool root is not running");
|
|
61918
|
-
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
63256
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
|
|
61919
63257
|
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
61920
63258
|
return receipt;
|
|
61921
63259
|
});
|
|
63260
|
+
checkpoints.commit();
|
|
63261
|
+
return receipt;
|
|
61922
63262
|
}
|
|
61923
63263
|
async appendSession(tx, row, mutation, now) {
|
|
61924
63264
|
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
61925
63265
|
}
|
|
61926
|
-
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
63266
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
|
|
61927
63267
|
const checkpoint = mutation.checkpoint;
|
|
61928
|
-
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
63268
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
|
|
61929
63269
|
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
61930
63270
|
throw new Error("Pool checkpoint timestamp is in the future");
|
|
61931
63271
|
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
61932
63272
|
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
61933
63273
|
const payload = serializeJson(checkpoint);
|
|
61934
63274
|
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
61935
|
-
if (existing && existing.payload
|
|
63275
|
+
if (existing && !isRecordedCheckpoint(existing.payload, payload))
|
|
61936
63276
|
throw new Error("Pool checkpoint identity conflict");
|
|
61937
63277
|
if (existing && mutation.kind !== "checkpoint")
|
|
61938
63278
|
throw new Error("Pool session checkpoint identity conflict");
|
|
@@ -62030,22 +63370,37 @@ class SqlPoolQueue {
|
|
|
62030
63370
|
return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
|
|
62031
63371
|
}
|
|
62032
63372
|
}
|
|
63373
|
+
function isRecordedCheckpoint(stored, payload) {
|
|
63374
|
+
if (stored === payload)
|
|
63375
|
+
return true;
|
|
63376
|
+
const { completedAt } = JSON.parse(stored);
|
|
63377
|
+
if (typeof completedAt !== "number")
|
|
63378
|
+
return false;
|
|
63379
|
+
return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
|
|
63380
|
+
}
|
|
62033
63381
|
function openSqlPoolQueue(journal) {
|
|
62034
63382
|
const storage = sqlPoolStorage(journal);
|
|
62035
63383
|
if (!storage)
|
|
62036
63384
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
62037
63385
|
return SqlPoolQueue.open(storage);
|
|
62038
63386
|
}
|
|
62039
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
63387
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
|
|
62040
63388
|
var init_sql_pool_queue = __esm(() => {
|
|
62041
63389
|
init_serializable();
|
|
62042
63390
|
init_boundary_topology();
|
|
62043
63391
|
init_durable_hash();
|
|
62044
63392
|
init_internal();
|
|
63393
|
+
init_pool_checkpoint_index();
|
|
62045
63394
|
init_pool_mutations();
|
|
62046
63395
|
init_pool_stage_usage();
|
|
62047
63396
|
init_types();
|
|
62048
63397
|
init_values();
|
|
63398
|
+
PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
|
|
63399
|
+
constructor(message, options) {
|
|
63400
|
+
super(message, options);
|
|
63401
|
+
this.name = "PoolAppendRefusedError";
|
|
63402
|
+
}
|
|
63403
|
+
};
|
|
62049
63404
|
});
|
|
62050
63405
|
|
|
62051
63406
|
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
@@ -62068,15 +63423,22 @@ function validate(report) {
|
|
|
62068
63423
|
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
62069
63424
|
}
|
|
62070
63425
|
}
|
|
63426
|
+
function addCompensated(total, value2) {
|
|
63427
|
+
const sum = total.sum + value2;
|
|
63428
|
+
total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
|
|
63429
|
+
total.sum = sum;
|
|
63430
|
+
}
|
|
62071
63431
|
|
|
62072
63432
|
class SqlPoolUsage {
|
|
62073
63433
|
transaction;
|
|
62074
63434
|
journal;
|
|
62075
63435
|
table;
|
|
63436
|
+
totals;
|
|
62076
63437
|
constructor(transaction, journal) {
|
|
62077
63438
|
this.transaction = transaction;
|
|
62078
63439
|
this.journal = journal;
|
|
62079
63440
|
this.table = poolUsageTable(journal.runs);
|
|
63441
|
+
this.totals = poolUsageTotalsTable(journal.runs);
|
|
62080
63442
|
}
|
|
62081
63443
|
static async open(transaction, journal) {
|
|
62082
63444
|
const usage = new SqlPoolUsage(transaction, journal);
|
|
@@ -62084,9 +63446,27 @@ class SqlPoolUsage {
|
|
|
62084
63446
|
return usage;
|
|
62085
63447
|
}
|
|
62086
63448
|
async totalIn(tx, root) {
|
|
63449
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
63450
|
+
const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
|
|
63451
|
+
if (kept)
|
|
63452
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
|
|
63453
|
+
field,
|
|
63454
|
+
Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
|
|
63455
|
+
]));
|
|
62087
63456
|
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
62088
63457
|
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
62089
|
-
|
|
63458
|
+
const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
63459
|
+
if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
|
|
63460
|
+
return total;
|
|
63461
|
+
const sums = columns.map(() => ({ sum: 0, error: 0 }));
|
|
63462
|
+
const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
|
|
63463
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
63464
|
+
for (const usage of rows)
|
|
63465
|
+
for (const [index, column] of columns.entries())
|
|
63466
|
+
addCompensated(sums[index], Number(usage[column]));
|
|
63467
|
+
await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
|
|
63468
|
+
VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
|
|
63469
|
+
return total;
|
|
62090
63470
|
}
|
|
62091
63471
|
async report(lease, token, reports) {
|
|
62092
63472
|
validateText("runId", lease.runId);
|
|
@@ -63002,15 +64382,74 @@ function isWorkflowPendingStageDeliveryFailure(error) {
|
|
|
63002
64382
|
return true;
|
|
63003
64383
|
return error?.code === "pending_stage_delivery_failed";
|
|
63004
64384
|
}
|
|
63005
|
-
function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action()) {
|
|
64385
|
+
function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action(), options = {}) {
|
|
63006
64386
|
let resolveReady;
|
|
63007
64387
|
let rejectReady;
|
|
63008
64388
|
let readyPromise;
|
|
63009
64389
|
let drainError;
|
|
63010
64390
|
let terminalError;
|
|
63011
64391
|
let drainPromise;
|
|
64392
|
+
let drainedSessionId;
|
|
63012
64393
|
let currentRecipient;
|
|
63013
|
-
|
|
64394
|
+
let lastProgressAt = Date.now();
|
|
64395
|
+
let stallReported = false;
|
|
64396
|
+
const progress = () => {
|
|
64397
|
+
lastProgressAt = Date.now();
|
|
64398
|
+
};
|
|
64399
|
+
const progressingCommit = async (action) => {
|
|
64400
|
+
try {
|
|
64401
|
+
return await commit(action);
|
|
64402
|
+
} finally {
|
|
64403
|
+
progress();
|
|
64404
|
+
}
|
|
64405
|
+
};
|
|
64406
|
+
const reportStall = (sessionId) => {
|
|
64407
|
+
if (stallReported)
|
|
64408
|
+
return;
|
|
64409
|
+
stallReported = true;
|
|
64410
|
+
const undelivered = [
|
|
64411
|
+
...activeStore.pendingStageMessagesFor(runId, stageId),
|
|
64412
|
+
...stageId === stageName ? [] : activeStore.pendingStageMessagesFor(runId, stageName)
|
|
64413
|
+
].map((entry) => `${entry.message.id} (direct → ${entry.stageKey})`).concat(stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).map((entry) => `${entry.message.id} (sticky → ${entry.targetPath ?? entry.stageKey})`));
|
|
64414
|
+
const listed = undelivered.slice(0, 10).join(", ");
|
|
64415
|
+
const more = undelivered.length > 10 ? `, and ${undelivered.length - 10} more` : "";
|
|
64416
|
+
const message = `atomic-workflows: stage "${stageName}" (${stageId}) is starting without its queued Intercom messages: ` + `no delivery progress for ${PENDING_STAGE_READY_STALL_MS / 1000}s; ` + `drain ${drainPromise === undefined ? "not running" : "running but hung"}; ` + `undelivered: ${listed.length === 0 ? "none" : `${listed}${more}`}. ` + "They stay queued and arrive live or on the stage's next drain.";
|
|
64417
|
+
try {
|
|
64418
|
+
(options.reportStall ?? console.warn)(message);
|
|
64419
|
+
} catch {}
|
|
64420
|
+
};
|
|
64421
|
+
const watchStall = (pending, sessionId, onStall) => {
|
|
64422
|
+
let timer;
|
|
64423
|
+
let settled = false;
|
|
64424
|
+
const check = () => {
|
|
64425
|
+
if (settled)
|
|
64426
|
+
return;
|
|
64427
|
+
const remaining = lastProgressAt + PENDING_STAGE_READY_STALL_MS - Date.now();
|
|
64428
|
+
if (remaining > 0) {
|
|
64429
|
+
timer = setTimeout(check, remaining);
|
|
64430
|
+
timer.unref?.();
|
|
64431
|
+
return;
|
|
64432
|
+
}
|
|
64433
|
+
reportStall(sessionId);
|
|
64434
|
+
onStall();
|
|
64435
|
+
};
|
|
64436
|
+
const stop = () => {
|
|
64437
|
+
settled = true;
|
|
64438
|
+
clearTimeout(timer);
|
|
64439
|
+
};
|
|
64440
|
+
pending.then(stop, stop);
|
|
64441
|
+
check();
|
|
64442
|
+
};
|
|
64443
|
+
const boundedDrain = (drain, sessionId) => {
|
|
64444
|
+
const waiting = new Promise((resolve, reject) => {
|
|
64445
|
+
drain.then(resolve, reject);
|
|
64446
|
+
watchStall(drain, sessionId, resolve);
|
|
64447
|
+
});
|
|
64448
|
+
waiting.catch(() => {});
|
|
64449
|
+
return waiting;
|
|
64450
|
+
};
|
|
64451
|
+
const hasUndeliveredExact = () => activeStore.pendingStageMessagesFor(runId, stageId).length > 0 || stageId !== stageName && activeStore.pendingStageMessagesFor(runId, stageName).length > 0;
|
|
64452
|
+
const hasUndelivered = (sessionId) => hasUndeliveredExact() || stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).length > 0;
|
|
63014
64453
|
const pendingReady = () => {
|
|
63015
64454
|
if (readyPromise === undefined) {
|
|
63016
64455
|
readyPromise = new Promise((resolve, reject) => {
|
|
@@ -63020,35 +64459,64 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
63020
64459
|
}
|
|
63021
64460
|
return readyPromise;
|
|
63022
64461
|
};
|
|
64462
|
+
let followUp;
|
|
64463
|
+
const startDrain = (deliver, recipient) => {
|
|
64464
|
+
if (terminalError !== undefined)
|
|
64465
|
+
return Promise.resolve();
|
|
64466
|
+
drainError = undefined;
|
|
64467
|
+
progress();
|
|
64468
|
+
const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, async (from, message) => {
|
|
64469
|
+
progress();
|
|
64470
|
+
await deliver(from, message);
|
|
64471
|
+
progress();
|
|
64472
|
+
}, progressingCommit, recipient);
|
|
64473
|
+
const drain = attempt.then(() => {
|
|
64474
|
+
progress();
|
|
64475
|
+
drainedSessionId = recipient?.sessionId;
|
|
64476
|
+
resolveReady?.();
|
|
64477
|
+
readyPromise = undefined;
|
|
64478
|
+
resolveReady = undefined;
|
|
64479
|
+
rejectReady = undefined;
|
|
64480
|
+
if (drainPromise === drain)
|
|
64481
|
+
drainPromise = undefined;
|
|
64482
|
+
}, (error) => {
|
|
64483
|
+
progress();
|
|
64484
|
+
drainError = error;
|
|
64485
|
+
rejectReady?.(error);
|
|
64486
|
+
readyPromise = undefined;
|
|
64487
|
+
resolveReady = undefined;
|
|
64488
|
+
rejectReady = undefined;
|
|
64489
|
+
if (drainPromise === drain)
|
|
64490
|
+
drainPromise = undefined;
|
|
64491
|
+
throw error;
|
|
64492
|
+
});
|
|
64493
|
+
drainPromise = drain;
|
|
64494
|
+
return boundedDrain(drain, recipient?.sessionId);
|
|
64495
|
+
};
|
|
64496
|
+
const requestDrain = (deliver, recipient) => {
|
|
64497
|
+
const running = drainPromise;
|
|
64498
|
+
if (running === undefined)
|
|
64499
|
+
return startDrain(deliver, recipient);
|
|
64500
|
+
if (followUp !== undefined) {
|
|
64501
|
+
followUp.deliver = deliver;
|
|
64502
|
+
followUp.recipient = recipient;
|
|
64503
|
+
return boundedDrain(followUp.settled.promise, recipient?.sessionId);
|
|
64504
|
+
}
|
|
64505
|
+
const next = { deliver, recipient, settled: Promise.withResolvers() };
|
|
64506
|
+
next.settled.promise.catch(() => {});
|
|
64507
|
+
followUp = next;
|
|
64508
|
+
running.catch(() => {}).then(() => {
|
|
64509
|
+
if (followUp === next)
|
|
64510
|
+
followUp = undefined;
|
|
64511
|
+
return requestDrain(next.deliver, next.recipient);
|
|
64512
|
+
}).then(next.settled.resolve, next.settled.reject);
|
|
64513
|
+
return boundedDrain(next.settled.promise, recipient?.sessionId);
|
|
64514
|
+
};
|
|
63023
64515
|
return {
|
|
63024
64516
|
routeCapability: workflowPendingStageRouteCapability(activeStore, runId),
|
|
63025
64517
|
deliverPending(deliver, recipient) {
|
|
63026
64518
|
currentRecipient = recipient;
|
|
63027
|
-
|
|
63028
|
-
return drainPromise;
|
|
63029
|
-
if (terminalError !== undefined)
|
|
63030
|
-
return Promise.resolve();
|
|
63031
|
-
drainError = undefined;
|
|
63032
|
-
const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, deliver, commit, recipient);
|
|
63033
|
-
const drain = attempt.then(() => {
|
|
63034
|
-
resolveReady?.();
|
|
63035
|
-
readyPromise = undefined;
|
|
63036
|
-
resolveReady = undefined;
|
|
63037
|
-
rejectReady = undefined;
|
|
63038
|
-
if (drainPromise === drain)
|
|
63039
|
-
drainPromise = undefined;
|
|
63040
|
-
}, (error) => {
|
|
63041
|
-
drainError = error;
|
|
63042
|
-
rejectReady?.(error);
|
|
63043
|
-
readyPromise = undefined;
|
|
63044
|
-
resolveReady = undefined;
|
|
63045
|
-
rejectReady = undefined;
|
|
63046
|
-
if (drainPromise === drain)
|
|
63047
|
-
drainPromise = undefined;
|
|
63048
|
-
throw error;
|
|
63049
|
-
});
|
|
63050
|
-
drainPromise = drain;
|
|
63051
|
-
return drainPromise;
|
|
64519
|
+
return requestDrain(deliver, recipient);
|
|
63052
64520
|
},
|
|
63053
64521
|
ready(sessionId) {
|
|
63054
64522
|
if (!hasUndelivered(sessionId)) {
|
|
@@ -63058,6 +64526,8 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
63058
64526
|
return Promise.reject(terminalError);
|
|
63059
64527
|
if (drainError !== undefined)
|
|
63060
64528
|
return Promise.reject(drainError);
|
|
64529
|
+
if (drainPromise === undefined && sessionId !== undefined && drainedSessionId === sessionId && !hasUndeliveredExact())
|
|
64530
|
+
return;
|
|
63061
64531
|
const ready = pendingReady();
|
|
63062
64532
|
if (sessionId !== undefined && activeStore.pendingStageMessagesFor(runId, stageId).length === 0 && activeStore.pendingStageMessagesFor(runId, stageName).length === 0) {
|
|
63063
64533
|
const entries = stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, { sessionId });
|
|
@@ -63081,6 +64551,14 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
63081
64551
|
rejectReady?.(error);
|
|
63082
64552
|
});
|
|
63083
64553
|
}
|
|
64554
|
+
watchStall(ready, sessionId, () => {
|
|
64555
|
+
if (readyPromise !== ready)
|
|
64556
|
+
return;
|
|
64557
|
+
resolveReady?.();
|
|
64558
|
+
readyPromise = undefined;
|
|
64559
|
+
resolveReady = undefined;
|
|
64560
|
+
rejectReady = undefined;
|
|
64561
|
+
});
|
|
63084
64562
|
return ready;
|
|
63085
64563
|
},
|
|
63086
64564
|
fail(reason) {
|
|
@@ -63208,7 +64686,7 @@ function toPendingStageSender(entry) {
|
|
|
63208
64686
|
lastActivity: entry.from.lastActivity ?? entry.message.timestamp
|
|
63209
64687
|
};
|
|
63210
64688
|
}
|
|
63211
|
-
var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError;
|
|
64689
|
+
var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError, PENDING_STAGE_READY_STALL_MS = 60000;
|
|
63212
64690
|
var init_pending_stage_delivery2 = __esm(() => {
|
|
63213
64691
|
init_run_owner_backend();
|
|
63214
64692
|
init_store_backend();
|
|
@@ -65630,10 +67108,120 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
|
65630
67108
|
}
|
|
65631
67109
|
var init_stage_session_store = () => {};
|
|
65632
67110
|
|
|
67111
|
+
// dist/builtin/workflows/src/durable/completed-catalog-index.ts
|
|
67112
|
+
class CheckpointCatalogIndex {
|
|
67113
|
+
checkpoints;
|
|
67114
|
+
stages;
|
|
67115
|
+
positions = new Map;
|
|
67116
|
+
groupKeys = new Map;
|
|
67117
|
+
groups = new Map;
|
|
67118
|
+
stagesByRunId = new Map;
|
|
67119
|
+
boundariesByReplayKey = new Map;
|
|
67120
|
+
boundariesByStageId = new Map;
|
|
67121
|
+
boundariesByChildRunId = new Map;
|
|
67122
|
+
outputsByReplayKey = new Map;
|
|
67123
|
+
evidence;
|
|
67124
|
+
constructor(checkpoints) {
|
|
67125
|
+
this.checkpoints = checkpoints;
|
|
67126
|
+
const stages = [];
|
|
67127
|
+
checkpoints.forEach((checkpoint, index) => {
|
|
67128
|
+
if (!this.positions.has(checkpoint))
|
|
67129
|
+
this.positions.set(checkpoint, index);
|
|
67130
|
+
if (checkpoint.kind !== "stage")
|
|
67131
|
+
return;
|
|
67132
|
+
stages.push(checkpoint);
|
|
67133
|
+
const key = durableStageGroupKey(checkpoint);
|
|
67134
|
+
this.groupKeys.set(checkpoint, key);
|
|
67135
|
+
append(this.groups, key, checkpoint);
|
|
67136
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
67137
|
+
if (runId !== undefined)
|
|
67138
|
+
append(this.stagesByRunId, runId, checkpoint);
|
|
67139
|
+
if (checkpoint.output !== undefined)
|
|
67140
|
+
append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
|
|
67141
|
+
const boundary = checkpoint.topology?.boundary;
|
|
67142
|
+
if (boundary === undefined)
|
|
67143
|
+
return;
|
|
67144
|
+
append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
|
|
67145
|
+
append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
|
|
67146
|
+
append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
|
|
67147
|
+
});
|
|
67148
|
+
this.stages = stages;
|
|
67149
|
+
}
|
|
67150
|
+
sequence(checkpoint) {
|
|
67151
|
+
return (this.positions.get(checkpoint) ?? -1) + 1;
|
|
67152
|
+
}
|
|
67153
|
+
stageGroups() {
|
|
67154
|
+
return this.groups.values();
|
|
67155
|
+
}
|
|
67156
|
+
boundaryLifecycle(start, requireTerminal) {
|
|
67157
|
+
const topology = start.topology;
|
|
67158
|
+
const childRunId = topology?.boundary?.child.runId;
|
|
67159
|
+
const related = this.inCheckpointOrder([
|
|
67160
|
+
this.boundariesByReplayKey.get(start.replayKey),
|
|
67161
|
+
topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
|
|
67162
|
+
childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
|
|
67163
|
+
]);
|
|
67164
|
+
return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
|
|
67165
|
+
}
|
|
67166
|
+
directChildRecords(childRunId) {
|
|
67167
|
+
return {
|
|
67168
|
+
claiming: this.stagesByRunId.get(childRunId) ?? [],
|
|
67169
|
+
direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
|
|
67170
|
+
};
|
|
67171
|
+
}
|
|
67172
|
+
childScopedEvidenceExists(boundaryReplayKey, childRunId) {
|
|
67173
|
+
this.evidence ??= scopedEvidenceIndex(this.checkpoints);
|
|
67174
|
+
return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
|
|
67175
|
+
}
|
|
67176
|
+
inCheckpointOrder(lists) {
|
|
67177
|
+
const present = lists.filter((list) => list !== undefined);
|
|
67178
|
+
if (present.length === 1)
|
|
67179
|
+
return present[0];
|
|
67180
|
+
return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
|
|
67181
|
+
}
|
|
67182
|
+
}
|
|
67183
|
+
function scopedEvidenceIndex(checkpoints) {
|
|
67184
|
+
const runIds = new Set;
|
|
67185
|
+
const identities = [];
|
|
67186
|
+
for (const checkpoint of checkpoints) {
|
|
67187
|
+
identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
|
|
67188
|
+
if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
|
|
67189
|
+
runIds.add(checkpoint.topology.run.runId);
|
|
67190
|
+
}
|
|
67191
|
+
}
|
|
67192
|
+
identities.sort(compareCodeUnits);
|
|
67193
|
+
return { runIds, identities };
|
|
67194
|
+
}
|
|
67195
|
+
function hasPrefix(sorted, prefix) {
|
|
67196
|
+
let low = 0;
|
|
67197
|
+
let high = sorted.length;
|
|
67198
|
+
while (low < high) {
|
|
67199
|
+
const middle = low + high >>> 1;
|
|
67200
|
+
if (compareCodeUnits(sorted[middle], prefix) < 0)
|
|
67201
|
+
low = middle + 1;
|
|
67202
|
+
else
|
|
67203
|
+
high = middle;
|
|
67204
|
+
}
|
|
67205
|
+
return low < sorted.length && sorted[low].startsWith(prefix);
|
|
67206
|
+
}
|
|
67207
|
+
function compareCodeUnits(left, right) {
|
|
67208
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
67209
|
+
}
|
|
67210
|
+
function append(map, key, value2) {
|
|
67211
|
+
const list = map.get(key);
|
|
67212
|
+
if (list === undefined)
|
|
67213
|
+
map.set(key, [value2]);
|
|
67214
|
+
else
|
|
67215
|
+
list.push(value2);
|
|
67216
|
+
}
|
|
67217
|
+
var init_completed_catalog_index = __esm(() => {
|
|
67218
|
+
init_boundary_lifecycle();
|
|
67219
|
+
init_stage_topology_validation();
|
|
67220
|
+
});
|
|
67221
|
+
|
|
65633
67222
|
// dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
|
|
65634
|
-
function validBoundaryRecordSet(
|
|
65635
|
-
const
|
|
65636
|
-
const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
|
|
67223
|
+
function validBoundaryRecordSet(index, strict) {
|
|
67224
|
+
const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
|
|
65637
67225
|
const startsByReplay = new Map;
|
|
65638
67226
|
for (const checkpoint of boundaries) {
|
|
65639
67227
|
if (checkpoint.topology.boundary.event !== "start")
|
|
@@ -65648,7 +67236,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
|
|
|
65648
67236
|
return false;
|
|
65649
67237
|
return [...startsByReplay.values()].every((starts) => {
|
|
65650
67238
|
const start = starts[0];
|
|
65651
|
-
if (
|
|
67239
|
+
if ("error" in index.boundaryLifecycle(start, strict))
|
|
65652
67240
|
return false;
|
|
65653
67241
|
return start.topology.boundary.invocationFingerprint !== undefined;
|
|
65654
67242
|
});
|
|
@@ -65712,8 +67300,8 @@ function compareDraftSourceOrder(left, right) {
|
|
|
65712
67300
|
function runTopologyFor(drafts, tools) {
|
|
65713
67301
|
return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
|
|
65714
67302
|
}
|
|
65715
|
-
function validateRunGroups(grouped, rootRunId,
|
|
65716
|
-
const toolsFor = (runId) =>
|
|
67303
|
+
function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
|
|
67304
|
+
const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
|
|
65717
67305
|
const owners = new Map;
|
|
65718
67306
|
const syntheticChildIds = new Set;
|
|
65719
67307
|
for (const [parentRunId, drafts] of grouped) {
|
|
@@ -65724,12 +67312,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
|
|
|
65724
67312
|
if (childRunId === parentRunId || owners.has(childRunId))
|
|
65725
67313
|
return false;
|
|
65726
67314
|
if (!grouped.has(childRunId)) {
|
|
65727
|
-
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(
|
|
67315
|
+
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
|
|
65728
67316
|
syntheticChildIds.add(childRunId);
|
|
65729
67317
|
owners.set(childRunId, draft);
|
|
65730
67318
|
continue;
|
|
65731
67319
|
}
|
|
65732
|
-
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(
|
|
67320
|
+
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
|
|
65733
67321
|
return false;
|
|
65734
67322
|
syntheticChildIds.add(childRunId);
|
|
65735
67323
|
owners.set(childRunId, draft);
|
|
@@ -65770,13 +67358,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
|
|
|
65770
67358
|
const boundary = draft.topology?.boundary;
|
|
65771
67359
|
return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
|
|
65772
67360
|
}
|
|
65773
|
-
function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
65774
|
-
const prefix = `${boundaryReplayKey}:`;
|
|
65775
|
-
return checkpoints.some((checkpoint) => {
|
|
65776
|
-
const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
|
|
65777
|
-
return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
|
|
65778
|
-
});
|
|
65779
|
-
}
|
|
65780
67361
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
65781
67362
|
const reachable = new Set([rootRunId]);
|
|
65782
67363
|
const settled = new Set;
|
|
@@ -65817,14 +67398,6 @@ function childRunIdFromDraft(draft) {
|
|
|
65817
67398
|
return;
|
|
65818
67399
|
return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
|
|
65819
67400
|
}
|
|
65820
|
-
function boundaryOwner(grouped, childRunId) {
|
|
65821
|
-
for (const drafts of grouped.values()) {
|
|
65822
|
-
const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
|
|
65823
|
-
if (owner !== undefined)
|
|
65824
|
-
return owner;
|
|
65825
|
-
}
|
|
65826
|
-
return;
|
|
65827
|
-
}
|
|
65828
67401
|
function childRunStatus(owner) {
|
|
65829
67402
|
const child = workflowChildFromDraft(owner);
|
|
65830
67403
|
if (child !== undefined)
|
|
@@ -65832,15 +67405,15 @@ function childRunStatus(owner) {
|
|
|
65832
67405
|
const status = owner.topology?.boundary?.status;
|
|
65833
67406
|
return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
|
|
65834
67407
|
}
|
|
65835
|
-
function mergeStageGroup(records,
|
|
67408
|
+
function mergeStageGroup(records, index) {
|
|
65836
67409
|
let merged;
|
|
65837
67410
|
for (const record of records)
|
|
65838
|
-
merged = mergeStageDraft(merged, record,
|
|
67411
|
+
merged = mergeStageDraft(merged, record, index.sequence(record));
|
|
65839
67412
|
const draft = merged;
|
|
65840
67413
|
const start = records.find((record) => record.topology?.boundary?.event === "start");
|
|
65841
67414
|
if (start === undefined)
|
|
65842
67415
|
return draft;
|
|
65843
|
-
const resolved =
|
|
67416
|
+
const resolved = index.boundaryLifecycle(start, false);
|
|
65844
67417
|
if ("error" in resolved)
|
|
65845
67418
|
return draft;
|
|
65846
67419
|
const terminal = resolved.lifecycle.latestTerminal;
|
|
@@ -65903,9 +67476,7 @@ function workflowChildFromDraft(draft) {
|
|
|
65903
67476
|
...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
|
|
65904
67477
|
};
|
|
65905
67478
|
}
|
|
65906
|
-
var init_completed_catalog_stage_groups =
|
|
65907
|
-
init_boundary_lifecycle();
|
|
65908
|
-
});
|
|
67479
|
+
var init_completed_catalog_stage_groups = () => {};
|
|
65909
67480
|
|
|
65910
67481
|
// dist/builtin/workflows/src/engine/run-tool-control-registry.ts
|
|
65911
67482
|
function createToolControlRegistry() {
|
|
@@ -66110,11 +67681,68 @@ var init_resume_runtime = __esm(() => {
|
|
|
66110
67681
|
function isReservedRunCheckpointHash(argsHash) {
|
|
66111
67682
|
return [RUN_TIMING_CHECKPOINT_NAME, RUN_USAGE_CHECKPOINT_NAME, RUN_RESULT_CHECKPOINT_NAME].includes(argsHash) || argsHash === WORKFLOW_HEARTBEAT_ANCHOR_CHECKPOINT_NAME;
|
|
66112
67683
|
}
|
|
66113
|
-
function
|
|
67684
|
+
function durableNestedRunCatalog(backend, rootWorkflowId) {
|
|
66114
67685
|
const handle = backend.getWorkflow(rootWorkflowId);
|
|
66115
67686
|
if (handle === undefined)
|
|
66116
|
-
return
|
|
66117
|
-
|
|
67687
|
+
return EMPTY_NESTED_RUN_CATALOG;
|
|
67688
|
+
const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
|
|
67689
|
+
const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
|
|
67690
|
+
if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
|
|
67691
|
+
return cached.catalog;
|
|
67692
|
+
}
|
|
67693
|
+
const checkpoints = backend.listCheckpoints(rootWorkflowId);
|
|
67694
|
+
const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
|
|
67695
|
+
const catalog = nestedRunCatalog(runs, checkpoints);
|
|
67696
|
+
if (version !== undefined) {
|
|
67697
|
+
const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
|
|
67698
|
+
nestedRunCatalogs.set(backend, byRoot);
|
|
67699
|
+
byRoot.delete(rootWorkflowId);
|
|
67700
|
+
byRoot.set(rootWorkflowId, {
|
|
67701
|
+
version,
|
|
67702
|
+
name: handle.name,
|
|
67703
|
+
updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
|
|
67704
|
+
catalog
|
|
67705
|
+
});
|
|
67706
|
+
if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
|
|
67707
|
+
byRoot.delete(byRoot.keys().next().value);
|
|
67708
|
+
}
|
|
67709
|
+
return catalog;
|
|
67710
|
+
}
|
|
67711
|
+
function nestedRunCatalog(runs, checkpoints) {
|
|
67712
|
+
const positionsById = new Map;
|
|
67713
|
+
const childIdsByParent = new Map;
|
|
67714
|
+
runs.forEach((run, position) => {
|
|
67715
|
+
const positions = positionsById.get(run.id);
|
|
67716
|
+
if (positions === undefined)
|
|
67717
|
+
positionsById.set(run.id, [position]);
|
|
67718
|
+
else
|
|
67719
|
+
positions.push(position);
|
|
67720
|
+
if (run.parentRunId === undefined)
|
|
67721
|
+
return;
|
|
67722
|
+
const children = childIdsByParent.get(run.parentRunId);
|
|
67723
|
+
if (children === undefined)
|
|
67724
|
+
childIdsByParent.set(run.parentRunId, [run.id]);
|
|
67725
|
+
else
|
|
67726
|
+
children.push(run.id);
|
|
67727
|
+
});
|
|
67728
|
+
const intentionalExitStatuses = new Map;
|
|
67729
|
+
for (const checkpoint of checkpoints) {
|
|
67730
|
+
if (checkpoint.kind !== "stage")
|
|
67731
|
+
continue;
|
|
67732
|
+
const child = parseWorkflowChildResult(checkpoint.output);
|
|
67733
|
+
if (child?.exited === true)
|
|
67734
|
+
intentionalExitStatuses.set(child.runId, child.status);
|
|
67735
|
+
}
|
|
67736
|
+
return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
|
|
67737
|
+
}
|
|
67738
|
+
function readsFallbackCompletedAt(run) {
|
|
67739
|
+
return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
|
|
67740
|
+
}
|
|
67741
|
+
function cloneRunSnapshot(run) {
|
|
67742
|
+
const copy = structuredClone(run);
|
|
67743
|
+
for (const tool of copy.toolNodes)
|
|
67744
|
+
Object.freeze(tool.parentIds);
|
|
67745
|
+
return copy;
|
|
66118
67746
|
}
|
|
66119
67747
|
function checkpointDrafts(checkpoints) {
|
|
66120
67748
|
const stageByReplayKey = new Map;
|
|
@@ -66138,11 +67766,8 @@ function checkpointDrafts(checkpoints) {
|
|
|
66138
67766
|
tools: [...toolByHash.values()]
|
|
66139
67767
|
};
|
|
66140
67768
|
}
|
|
66141
|
-
function completedToolNodes(
|
|
66142
|
-
return
|
|
66143
|
-
const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
66144
|
-
if (topologyRunId !== runId)
|
|
66145
|
-
return [];
|
|
67769
|
+
function completedToolNodes(ownedCheckpoints, sourceIds) {
|
|
67770
|
+
return ownedCheckpoints.flatMap((checkpoint) => {
|
|
66146
67771
|
const topology = checkpoint.topology;
|
|
66147
67772
|
if (topology === undefined)
|
|
66148
67773
|
return [];
|
|
@@ -66179,18 +67804,34 @@ function summarizeCompletedToolResult(value2) {
|
|
|
66179
67804
|
return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
|
|
66180
67805
|
}
|
|
66181
67806
|
function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
|
|
66182
|
-
|
|
67807
|
+
const index = new CheckpointCatalogIndex(checkpoints);
|
|
67808
|
+
if (!validBoundaryRecordSet(index, policy.requireTerminal))
|
|
66183
67809
|
return [];
|
|
66184
|
-
const stageRecords =
|
|
67810
|
+
const stageRecords = index.stages;
|
|
66185
67811
|
const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
|
|
66186
67812
|
if (immutableStageGroupError(identityRecords) !== undefined)
|
|
66187
67813
|
return [];
|
|
66188
|
-
const
|
|
66189
|
-
const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
67814
|
+
const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
66190
67815
|
const drafts = checkpointDrafts(checkpoints);
|
|
66191
67816
|
const toolCheckpoints = drafts.tools;
|
|
66192
67817
|
if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
|
|
66193
67818
|
return [];
|
|
67819
|
+
const toolNodeIdsByClaimedRun = new Map;
|
|
67820
|
+
const toolsByRun = new Map;
|
|
67821
|
+
for (const checkpoint of toolCheckpoints) {
|
|
67822
|
+
const claimedRunId = checkpoint.topology?.run?.runId;
|
|
67823
|
+
if (claimedRunId !== undefined) {
|
|
67824
|
+
const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
|
|
67825
|
+
ids.add(checkpoint.topology.nodeId);
|
|
67826
|
+
toolNodeIdsByClaimedRun.set(claimedRunId, ids);
|
|
67827
|
+
}
|
|
67828
|
+
const ownerRunId = claimedRunId ?? rootRunId;
|
|
67829
|
+
const owned = toolsByRun.get(ownerRunId);
|
|
67830
|
+
if (owned === undefined)
|
|
67831
|
+
toolsByRun.set(ownerRunId, [checkpoint]);
|
|
67832
|
+
else
|
|
67833
|
+
owned.push(checkpoint);
|
|
67834
|
+
}
|
|
66194
67835
|
for (const draft of candidates) {
|
|
66195
67836
|
const boundary = draft.topology?.boundary;
|
|
66196
67837
|
if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
|
|
@@ -66204,8 +67845,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66204
67845
|
parentStageId: boundary.child.parentStageId,
|
|
66205
67846
|
rootRunId: boundary.child.rootRunId
|
|
66206
67847
|
};
|
|
66207
|
-
const childToolIds =
|
|
66208
|
-
if (
|
|
67848
|
+
const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
|
|
67849
|
+
if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
|
|
66209
67850
|
return [];
|
|
66210
67851
|
}
|
|
66211
67852
|
if (candidates.length === 0 && toolCheckpoints.length === 0) {
|
|
@@ -66233,7 +67874,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66233
67874
|
group.sort(compareDraftSourceOrder);
|
|
66234
67875
|
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
66235
67876
|
return [];
|
|
66236
|
-
if (!validateRunGroups(grouped, rootRunId,
|
|
67877
|
+
if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
|
|
66237
67878
|
return [];
|
|
66238
67879
|
const syntheticChildren = [];
|
|
66239
67880
|
const pendingChildren = [];
|
|
@@ -66247,7 +67888,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66247
67888
|
const child = workflowChildFromDraft(draft);
|
|
66248
67889
|
if (child?.exited !== true || grouped.has(child.runId))
|
|
66249
67890
|
continue;
|
|
66250
|
-
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(
|
|
67891
|
+
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
|
|
66251
67892
|
return [];
|
|
66252
67893
|
}
|
|
66253
67894
|
syntheticChildren.push({ parentRunId, owner: draft, child });
|
|
@@ -66255,7 +67896,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66255
67896
|
}
|
|
66256
67897
|
const idMaps = new Map;
|
|
66257
67898
|
for (const [runId, runDrafts] of grouped) {
|
|
66258
|
-
const ownedTools =
|
|
67899
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
66259
67900
|
if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
|
|
66260
67901
|
return [];
|
|
66261
67902
|
const ids = new Map;
|
|
@@ -66273,32 +67914,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66273
67914
|
});
|
|
66274
67915
|
if (hasIdentityConflict)
|
|
66275
67916
|
return [];
|
|
66276
|
-
for (const checkpoint of
|
|
66277
|
-
const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
67917
|
+
for (const checkpoint of ownedTools) {
|
|
66278
67918
|
const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
|
|
66279
|
-
|
|
66280
|
-
ids.set(sourceId, sourceId);
|
|
67919
|
+
ids.set(sourceId, sourceId);
|
|
66281
67920
|
}
|
|
66282
67921
|
idMaps.set(runId, ids);
|
|
66283
67922
|
}
|
|
67923
|
+
const boundaryOwners = new Map;
|
|
67924
|
+
const boundarySourcesByParent = new Map;
|
|
67925
|
+
for (const [parentRunId, runDrafts] of grouped) {
|
|
67926
|
+
const sources = new Map;
|
|
67927
|
+
for (const draft of runDrafts) {
|
|
67928
|
+
const childRunId = childRunIdFromDraft(draft);
|
|
67929
|
+
if (childRunId === undefined)
|
|
67930
|
+
continue;
|
|
67931
|
+
if (!boundaryOwners.has(childRunId))
|
|
67932
|
+
boundaryOwners.set(childRunId, draft);
|
|
67933
|
+
if (!sources.has(childRunId))
|
|
67934
|
+
sources.set(childRunId, draft);
|
|
67935
|
+
}
|
|
67936
|
+
boundarySourcesByParent.set(parentRunId, sources);
|
|
67937
|
+
}
|
|
66284
67938
|
const runs = [];
|
|
66285
67939
|
const emittedRunIds = new Set;
|
|
66286
67940
|
for (const [runId, runDrafts] of grouped) {
|
|
66287
67941
|
const ids = idMaps.get(runId);
|
|
66288
|
-
const ownedTools =
|
|
67942
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
66289
67943
|
const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
|
|
66290
67944
|
if (hasUnknownParents)
|
|
66291
67945
|
return [];
|
|
66292
67946
|
const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
|
|
66293
|
-
const toolNodes = completedToolNodes(
|
|
67947
|
+
const toolNodes = completedToolNodes(ownedTools, ids);
|
|
66294
67948
|
const run = runTopologyFor(runDrafts, ownedTools);
|
|
66295
67949
|
const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
|
|
66296
67950
|
const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
|
|
66297
|
-
const owner = runId === rootRunId ? undefined :
|
|
67951
|
+
const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
|
|
66298
67952
|
const status = owner === undefined ? "completed" : childRunStatus(owner);
|
|
66299
67953
|
const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
|
|
66300
67954
|
const parentRunId = run?.parentRunId ?? rootRunId;
|
|
66301
|
-
const boundarySourceId =
|
|
67955
|
+
const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
|
|
66302
67956
|
const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
|
|
66303
67957
|
const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
|
|
66304
67958
|
const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
|
|
@@ -66434,10 +68088,11 @@ function emptyGraphRun(runId, runName, completedAt) {
|
|
|
66434
68088
|
resumable: false
|
|
66435
68089
|
};
|
|
66436
68090
|
}
|
|
66437
|
-
var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
68091
|
+
var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
66438
68092
|
var init_completed_catalog = __esm(() => {
|
|
66439
68093
|
init_stage_session_store();
|
|
66440
68094
|
init_tool_payload_bounds();
|
|
68095
|
+
init_completed_catalog_index();
|
|
66441
68096
|
init_completed_catalog_stage_groups();
|
|
66442
68097
|
init_resume_eligibility();
|
|
66443
68098
|
init_resume_runtime();
|
|
@@ -66446,6 +68101,13 @@ var init_completed_catalog = __esm(() => {
|
|
|
66446
68101
|
init_workflow_heartbeat_anchor();
|
|
66447
68102
|
init_stage_topology_validation();
|
|
66448
68103
|
init_tool_outcome();
|
|
68104
|
+
nestedRunCatalogs = new WeakMap;
|
|
68105
|
+
EMPTY_NESTED_RUN_CATALOG = {
|
|
68106
|
+
runs: [],
|
|
68107
|
+
positionsById: new Map,
|
|
68108
|
+
childIdsByParent: new Map,
|
|
68109
|
+
intentionalExitStatuses: new Map
|
|
68110
|
+
};
|
|
66449
68111
|
COMPLETED_RECONSTRUCTION = {
|
|
66450
68112
|
requireTerminal: true,
|
|
66451
68113
|
retainActive: false,
|
|
@@ -66460,29 +68122,23 @@ var init_completed_catalog = __esm(() => {
|
|
|
66460
68122
|
|
|
66461
68123
|
// dist/builtin/workflows/src/durable/completed-subtree.ts
|
|
66462
68124
|
function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
|
|
66463
|
-
const
|
|
66464
|
-
if (!
|
|
68125
|
+
const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
|
|
68126
|
+
if (!catalog.positionsById.has(childRunId))
|
|
66465
68127
|
return;
|
|
66466
|
-
const intentionalExitStatuses = new Map;
|
|
66467
|
-
for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
|
|
66468
|
-
if (checkpoint.kind !== "stage")
|
|
66469
|
-
continue;
|
|
66470
|
-
const child = parseWorkflowChildResult(checkpoint.output);
|
|
66471
|
-
if (child?.exited === true)
|
|
66472
|
-
intentionalExitStatuses.set(child.runId, child.status);
|
|
66473
|
-
}
|
|
66474
68128
|
const included = new Set([childRunId]);
|
|
66475
|
-
|
|
66476
|
-
|
|
66477
|
-
|
|
66478
|
-
|
|
66479
|
-
|
|
66480
|
-
|
|
68129
|
+
const pending = [childRunId];
|
|
68130
|
+
for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
|
|
68131
|
+
for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
|
|
68132
|
+
if (included.has(childId))
|
|
68133
|
+
continue;
|
|
68134
|
+
included.add(childId);
|
|
68135
|
+
pending.push(childId);
|
|
66481
68136
|
}
|
|
66482
68137
|
}
|
|
66483
|
-
const
|
|
66484
|
-
const
|
|
66485
|
-
|
|
68138
|
+
const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
|
|
68139
|
+
const subtree = positions.map((position) => catalog.runs[position]);
|
|
68140
|
+
const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
|
|
68141
|
+
return complete ? subtree.map(cloneRunSnapshot) : undefined;
|
|
66486
68142
|
}
|
|
66487
68143
|
function isCompleteCachedRun(run, intentionalExitStatuses) {
|
|
66488
68144
|
if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
|
|
@@ -66603,7 +68259,7 @@ function createDurableCachedStageRecorder(input) {
|
|
|
66603
68259
|
const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
|
|
66604
68260
|
kind: "stage",
|
|
66605
68261
|
workflowId: input.run.id,
|
|
66606
|
-
checkpointId:
|
|
68262
|
+
checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
|
|
66607
68263
|
name,
|
|
66608
68264
|
replayKey,
|
|
66609
68265
|
completedAt: Date.now(),
|
|
@@ -74229,6 +75885,7 @@ async function runWorker(start) {
|
|
|
74229
75885
|
"getStageOutput",
|
|
74230
75886
|
"getStageSession",
|
|
74231
75887
|
"listCheckpoints",
|
|
75888
|
+
"nestedSnapshotVersion",
|
|
74232
75889
|
"getWorkflow",
|
|
74233
75890
|
"getLoadableWorkflow",
|
|
74234
75891
|
"toMetadata",
|