@bastani/atomic 0.9.44-alpha.5 → 0.9.44-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/builtin/intercom/CHANGELOG.md +6 -0
- package/dist/builtin/intercom/broker/broker.ts +13 -1
- package/dist/builtin/intercom/index.bundle.mjs +41 -5
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/types.ts +7 -0
- package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +20 -0
- package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
- package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
- package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +2514 -741
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2384 -727
- package/dist/builtin/workflows/src/index.js +2383 -727
- package/dist/builtin/workflows/src/local-worker-entry.mjs +2384 -727
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +16 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/storage/node-sqlite.d.ts.map +1 -1
- package/dist/core/storage/node-sqlite.js +130 -13
- package/dist/core/storage/node-sqlite.js.map +1 -1
- package/docs/intercom/reference.md +2 -0
- package/docs/workflows/operations.md +1 -1
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -983,13 +983,19 @@ function registerRunJournalVersions(journal, versions) {
|
|
|
983
983
|
function runJournalVersions(journal) {
|
|
984
984
|
return versionReaders.get(journal);
|
|
985
985
|
}
|
|
986
|
+
function registerRunJournalAppendMany(journal, appendMany) {
|
|
987
|
+
appendManyWriters.set(journal, appendMany);
|
|
988
|
+
}
|
|
989
|
+
function runJournalAppendMany(journal) {
|
|
990
|
+
return appendManyWriters.get(journal);
|
|
991
|
+
}
|
|
986
992
|
function registerSqlPoolStorage(journal, storage) {
|
|
987
993
|
poolStorage.set(journal, storage);
|
|
988
994
|
}
|
|
989
995
|
function sqlPoolStorage(journal) {
|
|
990
996
|
return poolStorage.get(journal);
|
|
991
997
|
}
|
|
992
|
-
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
|
|
998
|
+
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
|
|
993
999
|
var init_internal = __esm(() => {
|
|
994
1000
|
init_types();
|
|
995
1001
|
NON_SNAPSHOT_EVENT_TYPES = [
|
|
@@ -998,6 +1004,7 @@ var init_internal = __esm(() => {
|
|
|
998
1004
|
RUN_EVENT_TYPES.messageDelivered
|
|
999
1005
|
];
|
|
1000
1006
|
versionReaders = new WeakMap;
|
|
1007
|
+
appendManyWriters = new WeakMap;
|
|
1001
1008
|
poolStorage = new WeakMap;
|
|
1002
1009
|
});
|
|
1003
1010
|
|
|
@@ -1381,6 +1388,7 @@ class SqlRunJournal {
|
|
|
1381
1388
|
runVersions: () => this.runVersions(),
|
|
1382
1389
|
snapshotTail: (runId) => this.snapshotTail(runId)
|
|
1383
1390
|
});
|
|
1391
|
+
registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
|
|
1384
1392
|
registerSqlPoolStorage(this, {
|
|
1385
1393
|
transaction: (body) => this.operation(() => this.database.transaction(body)),
|
|
1386
1394
|
withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
|
|
@@ -1645,38 +1653,76 @@ class SqlRunJournal {
|
|
|
1645
1653
|
this.assertOpen();
|
|
1646
1654
|
validateText("runId", lease.runId);
|
|
1647
1655
|
validateText("holderId", lease.holderId);
|
|
1648
|
-
if (options?.key !== undefined)
|
|
1649
|
-
validateText("key", options.key);
|
|
1650
1656
|
const owned = { ...lease };
|
|
1651
|
-
const
|
|
1657
|
+
const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
|
|
1652
1658
|
const expected = options?.expectSeq;
|
|
1659
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
1660
|
+
await this.checkLease(tx, owned);
|
|
1661
|
+
return this.insert(tx, owned, prepared, expected);
|
|
1662
|
+
}));
|
|
1663
|
+
}
|
|
1664
|
+
async appendMany(lease, appends) {
|
|
1665
|
+
this.assertOpen();
|
|
1666
|
+
validateText("runId", lease.runId);
|
|
1667
|
+
validateText("holderId", lease.holderId);
|
|
1668
|
+
const owned = { ...lease };
|
|
1669
|
+
const prepared = appends.map((append) => {
|
|
1670
|
+
try {
|
|
1671
|
+
return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
|
|
1672
|
+
} catch (error) {
|
|
1673
|
+
return error;
|
|
1674
|
+
}
|
|
1675
|
+
});
|
|
1676
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
1677
|
+
await this.checkLease(tx, owned);
|
|
1678
|
+
const results = [];
|
|
1679
|
+
for (const append of prepared) {
|
|
1680
|
+
if (append instanceof Error) {
|
|
1681
|
+
results.push({ status: "rejected", reason: append });
|
|
1682
|
+
continue;
|
|
1683
|
+
}
|
|
1684
|
+
await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
|
|
1685
|
+
try {
|
|
1686
|
+
results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
|
|
1687
|
+
} catch (error) {
|
|
1688
|
+
await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
|
|
1689
|
+
results.push({ status: "rejected", reason: error });
|
|
1690
|
+
}
|
|
1691
|
+
await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
|
|
1692
|
+
}
|
|
1693
|
+
return results;
|
|
1694
|
+
}));
|
|
1695
|
+
}
|
|
1696
|
+
prepare(event, key, appendGuard) {
|
|
1697
|
+
if (key !== undefined)
|
|
1698
|
+
validateText("key", key);
|
|
1653
1699
|
const type = event.type;
|
|
1654
1700
|
const payload = this.snapshotJson(event.payload);
|
|
1655
|
-
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ?
|
|
1701
|
+
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
|
|
1656
1702
|
const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
|
|
1657
1703
|
if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
|
|
1658
1704
|
validateText("status", status);
|
|
1659
|
-
return
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
}
|
|
1705
|
+
return { key, type, payload, guard, status };
|
|
1706
|
+
}
|
|
1707
|
+
async insert(tx, owned, { key, type, payload, guard, status }, expected) {
|
|
1708
|
+
await guard?.(tx, owned.runId);
|
|
1709
|
+
if (key !== undefined) {
|
|
1710
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
|
|
1711
|
+
if (row)
|
|
1712
|
+
return { status: "exists", entry: entryFromRow(row) };
|
|
1713
|
+
}
|
|
1714
|
+
const actual = await this.sequence(tx, owned.runId);
|
|
1715
|
+
if (expected !== undefined && expected !== actual)
|
|
1716
|
+
throw new JournalCasConflictError(owned.runId, expected, actual);
|
|
1717
|
+
const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
|
|
1718
|
+
if (payload instanceof StorageUnavailableError2)
|
|
1719
|
+
throw payload;
|
|
1720
|
+
await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
|
|
1721
|
+
if (status !== undefined)
|
|
1722
|
+
await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
|
|
1723
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
|
|
1724
|
+
await guard?.(tx, owned.runId);
|
|
1725
|
+
return { status: "appended", entry: entryFromRow(row) };
|
|
1680
1726
|
}
|
|
1681
1727
|
async* read(runId, fromSeq = 1) {
|
|
1682
1728
|
this.assertOpen();
|
|
@@ -1744,7 +1790,7 @@ class SqlRunJournal {
|
|
|
1744
1790
|
return this.closePromise;
|
|
1745
1791
|
}
|
|
1746
1792
|
}
|
|
1747
|
-
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
|
|
1793
|
+
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
|
|
1748
1794
|
var init_sql_run_journal = __esm(() => {
|
|
1749
1795
|
init_internal();
|
|
1750
1796
|
init_legacy_journal_tables();
|
|
@@ -2323,6 +2369,12 @@ class ScopedDurableBackend {
|
|
|
2323
2369
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
2324
2370
|
return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
|
|
2325
2371
|
}
|
|
2372
|
+
pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
|
|
2373
|
+
return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
|
|
2374
|
+
}
|
|
2375
|
+
pendingStageDeliveryCountVersion(_workflowId) {
|
|
2376
|
+
return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
|
|
2377
|
+
}
|
|
2326
2378
|
async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
2327
2379
|
return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
|
|
2328
2380
|
}
|
|
@@ -3617,8 +3669,11 @@ var init_pending_stage_delivery = __esm(() => {
|
|
|
3617
3669
|
});
|
|
3618
3670
|
|
|
3619
3671
|
// dist/builtin/workflows/src/shared/pending-stage-retention.ts
|
|
3672
|
+
function isSettledPendingStageMessage(entry) {
|
|
3673
|
+
return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
|
|
3674
|
+
}
|
|
3620
3675
|
async function retainPendingStageMessages(messages, archive) {
|
|
3621
|
-
const settled = messages.filter(
|
|
3676
|
+
const settled = messages.filter(isSettledPendingStageMessage);
|
|
3622
3677
|
const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
|
|
3623
3678
|
for (const entry of expired)
|
|
3624
3679
|
await archive(entry);
|
|
@@ -3626,53 +3681,210 @@ async function retainPendingStageMessages(messages, archive) {
|
|
|
3626
3681
|
}
|
|
3627
3682
|
var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
|
|
3628
3683
|
|
|
3684
|
+
// dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
|
|
3685
|
+
class PendingStageTransitionLedger {
|
|
3686
|
+
compacted = [];
|
|
3687
|
+
settled = 0;
|
|
3688
|
+
stages = new Map;
|
|
3689
|
+
fixed = new WeakSet;
|
|
3690
|
+
reconciled = [];
|
|
3691
|
+
reconciledVersion;
|
|
3692
|
+
reconcile(backend, runId, messages) {
|
|
3693
|
+
if (backend.getPendingStageMessageDeliveryCount === undefined)
|
|
3694
|
+
return messages;
|
|
3695
|
+
const version = backend.pendingStageDeliveryCountVersion?.(runId);
|
|
3696
|
+
const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
|
|
3697
|
+
let recovered;
|
|
3698
|
+
for (let index = 0;index < messages.length; index++) {
|
|
3699
|
+
const entry = messages[index];
|
|
3700
|
+
if (entry.sticky !== true || base !== undefined && base[index] === entry)
|
|
3701
|
+
continue;
|
|
3702
|
+
const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
|
|
3703
|
+
if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
|
|
3704
|
+
continue;
|
|
3705
|
+
recovered ??= [...messages];
|
|
3706
|
+
recovered[index] = { ...entry, deliveryCount: count };
|
|
3707
|
+
}
|
|
3708
|
+
const result = recovered ?? messages;
|
|
3709
|
+
this.reconciled = result;
|
|
3710
|
+
this.reconciledVersion = version;
|
|
3711
|
+
return result;
|
|
3712
|
+
}
|
|
3713
|
+
async compact(backend, runId, messages, runs, archiveSettled) {
|
|
3714
|
+
const stageStatus = createStageStatusLookup(runs);
|
|
3715
|
+
const endedStages = [];
|
|
3716
|
+
const dirty = new Set;
|
|
3717
|
+
for (const [deliveryRunId, byStage] of this.stages) {
|
|
3718
|
+
for (const [stageId, entries] of byStage) {
|
|
3719
|
+
if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
|
|
3720
|
+
continue;
|
|
3721
|
+
endedStages.push([deliveryRunId, stageId]);
|
|
3722
|
+
for (const entry of entries)
|
|
3723
|
+
dirty.add(entry);
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
3726
|
+
const base = this.compacted;
|
|
3727
|
+
const output = [];
|
|
3728
|
+
const removed = [];
|
|
3729
|
+
const added = [];
|
|
3730
|
+
let settled = this.settled;
|
|
3731
|
+
const length = Math.max(base.length, messages.length);
|
|
3732
|
+
for (let index = 0;index < length; index++) {
|
|
3733
|
+
const previous = base[index];
|
|
3734
|
+
const entry = messages[index];
|
|
3735
|
+
if (previous === entry && !dirty.has(entry)) {
|
|
3736
|
+
output.push(entry);
|
|
3737
|
+
continue;
|
|
3738
|
+
}
|
|
3739
|
+
if (previous !== undefined) {
|
|
3740
|
+
removed.push(previous);
|
|
3741
|
+
if (isSettledPendingStageMessage(previous))
|
|
3742
|
+
settled--;
|
|
3743
|
+
}
|
|
3744
|
+
if (entry === undefined)
|
|
3745
|
+
continue;
|
|
3746
|
+
if (isSettledPendingStageMessage(entry))
|
|
3747
|
+
settled++;
|
|
3748
|
+
const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
|
|
3749
|
+
output.push(next);
|
|
3750
|
+
added.push(next);
|
|
3751
|
+
}
|
|
3752
|
+
let retained = output;
|
|
3753
|
+
if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
|
|
3754
|
+
retained = await retainPendingStageMessages(output, archiveSettled);
|
|
3755
|
+
settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
|
|
3756
|
+
}
|
|
3757
|
+
return {
|
|
3758
|
+
messages: retained,
|
|
3759
|
+
commit: () => {
|
|
3760
|
+
for (const [deliveryRunId, stageId] of endedStages) {
|
|
3761
|
+
const byStage = this.stages.get(deliveryRunId);
|
|
3762
|
+
byStage?.delete(stageId);
|
|
3763
|
+
if (byStage?.size === 0)
|
|
3764
|
+
this.stages.delete(deliveryRunId);
|
|
3765
|
+
}
|
|
3766
|
+
for (const entry of dirty)
|
|
3767
|
+
this.fixed.delete(entry);
|
|
3768
|
+
for (const entry of removed)
|
|
3769
|
+
this.unindex(entry);
|
|
3770
|
+
for (const entry of added)
|
|
3771
|
+
this.index(entry);
|
|
3772
|
+
this.compacted = retained;
|
|
3773
|
+
this.settled = settled;
|
|
3774
|
+
}
|
|
3775
|
+
};
|
|
3776
|
+
}
|
|
3777
|
+
index(entry) {
|
|
3778
|
+
this.fixed.add(entry);
|
|
3779
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3780
|
+
let byStage = this.stages.get(delivery.runId);
|
|
3781
|
+
if (byStage === undefined) {
|
|
3782
|
+
byStage = new Map;
|
|
3783
|
+
this.stages.set(delivery.runId, byStage);
|
|
3784
|
+
}
|
|
3785
|
+
let entries = byStage.get(delivery.stageId);
|
|
3786
|
+
if (entries === undefined) {
|
|
3787
|
+
entries = new Set;
|
|
3788
|
+
byStage.set(delivery.stageId, entries);
|
|
3789
|
+
}
|
|
3790
|
+
entries.add(entry);
|
|
3791
|
+
}
|
|
3792
|
+
}
|
|
3793
|
+
unindex(entry) {
|
|
3794
|
+
this.fixed.delete(entry);
|
|
3795
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3796
|
+
const byStage = this.stages.get(delivery.runId);
|
|
3797
|
+
const entries = byStage?.get(delivery.stageId);
|
|
3798
|
+
if (entries === undefined)
|
|
3799
|
+
continue;
|
|
3800
|
+
entries.delete(entry);
|
|
3801
|
+
if (entries.size > 0)
|
|
3802
|
+
continue;
|
|
3803
|
+
byStage.delete(delivery.stageId);
|
|
3804
|
+
if (byStage.size === 0)
|
|
3805
|
+
this.stages.delete(delivery.runId);
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
}
|
|
3809
|
+
function isCompactingBackend(backend) {
|
|
3810
|
+
return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
|
|
3811
|
+
}
|
|
3812
|
+
function createStageStatusLookup(runs) {
|
|
3813
|
+
let runById;
|
|
3814
|
+
return (runId, stageId) => {
|
|
3815
|
+
if (runById === undefined) {
|
|
3816
|
+
runById = new Map;
|
|
3817
|
+
for (const run of runs)
|
|
3818
|
+
if (!runById.has(run.id))
|
|
3819
|
+
runById.set(run.id, run);
|
|
3820
|
+
}
|
|
3821
|
+
return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
|
|
3822
|
+
};
|
|
3823
|
+
}
|
|
3824
|
+
function isEndedStageStatus(status) {
|
|
3825
|
+
return status === "completed" || status === "skipped" || status === "failed";
|
|
3826
|
+
}
|
|
3827
|
+
async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
|
|
3828
|
+
if (entry.sticky !== true || entry.deliveries === undefined)
|
|
3829
|
+
return entry;
|
|
3830
|
+
const latest = new Map;
|
|
3831
|
+
entry.deliveries.forEach((delivery, index) => {
|
|
3832
|
+
latest.set(deliverySlot(delivery), index);
|
|
3833
|
+
});
|
|
3834
|
+
const deliveries = [];
|
|
3835
|
+
let archivedThrough = -1;
|
|
3836
|
+
for (let index = 0;index < entry.deliveries.length; index++) {
|
|
3837
|
+
const delivery = entry.deliveries[index];
|
|
3838
|
+
if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
|
|
3839
|
+
while (archivedThrough < index) {
|
|
3840
|
+
const prefixIndex = archivedThrough + 1;
|
|
3841
|
+
const receipt = entry.deliveries[prefixIndex];
|
|
3842
|
+
if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
|
|
3843
|
+
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
|
|
3844
|
+
messageRunId: entry.runId,
|
|
3845
|
+
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
3846
|
+
});
|
|
3847
|
+
archivedThrough = prefixIndex;
|
|
3848
|
+
}
|
|
3849
|
+
} else {
|
|
3850
|
+
deliveries.push(delivery);
|
|
3851
|
+
}
|
|
3852
|
+
}
|
|
3853
|
+
return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
|
|
3854
|
+
}
|
|
3855
|
+
function deliverySlot(delivery) {
|
|
3856
|
+
return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
|
|
3857
|
+
}
|
|
3858
|
+
var init_pending_stage_transition_ledger = () => {};
|
|
3859
|
+
|
|
3629
3860
|
// dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
|
|
3630
3861
|
function createPendingStageDeliveryStoreMethods(context) {
|
|
3631
|
-
const
|
|
3632
|
-
const
|
|
3633
|
-
const previous = transitions.get(runId) ?? Promise.resolve();
|
|
3634
|
-
const result = previous.catch(() => {
|
|
3635
|
-
return;
|
|
3636
|
-
}).then(transition);
|
|
3637
|
-
const settled = result.then(() => {
|
|
3638
|
-
return;
|
|
3639
|
-
}, () => {
|
|
3640
|
-
return;
|
|
3641
|
-
});
|
|
3642
|
-
transitions.set(runId, settled);
|
|
3643
|
-
settled.finally(() => {
|
|
3644
|
-
if (transitions.get(runId) === settled)
|
|
3645
|
-
transitions.delete(runId);
|
|
3646
|
-
});
|
|
3647
|
-
return await result;
|
|
3648
|
-
};
|
|
3862
|
+
const ledgers = new PendingStageLedgers;
|
|
3863
|
+
const batches = new PendingStageTransitionBatches(context, ledgers);
|
|
3649
3864
|
return {
|
|
3650
3865
|
async queueStageMessage(input, senderGroup, runGroup, backend) {
|
|
3651
|
-
return await
|
|
3866
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3652
3867
|
const run = context.findRun(input.runId);
|
|
3653
3868
|
if (run === undefined)
|
|
3654
3869
|
return;
|
|
3655
3870
|
const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
|
|
3656
3871
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3657
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3872
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
|
|
3658
3873
|
if (result.ok && !result.deduplicated) {
|
|
3659
|
-
|
|
3660
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3661
|
-
live.pendingStageMessages = [...messages];
|
|
3662
|
-
context.bumpAndNotify();
|
|
3874
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3663
3875
|
}
|
|
3664
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3876
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3665
3877
|
});
|
|
3666
3878
|
},
|
|
3667
3879
|
async validateLiveStageMessage(input) {
|
|
3668
|
-
return await
|
|
3880
|
+
return await batches.run(input.runId, undefined, async (batch) => {
|
|
3669
3881
|
const run = context.findRun(input.runId);
|
|
3670
3882
|
if (run === undefined)
|
|
3671
3883
|
return;
|
|
3672
3884
|
const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
|
|
3673
3885
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
|
|
3674
3886
|
const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3675
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3887
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
|
|
3676
3888
|
if (!result.ok) {
|
|
3677
3889
|
return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
|
|
3678
3890
|
}
|
|
@@ -3694,77 +3906,65 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3694
3906
|
return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
|
|
3695
3907
|
},
|
|
3696
3908
|
async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
|
|
3697
|
-
return await
|
|
3909
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3698
3910
|
const run = context.findRun(runId);
|
|
3699
3911
|
if (run === undefined)
|
|
3700
3912
|
return false;
|
|
3701
|
-
const current = run
|
|
3913
|
+
const current = batch.messages(run);
|
|
3702
3914
|
const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
|
|
3703
3915
|
if (next === current)
|
|
3704
3916
|
return false;
|
|
3705
|
-
|
|
3706
|
-
const live = context.findRun(runId) ?? run;
|
|
3707
|
-
live.pendingStageMessages = [...messages];
|
|
3708
|
-
context.bumpAndNotify();
|
|
3917
|
+
await batch.apply(backend, runId, next);
|
|
3709
3918
|
return true;
|
|
3710
3919
|
});
|
|
3711
3920
|
},
|
|
3712
3921
|
async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
|
|
3713
|
-
return await
|
|
3922
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3714
3923
|
const run = context.findRun(runId);
|
|
3715
3924
|
if (run === undefined)
|
|
3716
3925
|
return false;
|
|
3717
|
-
const current = run
|
|
3926
|
+
const current = batch.messages(run);
|
|
3718
3927
|
const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
|
|
3719
3928
|
if (next === current)
|
|
3720
3929
|
return false;
|
|
3721
|
-
|
|
3722
|
-
const live = context.findRun(runId) ?? run;
|
|
3723
|
-
live.pendingStageMessages = [...messages];
|
|
3724
|
-
context.bumpAndNotify();
|
|
3930
|
+
await batch.apply(backend, runId, next);
|
|
3725
3931
|
return true;
|
|
3726
3932
|
});
|
|
3727
3933
|
},
|
|
3728
3934
|
async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
|
|
3729
|
-
return await
|
|
3935
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3730
3936
|
const run = context.findRun(runId);
|
|
3731
3937
|
if (run === undefined)
|
|
3732
3938
|
return false;
|
|
3733
|
-
const current = run
|
|
3939
|
+
const current = batch.messages(run);
|
|
3734
3940
|
const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
|
|
3735
3941
|
if (next === current)
|
|
3736
3942
|
return false;
|
|
3737
|
-
|
|
3738
|
-
const live = context.findRun(runId) ?? run;
|
|
3739
|
-
live.pendingStageMessages = [...messages];
|
|
3740
|
-
context.bumpAndNotify();
|
|
3943
|
+
await batch.apply(backend, runId, next);
|
|
3741
3944
|
return true;
|
|
3742
3945
|
});
|
|
3743
3946
|
},
|
|
3744
3947
|
async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
|
|
3745
|
-
return await
|
|
3948
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3746
3949
|
const run = context.findRun(input.runId);
|
|
3747
3950
|
if (run === undefined)
|
|
3748
3951
|
return;
|
|
3749
3952
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3750
|
-
const result = queueStickyStageMessage(receipt === undefined ? run
|
|
3953
|
+
const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
|
|
3751
3954
|
if (result.ok && !result.deduplicated) {
|
|
3752
|
-
|
|
3753
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3754
|
-
live.pendingStageMessages = [...messages];
|
|
3755
|
-
context.bumpAndNotify();
|
|
3955
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3756
3956
|
}
|
|
3757
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3957
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3758
3958
|
});
|
|
3759
3959
|
},
|
|
3760
3960
|
async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
|
|
3761
|
-
return await
|
|
3961
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3762
3962
|
const run = context.findRun(runId);
|
|
3763
3963
|
if (run === undefined)
|
|
3764
3964
|
return false;
|
|
3765
3965
|
if (records.length === 0)
|
|
3766
3966
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3767
|
-
let current =
|
|
3967
|
+
let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3768
3968
|
const unconfirmedRecords = [];
|
|
3769
3969
|
for (const record of records) {
|
|
3770
3970
|
const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
|
|
@@ -3772,35 +3972,26 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3772
3972
|
unconfirmedRecords.push(record);
|
|
3773
3973
|
}
|
|
3774
3974
|
}
|
|
3775
|
-
current =
|
|
3776
|
-
|
|
3777
|
-
run.pendingStageMessages = [...current];
|
|
3778
|
-
context.bumpAndNotify();
|
|
3779
|
-
}
|
|
3975
|
+
current = ledgers.for(run).reconcile(backend, runId, current);
|
|
3976
|
+
batch.reconciled(run, current);
|
|
3780
3977
|
const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
|
|
3781
3978
|
if (next === current)
|
|
3782
3979
|
return false;
|
|
3783
|
-
|
|
3784
|
-
const live = context.findRun(runId) ?? run;
|
|
3785
|
-
live.pendingStageMessages = [...messages];
|
|
3786
|
-
context.bumpAndNotify();
|
|
3980
|
+
await batch.apply(backend, runId, next);
|
|
3787
3981
|
return true;
|
|
3788
3982
|
});
|
|
3789
3983
|
},
|
|
3790
3984
|
async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
|
|
3791
|
-
return await
|
|
3985
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3792
3986
|
const run = context.findRun(runId);
|
|
3793
3987
|
if (run === undefined)
|
|
3794
3988
|
return false;
|
|
3795
3989
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3796
|
-
const current =
|
|
3990
|
+
const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3797
3991
|
const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
|
|
3798
3992
|
if (next === current)
|
|
3799
3993
|
return false;
|
|
3800
|
-
|
|
3801
|
-
const live = context.findRun(runId) ?? run;
|
|
3802
|
-
live.pendingStageMessages = [...messages];
|
|
3803
|
-
context.bumpAndNotify();
|
|
3994
|
+
await batch.apply(backend, runId, next);
|
|
3804
3995
|
return true;
|
|
3805
3996
|
});
|
|
3806
3997
|
}
|
|
@@ -3818,78 +4009,186 @@ function resolvePendingStageIdentity(run, stageKey) {
|
|
|
3818
4009
|
aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
|
|
3819
4010
|
};
|
|
3820
4011
|
}
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
4012
|
+
|
|
4013
|
+
class PendingStageTransitionBatches {
|
|
4014
|
+
queues = new Map;
|
|
4015
|
+
context;
|
|
4016
|
+
ledgers;
|
|
4017
|
+
constructor(context, ledgers) {
|
|
4018
|
+
this.context = context;
|
|
4019
|
+
this.ledgers = ledgers;
|
|
4020
|
+
}
|
|
4021
|
+
run(runId, backend, step) {
|
|
4022
|
+
return new Promise((resolve, reject) => {
|
|
4023
|
+
const transition = {
|
|
4024
|
+
backend,
|
|
4025
|
+
step,
|
|
4026
|
+
resolve: (value2) => resolve(value2),
|
|
4027
|
+
reject
|
|
4028
|
+
};
|
|
4029
|
+
const queue = this.queues.get(runId);
|
|
4030
|
+
if (queue !== undefined) {
|
|
4031
|
+
queue.push(transition);
|
|
4032
|
+
return;
|
|
4033
|
+
}
|
|
4034
|
+
const created = [transition];
|
|
4035
|
+
this.queues.set(runId, created);
|
|
4036
|
+
Promise.resolve().then(() => this.drain(runId, created));
|
|
4037
|
+
});
|
|
4038
|
+
}
|
|
4039
|
+
async drain(runId, queue) {
|
|
4040
|
+
while (queue.length > 0)
|
|
4041
|
+
await this.runBatch(runId, takeBatch(queue));
|
|
4042
|
+
this.queues.delete(runId);
|
|
4043
|
+
}
|
|
4044
|
+
async runBatch(runId, transitions) {
|
|
4045
|
+
const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
|
|
4046
|
+
const outcomes = [];
|
|
4047
|
+
for (const transition of transitions) {
|
|
4048
|
+
const changedBefore = batch.changed;
|
|
4049
|
+
try {
|
|
4050
|
+
const value2 = await transition.step(batch);
|
|
4051
|
+
outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
|
|
4052
|
+
} catch (error) {
|
|
4053
|
+
outcomes.push({ ok: false, error });
|
|
3854
4054
|
}
|
|
3855
|
-
retained = compacted;
|
|
3856
4055
|
}
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
4056
|
+
let failure;
|
|
4057
|
+
try {
|
|
4058
|
+
await batch.persist();
|
|
4059
|
+
} catch (error) {
|
|
4060
|
+
failure = { error };
|
|
4061
|
+
}
|
|
4062
|
+
transitions.forEach((transition, index) => {
|
|
4063
|
+
const outcome = outcomes[index];
|
|
4064
|
+
if (!outcome.ok)
|
|
4065
|
+
transition.reject(outcome.error);
|
|
4066
|
+
else if (failure !== undefined && outcome.uncommitted)
|
|
4067
|
+
transition.reject(failure.error);
|
|
4068
|
+
else
|
|
4069
|
+
transition.resolve(outcome.value);
|
|
4070
|
+
});
|
|
4071
|
+
}
|
|
4072
|
+
}
|
|
4073
|
+
function takeBatch(queue) {
|
|
4074
|
+
let count = 1;
|
|
4075
|
+
let backend = queue[0].backend;
|
|
4076
|
+
while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
|
|
4077
|
+
const next = queue[count].backend;
|
|
4078
|
+
if (next !== undefined && backend !== undefined && next !== backend)
|
|
4079
|
+
break;
|
|
4080
|
+
backend ??= next;
|
|
4081
|
+
count++;
|
|
4082
|
+
}
|
|
4083
|
+
return queue.splice(0, count);
|
|
4084
|
+
}
|
|
4085
|
+
|
|
4086
|
+
class PendingStageTransitionBatch {
|
|
4087
|
+
changed = false;
|
|
4088
|
+
list;
|
|
4089
|
+
pending;
|
|
4090
|
+
context;
|
|
4091
|
+
ledgers;
|
|
4092
|
+
runId;
|
|
4093
|
+
constructor(context, ledgers, runId) {
|
|
4094
|
+
this.context = context;
|
|
4095
|
+
this.ledgers = ledgers;
|
|
4096
|
+
this.runId = runId;
|
|
4097
|
+
}
|
|
4098
|
+
messages(run) {
|
|
4099
|
+
return this.list ?? run.pendingStageMessages ?? [];
|
|
4100
|
+
}
|
|
4101
|
+
reconciled(run, messages) {
|
|
4102
|
+
if (this.changed) {
|
|
4103
|
+
this.list = messages;
|
|
4104
|
+
return;
|
|
3860
4105
|
}
|
|
3861
|
-
if (
|
|
3862
|
-
|
|
4106
|
+
if (messages !== run.pendingStageMessages) {
|
|
4107
|
+
run.pendingStageMessages = [...messages];
|
|
4108
|
+
this.context.bumpAndNotify();
|
|
3863
4109
|
}
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
const run = context.findRun(runId);
|
|
3867
|
-
|
|
3868
|
-
const
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
4110
|
+
}
|
|
4111
|
+
async apply(backend, runId, messages) {
|
|
4112
|
+
const run = this.context.findRun(runId);
|
|
4113
|
+
try {
|
|
4114
|
+
const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
|
|
4115
|
+
this.list = compacted.messages;
|
|
4116
|
+
this.changed = true;
|
|
4117
|
+
this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
|
|
4118
|
+
return compacted.messages;
|
|
4119
|
+
} catch (error) {
|
|
4120
|
+
if (this.changed && this.list !== undefined) {
|
|
4121
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
|
|
4122
|
+
this.list = ledger.reconcile(backend, runId, this.list);
|
|
4123
|
+
} else
|
|
4124
|
+
reconcileLive(backend, runId, this.context, this.ledgers);
|
|
4125
|
+
throw error;
|
|
3874
4126
|
}
|
|
3875
4127
|
}
|
|
4128
|
+
async persist() {
|
|
4129
|
+
if (this.pending === undefined)
|
|
4130
|
+
return;
|
|
4131
|
+
const { backend, messages, commit, run } = this.pending;
|
|
4132
|
+
const published = this.list ?? messages;
|
|
4133
|
+
try {
|
|
4134
|
+
if (!await backend.persistPendingStageMessages(this.runId, messages)) {
|
|
4135
|
+
throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
|
|
4136
|
+
}
|
|
4137
|
+
commit?.();
|
|
4138
|
+
} finally {
|
|
4139
|
+
reconcileLive(backend, this.runId, this.context, this.ledgers);
|
|
4140
|
+
}
|
|
4141
|
+
const live = this.context.findRun(this.runId) ?? run;
|
|
4142
|
+
if (live !== undefined)
|
|
4143
|
+
live.pendingStageMessages = [...published];
|
|
4144
|
+
this.context.bumpAndNotify();
|
|
4145
|
+
}
|
|
4146
|
+
}
|
|
4147
|
+
async function compactTransition(backend, runId, messages, context, ledgers, run) {
|
|
4148
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
|
|
4149
|
+
const reconciled = ledger.reconcile(backend, runId, messages);
|
|
4150
|
+
const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
|
|
4151
|
+
if (isCompactingBackend(backend)) {
|
|
4152
|
+
return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
|
|
4153
|
+
}
|
|
4154
|
+
let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
|
|
4155
|
+
if (archiveSettled !== undefined)
|
|
4156
|
+
retained = await retainPendingStageMessages(retained, archiveSettled);
|
|
4157
|
+
if (run !== undefined)
|
|
4158
|
+
ledgers.reset(run);
|
|
4159
|
+
return { messages: retained };
|
|
4160
|
+
}
|
|
4161
|
+
function reconcileLive(backend, runId, context, ledgers) {
|
|
4162
|
+
const current = context.findRun(runId);
|
|
4163
|
+
if (current === undefined)
|
|
4164
|
+
return;
|
|
4165
|
+
const messages = current.pendingStageMessages ?? [];
|
|
4166
|
+
const recovered = ledgers.for(current).reconcile(backend, runId, messages);
|
|
4167
|
+
if (recovered !== messages) {
|
|
4168
|
+
current.pendingStageMessages = [...recovered];
|
|
4169
|
+
context.bumpAndNotify();
|
|
4170
|
+
}
|
|
3876
4171
|
}
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
return
|
|
3887
|
-
}
|
|
3888
|
-
|
|
4172
|
+
|
|
4173
|
+
class PendingStageLedgers {
|
|
4174
|
+
ledgers = new WeakMap;
|
|
4175
|
+
for(run) {
|
|
4176
|
+
let ledger = this.ledgers.get(run);
|
|
4177
|
+
if (ledger === undefined) {
|
|
4178
|
+
ledger = new PendingStageTransitionLedger;
|
|
4179
|
+
this.ledgers.set(run, ledger);
|
|
4180
|
+
}
|
|
4181
|
+
return ledger;
|
|
4182
|
+
}
|
|
4183
|
+
reset(run) {
|
|
4184
|
+
this.ledgers.delete(run);
|
|
4185
|
+
}
|
|
3889
4186
|
}
|
|
4187
|
+
var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
|
|
3890
4188
|
var init_store_pending_stage_delivery_methods = __esm(() => {
|
|
3891
4189
|
init_run_owner_backend();
|
|
3892
4190
|
init_pending_stage_delivery();
|
|
4191
|
+
init_pending_stage_transition_ledger();
|
|
3893
4192
|
});
|
|
3894
4193
|
|
|
3895
4194
|
// dist/builtin/workflows/src/shared/store-prompt-methods.ts
|
|
@@ -5843,8 +6142,59 @@ function firstError(schema, value2) {
|
|
|
5843
6142
|
}
|
|
5844
6143
|
return;
|
|
5845
6144
|
}
|
|
6145
|
+
function certainlySerializable(root, rootMustBeObject) {
|
|
6146
|
+
if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
|
|
6147
|
+
return false;
|
|
6148
|
+
const pending = [root];
|
|
6149
|
+
const ancestors = new Set;
|
|
6150
|
+
const exits = [undefined];
|
|
6151
|
+
while (pending.length > 0) {
|
|
6152
|
+
const exit = exits.pop();
|
|
6153
|
+
const value2 = pending.pop();
|
|
6154
|
+
if (exit !== undefined) {
|
|
6155
|
+
ancestors.delete(exit);
|
|
6156
|
+
continue;
|
|
6157
|
+
}
|
|
6158
|
+
if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
|
|
6159
|
+
continue;
|
|
6160
|
+
if (typeof value2 === "number") {
|
|
6161
|
+
if (!Number.isFinite(value2))
|
|
6162
|
+
return false;
|
|
6163
|
+
continue;
|
|
6164
|
+
}
|
|
6165
|
+
if (typeof value2 !== "object" || ancestors.has(value2))
|
|
6166
|
+
return false;
|
|
6167
|
+
if (Object.getOwnPropertySymbols(value2).length > 0)
|
|
6168
|
+
return false;
|
|
6169
|
+
const names = Object.getOwnPropertyNames(value2);
|
|
6170
|
+
let children;
|
|
6171
|
+
if (Array.isArray(value2)) {
|
|
6172
|
+
if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
|
|
6173
|
+
return false;
|
|
6174
|
+
children = [];
|
|
6175
|
+
for (let index = 0;index < value2.length; index++)
|
|
6176
|
+
children.push(value2[index]);
|
|
6177
|
+
} else {
|
|
6178
|
+
const proto = Object.getPrototypeOf(value2);
|
|
6179
|
+
if (proto !== Object.prototype && proto !== null)
|
|
6180
|
+
return false;
|
|
6181
|
+
const keys = Object.keys(value2);
|
|
6182
|
+
if (keys.length !== names.length)
|
|
6183
|
+
return false;
|
|
6184
|
+
children = keys.map((key) => value2[key]);
|
|
6185
|
+
}
|
|
6186
|
+
ancestors.add(value2);
|
|
6187
|
+
pending.push(value2);
|
|
6188
|
+
exits.push(value2);
|
|
6189
|
+
for (const child of children) {
|
|
6190
|
+
pending.push(child);
|
|
6191
|
+
exits.push(undefined);
|
|
6192
|
+
}
|
|
6193
|
+
}
|
|
6194
|
+
return true;
|
|
6195
|
+
}
|
|
5846
6196
|
function workflowSerializableValidationError(value2, label) {
|
|
5847
|
-
if (Value.Check(workflowSerializableValueSchema, value2))
|
|
6197
|
+
if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
|
|
5848
6198
|
return;
|
|
5849
6199
|
const issue = firstError(workflowSerializableValueSchema, value2);
|
|
5850
6200
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -5853,7 +6203,7 @@ function workflowSerializableValidationError(value2, label) {
|
|
|
5853
6203
|
return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
|
|
5854
6204
|
}
|
|
5855
6205
|
function workflowSerializableObjectValidationError(value2, label) {
|
|
5856
|
-
if (Value.Check(workflowSerializableObjectSchema, value2))
|
|
6206
|
+
if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
|
|
5857
6207
|
return;
|
|
5858
6208
|
const issue = firstError(workflowSerializableObjectSchema, value2);
|
|
5859
6209
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -6882,6 +7232,204 @@ var init_resume_eligibility = () => {};
|
|
|
6882
7232
|
// dist/builtin/workflows/src/durable/types.ts
|
|
6883
7233
|
var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
|
|
6884
7234
|
|
|
7235
|
+
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
7236
|
+
function immutableStageGroupError(stages) {
|
|
7237
|
+
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
7238
|
+
if (occurrenceError !== undefined)
|
|
7239
|
+
return occurrenceError;
|
|
7240
|
+
const byReplayKey = groupByDurableStageKey(stages);
|
|
7241
|
+
for (const group of byReplayKey.values()) {
|
|
7242
|
+
const firstName = group[0]?.name;
|
|
7243
|
+
if (group.some((stage) => stage.name !== firstName))
|
|
7244
|
+
return "stage name drift within one replay-key group";
|
|
7245
|
+
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
7246
|
+
const first = topologies[0];
|
|
7247
|
+
if (first === undefined)
|
|
7248
|
+
continue;
|
|
7249
|
+
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
7250
|
+
return "immutable stage identity drift within one replay-key group";
|
|
7251
|
+
}
|
|
7252
|
+
}
|
|
7253
|
+
return;
|
|
7254
|
+
}
|
|
7255
|
+
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
7256
|
+
return directChildTopologyErrorFromRecords({
|
|
7257
|
+
claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
|
|
7258
|
+
direct: (claiming) => {
|
|
7259
|
+
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
7260
|
+
return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
7261
|
+
}
|
|
7262
|
+
}, replayScope, expectedRun, allowedNodeIds);
|
|
7263
|
+
}
|
|
7264
|
+
function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
7265
|
+
const prefix = `${replayScope}:`;
|
|
7266
|
+
const claiming = records.claiming;
|
|
7267
|
+
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
7268
|
+
return "child run topology exists outside its durable invocation scope";
|
|
7269
|
+
}
|
|
7270
|
+
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
7271
|
+
return "child checkpoint ownership is not reciprocal";
|
|
7272
|
+
}
|
|
7273
|
+
const direct = records.direct(claiming);
|
|
7274
|
+
const immutableError = immutableStageGroupError(direct);
|
|
7275
|
+
if (immutableError !== undefined)
|
|
7276
|
+
return immutableError;
|
|
7277
|
+
const merged = [];
|
|
7278
|
+
for (const group of groupByDurableStageKey(direct).values()) {
|
|
7279
|
+
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
7280
|
+
if (topology === undefined)
|
|
7281
|
+
return "direct child stage group lacks owning-run topology";
|
|
7282
|
+
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
7283
|
+
return "direct child stage topology lacks current source identity";
|
|
7284
|
+
}
|
|
7285
|
+
if (topology.stageId.length === 0)
|
|
7286
|
+
return "direct child stage id is empty";
|
|
7287
|
+
merged.push(topology);
|
|
7288
|
+
}
|
|
7289
|
+
const ids = new Set(allowedNodeIds);
|
|
7290
|
+
const stageIds = new Set;
|
|
7291
|
+
const orders = new Set;
|
|
7292
|
+
for (const topology of merged) {
|
|
7293
|
+
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
7294
|
+
return "direct child stage ids are not unique";
|
|
7295
|
+
stageIds.add(topology.stageId);
|
|
7296
|
+
ids.add(topology.stageId);
|
|
7297
|
+
if (topology.sourceOrder !== undefined) {
|
|
7298
|
+
if (orders.has(topology.sourceOrder))
|
|
7299
|
+
return "direct child source orders are not unique";
|
|
7300
|
+
orders.add(topology.sourceOrder);
|
|
7301
|
+
}
|
|
7302
|
+
}
|
|
7303
|
+
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
7304
|
+
return "direct child stage references an unknown parent";
|
|
7305
|
+
}
|
|
7306
|
+
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
7307
|
+
}
|
|
7308
|
+
function owningBoundaryIdError(stages, owningRunId) {
|
|
7309
|
+
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
7310
|
+
const seen = new Map;
|
|
7311
|
+
for (const start of starts) {
|
|
7312
|
+
const stageId = start.topology.stageId;
|
|
7313
|
+
if (stageId.length === 0)
|
|
7314
|
+
return "owning-run boundary stage id is empty";
|
|
7315
|
+
const prior = seen.get(stageId);
|
|
7316
|
+
if (prior !== undefined && prior !== start.replayKey)
|
|
7317
|
+
return "owning-run boundary stage id is aliased";
|
|
7318
|
+
seen.set(stageId, start.replayKey);
|
|
7319
|
+
}
|
|
7320
|
+
return;
|
|
7321
|
+
}
|
|
7322
|
+
function sameRun(actual, expected) {
|
|
7323
|
+
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
7324
|
+
}
|
|
7325
|
+
function sameStrings(left, right) {
|
|
7326
|
+
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
7327
|
+
}
|
|
7328
|
+
function sameOptionalRun(actual, expected) {
|
|
7329
|
+
if (actual === undefined || expected === undefined)
|
|
7330
|
+
return actual === expected;
|
|
7331
|
+
return sameRun(actual, expected);
|
|
7332
|
+
}
|
|
7333
|
+
function promptOccurrenceIdentityError(stages) {
|
|
7334
|
+
const replayByOccurrence = new Map;
|
|
7335
|
+
const occurrenceByReplayStage = new Map;
|
|
7336
|
+
const stageByReplayOccurrence = new Map;
|
|
7337
|
+
for (const stage of stages) {
|
|
7338
|
+
const topology = stage.topology;
|
|
7339
|
+
const occurrence = topology?.occurrenceKey;
|
|
7340
|
+
if (topology === undefined || occurrence === undefined)
|
|
7341
|
+
continue;
|
|
7342
|
+
const priorReplay = replayByOccurrence.get(occurrence);
|
|
7343
|
+
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
7344
|
+
return "prompt occurrence is bound to multiple replay keys";
|
|
7345
|
+
}
|
|
7346
|
+
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
7347
|
+
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
7348
|
+
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
7349
|
+
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
7350
|
+
return "prompt stage identity is bound to multiple occurrences";
|
|
7351
|
+
}
|
|
7352
|
+
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
7353
|
+
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7354
|
+
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
7355
|
+
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
7356
|
+
return "prompt occurrence is bound to multiple stage identities";
|
|
7357
|
+
}
|
|
7358
|
+
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
7359
|
+
}
|
|
7360
|
+
return;
|
|
7361
|
+
}
|
|
7362
|
+
function durableStageGroupKey(stage) {
|
|
7363
|
+
const occurrence = stage.topology?.occurrenceKey;
|
|
7364
|
+
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7365
|
+
}
|
|
7366
|
+
function groupByDurableStageKey(stages) {
|
|
7367
|
+
const groups = new Map;
|
|
7368
|
+
for (const stage of stages) {
|
|
7369
|
+
const key = durableStageGroupKey(stage);
|
|
7370
|
+
const group = groups.get(key) ?? [];
|
|
7371
|
+
group.push(stage);
|
|
7372
|
+
groups.set(key, group);
|
|
7373
|
+
}
|
|
7374
|
+
return groups;
|
|
7375
|
+
}
|
|
7376
|
+
function compositeKey(domain, ...parts) {
|
|
7377
|
+
return JSON.stringify([domain, ...parts]);
|
|
7378
|
+
}
|
|
7379
|
+
function hasCycle(topologies) {
|
|
7380
|
+
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
7381
|
+
const visiting = new Set;
|
|
7382
|
+
const visited = new Set;
|
|
7383
|
+
const cyclic = (stageId) => {
|
|
7384
|
+
if (visiting.has(stageId))
|
|
7385
|
+
return true;
|
|
7386
|
+
if (visited.has(stageId))
|
|
7387
|
+
return false;
|
|
7388
|
+
visiting.add(stageId);
|
|
7389
|
+
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
7390
|
+
return true;
|
|
7391
|
+
visiting.delete(stageId);
|
|
7392
|
+
visited.add(stageId);
|
|
7393
|
+
return false;
|
|
7394
|
+
};
|
|
7395
|
+
return topologies.some((topology) => cyclic(topology.stageId));
|
|
7396
|
+
}
|
|
7397
|
+
var init_stage_topology_validation = () => {};
|
|
7398
|
+
|
|
7399
|
+
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
7400
|
+
function parseWorkflowChildResult(value2) {
|
|
7401
|
+
if (!isOrdinaryObject(value2))
|
|
7402
|
+
return;
|
|
7403
|
+
const workflow = value2.workflow;
|
|
7404
|
+
const runId = value2.runId;
|
|
7405
|
+
const status = value2.status;
|
|
7406
|
+
const exited = value2.exited;
|
|
7407
|
+
const outputs = value2.outputs;
|
|
7408
|
+
const exitReason = value2.exitReason;
|
|
7409
|
+
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
7410
|
+
return;
|
|
7411
|
+
if (exited === false) {
|
|
7412
|
+
if (status !== "completed" || exitReason !== undefined)
|
|
7413
|
+
return;
|
|
7414
|
+
return value2;
|
|
7415
|
+
}
|
|
7416
|
+
if (exited !== true || !isWorkflowExitStatus2(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
7417
|
+
return;
|
|
7418
|
+
return value2;
|
|
7419
|
+
}
|
|
7420
|
+
function isWorkflowChildResult(value2) {
|
|
7421
|
+
return parseWorkflowChildResult(value2) !== undefined;
|
|
7422
|
+
}
|
|
7423
|
+
function isWorkflowExitStatus2(value2) {
|
|
7424
|
+
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
7425
|
+
}
|
|
7426
|
+
function isOrdinaryObject(value2) {
|
|
7427
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
7428
|
+
return false;
|
|
7429
|
+
const prototype = Object.getPrototypeOf(value2);
|
|
7430
|
+
return prototype === Object.prototype || prototype === null;
|
|
7431
|
+
}
|
|
7432
|
+
|
|
6885
7433
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
6886
7434
|
import { createHash as createHash2 } from "node:crypto";
|
|
6887
7435
|
function durableHash(value2) {
|
|
@@ -6948,6 +7496,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
|
|
|
6948
7496
|
function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
|
|
6949
7497
|
return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
|
|
6950
7498
|
}
|
|
7499
|
+
|
|
7500
|
+
class StageReceipts {
|
|
7501
|
+
exact = new Map;
|
|
7502
|
+
anySession = new Map;
|
|
7503
|
+
has(logicalRunId, sessionId, admission) {
|
|
7504
|
+
return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
|
|
7505
|
+
}
|
|
7506
|
+
hasAnySession(logicalRunId, admission) {
|
|
7507
|
+
return this.anySession.get(logicalRunId)?.has(admission) ?? false;
|
|
7508
|
+
}
|
|
7509
|
+
add(logicalRunId, sessionId, admission) {
|
|
7510
|
+
let bySession = this.exact.get(logicalRunId);
|
|
7511
|
+
if (bySession === undefined) {
|
|
7512
|
+
bySession = new Map;
|
|
7513
|
+
this.exact.set(logicalRunId, bySession);
|
|
7514
|
+
}
|
|
7515
|
+
let admissions = bySession.get(sessionId);
|
|
7516
|
+
if (admissions === undefined) {
|
|
7517
|
+
admissions = new Set;
|
|
7518
|
+
bySession.set(sessionId, admissions);
|
|
7519
|
+
}
|
|
7520
|
+
if (admissions.has(admission))
|
|
7521
|
+
return false;
|
|
7522
|
+
admissions.add(admission);
|
|
7523
|
+
return true;
|
|
7524
|
+
}
|
|
7525
|
+
addAnySession(logicalRunId, admission) {
|
|
7526
|
+
let admissions = this.anySession.get(logicalRunId);
|
|
7527
|
+
if (admissions === undefined) {
|
|
7528
|
+
admissions = new Set;
|
|
7529
|
+
this.anySession.set(logicalRunId, admissions);
|
|
7530
|
+
}
|
|
7531
|
+
if (admissions.has(admission))
|
|
7532
|
+
return false;
|
|
7533
|
+
admissions.add(admission);
|
|
7534
|
+
return true;
|
|
7535
|
+
}
|
|
7536
|
+
}
|
|
7537
|
+
|
|
7538
|
+
class PendingStageReceiptIndex {
|
|
7539
|
+
workflows = new Map;
|
|
7540
|
+
add(workflowId, messageId, delivery, logicalRunId) {
|
|
7541
|
+
let workflow = this.workflows.get(workflowId);
|
|
7542
|
+
if (workflow === undefined) {
|
|
7543
|
+
workflow = { byMessage: new Map, log: [] };
|
|
7544
|
+
this.workflows.set(workflowId, workflow);
|
|
7545
|
+
}
|
|
7546
|
+
let byRun = workflow.byMessage.get(messageId);
|
|
7547
|
+
if (byRun === undefined) {
|
|
7548
|
+
byRun = new Map;
|
|
7549
|
+
workflow.byMessage.set(messageId, byRun);
|
|
7550
|
+
}
|
|
7551
|
+
let byStage = byRun.get(delivery.runId);
|
|
7552
|
+
if (byStage === undefined) {
|
|
7553
|
+
byStage = new Map;
|
|
7554
|
+
byRun.set(delivery.runId, byStage);
|
|
7555
|
+
}
|
|
7556
|
+
let receipts = byStage.get(delivery.stageId);
|
|
7557
|
+
if (receipts === undefined) {
|
|
7558
|
+
receipts = new StageReceipts;
|
|
7559
|
+
byStage.set(delivery.stageId, receipts);
|
|
7560
|
+
}
|
|
7561
|
+
const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
|
|
7562
|
+
let changed = added;
|
|
7563
|
+
if (receipts.add(logicalRunId, undefined, undefined))
|
|
7564
|
+
changed = true;
|
|
7565
|
+
if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
|
|
7566
|
+
changed = true;
|
|
7567
|
+
if (changed)
|
|
7568
|
+
workflow.log.push(messageId);
|
|
7569
|
+
return added;
|
|
7570
|
+
}
|
|
7571
|
+
has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
|
|
7572
|
+
const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
|
|
7573
|
+
if (receipts === undefined)
|
|
7574
|
+
return false;
|
|
7575
|
+
if (anySession && delivery.admission === "context")
|
|
7576
|
+
return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
|
|
7577
|
+
return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
|
|
7578
|
+
}
|
|
7579
|
+
changesSince(workflowId, cursor) {
|
|
7580
|
+
let workflow = this.workflows.get(workflowId);
|
|
7581
|
+
if (workflow === undefined) {
|
|
7582
|
+
workflow = { byMessage: new Map, log: [] };
|
|
7583
|
+
this.workflows.set(workflowId, workflow);
|
|
7584
|
+
}
|
|
7585
|
+
const next = { log: workflow.log, length: workflow.log.length };
|
|
7586
|
+
if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
|
|
7587
|
+
return { cursor: next, messageIds: undefined };
|
|
7588
|
+
return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
|
|
7589
|
+
}
|
|
7590
|
+
delete(workflowId) {
|
|
7591
|
+
this.workflows.delete(workflowId);
|
|
7592
|
+
}
|
|
7593
|
+
clear() {
|
|
7594
|
+
this.workflows.clear();
|
|
7595
|
+
}
|
|
7596
|
+
}
|
|
7597
|
+
function nextVersion() {
|
|
7598
|
+
lastVersion += 1;
|
|
7599
|
+
return lastVersion;
|
|
7600
|
+
}
|
|
7601
|
+
|
|
7602
|
+
class NestedSnapshotVersion {
|
|
7603
|
+
rootWorkflowId;
|
|
7604
|
+
current = nextVersion();
|
|
7605
|
+
latestCompletedAt = Number.NEGATIVE_INFINITY;
|
|
7606
|
+
replayGroups = new Map;
|
|
7607
|
+
boundaryReplayKeys = new Set;
|
|
7608
|
+
constructor(rootWorkflowId) {
|
|
7609
|
+
this.rootWorkflowId = rootWorkflowId;
|
|
7610
|
+
}
|
|
7611
|
+
get version() {
|
|
7612
|
+
return this.current;
|
|
7613
|
+
}
|
|
7614
|
+
record(checkpoint) {
|
|
7615
|
+
if (!this.isInert(checkpoint))
|
|
7616
|
+
this.current = nextVersion();
|
|
7617
|
+
this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
|
|
7618
|
+
if (checkpoint.kind !== "stage")
|
|
7619
|
+
return;
|
|
7620
|
+
if (checkpoint.topology?.boundary !== undefined)
|
|
7621
|
+
this.boundaryReplayKeys.add(checkpoint.replayKey);
|
|
7622
|
+
if (checkpoint.topology?.occurrenceKey !== undefined)
|
|
7623
|
+
return;
|
|
7624
|
+
const identity = stageIdentity(checkpoint);
|
|
7625
|
+
const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
|
|
7626
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
7627
|
+
if (summary === undefined) {
|
|
7628
|
+
this.replayGroups.set(checkpoint.replayKey, {
|
|
7629
|
+
name: checkpoint.name,
|
|
7630
|
+
identity,
|
|
7631
|
+
consistent: true,
|
|
7632
|
+
hasOutput: checkpoint.output !== undefined,
|
|
7633
|
+
hasExitedChildOutput: exited
|
|
7634
|
+
});
|
|
7635
|
+
return;
|
|
7636
|
+
}
|
|
7637
|
+
summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
|
|
7638
|
+
summary.hasOutput ||= checkpoint.output !== undefined;
|
|
7639
|
+
summary.hasExitedChildOutput ||= exited;
|
|
7640
|
+
}
|
|
7641
|
+
isInert(checkpoint) {
|
|
7642
|
+
if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
|
|
7643
|
+
return false;
|
|
7644
|
+
}
|
|
7645
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
7646
|
+
if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
|
|
7647
|
+
return false;
|
|
7648
|
+
}
|
|
7649
|
+
const id = checkpoint.checkpointId;
|
|
7650
|
+
for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
|
|
7651
|
+
if (this.boundaryReplayKeys.has(id.slice(0, colon)))
|
|
7652
|
+
return false;
|
|
7653
|
+
}
|
|
7654
|
+
return true;
|
|
7655
|
+
}
|
|
7656
|
+
isRootOwned(checkpoint) {
|
|
7657
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
7658
|
+
return runId === undefined || runId === this.rootWorkflowId;
|
|
7659
|
+
}
|
|
7660
|
+
}
|
|
7661
|
+
function stageIdentity(checkpoint) {
|
|
7662
|
+
const topology = checkpoint.topology;
|
|
7663
|
+
return JSON.stringify([
|
|
7664
|
+
durableStageGroupKey(checkpoint),
|
|
7665
|
+
topology?.version,
|
|
7666
|
+
topology?.stageId,
|
|
7667
|
+
topology?.parentIds,
|
|
7668
|
+
topology?.sourceOrder,
|
|
7669
|
+
topology?.run === undefined ? undefined : [
|
|
7670
|
+
topology.run.runId,
|
|
7671
|
+
topology.run.runName,
|
|
7672
|
+
topology.run.parentRunId,
|
|
7673
|
+
topology.run.parentStageId,
|
|
7674
|
+
topology.run.rootRunId
|
|
7675
|
+
]
|
|
7676
|
+
]);
|
|
7677
|
+
}
|
|
6951
7678
|
function checkpointKey(c) {
|
|
6952
7679
|
return `${c.kind}:${c.checkpointId}`;
|
|
6953
7680
|
}
|
|
@@ -6959,7 +7686,7 @@ class JournalProjection {
|
|
|
6959
7686
|
workflows = new Map;
|
|
6960
7687
|
deletedWorkflowIds = new Set;
|
|
6961
7688
|
pendingReceipts = new Map;
|
|
6962
|
-
stickyReceipts = new
|
|
7689
|
+
stickyReceipts = new PendingStageReceiptIndex;
|
|
6963
7690
|
registerWorkflow(handle, authoritativeOwner = false) {
|
|
6964
7691
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
6965
7692
|
const existing = this.workflows.get(handle.workflowId);
|
|
@@ -7007,7 +7734,8 @@ class JournalProjection {
|
|
|
7007
7734
|
toolByHash: new Map,
|
|
7008
7735
|
uiByHash: new Map,
|
|
7009
7736
|
stageOutputByReplayKey: new Map,
|
|
7010
|
-
stageSessionByReplayKey: new Map
|
|
7737
|
+
stageSessionByReplayKey: new Map,
|
|
7738
|
+
nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
|
|
7011
7739
|
});
|
|
7012
7740
|
}
|
|
7013
7741
|
applyWorkflowSnapshot(handle) {
|
|
@@ -7033,39 +7761,43 @@ class JournalProjection {
|
|
|
7033
7761
|
return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
|
|
7034
7762
|
}
|
|
7035
7763
|
stickyDeliveryCounts = new Map;
|
|
7764
|
+
stickyDeliveryCountVersions = new Map;
|
|
7036
7765
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
7037
|
-
return this.stickyDeliveryCounts.get(workflowId)?.get(
|
|
7766
|
+
return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
|
|
7767
|
+
}
|
|
7768
|
+
pendingStageDeliveryCountVersion(workflowId) {
|
|
7769
|
+
return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
|
|
7770
|
+
}
|
|
7771
|
+
bumpDeliveryCountVersion(workflowId) {
|
|
7772
|
+
this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
|
|
7038
7773
|
}
|
|
7039
7774
|
async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
|
|
7040
|
-
|
|
7041
|
-
if (
|
|
7042
|
-
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7046
|
-
|
|
7047
|
-
|
|
7048
|
-
receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
|
|
7049
|
-
if (delivery.admission === "context" && delivery.sessionId !== undefined)
|
|
7050
|
-
receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
|
|
7051
|
-
if (accounting !== undefined && !existing) {
|
|
7052
|
-
let counts = this.stickyDeliveryCounts.get(workflowId);
|
|
7775
|
+
const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
|
|
7776
|
+
if (accounting !== undefined && added) {
|
|
7777
|
+
let byRun = this.stickyDeliveryCounts.get(workflowId);
|
|
7778
|
+
if (byRun === undefined) {
|
|
7779
|
+
byRun = new Map;
|
|
7780
|
+
this.stickyDeliveryCounts.set(workflowId, byRun);
|
|
7781
|
+
}
|
|
7782
|
+
let counts = byRun.get(accounting.messageRunId);
|
|
7053
7783
|
if (counts === undefined) {
|
|
7054
7784
|
counts = new Map;
|
|
7055
|
-
|
|
7785
|
+
byRun.set(accounting.messageRunId, counts);
|
|
7056
7786
|
}
|
|
7057
|
-
const
|
|
7058
|
-
counts.set(
|
|
7787
|
+
const previous = counts.get(messageId) ?? 0;
|
|
7788
|
+
counts.set(messageId, Math.max(previous, accounting.deliveryCount));
|
|
7789
|
+
if (accounting.deliveryCount > previous)
|
|
7790
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
7059
7791
|
}
|
|
7060
7792
|
}
|
|
7061
7793
|
async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
|
|
7062
|
-
return
|
|
7794
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
|
|
7063
7795
|
}
|
|
7064
7796
|
hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
|
|
7065
|
-
return
|
|
7066
|
-
|
|
7067
|
-
|
|
7068
|
-
|
|
7797
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
|
|
7798
|
+
}
|
|
7799
|
+
pendingStageDeliveryReceiptChanges(workflowId, cursor) {
|
|
7800
|
+
return this.stickyReceipts.changesSince(workflowId, cursor);
|
|
7069
7801
|
}
|
|
7070
7802
|
recordCheckpoint(recorded) {
|
|
7071
7803
|
const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
|
|
@@ -7084,6 +7816,7 @@ class JournalProjection {
|
|
|
7084
7816
|
if (rec.checkpoints.has(key))
|
|
7085
7817
|
return;
|
|
7086
7818
|
rec.checkpoints.set(key, currentCheckpoint);
|
|
7819
|
+
rec.nestedSnapshotVersion.record(currentCheckpoint);
|
|
7087
7820
|
if (currentCheckpoint.kind === "tool") {
|
|
7088
7821
|
const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
|
|
7089
7822
|
if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
|
|
@@ -7141,6 +7874,9 @@ class JournalProjection {
|
|
|
7141
7874
|
getWorkflow(workflowId) {
|
|
7142
7875
|
return this.workflows.get(workflowId)?.handle;
|
|
7143
7876
|
}
|
|
7877
|
+
nestedSnapshotVersion(workflowId) {
|
|
7878
|
+
return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
|
|
7879
|
+
}
|
|
7144
7880
|
getLoadableWorkflow(workflowId) {
|
|
7145
7881
|
return this.getWorkflow(workflowId);
|
|
7146
7882
|
}
|
|
@@ -7218,6 +7954,7 @@ class JournalProjection {
|
|
|
7218
7954
|
this.pendingReceipts.delete(workflowId);
|
|
7219
7955
|
this.stickyReceipts.delete(workflowId);
|
|
7220
7956
|
this.stickyDeliveryCounts.delete(workflowId);
|
|
7957
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
7221
7958
|
this.deletedWorkflowIds.add(workflowId);
|
|
7222
7959
|
}
|
|
7223
7960
|
isWorkflowLoadable(workflowId) {
|
|
@@ -7228,6 +7965,8 @@ class JournalProjection {
|
|
|
7228
7965
|
this.pendingReceipts.clear();
|
|
7229
7966
|
this.stickyReceipts.clear();
|
|
7230
7967
|
this.stickyDeliveryCounts.clear();
|
|
7968
|
+
for (const workflowId of this.stickyDeliveryCountVersions.keys())
|
|
7969
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
7231
7970
|
this.deletedWorkflowIds.clear();
|
|
7232
7971
|
}
|
|
7233
7972
|
}
|
|
@@ -7281,13 +8020,10 @@ function toResumableEntry(handle) {
|
|
|
7281
8020
|
updatedAt: handle.updatedAt
|
|
7282
8021
|
};
|
|
7283
8022
|
}
|
|
7284
|
-
|
|
7285
|
-
const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
|
|
7286
|
-
return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
|
|
7287
|
-
}
|
|
7288
|
-
var SESSION_HEADER_READ_LIMIT_BYTES;
|
|
8023
|
+
var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
|
|
7289
8024
|
var init_backend = __esm(() => {
|
|
7290
8025
|
init_resume_eligibility();
|
|
8026
|
+
init_stage_topology_validation();
|
|
7291
8027
|
init_durable_hash();
|
|
7292
8028
|
SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
|
|
7293
8029
|
});
|
|
@@ -7348,205 +8084,24 @@ var init_executor_child_helpers = __esm(() => {
|
|
|
7348
8084
|
init_workflow();
|
|
7349
8085
|
});
|
|
7350
8086
|
|
|
7351
|
-
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
7352
|
-
function immutableStageGroupError(stages) {
|
|
7353
|
-
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
7354
|
-
if (occurrenceError !== undefined)
|
|
7355
|
-
return occurrenceError;
|
|
7356
|
-
const byReplayKey = groupByDurableStageKey(stages);
|
|
7357
|
-
for (const group of byReplayKey.values()) {
|
|
7358
|
-
const firstName = group[0]?.name;
|
|
7359
|
-
if (group.some((stage) => stage.name !== firstName))
|
|
7360
|
-
return "stage name drift within one replay-key group";
|
|
7361
|
-
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
7362
|
-
const first = topologies[0];
|
|
7363
|
-
if (first === undefined)
|
|
7364
|
-
continue;
|
|
7365
|
-
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
7366
|
-
return "immutable stage identity drift within one replay-key group";
|
|
7367
|
-
}
|
|
7368
|
-
}
|
|
7369
|
-
return;
|
|
7370
|
-
}
|
|
7371
|
-
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
7372
|
-
const prefix = `${replayScope}:`;
|
|
7373
|
-
const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
|
|
7374
|
-
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
7375
|
-
return "child run topology exists outside its durable invocation scope";
|
|
7376
|
-
}
|
|
7377
|
-
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
7378
|
-
return "child checkpoint ownership is not reciprocal";
|
|
7379
|
-
}
|
|
7380
|
-
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
7381
|
-
const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
7382
|
-
const immutableError = immutableStageGroupError(direct);
|
|
7383
|
-
if (immutableError !== undefined)
|
|
7384
|
-
return immutableError;
|
|
7385
|
-
const merged = [];
|
|
7386
|
-
for (const group of groupByDurableStageKey(direct).values()) {
|
|
7387
|
-
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
7388
|
-
if (topology === undefined)
|
|
7389
|
-
return "direct child stage group lacks owning-run topology";
|
|
7390
|
-
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
7391
|
-
return "direct child stage topology lacks current source identity";
|
|
7392
|
-
}
|
|
7393
|
-
if (topology.stageId.length === 0)
|
|
7394
|
-
return "direct child stage id is empty";
|
|
7395
|
-
merged.push(topology);
|
|
7396
|
-
}
|
|
7397
|
-
const ids = new Set(allowedNodeIds);
|
|
7398
|
-
const stageIds = new Set;
|
|
7399
|
-
const orders = new Set;
|
|
7400
|
-
for (const topology of merged) {
|
|
7401
|
-
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
7402
|
-
return "direct child stage ids are not unique";
|
|
7403
|
-
stageIds.add(topology.stageId);
|
|
7404
|
-
ids.add(topology.stageId);
|
|
7405
|
-
if (topology.sourceOrder !== undefined) {
|
|
7406
|
-
if (orders.has(topology.sourceOrder))
|
|
7407
|
-
return "direct child source orders are not unique";
|
|
7408
|
-
orders.add(topology.sourceOrder);
|
|
7409
|
-
}
|
|
7410
|
-
}
|
|
7411
|
-
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
7412
|
-
return "direct child stage references an unknown parent";
|
|
7413
|
-
}
|
|
7414
|
-
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
7415
|
-
}
|
|
7416
|
-
function owningBoundaryIdError(stages, owningRunId) {
|
|
7417
|
-
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
7418
|
-
const seen = new Map;
|
|
7419
|
-
for (const start of starts) {
|
|
7420
|
-
const stageId = start.topology.stageId;
|
|
7421
|
-
if (stageId.length === 0)
|
|
7422
|
-
return "owning-run boundary stage id is empty";
|
|
7423
|
-
const prior = seen.get(stageId);
|
|
7424
|
-
if (prior !== undefined && prior !== start.replayKey)
|
|
7425
|
-
return "owning-run boundary stage id is aliased";
|
|
7426
|
-
seen.set(stageId, start.replayKey);
|
|
7427
|
-
}
|
|
7428
|
-
return;
|
|
7429
|
-
}
|
|
7430
|
-
function sameRun(actual, expected) {
|
|
7431
|
-
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
7432
|
-
}
|
|
7433
|
-
function sameStrings(left, right) {
|
|
7434
|
-
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
7435
|
-
}
|
|
7436
|
-
function sameOptionalRun(actual, expected) {
|
|
7437
|
-
if (actual === undefined || expected === undefined)
|
|
7438
|
-
return actual === expected;
|
|
7439
|
-
return sameRun(actual, expected);
|
|
7440
|
-
}
|
|
7441
|
-
function promptOccurrenceIdentityError(stages) {
|
|
7442
|
-
const replayByOccurrence = new Map;
|
|
7443
|
-
const occurrenceByReplayStage = new Map;
|
|
7444
|
-
const stageByReplayOccurrence = new Map;
|
|
7445
|
-
for (const stage of stages) {
|
|
7446
|
-
const topology = stage.topology;
|
|
7447
|
-
const occurrence = topology?.occurrenceKey;
|
|
7448
|
-
if (topology === undefined || occurrence === undefined)
|
|
7449
|
-
continue;
|
|
7450
|
-
const priorReplay = replayByOccurrence.get(occurrence);
|
|
7451
|
-
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
7452
|
-
return "prompt occurrence is bound to multiple replay keys";
|
|
7453
|
-
}
|
|
7454
|
-
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
7455
|
-
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
7456
|
-
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
7457
|
-
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
7458
|
-
return "prompt stage identity is bound to multiple occurrences";
|
|
7459
|
-
}
|
|
7460
|
-
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
7461
|
-
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7462
|
-
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
7463
|
-
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
7464
|
-
return "prompt occurrence is bound to multiple stage identities";
|
|
7465
|
-
}
|
|
7466
|
-
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
7467
|
-
}
|
|
7468
|
-
return;
|
|
7469
|
-
}
|
|
7470
|
-
function durableStageGroupKey(stage) {
|
|
7471
|
-
const occurrence = stage.topology?.occurrenceKey;
|
|
7472
|
-
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7473
|
-
}
|
|
7474
|
-
function groupByDurableStageKey(stages) {
|
|
7475
|
-
const groups = new Map;
|
|
7476
|
-
for (const stage of stages) {
|
|
7477
|
-
const key = durableStageGroupKey(stage);
|
|
7478
|
-
const group = groups.get(key) ?? [];
|
|
7479
|
-
group.push(stage);
|
|
7480
|
-
groups.set(key, group);
|
|
7481
|
-
}
|
|
7482
|
-
return groups;
|
|
7483
|
-
}
|
|
7484
|
-
function compositeKey(domain, ...parts) {
|
|
7485
|
-
return JSON.stringify([domain, ...parts]);
|
|
7486
|
-
}
|
|
7487
|
-
function hasCycle(topologies) {
|
|
7488
|
-
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
7489
|
-
const visiting = new Set;
|
|
7490
|
-
const visited = new Set;
|
|
7491
|
-
const cyclic = (stageId) => {
|
|
7492
|
-
if (visiting.has(stageId))
|
|
7493
|
-
return true;
|
|
7494
|
-
if (visited.has(stageId))
|
|
7495
|
-
return false;
|
|
7496
|
-
visiting.add(stageId);
|
|
7497
|
-
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
7498
|
-
return true;
|
|
7499
|
-
visiting.delete(stageId);
|
|
7500
|
-
visited.add(stageId);
|
|
7501
|
-
return false;
|
|
7502
|
-
};
|
|
7503
|
-
return topologies.some((topology) => cyclic(topology.stageId));
|
|
7504
|
-
}
|
|
7505
|
-
var init_stage_topology_validation = () => {};
|
|
7506
|
-
|
|
7507
|
-
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
7508
|
-
function parseWorkflowChildResult(value2) {
|
|
7509
|
-
if (!isOrdinaryObject(value2))
|
|
7510
|
-
return;
|
|
7511
|
-
const workflow = value2.workflow;
|
|
7512
|
-
const runId = value2.runId;
|
|
7513
|
-
const status = value2.status;
|
|
7514
|
-
const exited = value2.exited;
|
|
7515
|
-
const outputs = value2.outputs;
|
|
7516
|
-
const exitReason = value2.exitReason;
|
|
7517
|
-
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
7518
|
-
return;
|
|
7519
|
-
if (exited === false) {
|
|
7520
|
-
if (status !== "completed" || exitReason !== undefined)
|
|
7521
|
-
return;
|
|
7522
|
-
return value2;
|
|
7523
|
-
}
|
|
7524
|
-
if (exited !== true || !isWorkflowExitStatus2(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
7525
|
-
return;
|
|
7526
|
-
return value2;
|
|
7527
|
-
}
|
|
7528
|
-
function isWorkflowChildResult(value2) {
|
|
7529
|
-
return parseWorkflowChildResult(value2) !== undefined;
|
|
7530
|
-
}
|
|
7531
|
-
function isWorkflowExitStatus2(value2) {
|
|
7532
|
-
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
7533
|
-
}
|
|
7534
|
-
function isOrdinaryObject(value2) {
|
|
7535
|
-
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
7536
|
-
return false;
|
|
7537
|
-
const prototype = Object.getPrototypeOf(value2);
|
|
7538
|
-
return prototype === Object.prototype || prototype === null;
|
|
7539
|
-
}
|
|
7540
|
-
|
|
7541
8087
|
// dist/builtin/workflows/src/durable/boundary-lifecycle.ts
|
|
7542
8088
|
function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
7543
8089
|
const startTopology = start.topology;
|
|
7544
8090
|
const startBoundary = startTopology?.boundary;
|
|
7545
|
-
const startRun = startTopology?.run;
|
|
7546
8091
|
const related = stages.filter((stage) => {
|
|
7547
8092
|
const boundary = stage.topology?.boundary;
|
|
7548
8093
|
return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
|
|
7549
8094
|
});
|
|
8095
|
+
return resolveBoundaryLifecycleFromRecords(start, {
|
|
8096
|
+
related,
|
|
8097
|
+
outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
|
|
8098
|
+
}, requireTerminal);
|
|
8099
|
+
}
|
|
8100
|
+
function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
|
|
8101
|
+
const startTopology = start.topology;
|
|
8102
|
+
const startBoundary = startTopology?.boundary;
|
|
8103
|
+
const startRun = startTopology?.run;
|
|
8104
|
+
const related = records.related;
|
|
7550
8105
|
const starts = related.filter((stage) => stage.topology.boundary.event === "start");
|
|
7551
8106
|
if (starts.length !== 1 || starts[0] !== start)
|
|
7552
8107
|
return failure("boundary has no unique start record");
|
|
@@ -7583,7 +8138,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
|
7583
8138
|
return failure("boundary has an illegal terminal transition");
|
|
7584
8139
|
}
|
|
7585
8140
|
const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
|
|
7586
|
-
const outputRecords =
|
|
8141
|
+
const outputRecords = records.outputRecords();
|
|
7587
8142
|
if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
|
|
7588
8143
|
return failure("boundary replay key has conflicting output records");
|
|
7589
8144
|
}
|
|
@@ -9892,9 +10447,19 @@ function* invoke(action) {
|
|
|
9892
10447
|
} catch (error) {
|
|
9893
10448
|
if (error instanceof JournalError || error instanceof StorageError)
|
|
9894
10449
|
throw error;
|
|
9895
|
-
throw
|
|
10450
|
+
throw customJournalUnavailable(error);
|
|
9896
10451
|
}
|
|
9897
10452
|
}
|
|
10453
|
+
function customJournalUnavailable(failure) {
|
|
10454
|
+
const cause = failure instanceof Error ? failure : new Error(String(failure));
|
|
10455
|
+
const code = hasErrorCode(cause) ? cause.code : undefined;
|
|
10456
|
+
const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
|
|
10457
|
+
return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
|
|
10458
|
+
}
|
|
10459
|
+
function hasErrorCode(error) {
|
|
10460
|
+
const code = error.code;
|
|
10461
|
+
return typeof code === "string" || typeof code === "number";
|
|
10462
|
+
}
|
|
9898
10463
|
async function execute(operation) {
|
|
9899
10464
|
let step = operation.next();
|
|
9900
10465
|
while (!step.done) {
|
|
@@ -9922,6 +10487,8 @@ class JournalDurableBackend {
|
|
|
9922
10487
|
executions = new Map;
|
|
9923
10488
|
executionContext = new AsyncLocalStorage3;
|
|
9924
10489
|
pending = new Map;
|
|
10490
|
+
reconciledPendingStages = new Map;
|
|
10491
|
+
appendBatches = new Map;
|
|
9925
10492
|
catalogQueue = Symbol("catalog");
|
|
9926
10493
|
lost = new Map;
|
|
9927
10494
|
prompts = new Map;
|
|
@@ -9954,7 +10521,7 @@ class JournalDurableBackend {
|
|
|
9954
10521
|
await backend.close();
|
|
9955
10522
|
if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
|
|
9956
10523
|
throw error;
|
|
9957
|
-
throw
|
|
10524
|
+
throw customJournalUnavailable(error);
|
|
9958
10525
|
}
|
|
9959
10526
|
}
|
|
9960
10527
|
serialize(runId, action, requiresActiveExecution = true) {
|
|
@@ -9969,6 +10536,8 @@ class JournalDurableBackend {
|
|
|
9969
10536
|
return this.enqueue(runId, action, admitted);
|
|
9970
10537
|
}
|
|
9971
10538
|
enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
|
|
10539
|
+
if (typeof runId === "string")
|
|
10540
|
+
this.appendBatches.delete(runId);
|
|
9972
10541
|
const prior = this.pending.get(runId) ?? Promise.resolve();
|
|
9973
10542
|
const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
|
|
9974
10543
|
const settled = result.then(() => {}, () => {});
|
|
@@ -9983,6 +10552,7 @@ class JournalDurableBackend {
|
|
|
9983
10552
|
if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
|
|
9984
10553
|
return;
|
|
9985
10554
|
const event = entry.event;
|
|
10555
|
+
let raisedCount;
|
|
9986
10556
|
if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
|
|
9987
10557
|
const checkpoint = event.payload;
|
|
9988
10558
|
validateCheckpointInvocationFingerprint(checkpoint);
|
|
@@ -9991,7 +10561,10 @@ class JournalDurableBackend {
|
|
|
9991
10561
|
this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
|
|
9992
10562
|
} else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
|
|
9993
10563
|
const payload = event.payload;
|
|
10564
|
+
const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
|
|
9994
10565
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
10566
|
+
if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
|
|
10567
|
+
raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
|
|
9995
10568
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
9996
10569
|
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
9997
10570
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
@@ -10002,20 +10575,80 @@ class JournalDurableBackend {
|
|
|
10002
10575
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
10003
10576
|
this.prompts.delete(entry.runId);
|
|
10004
10577
|
}
|
|
10005
|
-
|
|
10006
|
-
if (handle?.pendingStageMessages !== undefined) {
|
|
10007
|
-
const messages = handle.pendingStageMessages.map((message) => {
|
|
10008
|
-
if (message.sticky !== true)
|
|
10009
|
-
return message;
|
|
10010
|
-
const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
|
|
10011
|
-
return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
|
|
10012
|
-
});
|
|
10013
|
-
if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
|
|
10014
|
-
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
|
|
10015
|
-
}
|
|
10578
|
+
this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
|
|
10016
10579
|
this.sequences.set(entry.runId, entry.seq);
|
|
10017
10580
|
this.foldedAt.set(entry.runId, entry.at);
|
|
10018
10581
|
}
|
|
10582
|
+
reconcileFoldedDeliveryCounts(runId, raisedCount) {
|
|
10583
|
+
const handle = this.projection.getWorkflow(runId);
|
|
10584
|
+
const list = handle?.pendingStageMessages;
|
|
10585
|
+
if (handle === undefined || list === undefined) {
|
|
10586
|
+
this.reconciledPendingStages.delete(runId);
|
|
10587
|
+
return;
|
|
10588
|
+
}
|
|
10589
|
+
let state = this.reconciledPendingStages.get(runId);
|
|
10590
|
+
let next;
|
|
10591
|
+
const raise = (index, count) => {
|
|
10592
|
+
const message = (next ?? list)[index];
|
|
10593
|
+
if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
|
|
10594
|
+
return;
|
|
10595
|
+
next ??= [...list];
|
|
10596
|
+
next[index] = { ...message, deliveryCount: count };
|
|
10597
|
+
};
|
|
10598
|
+
const reconcile = (index) => {
|
|
10599
|
+
const message = list[index];
|
|
10600
|
+
if (message.sticky !== true)
|
|
10601
|
+
return;
|
|
10602
|
+
raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
|
|
10603
|
+
};
|
|
10604
|
+
if (state?.list !== list) {
|
|
10605
|
+
const previous = state?.list ?? [];
|
|
10606
|
+
let aligned = list.length >= previous.length;
|
|
10607
|
+
const changed = [];
|
|
10608
|
+
for (let index = 0;index < list.length; index++) {
|
|
10609
|
+
const message = list[index];
|
|
10610
|
+
if (previous[index] === message)
|
|
10611
|
+
continue;
|
|
10612
|
+
if (index < previous.length && previous[index].id !== message.id)
|
|
10613
|
+
aligned = false;
|
|
10614
|
+
changed.push(index);
|
|
10615
|
+
}
|
|
10616
|
+
const reconciled = aligned ? undefined : new Set(previous);
|
|
10617
|
+
for (const index of changed)
|
|
10618
|
+
if (!reconciled?.has(list[index]))
|
|
10619
|
+
reconcile(index);
|
|
10620
|
+
const positions = aligned ? state?.positions : undefined;
|
|
10621
|
+
if (positions !== undefined) {
|
|
10622
|
+
for (let index = previous.length;index < list.length; index++) {
|
|
10623
|
+
const indexes = positions.get(list[index].id);
|
|
10624
|
+
if (indexes === undefined)
|
|
10625
|
+
positions.set(list[index].id, [index]);
|
|
10626
|
+
else
|
|
10627
|
+
indexes.push(index);
|
|
10628
|
+
}
|
|
10629
|
+
}
|
|
10630
|
+
state = { list, positions };
|
|
10631
|
+
}
|
|
10632
|
+
if (raisedCount !== undefined) {
|
|
10633
|
+
const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
|
|
10634
|
+
state.positions ??= messagePositions(list);
|
|
10635
|
+
for (const index of state.positions.get(raisedCount.messageId) ?? []) {
|
|
10636
|
+
const message = list[index];
|
|
10637
|
+
if (message.sticky === true && message.runId === raisedCount.messageRunId)
|
|
10638
|
+
raise(index, count);
|
|
10639
|
+
}
|
|
10640
|
+
}
|
|
10641
|
+
if (next === undefined) {
|
|
10642
|
+
this.reconciledPendingStages.set(runId, state);
|
|
10643
|
+
return;
|
|
10644
|
+
}
|
|
10645
|
+
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
|
|
10646
|
+
const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
|
|
10647
|
+
this.reconciledPendingStages.set(runId, {
|
|
10648
|
+
list: applied,
|
|
10649
|
+
positions: applied.length === list.length ? state.positions : undefined
|
|
10650
|
+
});
|
|
10651
|
+
}
|
|
10019
10652
|
summarize(runId, tail) {
|
|
10020
10653
|
const event = tail.snapshot.event;
|
|
10021
10654
|
const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
|
|
@@ -10039,6 +10672,7 @@ class JournalDurableBackend {
|
|
|
10039
10672
|
*rebuild(runId) {
|
|
10040
10673
|
const entries = yield* this.entries(runId);
|
|
10041
10674
|
this.projection.deleteWorkflow(runId);
|
|
10675
|
+
this.reconciledPendingStages.delete(runId);
|
|
10042
10676
|
this.sequences.delete(runId);
|
|
10043
10677
|
this.foldedAt.delete(runId);
|
|
10044
10678
|
this.summarized.delete(runId);
|
|
@@ -10134,9 +10768,9 @@ class JournalDurableBackend {
|
|
|
10134
10768
|
}
|
|
10135
10769
|
yield* this.write(runId, lease, event, options);
|
|
10136
10770
|
}
|
|
10137
|
-
*write(runId, lease, event, options) {
|
|
10771
|
+
*write(runId, lease, event, options, context = (action) => action()) {
|
|
10138
10772
|
try {
|
|
10139
|
-
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
10773
|
+
const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
|
|
10140
10774
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
10141
10775
|
if (outcome.entry.seq > nextSeq) {
|
|
10142
10776
|
for (const entry of yield* this.entries(runId, nextSeq))
|
|
@@ -10320,13 +10954,80 @@ class JournalDurableBackend {
|
|
|
10320
10954
|
return true;
|
|
10321
10955
|
}
|
|
10322
10956
|
async archivePendingStageMessage(runId, message) {
|
|
10323
|
-
await this.
|
|
10957
|
+
await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
|
|
10324
10958
|
}
|
|
10325
10959
|
async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
|
|
10326
|
-
await this.
|
|
10960
|
+
await this.queueAppend(runId, {
|
|
10327
10961
|
type: RUN_EVENT_TYPES.messageDelivered,
|
|
10328
10962
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
10329
|
-
})
|
|
10963
|
+
});
|
|
10964
|
+
}
|
|
10965
|
+
queueAppend(runId, event, key) {
|
|
10966
|
+
try {
|
|
10967
|
+
this.ensureOpen();
|
|
10968
|
+
this.assertExecutionActive();
|
|
10969
|
+
} catch (error) {
|
|
10970
|
+
return Promise.reject(error);
|
|
10971
|
+
}
|
|
10972
|
+
const open = this.appendBatches.get(runId);
|
|
10973
|
+
const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
|
|
10974
|
+
const queued = new Promise((resolve, reject) => {
|
|
10975
|
+
batch.push({
|
|
10976
|
+
event,
|
|
10977
|
+
...key === undefined ? {} : { key },
|
|
10978
|
+
context: AsyncLocalStorage3.snapshot(),
|
|
10979
|
+
resolve,
|
|
10980
|
+
reject
|
|
10981
|
+
});
|
|
10982
|
+
});
|
|
10983
|
+
if (batch !== open) {
|
|
10984
|
+
this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
|
|
10985
|
+
for (const append of batch)
|
|
10986
|
+
append.reject(error);
|
|
10987
|
+
});
|
|
10988
|
+
this.appendBatches.set(runId, batch);
|
|
10989
|
+
}
|
|
10990
|
+
return queued;
|
|
10991
|
+
}
|
|
10992
|
+
*appendBatch(runId, batch) {
|
|
10993
|
+
if (this.appendBatches.get(runId) === batch)
|
|
10994
|
+
this.appendBatches.delete(runId);
|
|
10995
|
+
const appendMany = runJournalAppendMany(this.journal);
|
|
10996
|
+
if (appendMany === undefined || batch.length === 1) {
|
|
10997
|
+
for (const append of batch) {
|
|
10998
|
+
try {
|
|
10999
|
+
const lease = yield* this.ownership(runId);
|
|
11000
|
+
yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
|
|
11001
|
+
append.resolve();
|
|
11002
|
+
} catch (error) {
|
|
11003
|
+
append.reject(error);
|
|
11004
|
+
}
|
|
11005
|
+
}
|
|
11006
|
+
return;
|
|
11007
|
+
}
|
|
11008
|
+
let settled = 0;
|
|
11009
|
+
try {
|
|
11010
|
+
const lease = yield* this.ownership(runId);
|
|
11011
|
+
const results = yield* invoke(() => appendMany(lease, batch));
|
|
11012
|
+
for (const result of results) {
|
|
11013
|
+
const append = batch[settled];
|
|
11014
|
+
if (result.status === "rejected") {
|
|
11015
|
+
append.reject(yield* this.mapOwnership(result.reason, runId));
|
|
11016
|
+
} else {
|
|
11017
|
+
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
11018
|
+
if (result.value.entry.seq > nextSeq)
|
|
11019
|
+
for (const entry of yield* this.entries(runId, nextSeq))
|
|
11020
|
+
this.fold(entry);
|
|
11021
|
+
this.fold(result.value.entry);
|
|
11022
|
+
append.resolve();
|
|
11023
|
+
}
|
|
11024
|
+
settled++;
|
|
11025
|
+
}
|
|
11026
|
+
} catch (error) {
|
|
11027
|
+
const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
|
|
11028
|
+
for (const append of batch.slice(settled))
|
|
11029
|
+
append.reject(failure);
|
|
11030
|
+
}
|
|
10330
11031
|
}
|
|
10331
11032
|
getToolOutput(...args) {
|
|
10332
11033
|
return this.projection.getToolOutput(...args);
|
|
@@ -10346,6 +11047,9 @@ class JournalDurableBackend {
|
|
|
10346
11047
|
listCheckpoints(runId) {
|
|
10347
11048
|
return this.projection.listCheckpoints(runId);
|
|
10348
11049
|
}
|
|
11050
|
+
nestedSnapshotVersion(runId) {
|
|
11051
|
+
return this.projection.nestedSnapshotVersion(runId);
|
|
11052
|
+
}
|
|
10349
11053
|
getWorkflow(runId) {
|
|
10350
11054
|
return this.projection.getWorkflow(runId);
|
|
10351
11055
|
}
|
|
@@ -10370,6 +11074,12 @@ class JournalDurableBackend {
|
|
|
10370
11074
|
getPendingStageMessageDeliveryCount(...args) {
|
|
10371
11075
|
return this.projection.getPendingStageMessageDeliveryCount(...args);
|
|
10372
11076
|
}
|
|
11077
|
+
pendingStageDeliveryReceiptChanges(...args) {
|
|
11078
|
+
return this.projection.pendingStageDeliveryReceiptChanges(...args);
|
|
11079
|
+
}
|
|
11080
|
+
pendingStageDeliveryCountVersion(runId) {
|
|
11081
|
+
return this.projection.pendingStageDeliveryCountVersion(runId);
|
|
11082
|
+
}
|
|
10373
11083
|
hasPendingStageDeliveryReceipt(...args) {
|
|
10374
11084
|
return this.projection.hasPendingStageDeliveryReceipt(...args);
|
|
10375
11085
|
}
|
|
@@ -10454,6 +11164,7 @@ class JournalDurableBackend {
|
|
|
10454
11164
|
this.foldedAt.delete(runId);
|
|
10455
11165
|
this.summarized.delete(runId);
|
|
10456
11166
|
this.prompts.delete(runId);
|
|
11167
|
+
this.reconciledPendingStages.delete(runId);
|
|
10457
11168
|
this.projection.deleteWorkflow(runId);
|
|
10458
11169
|
}
|
|
10459
11170
|
async deleteWorkflow(runId) {
|
|
@@ -10636,8 +11347,24 @@ class JournalDurableBackend {
|
|
|
10636
11347
|
}
|
|
10637
11348
|
}
|
|
10638
11349
|
}
|
|
11350
|
+
function messagePositions(list) {
|
|
11351
|
+
const positions = new Map;
|
|
11352
|
+
list.forEach((message, index) => {
|
|
11353
|
+
const indexes = positions.get(message.id);
|
|
11354
|
+
if (indexes === undefined)
|
|
11355
|
+
positions.set(message.id, [index]);
|
|
11356
|
+
else
|
|
11357
|
+
indexes.push(index);
|
|
11358
|
+
});
|
|
11359
|
+
return positions;
|
|
11360
|
+
}
|
|
10639
11361
|
function identity(message) {
|
|
10640
|
-
|
|
11362
|
+
let key = entryIdentity.get(message);
|
|
11363
|
+
if (key === undefined) {
|
|
11364
|
+
key = JSON.stringify([message.runId, message.id]);
|
|
11365
|
+
entryIdentity.set(message, key);
|
|
11366
|
+
}
|
|
11367
|
+
return key;
|
|
10641
11368
|
}
|
|
10642
11369
|
function uniqueByIdentity(messages) {
|
|
10643
11370
|
const result = new Map;
|
|
@@ -10811,7 +11538,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
10811
11538
|
function hasPendingStageDeliveryDelta(messages) {
|
|
10812
11539
|
return messages?.some(isDeltaEncoded) === true;
|
|
10813
11540
|
}
|
|
10814
|
-
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
11541
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
|
|
10815
11542
|
var init_journal_backend = __esm(() => {
|
|
10816
11543
|
init_workflow_instance_ownership_error();
|
|
10817
11544
|
init_backend();
|
|
@@ -10823,6 +11550,7 @@ var init_journal_backend = __esm(() => {
|
|
|
10823
11550
|
init_types();
|
|
10824
11551
|
PROCESS_HOLDER_ID = randomUUID4();
|
|
10825
11552
|
processExitCleanups = new Set;
|
|
11553
|
+
CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
|
|
10826
11554
|
JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
|
|
10827
11555
|
runId;
|
|
10828
11556
|
holderId;
|
|
@@ -10869,6 +11597,7 @@ var init_journal_backend = __esm(() => {
|
|
|
10869
11597
|
this.name = "JournalMessageScopeError";
|
|
10870
11598
|
}
|
|
10871
11599
|
};
|
|
11600
|
+
entryIdentity = new WeakMap;
|
|
10872
11601
|
entryJson = new WeakMap;
|
|
10873
11602
|
});
|
|
10874
11603
|
|
|
@@ -57517,9 +58246,146 @@ var init_definitions = __esm(() => {
|
|
|
57517
58246
|
workerDefinitionSources = new WeakMap;
|
|
57518
58247
|
});
|
|
57519
58248
|
|
|
58249
|
+
// dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
|
|
58250
|
+
function push(map, key, value2) {
|
|
58251
|
+
if (key === undefined)
|
|
58252
|
+
return;
|
|
58253
|
+
const list = map.get(key);
|
|
58254
|
+
if (list)
|
|
58255
|
+
list.push(value2);
|
|
58256
|
+
else
|
|
58257
|
+
map.set(key, [value2]);
|
|
58258
|
+
}
|
|
58259
|
+
|
|
58260
|
+
class Partitions {
|
|
58261
|
+
byRun = new Map;
|
|
58262
|
+
startsByChild = new Map;
|
|
58263
|
+
stagesByGroup = new Map;
|
|
58264
|
+
stagesByReplayKey = new Map;
|
|
58265
|
+
boundariesByStageId = new Map;
|
|
58266
|
+
boundariesByChild = new Map;
|
|
58267
|
+
add(entry) {
|
|
58268
|
+
const checkpoint = entry.checkpoint;
|
|
58269
|
+
push(this.byRun, checkpoint.topology?.run?.runId, entry);
|
|
58270
|
+
if (checkpoint.kind !== "stage")
|
|
58271
|
+
return;
|
|
58272
|
+
const stage = entry;
|
|
58273
|
+
push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
|
|
58274
|
+
push(this.stagesByReplayKey, checkpoint.replayKey, stage);
|
|
58275
|
+
const boundary = checkpoint.topology?.boundary;
|
|
58276
|
+
if (!boundary)
|
|
58277
|
+
return;
|
|
58278
|
+
push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
|
|
58279
|
+
push(this.boundariesByChild, boundary.child?.runId, stage);
|
|
58280
|
+
if (boundary.event === "start")
|
|
58281
|
+
push(this.startsByChild, boundary.child?.runId, stage);
|
|
58282
|
+
}
|
|
58283
|
+
}
|
|
58284
|
+
function inKeyOrder(entries) {
|
|
58285
|
+
return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
|
|
58286
|
+
}
|
|
58287
|
+
|
|
58288
|
+
class PoolCheckpointView {
|
|
58289
|
+
base;
|
|
58290
|
+
from;
|
|
58291
|
+
delta;
|
|
58292
|
+
constructor(base, from, delta) {
|
|
58293
|
+
this.base = base;
|
|
58294
|
+
this.from = from;
|
|
58295
|
+
this.delta = delta;
|
|
58296
|
+
}
|
|
58297
|
+
entries(partition, key) {
|
|
58298
|
+
const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
|
|
58299
|
+
return [...committed, ...this.delta[partition].get(key) ?? []];
|
|
58300
|
+
}
|
|
58301
|
+
}
|
|
58302
|
+
|
|
58303
|
+
class PoolCheckpointRead {
|
|
58304
|
+
index;
|
|
58305
|
+
root;
|
|
58306
|
+
pending;
|
|
58307
|
+
constructor(index, root) {
|
|
58308
|
+
this.index = index;
|
|
58309
|
+
this.root = root;
|
|
58310
|
+
}
|
|
58311
|
+
read(tx, events) {
|
|
58312
|
+
return this.index.read(tx, events, this.root, (apply) => {
|
|
58313
|
+
this.pending = apply;
|
|
58314
|
+
});
|
|
58315
|
+
}
|
|
58316
|
+
commit() {
|
|
58317
|
+
this.pending?.();
|
|
58318
|
+
this.pending = undefined;
|
|
58319
|
+
}
|
|
58320
|
+
}
|
|
58321
|
+
|
|
58322
|
+
class PoolCheckpointIndex {
|
|
58323
|
+
roots = new Map;
|
|
58324
|
+
begin(root) {
|
|
58325
|
+
return new PoolCheckpointRead(this, root);
|
|
58326
|
+
}
|
|
58327
|
+
async read(tx, events, root, onCommit) {
|
|
58328
|
+
const current = this.roots.get(root);
|
|
58329
|
+
let base = current;
|
|
58330
|
+
const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
|
|
58331
|
+
if (base && base.seq > 0) {
|
|
58332
|
+
const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
|
|
58333
|
+
if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
|
|
58334
|
+
base = undefined;
|
|
58335
|
+
}
|
|
58336
|
+
const from = base?.seq ?? 0;
|
|
58337
|
+
const to = head ? Number(head.seq) : 0;
|
|
58338
|
+
const delta = new Partitions;
|
|
58339
|
+
const added = [];
|
|
58340
|
+
if (to > from) {
|
|
58341
|
+
const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
|
|
58342
|
+
for (const row of rows) {
|
|
58343
|
+
const checkpoint = JSON.parse(String(row.payload));
|
|
58344
|
+
if (checkpoint.kind === "ui")
|
|
58345
|
+
continue;
|
|
58346
|
+
const entry = {
|
|
58347
|
+
seq: Number(row.seq),
|
|
58348
|
+
order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
|
|
58349
|
+
checkpoint
|
|
58350
|
+
};
|
|
58351
|
+
delta.add(entry);
|
|
58352
|
+
added.push(entry);
|
|
58353
|
+
}
|
|
58354
|
+
}
|
|
58355
|
+
onCommit(() => {
|
|
58356
|
+
const latest = this.roots.get(root);
|
|
58357
|
+
if (!base) {
|
|
58358
|
+
if (latest !== current)
|
|
58359
|
+
return;
|
|
58360
|
+
const fresh = {
|
|
58361
|
+
seq: to,
|
|
58362
|
+
key: head?.key ?? null,
|
|
58363
|
+
at: Number(head?.at ?? 0),
|
|
58364
|
+
partitions: delta
|
|
58365
|
+
};
|
|
58366
|
+
this.roots.set(root, fresh);
|
|
58367
|
+
return;
|
|
58368
|
+
}
|
|
58369
|
+
if (latest !== base || to <= base.seq)
|
|
58370
|
+
return;
|
|
58371
|
+
for (const entry of added)
|
|
58372
|
+
if (entry.seq > base.seq)
|
|
58373
|
+
base.partitions.add(entry);
|
|
58374
|
+
base.seq = to;
|
|
58375
|
+
base.key = head?.key ?? null;
|
|
58376
|
+
base.at = Number(head?.at ?? 0);
|
|
58377
|
+
});
|
|
58378
|
+
return new PoolCheckpointView(base, from, delta);
|
|
58379
|
+
}
|
|
58380
|
+
}
|
|
58381
|
+
var init_pool_checkpoint_index = __esm(() => {
|
|
58382
|
+
init_stage_topology_validation();
|
|
58383
|
+
init_types();
|
|
58384
|
+
});
|
|
58385
|
+
|
|
57520
58386
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
57521
58387
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
57522
|
-
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
58388
|
+
async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
|
|
57523
58389
|
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
57524
58390
|
throw new Error("Invalid pool mutation");
|
|
57525
58391
|
const checkpoint = mutation.checkpoint;
|
|
@@ -57539,42 +58405,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
|
57539
58405
|
}
|
|
57540
58406
|
if (checkpoint.kind === "ui")
|
|
57541
58407
|
return;
|
|
57542
|
-
const
|
|
57543
|
-
const records = rows.map((row) => JSON.parse(String(row.payload)));
|
|
57544
|
-
const stages = records.filter((entry) => entry.kind === "stage");
|
|
58408
|
+
const view = await checkpoints.read(tx, journal.events);
|
|
57545
58409
|
const topology = checkpoint.topology;
|
|
57546
58410
|
const run = topology?.run;
|
|
57547
58411
|
if (!topology || !run || run.rootRunId !== root || run.runId === root)
|
|
57548
58412
|
throw new Error("Pool topology scope mismatch");
|
|
57549
|
-
const owner =
|
|
58413
|
+
const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
|
|
57550
58414
|
if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
|
|
57551
58415
|
throw new Error("Pool topology scope mismatch");
|
|
57552
58416
|
if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
|
|
57553
58417
|
throw new Error("Pool session scope mismatch");
|
|
58418
|
+
const runEntries = view.entries("byRun", run.runId);
|
|
57554
58419
|
if (checkpoint.kind === "tool") {
|
|
57555
58420
|
const tool = checkpoint.topology;
|
|
57556
|
-
const peers =
|
|
58421
|
+
const peers = runEntries.map((entry) => entry.checkpoint);
|
|
57557
58422
|
const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
|
|
57558
58423
|
if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
|
|
57559
58424
|
throw new Error("Pool tool topology scope mismatch");
|
|
57560
58425
|
return;
|
|
57561
58426
|
}
|
|
57562
|
-
const
|
|
57563
|
-
const
|
|
58427
|
+
const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
|
|
58428
|
+
const relevant = [...runStages];
|
|
58429
|
+
const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
|
|
58430
|
+
for (const group of groups)
|
|
58431
|
+
relevant.push(...view.entries("stagesByGroup", group));
|
|
58432
|
+
const boundary = checkpoint.topology?.boundary;
|
|
58433
|
+
let start;
|
|
58434
|
+
if (boundary) {
|
|
58435
|
+
const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
|
|
58436
|
+
start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
58437
|
+
relevant.push(...replayed);
|
|
58438
|
+
if (start) {
|
|
58439
|
+
const startTopology = start.topology;
|
|
58440
|
+
relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
|
|
58441
|
+
relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
|
|
58442
|
+
}
|
|
58443
|
+
}
|
|
58444
|
+
const next = [...stagesExcept(relevant, checkpoint), checkpoint];
|
|
58445
|
+
const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
|
|
57564
58446
|
const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
|
|
57565
58447
|
if (stageError)
|
|
57566
58448
|
throw new Error(`Pool topology scope: ${stageError}`);
|
|
57567
|
-
const boundary = checkpoint.topology?.boundary;
|
|
57568
58449
|
if (!boundary) {
|
|
57569
58450
|
if (parseWorkflowChildResult(checkpoint.output))
|
|
57570
58451
|
throw new Error("Pool boundary output without ownership");
|
|
57571
58452
|
return;
|
|
57572
58453
|
}
|
|
57573
|
-
const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
57574
58454
|
const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
|
|
57575
58455
|
if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
|
|
57576
58456
|
throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
|
|
57577
58457
|
}
|
|
58458
|
+
function stagesExcept(entries, checkpoint) {
|
|
58459
|
+
return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
|
|
58460
|
+
}
|
|
57578
58461
|
async function sessionStore(tx, prefix) {
|
|
57579
58462
|
const database = {
|
|
57580
58463
|
descriptor: { adapter: "pool-transaction", location: prefix },
|
|
@@ -57623,20 +58506,52 @@ var init_pool_mutations = __esm(() => {
|
|
|
57623
58506
|
init_boundary_lifecycle();
|
|
57624
58507
|
init_boundary_topology();
|
|
57625
58508
|
init_stage_topology_validation();
|
|
57626
|
-
|
|
58509
|
+
init_pool_checkpoint_index();
|
|
57627
58510
|
});
|
|
57628
58511
|
|
|
57629
58512
|
// dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
|
|
57630
58513
|
function poolUsageTable(runs) {
|
|
57631
58514
|
return `${runs}_pool_usage`;
|
|
57632
58515
|
}
|
|
58516
|
+
function poolUsageTotalsTable(runs) {
|
|
58517
|
+
return `${runs}_pool_usage_totals`;
|
|
58518
|
+
}
|
|
58519
|
+
function addToTotals(totals, runs, work, sign, row) {
|
|
58520
|
+
const assignments = POOL_USAGE_FIELDS.map((field) => {
|
|
58521
|
+
const total = field.toLowerCase();
|
|
58522
|
+
const value2 = `(${sign}${row}.${total})`;
|
|
58523
|
+
return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
|
|
58524
|
+
});
|
|
58525
|
+
return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
|
|
58526
|
+
}
|
|
57633
58527
|
function poolUsageSchema(runs) {
|
|
57634
|
-
|
|
58528
|
+
const usage = poolUsageTable(runs);
|
|
58529
|
+
const totals = poolUsageTotalsTable(runs);
|
|
58530
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
58531
|
+
return `CREATE TABLE IF NOT EXISTS ${usage}(
|
|
57635
58532
|
work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
|
|
57636
58533
|
generation INTEGER NOT NULL, key TEXT NOT NULL,
|
|
57637
58534
|
input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
|
|
57638
58535
|
PRIMARY KEY(work, generation, key)
|
|
57639
|
-
)
|
|
58536
|
+
);
|
|
58537
|
+
CREATE TABLE IF NOT EXISTS ${totals}(
|
|
58538
|
+
root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
|
|
58539
|
+
${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
|
|
58540
|
+
);
|
|
58541
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
|
|
58542
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
58543
|
+
END;
|
|
58544
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
|
|
58545
|
+
WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
|
|
58546
|
+
${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
|
|
58547
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
58548
|
+
END;
|
|
58549
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
|
|
58550
|
+
DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
|
|
58551
|
+
END;
|
|
58552
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
|
|
58553
|
+
DELETE FROM ${totals} WHERE root = OLD.root;
|
|
58554
|
+
END;`;
|
|
57640
58555
|
}
|
|
57641
58556
|
function poolStageUsageKey(runId, stageId) {
|
|
57642
58557
|
return `stage:${runId}:${stageId}`;
|
|
@@ -57663,7 +58578,7 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
57663
58578
|
|
|
57664
58579
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
57665
58580
|
import { createHash as createHash4, randomUUID as randomUUID10 } from "node:crypto";
|
|
57666
|
-
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
58581
|
+
import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
|
|
57667
58582
|
function requestIdentity(request) {
|
|
57668
58583
|
return createHash4("sha256").update(request).digest("hex");
|
|
57669
58584
|
}
|
|
@@ -57702,6 +58617,7 @@ class SqlPoolQueue {
|
|
|
57702
58617
|
operations;
|
|
57703
58618
|
transaction;
|
|
57704
58619
|
journal;
|
|
58620
|
+
checkpoints = new PoolCheckpointIndex;
|
|
57705
58621
|
constructor(storage) {
|
|
57706
58622
|
this.storage = storage;
|
|
57707
58623
|
const { transaction, tables: journal } = storage;
|
|
@@ -57813,17 +58729,23 @@ class SqlPoolQueue {
|
|
|
57813
58729
|
lease = snapshotLease(lease);
|
|
57814
58730
|
token = { ...token };
|
|
57815
58731
|
return this.storage.withAppendGuard(async (tx, runId) => {
|
|
57816
|
-
|
|
57817
|
-
|
|
57818
|
-
|
|
57819
|
-
|
|
57820
|
-
|
|
57821
|
-
|
|
57822
|
-
|
|
57823
|
-
|
|
57824
|
-
|
|
57825
|
-
|
|
57826
|
-
|
|
58732
|
+
try {
|
|
58733
|
+
if (runId !== lease.runId)
|
|
58734
|
+
throw new Error("Pool delivery cannot write another root");
|
|
58735
|
+
await this.authority(tx, lease).catch((error) => {
|
|
58736
|
+
if (error instanceof StaleLeaseError)
|
|
58737
|
+
throw new Error("Stale pool coordinator authority", { cause: error });
|
|
58738
|
+
throw error;
|
|
58739
|
+
});
|
|
58740
|
+
const row = await this.current(tx, lease, token);
|
|
58741
|
+
if (row.epoch !== lease.epoch)
|
|
58742
|
+
throw new Error(STALE_WORK_LEASE);
|
|
58743
|
+
assertSession();
|
|
58744
|
+
} catch (error) {
|
|
58745
|
+
if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
|
|
58746
|
+
throw error;
|
|
58747
|
+
throw new PoolAppendRefusedError(error.message, { cause: error });
|
|
58748
|
+
}
|
|
57827
58749
|
}, action);
|
|
57828
58750
|
}
|
|
57829
58751
|
async assertCurrent(lease, token) {
|
|
@@ -57936,7 +58858,8 @@ class SqlPoolQueue {
|
|
|
57936
58858
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
57937
58859
|
const identity = requestIdentity(request);
|
|
57938
58860
|
mutation = JSON.parse(request).mutation;
|
|
57939
|
-
|
|
58861
|
+
const checkpoints = this.checkpoints.begin(lease.runId);
|
|
58862
|
+
const receipt = await this.transaction(async (tx) => {
|
|
57940
58863
|
const { now, status } = await this.authority(tx, lease);
|
|
57941
58864
|
const row = await this.current(tx, lease, token);
|
|
57942
58865
|
if (row.epoch !== lease.epoch)
|
|
@@ -57951,24 +58874,26 @@ class SqlPoolQueue {
|
|
|
57951
58874
|
throw new Error(STALE_WORK_LEASE);
|
|
57952
58875
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
57953
58876
|
throw new Error("Pool root is not running");
|
|
57954
|
-
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
58877
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
|
|
57955
58878
|
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
57956
58879
|
return receipt;
|
|
57957
58880
|
});
|
|
58881
|
+
checkpoints.commit();
|
|
58882
|
+
return receipt;
|
|
57958
58883
|
}
|
|
57959
58884
|
async appendSession(tx, row, mutation, now) {
|
|
57960
58885
|
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
57961
58886
|
}
|
|
57962
|
-
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
58887
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
|
|
57963
58888
|
const checkpoint = mutation.checkpoint;
|
|
57964
|
-
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
58889
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
|
|
57965
58890
|
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
57966
58891
|
throw new Error("Pool checkpoint timestamp is in the future");
|
|
57967
58892
|
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
57968
58893
|
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
57969
58894
|
const payload = serializeJson(checkpoint);
|
|
57970
58895
|
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
57971
|
-
if (existing && existing.payload
|
|
58896
|
+
if (existing && !isRecordedCheckpoint(existing.payload, payload))
|
|
57972
58897
|
throw new Error("Pool checkpoint identity conflict");
|
|
57973
58898
|
if (existing && mutation.kind !== "checkpoint")
|
|
57974
58899
|
throw new Error("Pool session checkpoint identity conflict");
|
|
@@ -58066,22 +58991,37 @@ class SqlPoolQueue {
|
|
|
58066
58991
|
return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
|
|
58067
58992
|
}
|
|
58068
58993
|
}
|
|
58994
|
+
function isRecordedCheckpoint(stored, payload) {
|
|
58995
|
+
if (stored === payload)
|
|
58996
|
+
return true;
|
|
58997
|
+
const { completedAt } = JSON.parse(stored);
|
|
58998
|
+
if (typeof completedAt !== "number")
|
|
58999
|
+
return false;
|
|
59000
|
+
return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
|
|
59001
|
+
}
|
|
58069
59002
|
function openSqlPoolQueue(journal) {
|
|
58070
59003
|
const storage = sqlPoolStorage(journal);
|
|
58071
59004
|
if (!storage)
|
|
58072
59005
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
58073
59006
|
return SqlPoolQueue.open(storage);
|
|
58074
59007
|
}
|
|
58075
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
59008
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
|
|
58076
59009
|
var init_sql_pool_queue = __esm(() => {
|
|
58077
59010
|
init_serializable();
|
|
58078
59011
|
init_boundary_topology();
|
|
58079
59012
|
init_durable_hash();
|
|
58080
59013
|
init_internal();
|
|
59014
|
+
init_pool_checkpoint_index();
|
|
58081
59015
|
init_pool_mutations();
|
|
58082
59016
|
init_pool_stage_usage();
|
|
58083
59017
|
init_types();
|
|
58084
59018
|
init_values();
|
|
59019
|
+
PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
|
|
59020
|
+
constructor(message, options) {
|
|
59021
|
+
super(message, options);
|
|
59022
|
+
this.name = "PoolAppendRefusedError";
|
|
59023
|
+
}
|
|
59024
|
+
};
|
|
58085
59025
|
});
|
|
58086
59026
|
|
|
58087
59027
|
// dist/builtin/workflows/src/engine/session-closed-error.ts
|
|
@@ -58303,6 +59243,9 @@ var init_workflow_stage_target = __esm(() => {
|
|
|
58303
59243
|
});
|
|
58304
59244
|
|
|
58305
59245
|
// dist/builtin/workflows/src/shared/pending-stage-status.ts
|
|
59246
|
+
function isAddressableStageStatus(status) {
|
|
59247
|
+
return ADDRESSABLE_STAGE_STATUSES.has(status);
|
|
59248
|
+
}
|
|
58306
59249
|
function workflowBoundarySegments(runs, runId) {
|
|
58307
59250
|
return createWorkflowBoundarySegmentsResolver(runs)(runId);
|
|
58308
59251
|
}
|
|
@@ -58425,44 +59368,412 @@ function createStagePathPatternIndex(patterns) {
|
|
|
58425
59368
|
}
|
|
58426
59369
|
};
|
|
58427
59370
|
}
|
|
59371
|
+
var ADDRESSABLE_STAGE_STATUSES;
|
|
58428
59372
|
var init_pending_stage_status = __esm(() => {
|
|
58429
59373
|
init_store_internal();
|
|
58430
59374
|
init_workflow_stage_target();
|
|
59375
|
+
ADDRESSABLE_STAGE_STATUSES = new Set([
|
|
59376
|
+
"pending",
|
|
59377
|
+
"running",
|
|
59378
|
+
"awaiting_input",
|
|
59379
|
+
"paused",
|
|
59380
|
+
"blocked"
|
|
59381
|
+
]);
|
|
58431
59382
|
});
|
|
58432
59383
|
|
|
58433
|
-
// dist/builtin/workflows/src/shared/sticky-
|
|
58434
|
-
function
|
|
58435
|
-
|
|
58436
|
-
|
|
58437
|
-
const
|
|
58438
|
-
|
|
58439
|
-
const
|
|
58440
|
-
|
|
58441
|
-
const
|
|
58442
|
-
|
|
59384
|
+
// dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
|
|
59385
|
+
function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
|
|
59386
|
+
if (states.length === 0)
|
|
59387
|
+
return { deliveryStates: states };
|
|
59388
|
+
const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
|
|
59389
|
+
counts.set("delivered", 0);
|
|
59390
|
+
for (const delivery of states)
|
|
59391
|
+
counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
|
|
59392
|
+
const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
|
|
59393
|
+
if (states.length <= limit)
|
|
59394
|
+
return { deliveryStates: states, deliveryStateCounts };
|
|
59395
|
+
const exemplars = [];
|
|
59396
|
+
for (const state of UNRESOLVED_STATES) {
|
|
59397
|
+
for (const delivery of states) {
|
|
59398
|
+
if (exemplars.length === limit)
|
|
59399
|
+
break;
|
|
59400
|
+
if (delivery.state === state)
|
|
59401
|
+
exemplars.push(delivery);
|
|
59402
|
+
}
|
|
59403
|
+
}
|
|
59404
|
+
for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
|
|
59405
|
+
const delivery = states[index];
|
|
59406
|
+
if (delivery.state === "delivered")
|
|
59407
|
+
exemplars.push(delivery);
|
|
59408
|
+
}
|
|
59409
|
+
return {
|
|
59410
|
+
deliveryStates: exemplars,
|
|
59411
|
+
deliveryStateCounts,
|
|
59412
|
+
omittedDeliveryStates: states.length - exemplars.length
|
|
59413
|
+
};
|
|
59414
|
+
}
|
|
59415
|
+
var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
|
|
59416
|
+
var init_sticky_delivery_state_bound = __esm(() => {
|
|
59417
|
+
UNRESOLVED_STATES = [
|
|
59418
|
+
"queued",
|
|
59419
|
+
"receipt-unverified",
|
|
59420
|
+
"session-replaced",
|
|
59421
|
+
"delivery-unavailable",
|
|
59422
|
+
"root-unresolved",
|
|
59423
|
+
"stage-terminal"
|
|
59424
|
+
];
|
|
59425
|
+
});
|
|
59426
|
+
|
|
59427
|
+
// dist/builtin/workflows/src/shared/possible-stage-rows.ts
|
|
59428
|
+
function boundPossibleStageDeliveryStates(states) {
|
|
59429
|
+
const bounded = boundStickyDeliveryStates(states);
|
|
59430
|
+
return {
|
|
59431
|
+
...bounded,
|
|
59432
|
+
deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
|
|
59433
|
+
};
|
|
59434
|
+
}
|
|
59435
|
+
|
|
59436
|
+
class WorkflowTopologyIndex {
|
|
59437
|
+
lineage = [];
|
|
59438
|
+
runById = new Map;
|
|
59439
|
+
roots = new Map;
|
|
59440
|
+
hops = new Map;
|
|
59441
|
+
refresh(runs) {
|
|
59442
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
59443
|
+
const lineage = [];
|
|
59444
|
+
for (const run of runs) {
|
|
59445
|
+
lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
|
|
59446
|
+
if (run.parentRunId === undefined)
|
|
59447
|
+
continue;
|
|
59448
|
+
const parent = runById.get(run.parentRunId);
|
|
59449
|
+
const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
|
|
59450
|
+
lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
|
|
59451
|
+
}
|
|
59452
|
+
if (sameValues(lineage, this.lineage))
|
|
59453
|
+
return;
|
|
59454
|
+
this.lineage = lineage;
|
|
59455
|
+
this.runById = runById;
|
|
59456
|
+
this.roots.clear();
|
|
59457
|
+
this.hops.clear();
|
|
59458
|
+
}
|
|
59459
|
+
rootOf(runId) {
|
|
59460
|
+
if (this.roots.has(runId))
|
|
59461
|
+
return this.roots.get(runId);
|
|
59462
|
+
const root = reciprocalWorkflowRootRunId(this.runById, runId);
|
|
59463
|
+
this.roots.set(runId, root);
|
|
59464
|
+
return root;
|
|
59465
|
+
}
|
|
59466
|
+
hopsOf(runId) {
|
|
59467
|
+
if (this.hops.has(runId))
|
|
59468
|
+
return this.hops.get(runId);
|
|
59469
|
+
const hops = boundaryHops(this.runById, runId);
|
|
59470
|
+
this.hops.set(runId, hops);
|
|
59471
|
+
return hops;
|
|
59472
|
+
}
|
|
59473
|
+
}
|
|
59474
|
+
function boundaryHops(runById, runId) {
|
|
59475
|
+
const hops = [];
|
|
59476
|
+
let current = runById.get(runId);
|
|
59477
|
+
while (current !== undefined && current.parentRunId !== undefined) {
|
|
59478
|
+
const parent = runById.get(current.parentRunId);
|
|
59479
|
+
const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
|
|
59480
|
+
if (parent === undefined || boundary === undefined)
|
|
59481
|
+
return;
|
|
59482
|
+
hops.unshift({ name: boundary.name, runId: current.id });
|
|
59483
|
+
current = parent;
|
|
59484
|
+
}
|
|
59485
|
+
return current === undefined ? undefined : hops;
|
|
59486
|
+
}
|
|
59487
|
+
function sameValues(left, right) {
|
|
59488
|
+
if (left.length !== right.length)
|
|
59489
|
+
return false;
|
|
59490
|
+
for (let index = 0;index < left.length; index++)
|
|
59491
|
+
if (left[index] !== right[index])
|
|
59492
|
+
return false;
|
|
59493
|
+
return true;
|
|
59494
|
+
}
|
|
59495
|
+
|
|
59496
|
+
class PossibleStageRowsIndex {
|
|
59497
|
+
rootRunId;
|
|
59498
|
+
topology;
|
|
59499
|
+
targets = new Map;
|
|
59500
|
+
patterns = [];
|
|
59501
|
+
targetOf = new Map;
|
|
59502
|
+
targetsByMessage = new Map;
|
|
59503
|
+
list = [];
|
|
59504
|
+
stageFields = new WeakMap;
|
|
59505
|
+
backend;
|
|
59506
|
+
receiptCursor;
|
|
59507
|
+
scanMatches = new Map;
|
|
59508
|
+
rowCache = new Map;
|
|
59509
|
+
boundedStates = new WeakMap;
|
|
59510
|
+
deliveriesByEntry = new WeakMap;
|
|
59511
|
+
statusCache = new WeakMap;
|
|
59512
|
+
receiptGeneration = 0;
|
|
59513
|
+
receiptEpochs = new Map;
|
|
59514
|
+
constructor(rootRunId, topology) {
|
|
59515
|
+
this.rootRunId = rootRunId;
|
|
59516
|
+
this.topology = topology;
|
|
59517
|
+
}
|
|
59518
|
+
rows(runs, rootRun, availableBackend) {
|
|
59519
|
+
if (isTerminalRunStatus(rootRun.status))
|
|
59520
|
+
return [];
|
|
59521
|
+
this.refreshEntries(rootRun.pendingStageMessages ?? []);
|
|
59522
|
+
const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
|
|
59523
|
+
const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
|
|
59524
|
+
this.refreshReceipts(backend);
|
|
59525
|
+
this.refreshStages(runs);
|
|
59526
|
+
for (const state of this.patterns)
|
|
59527
|
+
if (state.dirty)
|
|
59528
|
+
this.recompute(state, backend);
|
|
59529
|
+
return this.assemble(runs, rootRun);
|
|
59530
|
+
}
|
|
59531
|
+
targetFor(entry) {
|
|
58443
59532
|
const target = entry.targetPath ?? entry.stageKey;
|
|
58444
|
-
let
|
|
58445
|
-
if (
|
|
59533
|
+
let state = this.targets.get(target);
|
|
59534
|
+
if (state === undefined) {
|
|
58446
59535
|
const parsed = parseWorkflowStageTarget(target);
|
|
58447
|
-
|
|
58448
|
-
|
|
58449
|
-
|
|
58450
|
-
|
|
59536
|
+
const owned = parsed?.rootRunId === this.rootRunId;
|
|
59537
|
+
state = {
|
|
59538
|
+
target,
|
|
59539
|
+
segments: owned ? parsed.segments : undefined,
|
|
59540
|
+
pattern: owned ? this.patterns.length : -1,
|
|
59541
|
+
entries: [],
|
|
59542
|
+
positions: [],
|
|
59543
|
+
queued: 0,
|
|
59544
|
+
stages: [],
|
|
59545
|
+
statuses: [],
|
|
59546
|
+
dirty: true
|
|
59547
|
+
};
|
|
59548
|
+
this.targets.set(target, state);
|
|
59549
|
+
if (owned)
|
|
59550
|
+
this.patterns.push(state);
|
|
59551
|
+
}
|
|
59552
|
+
return state;
|
|
59553
|
+
}
|
|
59554
|
+
refreshEntries(list) {
|
|
59555
|
+
const previous = this.list;
|
|
59556
|
+
if (list === previous)
|
|
59557
|
+
return;
|
|
59558
|
+
this.list = list;
|
|
59559
|
+
let incremental = list.length >= previous.length;
|
|
59560
|
+
const replaced = [];
|
|
59561
|
+
for (let index = 0;incremental && index < previous.length; index++) {
|
|
59562
|
+
if (list[index] === previous[index])
|
|
59563
|
+
continue;
|
|
59564
|
+
const before = previous[index];
|
|
59565
|
+
const after = list[index];
|
|
59566
|
+
if (before.sticky !== true && after.sticky !== true)
|
|
59567
|
+
continue;
|
|
59568
|
+
if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
|
|
59569
|
+
incremental = false;
|
|
59570
|
+
break;
|
|
59571
|
+
}
|
|
59572
|
+
replaced.push(index);
|
|
58451
59573
|
}
|
|
58452
|
-
if (
|
|
59574
|
+
if (!incremental) {
|
|
59575
|
+
this.rebuildEntries(list);
|
|
58453
59576
|
return;
|
|
58454
|
-
|
|
58455
|
-
|
|
58456
|
-
|
|
58457
|
-
|
|
58458
|
-
|
|
58459
|
-
|
|
58460
|
-
|
|
58461
|
-
|
|
58462
|
-
|
|
59577
|
+
}
|
|
59578
|
+
for (const index of replaced) {
|
|
59579
|
+
const before = previous[index];
|
|
59580
|
+
const after = list[index];
|
|
59581
|
+
const state = this.targetOf.get(before);
|
|
59582
|
+
const slot = state.entries.indexOf(before);
|
|
59583
|
+
state.entries[slot] = after;
|
|
59584
|
+
this.targetOf.delete(before);
|
|
59585
|
+
this.targetOf.set(after, state);
|
|
59586
|
+
state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
|
|
59587
|
+
if (before.id !== after.id)
|
|
59588
|
+
this.rebuildMessageIndex();
|
|
59589
|
+
state.dirty = true;
|
|
59590
|
+
}
|
|
59591
|
+
for (let index = previous.length;index < list.length; index++)
|
|
59592
|
+
this.addEntry(list[index], index);
|
|
59593
|
+
}
|
|
59594
|
+
addEntry(entry, position) {
|
|
59595
|
+
if (entry.sticky !== true)
|
|
59596
|
+
return;
|
|
59597
|
+
const state = this.targetFor(entry);
|
|
59598
|
+
state.entries.push(entry);
|
|
59599
|
+
state.positions.push(position);
|
|
59600
|
+
if (entry.status === "queued")
|
|
59601
|
+
state.queued++;
|
|
59602
|
+
state.dirty = true;
|
|
59603
|
+
this.targetOf.set(entry, state);
|
|
59604
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
59605
|
+
if (targets === undefined) {
|
|
59606
|
+
targets = new Set;
|
|
59607
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
59608
|
+
}
|
|
59609
|
+
targets.add(state);
|
|
59610
|
+
}
|
|
59611
|
+
rebuildEntries(list) {
|
|
59612
|
+
const before = new Map;
|
|
59613
|
+
for (const state of this.targets.values()) {
|
|
59614
|
+
before.set(state, state.entries);
|
|
59615
|
+
state.entries = [];
|
|
59616
|
+
state.positions = [];
|
|
59617
|
+
state.queued = 0;
|
|
59618
|
+
}
|
|
59619
|
+
this.targetOf.clear();
|
|
59620
|
+
this.targetsByMessage.clear();
|
|
59621
|
+
list.forEach((entry, position) => {
|
|
59622
|
+
this.addEntry(entry, position);
|
|
59623
|
+
});
|
|
59624
|
+
for (const state of this.targets.values()) {
|
|
59625
|
+
const previous = before.get(state);
|
|
59626
|
+
state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
|
|
59627
|
+
}
|
|
59628
|
+
}
|
|
59629
|
+
rebuildMessageIndex() {
|
|
59630
|
+
this.targetsByMessage.clear();
|
|
59631
|
+
for (const [entry, state] of this.targetOf) {
|
|
59632
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
59633
|
+
if (targets === undefined) {
|
|
59634
|
+
targets = new Set;
|
|
59635
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
59636
|
+
}
|
|
59637
|
+
targets.add(state);
|
|
59638
|
+
}
|
|
59639
|
+
}
|
|
59640
|
+
refreshReceipts(backend) {
|
|
59641
|
+
if (backend !== this.backend) {
|
|
59642
|
+
this.backend = backend;
|
|
59643
|
+
this.receiptCursor = undefined;
|
|
59644
|
+
this.invalidateReceipts();
|
|
59645
|
+
}
|
|
59646
|
+
if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
|
|
59647
|
+
return;
|
|
59648
|
+
const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
|
|
59649
|
+
if (changes === undefined) {
|
|
59650
|
+
this.invalidateReceipts();
|
|
59651
|
+
return;
|
|
59652
|
+
}
|
|
59653
|
+
const firstRead = this.receiptCursor === undefined;
|
|
59654
|
+
this.receiptCursor = changes.cursor;
|
|
59655
|
+
if (changes.messageIds === undefined) {
|
|
59656
|
+
if (!firstRead)
|
|
59657
|
+
this.invalidateReceipts();
|
|
59658
|
+
return;
|
|
59659
|
+
}
|
|
59660
|
+
for (const messageId of changes.messageIds) {
|
|
59661
|
+
this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
|
|
59662
|
+
for (const state of this.targetsByMessage.get(messageId) ?? [])
|
|
59663
|
+
state.dirty = true;
|
|
59664
|
+
}
|
|
59665
|
+
}
|
|
59666
|
+
invalidateReceipts() {
|
|
59667
|
+
this.receiptGeneration++;
|
|
59668
|
+
this.receiptEpochs.clear();
|
|
59669
|
+
for (const state of this.patterns)
|
|
59670
|
+
state.dirty = true;
|
|
59671
|
+
}
|
|
59672
|
+
refreshStages(runs) {
|
|
59673
|
+
const current = new Map;
|
|
59674
|
+
let ordinal = 0;
|
|
59675
|
+
for (const run of runs) {
|
|
59676
|
+
const root = this.topology.rootOf(run.id);
|
|
59677
|
+
if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
|
|
59678
|
+
continue;
|
|
59679
|
+
const resolved = root === this.rootRunId;
|
|
59680
|
+
const hops = this.topology.hopsOf(run.id);
|
|
59681
|
+
for (const stage of run.stages) {
|
|
59682
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
59683
|
+
continue;
|
|
59684
|
+
const fields = this.fieldsOf(run, stage, resolved, hops);
|
|
59685
|
+
const status = { run, stage, resolved, hops, ordinal: ordinal++ };
|
|
59686
|
+
for (const state of fields.matched) {
|
|
59687
|
+
const stages = current.get(state);
|
|
59688
|
+
if (stages === undefined)
|
|
59689
|
+
current.set(state, [status]);
|
|
59690
|
+
else
|
|
59691
|
+
stages.push(status);
|
|
59692
|
+
}
|
|
59693
|
+
}
|
|
59694
|
+
}
|
|
59695
|
+
for (const state of this.patterns) {
|
|
59696
|
+
const stages = current.get(state) ?? [];
|
|
59697
|
+
if (stages.length !== state.stages.length || stages.some((status, index) => {
|
|
59698
|
+
const previous = state.stages[index];
|
|
59699
|
+
return previous.stage !== status.stage || previous.run !== status.run;
|
|
59700
|
+
}))
|
|
59701
|
+
state.dirty = true;
|
|
59702
|
+
state.stages = stages;
|
|
59703
|
+
}
|
|
59704
|
+
}
|
|
59705
|
+
fieldsOf(run, stage, resolved, hops) {
|
|
59706
|
+
let fields = this.stageFields.get(stage);
|
|
59707
|
+
const matchable = resolved && hops !== undefined;
|
|
59708
|
+
let changed = false;
|
|
59709
|
+
if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
|
|
59710
|
+
const previous = fields;
|
|
59711
|
+
const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
|
|
59712
|
+
fields = {
|
|
59713
|
+
run,
|
|
59714
|
+
status: stage.status,
|
|
59715
|
+
error: stage.error,
|
|
59716
|
+
sessionId: stage.sessionId,
|
|
59717
|
+
pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
|
|
59718
|
+
resolved,
|
|
59719
|
+
hops,
|
|
59720
|
+
id: stage.id,
|
|
59721
|
+
name: stage.name,
|
|
59722
|
+
tested: sameMatching ? previous.tested : 0,
|
|
59723
|
+
matched: sameMatching ? previous.matched : []
|
|
59724
|
+
};
|
|
59725
|
+
this.stageFields.set(stage, fields);
|
|
59726
|
+
changed = true;
|
|
59727
|
+
}
|
|
59728
|
+
if (fields.tested < this.patterns.length) {
|
|
59729
|
+
const added = this.patterns.slice(fields.tested);
|
|
59730
|
+
if (!matchable)
|
|
59731
|
+
fields.matched.push(...added);
|
|
59732
|
+
else {
|
|
59733
|
+
const index = createStagePathPatternIndex(added.map((state) => state.segments));
|
|
59734
|
+
for (const position of index.matching(hops, [stage.id, stage.name]))
|
|
59735
|
+
fields.matched.push(added[position]);
|
|
59736
|
+
}
|
|
59737
|
+
fields.tested = this.patterns.length;
|
|
59738
|
+
}
|
|
59739
|
+
if (changed)
|
|
59740
|
+
for (const state of fields.matched)
|
|
59741
|
+
state.dirty = true;
|
|
59742
|
+
return fields;
|
|
59743
|
+
}
|
|
59744
|
+
recompute(state, backend) {
|
|
59745
|
+
state.dirty = false;
|
|
59746
|
+
const statuses = [];
|
|
59747
|
+
for (const { run, stage, resolved, hops } of state.stages) {
|
|
59748
|
+
const fields = this.stageFields.get(stage);
|
|
59749
|
+
for (const entry of state.entries) {
|
|
59750
|
+
let byStage = this.statusCache.get(entry);
|
|
59751
|
+
if (byStage === undefined) {
|
|
59752
|
+
byStage = new Map;
|
|
59753
|
+
this.statusCache.set(entry, byStage);
|
|
59754
|
+
}
|
|
59755
|
+
const epoch = this.receiptEpochs.get(entry.id) ?? 0;
|
|
59756
|
+
let cached = byStage.get(stage);
|
|
59757
|
+
if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
|
|
59758
|
+
cached = {
|
|
59759
|
+
fields,
|
|
59760
|
+
generation: this.receiptGeneration,
|
|
59761
|
+
epoch,
|
|
59762
|
+
status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
|
|
59763
|
+
};
|
|
59764
|
+
byStage.set(stage, cached);
|
|
59765
|
+
}
|
|
59766
|
+
statuses.push(cached.status);
|
|
59767
|
+
}
|
|
59768
|
+
}
|
|
59769
|
+
if (!sameStatuses(statuses, state.statuses))
|
|
59770
|
+
state.statuses = statuses;
|
|
59771
|
+
}
|
|
59772
|
+
receiptsOf(entry, runId, stageId) {
|
|
59773
|
+
let byStage = this.deliveriesByEntry.get(entry);
|
|
58463
59774
|
if (byStage === undefined) {
|
|
58464
59775
|
byStage = new Map;
|
|
58465
|
-
for (const delivery of
|
|
59776
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
58466
59777
|
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
58467
59778
|
const list = byStage.get(key);
|
|
58468
59779
|
if (list === undefined)
|
|
@@ -58470,75 +59781,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
58470
59781
|
else
|
|
58471
59782
|
list.push(delivery);
|
|
58472
59783
|
}
|
|
58473
|
-
deliveriesByEntry.set(
|
|
59784
|
+
this.deliveriesByEntry.set(entry, byStage);
|
|
58474
59785
|
}
|
|
58475
59786
|
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
58476
|
-
}
|
|
58477
|
-
|
|
58478
|
-
|
|
58479
|
-
|
|
58480
|
-
|
|
58481
|
-
|
|
58482
|
-
|
|
58483
|
-
if (
|
|
58484
|
-
|
|
58485
|
-
|
|
58486
|
-
|
|
58487
|
-
|
|
58488
|
-
|
|
59787
|
+
}
|
|
59788
|
+
status(entry, run, stage, matchable, backend) {
|
|
59789
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
59790
|
+
const receipts = this.receiptsOf(entry, run.id, stage.id);
|
|
59791
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
59792
|
+
let state;
|
|
59793
|
+
let reason;
|
|
59794
|
+
if (!matchable) {
|
|
59795
|
+
state = "root-unresolved";
|
|
59796
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
59797
|
+
} else if (delivered) {
|
|
59798
|
+
state = "delivered";
|
|
59799
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
59800
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
59801
|
+
state = "stage-terminal";
|
|
59802
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
59803
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
59804
|
+
state = "receipt-unverified";
|
|
59805
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
59806
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
59807
|
+
state = "delivery-unavailable";
|
|
59808
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
59809
|
+
} else if (receipts.length > 0) {
|
|
59810
|
+
state = "session-replaced";
|
|
59811
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
59812
|
+
} else {
|
|
59813
|
+
state = "queued";
|
|
59814
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
59815
|
+
}
|
|
59816
|
+
return {
|
|
59817
|
+
messageId: entry.message.id,
|
|
59818
|
+
target,
|
|
59819
|
+
runId: run.id,
|
|
59820
|
+
stageId: stage.id,
|
|
59821
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
59822
|
+
state,
|
|
59823
|
+
reason
|
|
59824
|
+
};
|
|
59825
|
+
}
|
|
59826
|
+
assemble(runs, rootRun) {
|
|
59827
|
+
const rows = [];
|
|
59828
|
+
const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
|
|
59829
|
+
let liveStageMatches;
|
|
59830
|
+
const rowTargets = new Set;
|
|
59831
|
+
for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
|
|
59832
|
+
const rowSegments = splitStagePathSegments(scanEntry);
|
|
59833
|
+
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
58489
59834
|
continue;
|
|
58490
|
-
|
|
58491
|
-
|
|
58492
|
-
|
|
58493
|
-
|
|
58494
|
-
const receipts = entryReceipts(index, run.id, stage.id);
|
|
58495
|
-
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
58496
|
-
runId: run.id,
|
|
58497
|
-
stageId: stage.id,
|
|
58498
|
-
sessionId: stage.sessionId,
|
|
58499
|
-
admission: "context"
|
|
58500
|
-
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
58501
|
-
let state;
|
|
58502
|
-
let reason;
|
|
58503
|
-
if (!resolved || !hops) {
|
|
58504
|
-
state = "root-unresolved";
|
|
58505
|
-
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
58506
|
-
} else if (delivered) {
|
|
58507
|
-
state = "delivered";
|
|
58508
|
-
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
58509
|
-
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
58510
|
-
state = "stage-terminal";
|
|
58511
|
-
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
58512
|
-
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
58513
|
-
state = "receipt-unverified";
|
|
58514
|
-
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
58515
|
-
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
58516
|
-
state = "delivery-unavailable";
|
|
58517
|
-
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
58518
|
-
} else if (receipts.length > 0) {
|
|
58519
|
-
state = "session-replaced";
|
|
58520
|
-
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
58521
|
-
} else {
|
|
58522
|
-
state = "queued";
|
|
58523
|
-
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
58524
|
-
}
|
|
58525
|
-
result.push({
|
|
58526
|
-
messageId: entry.message.id,
|
|
58527
|
-
target,
|
|
58528
|
-
runId: run.id,
|
|
58529
|
-
stageId: stage.id,
|
|
58530
|
-
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
58531
|
-
state,
|
|
58532
|
-
reason
|
|
58533
|
-
});
|
|
59835
|
+
if (!rowSegments.some((segment) => segment.includes("*"))) {
|
|
59836
|
+
liveStageMatches ??= this.materializedStageMatcher(runs);
|
|
59837
|
+
if (liveStageMatches(rowSegments))
|
|
59838
|
+
continue;
|
|
58534
59839
|
}
|
|
59840
|
+
let matches = this.scanMatches.get(scanEntry);
|
|
59841
|
+
if (matches === undefined) {
|
|
59842
|
+
matches = new Map;
|
|
59843
|
+
this.scanMatches.set(scanEntry, matches);
|
|
59844
|
+
}
|
|
59845
|
+
let queuedCount = 0;
|
|
59846
|
+
for (const state of scanTargets) {
|
|
59847
|
+
let match = matches.get(state);
|
|
59848
|
+
if (match === undefined) {
|
|
59849
|
+
const segments = state.segments;
|
|
59850
|
+
match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
59851
|
+
matches.set(state, match);
|
|
59852
|
+
}
|
|
59853
|
+
if (match)
|
|
59854
|
+
queuedCount += state.queued;
|
|
59855
|
+
}
|
|
59856
|
+
const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
|
|
59857
|
+
rows.push({ slot: `s${position}`, target, queuedCount });
|
|
59858
|
+
rowTargets.add(target);
|
|
59859
|
+
}
|
|
59860
|
+
let broadcastCount = 0;
|
|
59861
|
+
for (const state of this.targets.values()) {
|
|
59862
|
+
if (state.segments?.length === 1 && state.segments[0] === "**")
|
|
59863
|
+
broadcastCount += state.queued;
|
|
59864
|
+
}
|
|
59865
|
+
const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
|
|
59866
|
+
rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
|
|
59867
|
+
rowTargets.add(broadcast);
|
|
59868
|
+
const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
|
|
59869
|
+
const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
|
|
59870
|
+
for (const state of statusTargets) {
|
|
59871
|
+
if (rowTargets.has(state.target))
|
|
59872
|
+
continue;
|
|
59873
|
+
rowTargets.add(state.target);
|
|
59874
|
+
rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
|
|
58535
59875
|
}
|
|
59876
|
+
const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
|
|
59877
|
+
this.pruneRows(result);
|
|
59878
|
+
return result;
|
|
59879
|
+
}
|
|
59880
|
+
row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
|
|
59881
|
+
const cached = this.rowCache.get(slot);
|
|
59882
|
+
const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
|
|
59883
|
+
if (same && cached.statuses === deliveryStates)
|
|
59884
|
+
return cached.row;
|
|
59885
|
+
const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
|
|
59886
|
+
if (same && sameBoundedRow(cached.row, bounded)) {
|
|
59887
|
+
this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
|
|
59888
|
+
return cached.row;
|
|
59889
|
+
}
|
|
59890
|
+
const row = {
|
|
59891
|
+
target,
|
|
59892
|
+
queuedCount,
|
|
59893
|
+
...deliveryOnly === true ? { deliveryOnly } : {},
|
|
59894
|
+
...bounded
|
|
59895
|
+
};
|
|
59896
|
+
this.rowCache.set(slot, { row, statuses: deliveryStates });
|
|
59897
|
+
return row;
|
|
59898
|
+
}
|
|
59899
|
+
bounded(states) {
|
|
59900
|
+
let bounded = this.boundedStates.get(states);
|
|
59901
|
+
if (bounded === undefined) {
|
|
59902
|
+
bounded = boundPossibleStageDeliveryStates(states);
|
|
59903
|
+
this.boundedStates.set(states, bounded);
|
|
59904
|
+
}
|
|
59905
|
+
return bounded;
|
|
59906
|
+
}
|
|
59907
|
+
pruneRows(rows) {
|
|
59908
|
+
if (this.rowCache.size <= rows.length * 4 + 16)
|
|
59909
|
+
return;
|
|
59910
|
+
const kept = new Set(rows);
|
|
59911
|
+
for (const [key, cached] of this.rowCache)
|
|
59912
|
+
if (!kept.has(cached.row))
|
|
59913
|
+
this.rowCache.delete(key);
|
|
59914
|
+
}
|
|
59915
|
+
materializedStageMatcher(runs) {
|
|
59916
|
+
const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
|
|
59917
|
+
return (entrySegments) => {
|
|
59918
|
+
for (const run of invocationRuns) {
|
|
59919
|
+
const hops = this.topology.hopsOf(run.id);
|
|
59920
|
+
if (hops === undefined)
|
|
59921
|
+
continue;
|
|
59922
|
+
for (const stage of run.stages) {
|
|
59923
|
+
if (stage.pendingStageDeliveryAvailable !== true)
|
|
59924
|
+
continue;
|
|
59925
|
+
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
59926
|
+
continue;
|
|
59927
|
+
}
|
|
59928
|
+
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
59929
|
+
return true;
|
|
59930
|
+
}
|
|
59931
|
+
}
|
|
59932
|
+
return false;
|
|
59933
|
+
};
|
|
58536
59934
|
}
|
|
58537
|
-
return result;
|
|
58538
59935
|
}
|
|
58539
|
-
|
|
59936
|
+
function sameBoundedRow(row, bounded) {
|
|
59937
|
+
if (bounded === undefined)
|
|
59938
|
+
return row.deliveryStates === undefined;
|
|
59939
|
+
if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
|
|
59940
|
+
return false;
|
|
59941
|
+
const counts = Object.entries(row.deliveryStateCounts ?? {});
|
|
59942
|
+
const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
|
|
59943
|
+
if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
|
|
59944
|
+
return false;
|
|
59945
|
+
return sameStatuses(row.deliveryStates, bounded.deliveryStates);
|
|
59946
|
+
}
|
|
59947
|
+
function sameStatuses(left, right) {
|
|
59948
|
+
if (left.length !== right.length)
|
|
59949
|
+
return false;
|
|
59950
|
+
for (let index = 0;index < left.length; index++) {
|
|
59951
|
+
const a = left[index];
|
|
59952
|
+
const b = right[index];
|
|
59953
|
+
if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
|
|
59954
|
+
return false;
|
|
59955
|
+
}
|
|
59956
|
+
return true;
|
|
59957
|
+
}
|
|
59958
|
+
var init_possible_stage_rows = __esm(() => {
|
|
58540
59959
|
init_run_owner_backend();
|
|
58541
59960
|
init_pending_stage_status();
|
|
59961
|
+
init_sticky_delivery_state_bound();
|
|
59962
|
+
init_store_internal();
|
|
58542
59963
|
init_workflow_stage_target();
|
|
58543
59964
|
});
|
|
58544
59965
|
|
|
@@ -58603,11 +60024,32 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58603
60024
|
announceIfPending();
|
|
58604
60025
|
return routeCompletions.get(runId);
|
|
58605
60026
|
});
|
|
60027
|
+
const topology = new WorkflowTopologyIndex;
|
|
60028
|
+
const possibleStageRowIndexes = new Map;
|
|
60029
|
+
const rowIdentities = new WeakMap;
|
|
60030
|
+
let nextRowIdentity = 0;
|
|
60031
|
+
const rowIdentity = (row) => {
|
|
60032
|
+
let identity = rowIdentities.get(row);
|
|
60033
|
+
if (identity === undefined) {
|
|
60034
|
+
identity = nextRowIdentity++;
|
|
60035
|
+
rowIdentities.set(row, identity);
|
|
60036
|
+
}
|
|
60037
|
+
return identity;
|
|
60038
|
+
};
|
|
60039
|
+
const possibleStageRowsIndex = (rootRunId) => {
|
|
60040
|
+
let index = possibleStageRowIndexes.get(rootRunId);
|
|
60041
|
+
if (index === undefined) {
|
|
60042
|
+
index = new PossibleStageRowsIndex(rootRunId, topology);
|
|
60043
|
+
possibleStageRowIndexes.set(rootRunId, index);
|
|
60044
|
+
}
|
|
60045
|
+
return index;
|
|
60046
|
+
};
|
|
58606
60047
|
const announceRoutes = () => {
|
|
58607
60048
|
if (disposed)
|
|
58608
60049
|
return;
|
|
58609
60050
|
const ownedRunIds = new Set;
|
|
58610
60051
|
const runs = activeStore.runs();
|
|
60052
|
+
topology.refresh(runs);
|
|
58611
60053
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58612
60054
|
const childrenByBoundary = new Map;
|
|
58613
60055
|
for (const run of runs) {
|
|
@@ -58622,7 +60064,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58622
60064
|
for (const run of runs) {
|
|
58623
60065
|
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
58624
60066
|
continue;
|
|
58625
|
-
const rootRunId =
|
|
60067
|
+
const rootRunId = topology.rootOf(run.id);
|
|
58626
60068
|
if (rootRunId === undefined)
|
|
58627
60069
|
continue;
|
|
58628
60070
|
ownedRunIds.add(run.id);
|
|
@@ -58632,6 +60074,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58632
60074
|
runId: parentRun.id,
|
|
58633
60075
|
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
58634
60076
|
};
|
|
60077
|
+
let possibleStages;
|
|
60078
|
+
if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
|
|
60079
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
60080
|
+
possibleStages = [];
|
|
60081
|
+
} else if (run.id === rootRunId) {
|
|
60082
|
+
possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
|
|
60083
|
+
}
|
|
58635
60084
|
const announcement = {
|
|
58636
60085
|
runId: run.id,
|
|
58637
60086
|
group: workflowInvocationIntercomGroup(rootRunId),
|
|
@@ -58643,7 +60092,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58643
60092
|
stageName: stage.name,
|
|
58644
60093
|
target: stageRouteTarget(runs, rootRunId, run.id, stage.id, resolveBoundarySegments),
|
|
58645
60094
|
lifecycle: stage.sessionId === undefined && stage.sessionFile === undefined ? "pending" : "running",
|
|
58646
|
-
routeEligible: stage.pendingStageDeliveryAvailable === true && (stage.status
|
|
60095
|
+
routeEligible: stage.pendingStageDeliveryAvailable === true && isAddressableStageStatus(stage.status),
|
|
58647
60096
|
group: stage.intercomGroup ?? workflowInvocationIntercomGroup(rootRunId)
|
|
58648
60097
|
})),
|
|
58649
60098
|
...nonAgentNodes(run).map((node) => ({
|
|
@@ -58656,11 +60105,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58656
60105
|
group: workflowInvocationIntercomGroup(rootRunId)
|
|
58657
60106
|
}))
|
|
58658
60107
|
],
|
|
58659
|
-
...
|
|
58660
|
-
possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
|
|
58661
|
-
} : {}
|
|
60108
|
+
...possibleStages === undefined ? {} : { possibleStages }
|
|
58662
60109
|
};
|
|
58663
|
-
const signature = JSON.stringify(announcement)
|
|
60110
|
+
const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
|
|
58664
60111
|
if (announcedRoutes.get(run.id)?.signature === signature)
|
|
58665
60112
|
continue;
|
|
58666
60113
|
const entry = { signature };
|
|
@@ -58695,6 +60142,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58695
60142
|
if (!ownedRunIds.has(runId))
|
|
58696
60143
|
routeCompletions.delete(runId);
|
|
58697
60144
|
}
|
|
60145
|
+
for (const rootRunId of possibleStageRowIndexes.keys()) {
|
|
60146
|
+
if (!ownedRunIds.has(rootRunId))
|
|
60147
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
60148
|
+
}
|
|
58698
60149
|
sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
|
|
58699
60150
|
return;
|
|
58700
60151
|
}).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
|
|
@@ -58855,102 +60306,6 @@ function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegmen
|
|
|
58855
60306
|
const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
|
|
58856
60307
|
return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
|
|
58857
60308
|
}
|
|
58858
|
-
function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
58859
|
-
if (isTerminalRunStatus(rootRun.status))
|
|
58860
|
-
return [];
|
|
58861
|
-
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
58862
|
-
const segmentsByTarget = new Map;
|
|
58863
|
-
const stickySegments = (entry) => {
|
|
58864
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58865
|
-
if (segmentsByTarget.has(target))
|
|
58866
|
-
return segmentsByTarget.get(target);
|
|
58867
|
-
const parsed = parseWorkflowStageTarget(target);
|
|
58868
|
-
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
58869
|
-
segmentsByTarget.set(target, segments);
|
|
58870
|
-
return segments;
|
|
58871
|
-
};
|
|
58872
|
-
const scanEntries = stickyQueued.filter((entry) => {
|
|
58873
|
-
const segments = stickySegments(entry);
|
|
58874
|
-
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
58875
|
-
});
|
|
58876
|
-
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
58877
|
-
const rows = [];
|
|
58878
|
-
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
58879
|
-
const rowSegments = splitStagePathSegments(scanEntry);
|
|
58880
|
-
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
58881
|
-
continue;
|
|
58882
|
-
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
58883
|
-
continue;
|
|
58884
|
-
}
|
|
58885
|
-
rows.push({
|
|
58886
|
-
target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
|
|
58887
|
-
queuedCount: scanEntries.filter((entry) => {
|
|
58888
|
-
const segments = stickySegments(entry);
|
|
58889
|
-
if (segments === undefined)
|
|
58890
|
-
return false;
|
|
58891
|
-
return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
58892
|
-
}).length
|
|
58893
|
-
});
|
|
58894
|
-
}
|
|
58895
|
-
rows.push({
|
|
58896
|
-
target: formatWorkflowStageTarget(rootRunId, "**"),
|
|
58897
|
-
queuedCount: stickyQueued.filter((entry) => {
|
|
58898
|
-
const segments = stickySegments(entry);
|
|
58899
|
-
return segments !== undefined && segments.length === 1 && segments[0] === "**";
|
|
58900
|
-
}).length
|
|
58901
|
-
});
|
|
58902
|
-
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
58903
|
-
const statusesByTarget = new Map;
|
|
58904
|
-
for (const status of statuses) {
|
|
58905
|
-
const forTarget = statusesByTarget.get(status.target);
|
|
58906
|
-
if (forTarget === undefined)
|
|
58907
|
-
statusesByTarget.set(status.target, [status]);
|
|
58908
|
-
else
|
|
58909
|
-
forTarget.push(status);
|
|
58910
|
-
}
|
|
58911
|
-
const queuedByTarget = new Map;
|
|
58912
|
-
for (const entry of stickyQueued) {
|
|
58913
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58914
|
-
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
58915
|
-
}
|
|
58916
|
-
const rowTargets = new Set(rows.map((row) => row.target));
|
|
58917
|
-
for (const target of statusesByTarget.keys()) {
|
|
58918
|
-
if (rowTargets.has(target))
|
|
58919
|
-
continue;
|
|
58920
|
-
rowTargets.add(target);
|
|
58921
|
-
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
58922
|
-
}
|
|
58923
|
-
for (let index = 0;index < rows.length; index++) {
|
|
58924
|
-
const row = rows[index];
|
|
58925
|
-
const deliveryStates = statusesByTarget.get(row.target);
|
|
58926
|
-
if (deliveryStates !== undefined)
|
|
58927
|
-
rows[index] = { ...row, deliveryStates };
|
|
58928
|
-
}
|
|
58929
|
-
return rows;
|
|
58930
|
-
}
|
|
58931
|
-
function materializedStageMatcher(runs, rootRunId) {
|
|
58932
|
-
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58933
|
-
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
58934
|
-
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
58935
|
-
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
58936
|
-
}
|
|
58937
|
-
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
58938
|
-
for (const run of invocationRuns) {
|
|
58939
|
-
const hops = hopsFor(run.id);
|
|
58940
|
-
if (hops === undefined)
|
|
58941
|
-
continue;
|
|
58942
|
-
for (const stage of run.stages) {
|
|
58943
|
-
if (stage.pendingStageDeliveryAvailable !== true)
|
|
58944
|
-
continue;
|
|
58945
|
-
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
58946
|
-
continue;
|
|
58947
|
-
}
|
|
58948
|
-
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
58949
|
-
return true;
|
|
58950
|
-
}
|
|
58951
|
-
}
|
|
58952
|
-
return false;
|
|
58953
|
-
}
|
|
58954
60309
|
function isPendingStageMessageEvent(value2) {
|
|
58955
60310
|
if (typeof value2 !== "object" || value2 === null)
|
|
58956
60311
|
return false;
|
|
@@ -59240,7 +60595,7 @@ function knownUninitializedStage(run, stageKey) {
|
|
|
59240
60595
|
if (candidates.length !== 1)
|
|
59241
60596
|
return;
|
|
59242
60597
|
const stage = candidates[0];
|
|
59243
|
-
return (stage.status
|
|
60598
|
+
return isAddressableStageStatus(stage.status) && stage.sessionId === undefined && stage.sessionFile === undefined ? stage : undefined;
|
|
59244
60599
|
}
|
|
59245
60600
|
function pendingStageDestination(run, entry) {
|
|
59246
60601
|
if (entry.stageId !== undefined) {
|
|
@@ -59334,7 +60689,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
59334
60689
|
init_pending_stage_route_capability();
|
|
59335
60690
|
init_pending_stage_route_readiness();
|
|
59336
60691
|
init_pending_stage_status();
|
|
59337
|
-
|
|
60692
|
+
init_possible_stage_rows();
|
|
59338
60693
|
init_store_internal();
|
|
59339
60694
|
init_workflow_stage_target();
|
|
59340
60695
|
});
|
|
@@ -61881,15 +63236,22 @@ function validate(report) {
|
|
|
61881
63236
|
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
61882
63237
|
}
|
|
61883
63238
|
}
|
|
63239
|
+
function addCompensated(total, value2) {
|
|
63240
|
+
const sum = total.sum + value2;
|
|
63241
|
+
total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
|
|
63242
|
+
total.sum = sum;
|
|
63243
|
+
}
|
|
61884
63244
|
|
|
61885
63245
|
class SqlPoolUsage {
|
|
61886
63246
|
transaction;
|
|
61887
63247
|
journal;
|
|
61888
63248
|
table;
|
|
63249
|
+
totals;
|
|
61889
63250
|
constructor(transaction, journal) {
|
|
61890
63251
|
this.transaction = transaction;
|
|
61891
63252
|
this.journal = journal;
|
|
61892
63253
|
this.table = poolUsageTable(journal.runs);
|
|
63254
|
+
this.totals = poolUsageTotalsTable(journal.runs);
|
|
61893
63255
|
}
|
|
61894
63256
|
static async open(transaction, journal) {
|
|
61895
63257
|
const usage = new SqlPoolUsage(transaction, journal);
|
|
@@ -61897,9 +63259,27 @@ class SqlPoolUsage {
|
|
|
61897
63259
|
return usage;
|
|
61898
63260
|
}
|
|
61899
63261
|
async totalIn(tx, root) {
|
|
63262
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
63263
|
+
const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
|
|
63264
|
+
if (kept)
|
|
63265
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
|
|
63266
|
+
field,
|
|
63267
|
+
Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
|
|
63268
|
+
]));
|
|
61900
63269
|
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
61901
63270
|
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
61902
|
-
|
|
63271
|
+
const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
63272
|
+
if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
|
|
63273
|
+
return total;
|
|
63274
|
+
const sums = columns.map(() => ({ sum: 0, error: 0 }));
|
|
63275
|
+
const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
|
|
63276
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
63277
|
+
for (const usage of rows)
|
|
63278
|
+
for (const [index, column] of columns.entries())
|
|
63279
|
+
addCompensated(sums[index], Number(usage[column]));
|
|
63280
|
+
await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
|
|
63281
|
+
VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
|
|
63282
|
+
return total;
|
|
61903
63283
|
}
|
|
61904
63284
|
async report(lease, token, reports) {
|
|
61905
63285
|
validateText("runId", lease.runId);
|
|
@@ -62789,15 +64169,74 @@ function isWorkflowPendingStageDeliveryFailure(error) {
|
|
|
62789
64169
|
return true;
|
|
62790
64170
|
return error?.code === "pending_stage_delivery_failed";
|
|
62791
64171
|
}
|
|
62792
|
-
function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action()) {
|
|
64172
|
+
function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action(), options = {}) {
|
|
62793
64173
|
let resolveReady;
|
|
62794
64174
|
let rejectReady;
|
|
62795
64175
|
let readyPromise;
|
|
62796
64176
|
let drainError;
|
|
62797
64177
|
let terminalError;
|
|
62798
64178
|
let drainPromise;
|
|
64179
|
+
let drainedSessionId;
|
|
62799
64180
|
let currentRecipient;
|
|
62800
|
-
|
|
64181
|
+
let lastProgressAt = Date.now();
|
|
64182
|
+
let stallReported = false;
|
|
64183
|
+
const progress = () => {
|
|
64184
|
+
lastProgressAt = Date.now();
|
|
64185
|
+
};
|
|
64186
|
+
const progressingCommit = async (action) => {
|
|
64187
|
+
try {
|
|
64188
|
+
return await commit(action);
|
|
64189
|
+
} finally {
|
|
64190
|
+
progress();
|
|
64191
|
+
}
|
|
64192
|
+
};
|
|
64193
|
+
const reportStall = (sessionId) => {
|
|
64194
|
+
if (stallReported)
|
|
64195
|
+
return;
|
|
64196
|
+
stallReported = true;
|
|
64197
|
+
const undelivered = [
|
|
64198
|
+
...activeStore.pendingStageMessagesFor(runId, stageId),
|
|
64199
|
+
...stageId === stageName ? [] : activeStore.pendingStageMessagesFor(runId, stageName)
|
|
64200
|
+
].map((entry) => `${entry.message.id} (direct → ${entry.stageKey})`).concat(stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).map((entry) => `${entry.message.id} (sticky → ${entry.targetPath ?? entry.stageKey})`));
|
|
64201
|
+
const listed = undelivered.slice(0, 10).join(", ");
|
|
64202
|
+
const more = undelivered.length > 10 ? `, and ${undelivered.length - 10} more` : "";
|
|
64203
|
+
const message = `atomic-workflows: stage "${stageName}" (${stageId}) is starting without its queued Intercom messages: ` + `no delivery progress for ${PENDING_STAGE_READY_STALL_MS / 1000}s; ` + `drain ${drainPromise === undefined ? "not running" : "running but hung"}; ` + `undelivered: ${listed.length === 0 ? "none" : `${listed}${more}`}. ` + "They stay queued and arrive live or on the stage's next drain.";
|
|
64204
|
+
try {
|
|
64205
|
+
(options.reportStall ?? console.warn)(message);
|
|
64206
|
+
} catch {}
|
|
64207
|
+
};
|
|
64208
|
+
const watchStall = (pending, sessionId, onStall) => {
|
|
64209
|
+
let timer;
|
|
64210
|
+
let settled = false;
|
|
64211
|
+
const check = () => {
|
|
64212
|
+
if (settled)
|
|
64213
|
+
return;
|
|
64214
|
+
const remaining = lastProgressAt + PENDING_STAGE_READY_STALL_MS - Date.now();
|
|
64215
|
+
if (remaining > 0) {
|
|
64216
|
+
timer = setTimeout(check, remaining);
|
|
64217
|
+
timer.unref?.();
|
|
64218
|
+
return;
|
|
64219
|
+
}
|
|
64220
|
+
reportStall(sessionId);
|
|
64221
|
+
onStall();
|
|
64222
|
+
};
|
|
64223
|
+
const stop = () => {
|
|
64224
|
+
settled = true;
|
|
64225
|
+
clearTimeout(timer);
|
|
64226
|
+
};
|
|
64227
|
+
pending.then(stop, stop);
|
|
64228
|
+
check();
|
|
64229
|
+
};
|
|
64230
|
+
const boundedDrain = (drain, sessionId) => {
|
|
64231
|
+
const waiting = new Promise((resolve, reject) => {
|
|
64232
|
+
drain.then(resolve, reject);
|
|
64233
|
+
watchStall(drain, sessionId, resolve);
|
|
64234
|
+
});
|
|
64235
|
+
waiting.catch(() => {});
|
|
64236
|
+
return waiting;
|
|
64237
|
+
};
|
|
64238
|
+
const hasUndeliveredExact = () => activeStore.pendingStageMessagesFor(runId, stageId).length > 0 || stageId !== stageName && activeStore.pendingStageMessagesFor(runId, stageName).length > 0;
|
|
64239
|
+
const hasUndelivered = (sessionId) => hasUndeliveredExact() || stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).length > 0;
|
|
62801
64240
|
const pendingReady = () => {
|
|
62802
64241
|
if (readyPromise === undefined) {
|
|
62803
64242
|
readyPromise = new Promise((resolve, reject) => {
|
|
@@ -62807,35 +64246,64 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
62807
64246
|
}
|
|
62808
64247
|
return readyPromise;
|
|
62809
64248
|
};
|
|
64249
|
+
let followUp;
|
|
64250
|
+
const startDrain = (deliver, recipient) => {
|
|
64251
|
+
if (terminalError !== undefined)
|
|
64252
|
+
return Promise.resolve();
|
|
64253
|
+
drainError = undefined;
|
|
64254
|
+
progress();
|
|
64255
|
+
const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, async (from, message) => {
|
|
64256
|
+
progress();
|
|
64257
|
+
await deliver(from, message);
|
|
64258
|
+
progress();
|
|
64259
|
+
}, progressingCommit, recipient);
|
|
64260
|
+
const drain = attempt.then(() => {
|
|
64261
|
+
progress();
|
|
64262
|
+
drainedSessionId = recipient?.sessionId;
|
|
64263
|
+
resolveReady?.();
|
|
64264
|
+
readyPromise = undefined;
|
|
64265
|
+
resolveReady = undefined;
|
|
64266
|
+
rejectReady = undefined;
|
|
64267
|
+
if (drainPromise === drain)
|
|
64268
|
+
drainPromise = undefined;
|
|
64269
|
+
}, (error) => {
|
|
64270
|
+
progress();
|
|
64271
|
+
drainError = error;
|
|
64272
|
+
rejectReady?.(error);
|
|
64273
|
+
readyPromise = undefined;
|
|
64274
|
+
resolveReady = undefined;
|
|
64275
|
+
rejectReady = undefined;
|
|
64276
|
+
if (drainPromise === drain)
|
|
64277
|
+
drainPromise = undefined;
|
|
64278
|
+
throw error;
|
|
64279
|
+
});
|
|
64280
|
+
drainPromise = drain;
|
|
64281
|
+
return boundedDrain(drain, recipient?.sessionId);
|
|
64282
|
+
};
|
|
64283
|
+
const requestDrain = (deliver, recipient) => {
|
|
64284
|
+
const running = drainPromise;
|
|
64285
|
+
if (running === undefined)
|
|
64286
|
+
return startDrain(deliver, recipient);
|
|
64287
|
+
if (followUp !== undefined) {
|
|
64288
|
+
followUp.deliver = deliver;
|
|
64289
|
+
followUp.recipient = recipient;
|
|
64290
|
+
return boundedDrain(followUp.settled.promise, recipient?.sessionId);
|
|
64291
|
+
}
|
|
64292
|
+
const next = { deliver, recipient, settled: Promise.withResolvers() };
|
|
64293
|
+
next.settled.promise.catch(() => {});
|
|
64294
|
+
followUp = next;
|
|
64295
|
+
running.catch(() => {}).then(() => {
|
|
64296
|
+
if (followUp === next)
|
|
64297
|
+
followUp = undefined;
|
|
64298
|
+
return requestDrain(next.deliver, next.recipient);
|
|
64299
|
+
}).then(next.settled.resolve, next.settled.reject);
|
|
64300
|
+
return boundedDrain(next.settled.promise, recipient?.sessionId);
|
|
64301
|
+
};
|
|
62810
64302
|
return {
|
|
62811
64303
|
routeCapability: workflowPendingStageRouteCapability(activeStore, runId),
|
|
62812
64304
|
deliverPending(deliver, recipient) {
|
|
62813
64305
|
currentRecipient = recipient;
|
|
62814
|
-
|
|
62815
|
-
return drainPromise;
|
|
62816
|
-
if (terminalError !== undefined)
|
|
62817
|
-
return Promise.resolve();
|
|
62818
|
-
drainError = undefined;
|
|
62819
|
-
const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, deliver, commit, recipient);
|
|
62820
|
-
const drain = attempt.then(() => {
|
|
62821
|
-
resolveReady?.();
|
|
62822
|
-
readyPromise = undefined;
|
|
62823
|
-
resolveReady = undefined;
|
|
62824
|
-
rejectReady = undefined;
|
|
62825
|
-
if (drainPromise === drain)
|
|
62826
|
-
drainPromise = undefined;
|
|
62827
|
-
}, (error) => {
|
|
62828
|
-
drainError = error;
|
|
62829
|
-
rejectReady?.(error);
|
|
62830
|
-
readyPromise = undefined;
|
|
62831
|
-
resolveReady = undefined;
|
|
62832
|
-
rejectReady = undefined;
|
|
62833
|
-
if (drainPromise === drain)
|
|
62834
|
-
drainPromise = undefined;
|
|
62835
|
-
throw error;
|
|
62836
|
-
});
|
|
62837
|
-
drainPromise = drain;
|
|
62838
|
-
return drainPromise;
|
|
64306
|
+
return requestDrain(deliver, recipient);
|
|
62839
64307
|
},
|
|
62840
64308
|
ready(sessionId) {
|
|
62841
64309
|
if (!hasUndelivered(sessionId)) {
|
|
@@ -62845,6 +64313,8 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
62845
64313
|
return Promise.reject(terminalError);
|
|
62846
64314
|
if (drainError !== undefined)
|
|
62847
64315
|
return Promise.reject(drainError);
|
|
64316
|
+
if (drainPromise === undefined && sessionId !== undefined && drainedSessionId === sessionId && !hasUndeliveredExact())
|
|
64317
|
+
return;
|
|
62848
64318
|
const ready = pendingReady();
|
|
62849
64319
|
if (sessionId !== undefined && activeStore.pendingStageMessagesFor(runId, stageId).length === 0 && activeStore.pendingStageMessagesFor(runId, stageName).length === 0) {
|
|
62850
64320
|
const entries = stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, { sessionId });
|
|
@@ -62868,6 +64338,14 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
62868
64338
|
rejectReady?.(error);
|
|
62869
64339
|
});
|
|
62870
64340
|
}
|
|
64341
|
+
watchStall(ready, sessionId, () => {
|
|
64342
|
+
if (readyPromise !== ready)
|
|
64343
|
+
return;
|
|
64344
|
+
resolveReady?.();
|
|
64345
|
+
readyPromise = undefined;
|
|
64346
|
+
resolveReady = undefined;
|
|
64347
|
+
rejectReady = undefined;
|
|
64348
|
+
});
|
|
62871
64349
|
return ready;
|
|
62872
64350
|
},
|
|
62873
64351
|
fail(reason) {
|
|
@@ -62995,7 +64473,7 @@ function toPendingStageSender(entry) {
|
|
|
62995
64473
|
lastActivity: entry.from.lastActivity ?? entry.message.timestamp
|
|
62996
64474
|
};
|
|
62997
64475
|
}
|
|
62998
|
-
var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError;
|
|
64476
|
+
var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError, PENDING_STAGE_READY_STALL_MS = 60000;
|
|
62999
64477
|
var init_pending_stage_delivery2 = __esm(() => {
|
|
63000
64478
|
init_run_owner_backend();
|
|
63001
64479
|
init_store_backend();
|
|
@@ -65417,10 +66895,120 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
|
65417
66895
|
}
|
|
65418
66896
|
var init_stage_session_store = () => {};
|
|
65419
66897
|
|
|
66898
|
+
// dist/builtin/workflows/src/durable/completed-catalog-index.ts
|
|
66899
|
+
class CheckpointCatalogIndex {
|
|
66900
|
+
checkpoints;
|
|
66901
|
+
stages;
|
|
66902
|
+
positions = new Map;
|
|
66903
|
+
groupKeys = new Map;
|
|
66904
|
+
groups = new Map;
|
|
66905
|
+
stagesByRunId = new Map;
|
|
66906
|
+
boundariesByReplayKey = new Map;
|
|
66907
|
+
boundariesByStageId = new Map;
|
|
66908
|
+
boundariesByChildRunId = new Map;
|
|
66909
|
+
outputsByReplayKey = new Map;
|
|
66910
|
+
evidence;
|
|
66911
|
+
constructor(checkpoints) {
|
|
66912
|
+
this.checkpoints = checkpoints;
|
|
66913
|
+
const stages = [];
|
|
66914
|
+
checkpoints.forEach((checkpoint, index) => {
|
|
66915
|
+
if (!this.positions.has(checkpoint))
|
|
66916
|
+
this.positions.set(checkpoint, index);
|
|
66917
|
+
if (checkpoint.kind !== "stage")
|
|
66918
|
+
return;
|
|
66919
|
+
stages.push(checkpoint);
|
|
66920
|
+
const key = durableStageGroupKey(checkpoint);
|
|
66921
|
+
this.groupKeys.set(checkpoint, key);
|
|
66922
|
+
append(this.groups, key, checkpoint);
|
|
66923
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
66924
|
+
if (runId !== undefined)
|
|
66925
|
+
append(this.stagesByRunId, runId, checkpoint);
|
|
66926
|
+
if (checkpoint.output !== undefined)
|
|
66927
|
+
append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
|
|
66928
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66929
|
+
if (boundary === undefined)
|
|
66930
|
+
return;
|
|
66931
|
+
append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
|
|
66932
|
+
append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
|
|
66933
|
+
append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
|
|
66934
|
+
});
|
|
66935
|
+
this.stages = stages;
|
|
66936
|
+
}
|
|
66937
|
+
sequence(checkpoint) {
|
|
66938
|
+
return (this.positions.get(checkpoint) ?? -1) + 1;
|
|
66939
|
+
}
|
|
66940
|
+
stageGroups() {
|
|
66941
|
+
return this.groups.values();
|
|
66942
|
+
}
|
|
66943
|
+
boundaryLifecycle(start, requireTerminal) {
|
|
66944
|
+
const topology = start.topology;
|
|
66945
|
+
const childRunId = topology?.boundary?.child.runId;
|
|
66946
|
+
const related = this.inCheckpointOrder([
|
|
66947
|
+
this.boundariesByReplayKey.get(start.replayKey),
|
|
66948
|
+
topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
|
|
66949
|
+
childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
|
|
66950
|
+
]);
|
|
66951
|
+
return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
|
|
66952
|
+
}
|
|
66953
|
+
directChildRecords(childRunId) {
|
|
66954
|
+
return {
|
|
66955
|
+
claiming: this.stagesByRunId.get(childRunId) ?? [],
|
|
66956
|
+
direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
|
|
66957
|
+
};
|
|
66958
|
+
}
|
|
66959
|
+
childScopedEvidenceExists(boundaryReplayKey, childRunId) {
|
|
66960
|
+
this.evidence ??= scopedEvidenceIndex(this.checkpoints);
|
|
66961
|
+
return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
|
|
66962
|
+
}
|
|
66963
|
+
inCheckpointOrder(lists) {
|
|
66964
|
+
const present = lists.filter((list) => list !== undefined);
|
|
66965
|
+
if (present.length === 1)
|
|
66966
|
+
return present[0];
|
|
66967
|
+
return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
|
|
66968
|
+
}
|
|
66969
|
+
}
|
|
66970
|
+
function scopedEvidenceIndex(checkpoints) {
|
|
66971
|
+
const runIds = new Set;
|
|
66972
|
+
const identities = [];
|
|
66973
|
+
for (const checkpoint of checkpoints) {
|
|
66974
|
+
identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
|
|
66975
|
+
if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
|
|
66976
|
+
runIds.add(checkpoint.topology.run.runId);
|
|
66977
|
+
}
|
|
66978
|
+
}
|
|
66979
|
+
identities.sort(compareCodeUnits);
|
|
66980
|
+
return { runIds, identities };
|
|
66981
|
+
}
|
|
66982
|
+
function hasPrefix(sorted, prefix) {
|
|
66983
|
+
let low = 0;
|
|
66984
|
+
let high = sorted.length;
|
|
66985
|
+
while (low < high) {
|
|
66986
|
+
const middle = low + high >>> 1;
|
|
66987
|
+
if (compareCodeUnits(sorted[middle], prefix) < 0)
|
|
66988
|
+
low = middle + 1;
|
|
66989
|
+
else
|
|
66990
|
+
high = middle;
|
|
66991
|
+
}
|
|
66992
|
+
return low < sorted.length && sorted[low].startsWith(prefix);
|
|
66993
|
+
}
|
|
66994
|
+
function compareCodeUnits(left, right) {
|
|
66995
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
66996
|
+
}
|
|
66997
|
+
function append(map, key, value2) {
|
|
66998
|
+
const list = map.get(key);
|
|
66999
|
+
if (list === undefined)
|
|
67000
|
+
map.set(key, [value2]);
|
|
67001
|
+
else
|
|
67002
|
+
list.push(value2);
|
|
67003
|
+
}
|
|
67004
|
+
var init_completed_catalog_index = __esm(() => {
|
|
67005
|
+
init_boundary_lifecycle();
|
|
67006
|
+
init_stage_topology_validation();
|
|
67007
|
+
});
|
|
67008
|
+
|
|
65420
67009
|
// dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
|
|
65421
|
-
function validBoundaryRecordSet(
|
|
65422
|
-
const
|
|
65423
|
-
const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
|
|
67010
|
+
function validBoundaryRecordSet(index, strict) {
|
|
67011
|
+
const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
|
|
65424
67012
|
const startsByReplay = new Map;
|
|
65425
67013
|
for (const checkpoint of boundaries) {
|
|
65426
67014
|
if (checkpoint.topology.boundary.event !== "start")
|
|
@@ -65435,7 +67023,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
|
|
|
65435
67023
|
return false;
|
|
65436
67024
|
return [...startsByReplay.values()].every((starts) => {
|
|
65437
67025
|
const start = starts[0];
|
|
65438
|
-
if (
|
|
67026
|
+
if ("error" in index.boundaryLifecycle(start, strict))
|
|
65439
67027
|
return false;
|
|
65440
67028
|
return start.topology.boundary.invocationFingerprint !== undefined;
|
|
65441
67029
|
});
|
|
@@ -65499,8 +67087,8 @@ function compareDraftSourceOrder(left, right) {
|
|
|
65499
67087
|
function runTopologyFor(drafts, tools) {
|
|
65500
67088
|
return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
|
|
65501
67089
|
}
|
|
65502
|
-
function validateRunGroups(grouped, rootRunId,
|
|
65503
|
-
const toolsFor = (runId) =>
|
|
67090
|
+
function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
|
|
67091
|
+
const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
|
|
65504
67092
|
const owners = new Map;
|
|
65505
67093
|
const syntheticChildIds = new Set;
|
|
65506
67094
|
for (const [parentRunId, drafts] of grouped) {
|
|
@@ -65511,12 +67099,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
|
|
|
65511
67099
|
if (childRunId === parentRunId || owners.has(childRunId))
|
|
65512
67100
|
return false;
|
|
65513
67101
|
if (!grouped.has(childRunId)) {
|
|
65514
|
-
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(
|
|
67102
|
+
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
|
|
65515
67103
|
syntheticChildIds.add(childRunId);
|
|
65516
67104
|
owners.set(childRunId, draft);
|
|
65517
67105
|
continue;
|
|
65518
67106
|
}
|
|
65519
|
-
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(
|
|
67107
|
+
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
|
|
65520
67108
|
return false;
|
|
65521
67109
|
syntheticChildIds.add(childRunId);
|
|
65522
67110
|
owners.set(childRunId, draft);
|
|
@@ -65557,13 +67145,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
|
|
|
65557
67145
|
const boundary = draft.topology?.boundary;
|
|
65558
67146
|
return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
|
|
65559
67147
|
}
|
|
65560
|
-
function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
65561
|
-
const prefix = `${boundaryReplayKey}:`;
|
|
65562
|
-
return checkpoints.some((checkpoint) => {
|
|
65563
|
-
const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
|
|
65564
|
-
return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
|
|
65565
|
-
});
|
|
65566
|
-
}
|
|
65567
67148
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
65568
67149
|
const reachable = new Set([rootRunId]);
|
|
65569
67150
|
const settled = new Set;
|
|
@@ -65604,14 +67185,6 @@ function childRunIdFromDraft(draft) {
|
|
|
65604
67185
|
return;
|
|
65605
67186
|
return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
|
|
65606
67187
|
}
|
|
65607
|
-
function boundaryOwner(grouped, childRunId) {
|
|
65608
|
-
for (const drafts of grouped.values()) {
|
|
65609
|
-
const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
|
|
65610
|
-
if (owner !== undefined)
|
|
65611
|
-
return owner;
|
|
65612
|
-
}
|
|
65613
|
-
return;
|
|
65614
|
-
}
|
|
65615
67188
|
function childRunStatus(owner) {
|
|
65616
67189
|
const child = workflowChildFromDraft(owner);
|
|
65617
67190
|
if (child !== undefined)
|
|
@@ -65619,15 +67192,15 @@ function childRunStatus(owner) {
|
|
|
65619
67192
|
const status = owner.topology?.boundary?.status;
|
|
65620
67193
|
return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
|
|
65621
67194
|
}
|
|
65622
|
-
function mergeStageGroup(records,
|
|
67195
|
+
function mergeStageGroup(records, index) {
|
|
65623
67196
|
let merged;
|
|
65624
67197
|
for (const record of records)
|
|
65625
|
-
merged = mergeStageDraft(merged, record,
|
|
67198
|
+
merged = mergeStageDraft(merged, record, index.sequence(record));
|
|
65626
67199
|
const draft = merged;
|
|
65627
67200
|
const start = records.find((record) => record.topology?.boundary?.event === "start");
|
|
65628
67201
|
if (start === undefined)
|
|
65629
67202
|
return draft;
|
|
65630
|
-
const resolved =
|
|
67203
|
+
const resolved = index.boundaryLifecycle(start, false);
|
|
65631
67204
|
if ("error" in resolved)
|
|
65632
67205
|
return draft;
|
|
65633
67206
|
const terminal = resolved.lifecycle.latestTerminal;
|
|
@@ -65690,9 +67263,7 @@ function workflowChildFromDraft(draft) {
|
|
|
65690
67263
|
...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
|
|
65691
67264
|
};
|
|
65692
67265
|
}
|
|
65693
|
-
var init_completed_catalog_stage_groups =
|
|
65694
|
-
init_boundary_lifecycle();
|
|
65695
|
-
});
|
|
67266
|
+
var init_completed_catalog_stage_groups = () => {};
|
|
65696
67267
|
|
|
65697
67268
|
// dist/builtin/workflows/src/engine/run-tool-control-registry.ts
|
|
65698
67269
|
function createToolControlRegistry() {
|
|
@@ -65897,11 +67468,68 @@ var init_resume_runtime = __esm(() => {
|
|
|
65897
67468
|
function isReservedRunCheckpointHash(argsHash) {
|
|
65898
67469
|
return [RUN_TIMING_CHECKPOINT_NAME, RUN_USAGE_CHECKPOINT_NAME, RUN_RESULT_CHECKPOINT_NAME].includes(argsHash) || argsHash === WORKFLOW_HEARTBEAT_ANCHOR_CHECKPOINT_NAME;
|
|
65899
67470
|
}
|
|
65900
|
-
function
|
|
67471
|
+
function durableNestedRunCatalog(backend, rootWorkflowId) {
|
|
65901
67472
|
const handle = backend.getWorkflow(rootWorkflowId);
|
|
65902
67473
|
if (handle === undefined)
|
|
65903
|
-
return
|
|
65904
|
-
|
|
67474
|
+
return EMPTY_NESTED_RUN_CATALOG;
|
|
67475
|
+
const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
|
|
67476
|
+
const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
|
|
67477
|
+
if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
|
|
67478
|
+
return cached.catalog;
|
|
67479
|
+
}
|
|
67480
|
+
const checkpoints = backend.listCheckpoints(rootWorkflowId);
|
|
67481
|
+
const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
|
|
67482
|
+
const catalog = nestedRunCatalog(runs, checkpoints);
|
|
67483
|
+
if (version !== undefined) {
|
|
67484
|
+
const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
|
|
67485
|
+
nestedRunCatalogs.set(backend, byRoot);
|
|
67486
|
+
byRoot.delete(rootWorkflowId);
|
|
67487
|
+
byRoot.set(rootWorkflowId, {
|
|
67488
|
+
version,
|
|
67489
|
+
name: handle.name,
|
|
67490
|
+
updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
|
|
67491
|
+
catalog
|
|
67492
|
+
});
|
|
67493
|
+
if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
|
|
67494
|
+
byRoot.delete(byRoot.keys().next().value);
|
|
67495
|
+
}
|
|
67496
|
+
return catalog;
|
|
67497
|
+
}
|
|
67498
|
+
function nestedRunCatalog(runs, checkpoints) {
|
|
67499
|
+
const positionsById = new Map;
|
|
67500
|
+
const childIdsByParent = new Map;
|
|
67501
|
+
runs.forEach((run, position) => {
|
|
67502
|
+
const positions = positionsById.get(run.id);
|
|
67503
|
+
if (positions === undefined)
|
|
67504
|
+
positionsById.set(run.id, [position]);
|
|
67505
|
+
else
|
|
67506
|
+
positions.push(position);
|
|
67507
|
+
if (run.parentRunId === undefined)
|
|
67508
|
+
return;
|
|
67509
|
+
const children = childIdsByParent.get(run.parentRunId);
|
|
67510
|
+
if (children === undefined)
|
|
67511
|
+
childIdsByParent.set(run.parentRunId, [run.id]);
|
|
67512
|
+
else
|
|
67513
|
+
children.push(run.id);
|
|
67514
|
+
});
|
|
67515
|
+
const intentionalExitStatuses = new Map;
|
|
67516
|
+
for (const checkpoint of checkpoints) {
|
|
67517
|
+
if (checkpoint.kind !== "stage")
|
|
67518
|
+
continue;
|
|
67519
|
+
const child = parseWorkflowChildResult(checkpoint.output);
|
|
67520
|
+
if (child?.exited === true)
|
|
67521
|
+
intentionalExitStatuses.set(child.runId, child.status);
|
|
67522
|
+
}
|
|
67523
|
+
return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
|
|
67524
|
+
}
|
|
67525
|
+
function readsFallbackCompletedAt(run) {
|
|
67526
|
+
return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
|
|
67527
|
+
}
|
|
67528
|
+
function cloneRunSnapshot(run) {
|
|
67529
|
+
const copy = structuredClone(run);
|
|
67530
|
+
for (const tool of copy.toolNodes)
|
|
67531
|
+
Object.freeze(tool.parentIds);
|
|
67532
|
+
return copy;
|
|
65905
67533
|
}
|
|
65906
67534
|
function checkpointDrafts(checkpoints) {
|
|
65907
67535
|
const stageByReplayKey = new Map;
|
|
@@ -65925,11 +67553,8 @@ function checkpointDrafts(checkpoints) {
|
|
|
65925
67553
|
tools: [...toolByHash.values()]
|
|
65926
67554
|
};
|
|
65927
67555
|
}
|
|
65928
|
-
function completedToolNodes(
|
|
65929
|
-
return
|
|
65930
|
-
const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
65931
|
-
if (topologyRunId !== runId)
|
|
65932
|
-
return [];
|
|
67556
|
+
function completedToolNodes(ownedCheckpoints, sourceIds) {
|
|
67557
|
+
return ownedCheckpoints.flatMap((checkpoint) => {
|
|
65933
67558
|
const topology = checkpoint.topology;
|
|
65934
67559
|
if (topology === undefined)
|
|
65935
67560
|
return [];
|
|
@@ -65966,18 +67591,34 @@ function summarizeCompletedToolResult(value2) {
|
|
|
65966
67591
|
return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
|
|
65967
67592
|
}
|
|
65968
67593
|
function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
|
|
65969
|
-
|
|
67594
|
+
const index = new CheckpointCatalogIndex(checkpoints);
|
|
67595
|
+
if (!validBoundaryRecordSet(index, policy.requireTerminal))
|
|
65970
67596
|
return [];
|
|
65971
|
-
const stageRecords =
|
|
67597
|
+
const stageRecords = index.stages;
|
|
65972
67598
|
const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
|
|
65973
67599
|
if (immutableStageGroupError(identityRecords) !== undefined)
|
|
65974
67600
|
return [];
|
|
65975
|
-
const
|
|
65976
|
-
const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
67601
|
+
const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
65977
67602
|
const drafts = checkpointDrafts(checkpoints);
|
|
65978
67603
|
const toolCheckpoints = drafts.tools;
|
|
65979
67604
|
if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
|
|
65980
67605
|
return [];
|
|
67606
|
+
const toolNodeIdsByClaimedRun = new Map;
|
|
67607
|
+
const toolsByRun = new Map;
|
|
67608
|
+
for (const checkpoint of toolCheckpoints) {
|
|
67609
|
+
const claimedRunId = checkpoint.topology?.run?.runId;
|
|
67610
|
+
if (claimedRunId !== undefined) {
|
|
67611
|
+
const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
|
|
67612
|
+
ids.add(checkpoint.topology.nodeId);
|
|
67613
|
+
toolNodeIdsByClaimedRun.set(claimedRunId, ids);
|
|
67614
|
+
}
|
|
67615
|
+
const ownerRunId = claimedRunId ?? rootRunId;
|
|
67616
|
+
const owned = toolsByRun.get(ownerRunId);
|
|
67617
|
+
if (owned === undefined)
|
|
67618
|
+
toolsByRun.set(ownerRunId, [checkpoint]);
|
|
67619
|
+
else
|
|
67620
|
+
owned.push(checkpoint);
|
|
67621
|
+
}
|
|
65981
67622
|
for (const draft of candidates) {
|
|
65982
67623
|
const boundary = draft.topology?.boundary;
|
|
65983
67624
|
if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
|
|
@@ -65991,8 +67632,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
65991
67632
|
parentStageId: boundary.child.parentStageId,
|
|
65992
67633
|
rootRunId: boundary.child.rootRunId
|
|
65993
67634
|
};
|
|
65994
|
-
const childToolIds =
|
|
65995
|
-
if (
|
|
67635
|
+
const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
|
|
67636
|
+
if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
|
|
65996
67637
|
return [];
|
|
65997
67638
|
}
|
|
65998
67639
|
if (candidates.length === 0 && toolCheckpoints.length === 0) {
|
|
@@ -66020,7 +67661,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66020
67661
|
group.sort(compareDraftSourceOrder);
|
|
66021
67662
|
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
66022
67663
|
return [];
|
|
66023
|
-
if (!validateRunGroups(grouped, rootRunId,
|
|
67664
|
+
if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
|
|
66024
67665
|
return [];
|
|
66025
67666
|
const syntheticChildren = [];
|
|
66026
67667
|
const pendingChildren = [];
|
|
@@ -66034,7 +67675,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66034
67675
|
const child = workflowChildFromDraft(draft);
|
|
66035
67676
|
if (child?.exited !== true || grouped.has(child.runId))
|
|
66036
67677
|
continue;
|
|
66037
|
-
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(
|
|
67678
|
+
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
|
|
66038
67679
|
return [];
|
|
66039
67680
|
}
|
|
66040
67681
|
syntheticChildren.push({ parentRunId, owner: draft, child });
|
|
@@ -66042,7 +67683,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66042
67683
|
}
|
|
66043
67684
|
const idMaps = new Map;
|
|
66044
67685
|
for (const [runId, runDrafts] of grouped) {
|
|
66045
|
-
const ownedTools =
|
|
67686
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
66046
67687
|
if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
|
|
66047
67688
|
return [];
|
|
66048
67689
|
const ids = new Map;
|
|
@@ -66060,32 +67701,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
66060
67701
|
});
|
|
66061
67702
|
if (hasIdentityConflict)
|
|
66062
67703
|
return [];
|
|
66063
|
-
for (const checkpoint of
|
|
66064
|
-
const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
67704
|
+
for (const checkpoint of ownedTools) {
|
|
66065
67705
|
const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
|
|
66066
|
-
|
|
66067
|
-
ids.set(sourceId, sourceId);
|
|
67706
|
+
ids.set(sourceId, sourceId);
|
|
66068
67707
|
}
|
|
66069
67708
|
idMaps.set(runId, ids);
|
|
66070
67709
|
}
|
|
67710
|
+
const boundaryOwners = new Map;
|
|
67711
|
+
const boundarySourcesByParent = new Map;
|
|
67712
|
+
for (const [parentRunId, runDrafts] of grouped) {
|
|
67713
|
+
const sources = new Map;
|
|
67714
|
+
for (const draft of runDrafts) {
|
|
67715
|
+
const childRunId = childRunIdFromDraft(draft);
|
|
67716
|
+
if (childRunId === undefined)
|
|
67717
|
+
continue;
|
|
67718
|
+
if (!boundaryOwners.has(childRunId))
|
|
67719
|
+
boundaryOwners.set(childRunId, draft);
|
|
67720
|
+
if (!sources.has(childRunId))
|
|
67721
|
+
sources.set(childRunId, draft);
|
|
67722
|
+
}
|
|
67723
|
+
boundarySourcesByParent.set(parentRunId, sources);
|
|
67724
|
+
}
|
|
66071
67725
|
const runs = [];
|
|
66072
67726
|
const emittedRunIds = new Set;
|
|
66073
67727
|
for (const [runId, runDrafts] of grouped) {
|
|
66074
67728
|
const ids = idMaps.get(runId);
|
|
66075
|
-
const ownedTools =
|
|
67729
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
66076
67730
|
const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
|
|
66077
67731
|
if (hasUnknownParents)
|
|
66078
67732
|
return [];
|
|
66079
67733
|
const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
|
|
66080
|
-
const toolNodes = completedToolNodes(
|
|
67734
|
+
const toolNodes = completedToolNodes(ownedTools, ids);
|
|
66081
67735
|
const run = runTopologyFor(runDrafts, ownedTools);
|
|
66082
67736
|
const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
|
|
66083
67737
|
const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
|
|
66084
|
-
const owner = runId === rootRunId ? undefined :
|
|
67738
|
+
const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
|
|
66085
67739
|
const status = owner === undefined ? "completed" : childRunStatus(owner);
|
|
66086
67740
|
const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
|
|
66087
67741
|
const parentRunId = run?.parentRunId ?? rootRunId;
|
|
66088
|
-
const boundarySourceId =
|
|
67742
|
+
const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
|
|
66089
67743
|
const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
|
|
66090
67744
|
const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
|
|
66091
67745
|
const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
|
|
@@ -66221,10 +67875,11 @@ function emptyGraphRun(runId, runName, completedAt) {
|
|
|
66221
67875
|
resumable: false
|
|
66222
67876
|
};
|
|
66223
67877
|
}
|
|
66224
|
-
var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
67878
|
+
var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
66225
67879
|
var init_completed_catalog = __esm(() => {
|
|
66226
67880
|
init_stage_session_store();
|
|
66227
67881
|
init_tool_payload_bounds();
|
|
67882
|
+
init_completed_catalog_index();
|
|
66228
67883
|
init_completed_catalog_stage_groups();
|
|
66229
67884
|
init_resume_eligibility();
|
|
66230
67885
|
init_resume_runtime();
|
|
@@ -66233,6 +67888,13 @@ var init_completed_catalog = __esm(() => {
|
|
|
66233
67888
|
init_workflow_heartbeat_anchor();
|
|
66234
67889
|
init_stage_topology_validation();
|
|
66235
67890
|
init_tool_outcome();
|
|
67891
|
+
nestedRunCatalogs = new WeakMap;
|
|
67892
|
+
EMPTY_NESTED_RUN_CATALOG = {
|
|
67893
|
+
runs: [],
|
|
67894
|
+
positionsById: new Map,
|
|
67895
|
+
childIdsByParent: new Map,
|
|
67896
|
+
intentionalExitStatuses: new Map
|
|
67897
|
+
};
|
|
66236
67898
|
COMPLETED_RECONSTRUCTION = {
|
|
66237
67899
|
requireTerminal: true,
|
|
66238
67900
|
retainActive: false,
|
|
@@ -66247,29 +67909,23 @@ var init_completed_catalog = __esm(() => {
|
|
|
66247
67909
|
|
|
66248
67910
|
// dist/builtin/workflows/src/durable/completed-subtree.ts
|
|
66249
67911
|
function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
|
|
66250
|
-
const
|
|
66251
|
-
if (!
|
|
67912
|
+
const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
|
|
67913
|
+
if (!catalog.positionsById.has(childRunId))
|
|
66252
67914
|
return;
|
|
66253
|
-
const intentionalExitStatuses = new Map;
|
|
66254
|
-
for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
|
|
66255
|
-
if (checkpoint.kind !== "stage")
|
|
66256
|
-
continue;
|
|
66257
|
-
const child = parseWorkflowChildResult(checkpoint.output);
|
|
66258
|
-
if (child?.exited === true)
|
|
66259
|
-
intentionalExitStatuses.set(child.runId, child.status);
|
|
66260
|
-
}
|
|
66261
67915
|
const included = new Set([childRunId]);
|
|
66262
|
-
|
|
66263
|
-
|
|
66264
|
-
|
|
66265
|
-
|
|
66266
|
-
|
|
66267
|
-
|
|
67916
|
+
const pending = [childRunId];
|
|
67917
|
+
for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
|
|
67918
|
+
for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
|
|
67919
|
+
if (included.has(childId))
|
|
67920
|
+
continue;
|
|
67921
|
+
included.add(childId);
|
|
67922
|
+
pending.push(childId);
|
|
66268
67923
|
}
|
|
66269
67924
|
}
|
|
66270
|
-
const
|
|
66271
|
-
const
|
|
66272
|
-
|
|
67925
|
+
const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
|
|
67926
|
+
const subtree = positions.map((position) => catalog.runs[position]);
|
|
67927
|
+
const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
|
|
67928
|
+
return complete ? subtree.map(cloneRunSnapshot) : undefined;
|
|
66273
67929
|
}
|
|
66274
67930
|
function isCompleteCachedRun(run, intentionalExitStatuses) {
|
|
66275
67931
|
if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
|
|
@@ -66390,7 +68046,7 @@ function createDurableCachedStageRecorder(input) {
|
|
|
66390
68046
|
const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
|
|
66391
68047
|
kind: "stage",
|
|
66392
68048
|
workflowId: input.run.id,
|
|
66393
|
-
checkpointId:
|
|
68049
|
+
checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
|
|
66394
68050
|
name,
|
|
66395
68051
|
replayKey,
|
|
66396
68052
|
completedAt: Date.now(),
|