@bastani/atomic 0.9.44-alpha.4 → 0.9.44-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/builtin/intercom/CHANGELOG.md +12 -0
- package/dist/builtin/intercom/broker/broker.ts +13 -1
- package/dist/builtin/intercom/index.bundle.mjs +9 -1
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/types.ts +7 -0
- package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +26 -0
- package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
- package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
- package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +2457 -736
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2329 -730
- package/dist/builtin/workflows/src/index.js +2328 -730
- package/dist/builtin/workflows/src/local-worker-entry.mjs +2329 -730
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +16 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/storage/errors.d.ts +5 -1
- package/dist/core/storage/errors.d.ts.map +1 -1
- package/dist/core/storage/errors.js +8 -2
- package/dist/core/storage/errors.js.map +1 -1
- package/dist/core/storage/node-sqlite.d.ts +2 -1
- package/dist/core/storage/node-sqlite.d.ts.map +1 -1
- package/dist/core/storage/node-sqlite.js +213 -20
- package/dist/core/storage/node-sqlite.js.map +1 -1
- package/docs/intercom/reference.md +2 -0
- package/docs/workflows/operations.md +1 -1
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -983,13 +983,19 @@ function registerRunJournalVersions(journal, versions) {
|
|
|
983
983
|
function runJournalVersions(journal) {
|
|
984
984
|
return versionReaders.get(journal);
|
|
985
985
|
}
|
|
986
|
+
function registerRunJournalAppendMany(journal, appendMany) {
|
|
987
|
+
appendManyWriters.set(journal, appendMany);
|
|
988
|
+
}
|
|
989
|
+
function runJournalAppendMany(journal) {
|
|
990
|
+
return appendManyWriters.get(journal);
|
|
991
|
+
}
|
|
986
992
|
function registerSqlPoolStorage(journal, storage) {
|
|
987
993
|
poolStorage.set(journal, storage);
|
|
988
994
|
}
|
|
989
995
|
function sqlPoolStorage(journal) {
|
|
990
996
|
return poolStorage.get(journal);
|
|
991
997
|
}
|
|
992
|
-
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
|
|
998
|
+
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
|
|
993
999
|
var init_internal = __esm(() => {
|
|
994
1000
|
init_types();
|
|
995
1001
|
NON_SNAPSHOT_EVENT_TYPES = [
|
|
@@ -998,6 +1004,7 @@ var init_internal = __esm(() => {
|
|
|
998
1004
|
RUN_EVENT_TYPES.messageDelivered
|
|
999
1005
|
];
|
|
1000
1006
|
versionReaders = new WeakMap;
|
|
1007
|
+
appendManyWriters = new WeakMap;
|
|
1001
1008
|
poolStorage = new WeakMap;
|
|
1002
1009
|
});
|
|
1003
1010
|
|
|
@@ -1381,6 +1388,7 @@ class SqlRunJournal {
|
|
|
1381
1388
|
runVersions: () => this.runVersions(),
|
|
1382
1389
|
snapshotTail: (runId) => this.snapshotTail(runId)
|
|
1383
1390
|
});
|
|
1391
|
+
registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
|
|
1384
1392
|
registerSqlPoolStorage(this, {
|
|
1385
1393
|
transaction: (body) => this.operation(() => this.database.transaction(body)),
|
|
1386
1394
|
withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
|
|
@@ -1645,38 +1653,76 @@ class SqlRunJournal {
|
|
|
1645
1653
|
this.assertOpen();
|
|
1646
1654
|
validateText("runId", lease.runId);
|
|
1647
1655
|
validateText("holderId", lease.holderId);
|
|
1648
|
-
if (options?.key !== undefined)
|
|
1649
|
-
validateText("key", options.key);
|
|
1650
1656
|
const owned = { ...lease };
|
|
1651
|
-
const
|
|
1657
|
+
const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
|
|
1652
1658
|
const expected = options?.expectSeq;
|
|
1659
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
1660
|
+
await this.checkLease(tx, owned);
|
|
1661
|
+
return this.insert(tx, owned, prepared, expected);
|
|
1662
|
+
}));
|
|
1663
|
+
}
|
|
1664
|
+
async appendMany(lease, appends) {
|
|
1665
|
+
this.assertOpen();
|
|
1666
|
+
validateText("runId", lease.runId);
|
|
1667
|
+
validateText("holderId", lease.holderId);
|
|
1668
|
+
const owned = { ...lease };
|
|
1669
|
+
const prepared = appends.map((append) => {
|
|
1670
|
+
try {
|
|
1671
|
+
return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
|
|
1672
|
+
} catch (error) {
|
|
1673
|
+
return error;
|
|
1674
|
+
}
|
|
1675
|
+
});
|
|
1676
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
1677
|
+
await this.checkLease(tx, owned);
|
|
1678
|
+
const results = [];
|
|
1679
|
+
for (const append of prepared) {
|
|
1680
|
+
if (append instanceof Error) {
|
|
1681
|
+
results.push({ status: "rejected", reason: append });
|
|
1682
|
+
continue;
|
|
1683
|
+
}
|
|
1684
|
+
await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
|
|
1685
|
+
try {
|
|
1686
|
+
results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
|
|
1687
|
+
} catch (error) {
|
|
1688
|
+
await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
|
|
1689
|
+
results.push({ status: "rejected", reason: error });
|
|
1690
|
+
}
|
|
1691
|
+
await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
|
|
1692
|
+
}
|
|
1693
|
+
return results;
|
|
1694
|
+
}));
|
|
1695
|
+
}
|
|
1696
|
+
prepare(event, key, appendGuard) {
|
|
1697
|
+
if (key !== undefined)
|
|
1698
|
+
validateText("key", key);
|
|
1653
1699
|
const type = event.type;
|
|
1654
1700
|
const payload = this.snapshotJson(event.payload);
|
|
1655
|
-
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ?
|
|
1701
|
+
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
|
|
1656
1702
|
const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
|
|
1657
1703
|
if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
|
|
1658
1704
|
validateText("status", status);
|
|
1659
|
-
return
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
}
|
|
1705
|
+
return { key, type, payload, guard, status };
|
|
1706
|
+
}
|
|
1707
|
+
async insert(tx, owned, { key, type, payload, guard, status }, expected) {
|
|
1708
|
+
await guard?.(tx, owned.runId);
|
|
1709
|
+
if (key !== undefined) {
|
|
1710
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
|
|
1711
|
+
if (row)
|
|
1712
|
+
return { status: "exists", entry: entryFromRow(row) };
|
|
1713
|
+
}
|
|
1714
|
+
const actual = await this.sequence(tx, owned.runId);
|
|
1715
|
+
if (expected !== undefined && expected !== actual)
|
|
1716
|
+
throw new JournalCasConflictError(owned.runId, expected, actual);
|
|
1717
|
+
const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
|
|
1718
|
+
if (payload instanceof StorageUnavailableError2)
|
|
1719
|
+
throw payload;
|
|
1720
|
+
await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
|
|
1721
|
+
if (status !== undefined)
|
|
1722
|
+
await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
|
|
1723
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
|
|
1724
|
+
await guard?.(tx, owned.runId);
|
|
1725
|
+
return { status: "appended", entry: entryFromRow(row) };
|
|
1680
1726
|
}
|
|
1681
1727
|
async* read(runId, fromSeq = 1) {
|
|
1682
1728
|
this.assertOpen();
|
|
@@ -1744,7 +1790,7 @@ class SqlRunJournal {
|
|
|
1744
1790
|
return this.closePromise;
|
|
1745
1791
|
}
|
|
1746
1792
|
}
|
|
1747
|
-
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
|
|
1793
|
+
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
|
|
1748
1794
|
var init_sql_run_journal = __esm(() => {
|
|
1749
1795
|
init_internal();
|
|
1750
1796
|
init_legacy_journal_tables();
|
|
@@ -2323,6 +2369,12 @@ class ScopedDurableBackend {
|
|
|
2323
2369
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
2324
2370
|
return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
|
|
2325
2371
|
}
|
|
2372
|
+
pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
|
|
2373
|
+
return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
|
|
2374
|
+
}
|
|
2375
|
+
pendingStageDeliveryCountVersion(_workflowId) {
|
|
2376
|
+
return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
|
|
2377
|
+
}
|
|
2326
2378
|
async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
2327
2379
|
return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
|
|
2328
2380
|
}
|
|
@@ -3617,8 +3669,11 @@ var init_pending_stage_delivery = __esm(() => {
|
|
|
3617
3669
|
});
|
|
3618
3670
|
|
|
3619
3671
|
// dist/builtin/workflows/src/shared/pending-stage-retention.ts
|
|
3672
|
+
function isSettledPendingStageMessage(entry) {
|
|
3673
|
+
return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
|
|
3674
|
+
}
|
|
3620
3675
|
async function retainPendingStageMessages(messages, archive) {
|
|
3621
|
-
const settled = messages.filter(
|
|
3676
|
+
const settled = messages.filter(isSettledPendingStageMessage);
|
|
3622
3677
|
const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
|
|
3623
3678
|
for (const entry of expired)
|
|
3624
3679
|
await archive(entry);
|
|
@@ -3626,53 +3681,210 @@ async function retainPendingStageMessages(messages, archive) {
|
|
|
3626
3681
|
}
|
|
3627
3682
|
var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
|
|
3628
3683
|
|
|
3684
|
+
// dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
|
|
3685
|
+
class PendingStageTransitionLedger {
|
|
3686
|
+
compacted = [];
|
|
3687
|
+
settled = 0;
|
|
3688
|
+
stages = new Map;
|
|
3689
|
+
fixed = new WeakSet;
|
|
3690
|
+
reconciled = [];
|
|
3691
|
+
reconciledVersion;
|
|
3692
|
+
reconcile(backend, runId, messages) {
|
|
3693
|
+
if (backend.getPendingStageMessageDeliveryCount === undefined)
|
|
3694
|
+
return messages;
|
|
3695
|
+
const version = backend.pendingStageDeliveryCountVersion?.(runId);
|
|
3696
|
+
const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
|
|
3697
|
+
let recovered;
|
|
3698
|
+
for (let index = 0;index < messages.length; index++) {
|
|
3699
|
+
const entry = messages[index];
|
|
3700
|
+
if (entry.sticky !== true || base !== undefined && base[index] === entry)
|
|
3701
|
+
continue;
|
|
3702
|
+
const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
|
|
3703
|
+
if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
|
|
3704
|
+
continue;
|
|
3705
|
+
recovered ??= [...messages];
|
|
3706
|
+
recovered[index] = { ...entry, deliveryCount: count };
|
|
3707
|
+
}
|
|
3708
|
+
const result = recovered ?? messages;
|
|
3709
|
+
this.reconciled = result;
|
|
3710
|
+
this.reconciledVersion = version;
|
|
3711
|
+
return result;
|
|
3712
|
+
}
|
|
3713
|
+
async compact(backend, runId, messages, runs, archiveSettled) {
|
|
3714
|
+
const stageStatus = createStageStatusLookup(runs);
|
|
3715
|
+
const endedStages = [];
|
|
3716
|
+
const dirty = new Set;
|
|
3717
|
+
for (const [deliveryRunId, byStage] of this.stages) {
|
|
3718
|
+
for (const [stageId, entries] of byStage) {
|
|
3719
|
+
if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
|
|
3720
|
+
continue;
|
|
3721
|
+
endedStages.push([deliveryRunId, stageId]);
|
|
3722
|
+
for (const entry of entries)
|
|
3723
|
+
dirty.add(entry);
|
|
3724
|
+
}
|
|
3725
|
+
}
|
|
3726
|
+
const base = this.compacted;
|
|
3727
|
+
const output = [];
|
|
3728
|
+
const removed = [];
|
|
3729
|
+
const added = [];
|
|
3730
|
+
let settled = this.settled;
|
|
3731
|
+
const length = Math.max(base.length, messages.length);
|
|
3732
|
+
for (let index = 0;index < length; index++) {
|
|
3733
|
+
const previous = base[index];
|
|
3734
|
+
const entry = messages[index];
|
|
3735
|
+
if (previous === entry && !dirty.has(entry)) {
|
|
3736
|
+
output.push(entry);
|
|
3737
|
+
continue;
|
|
3738
|
+
}
|
|
3739
|
+
if (previous !== undefined) {
|
|
3740
|
+
removed.push(previous);
|
|
3741
|
+
if (isSettledPendingStageMessage(previous))
|
|
3742
|
+
settled--;
|
|
3743
|
+
}
|
|
3744
|
+
if (entry === undefined)
|
|
3745
|
+
continue;
|
|
3746
|
+
if (isSettledPendingStageMessage(entry))
|
|
3747
|
+
settled++;
|
|
3748
|
+
const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
|
|
3749
|
+
output.push(next);
|
|
3750
|
+
added.push(next);
|
|
3751
|
+
}
|
|
3752
|
+
let retained = output;
|
|
3753
|
+
if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
|
|
3754
|
+
retained = await retainPendingStageMessages(output, archiveSettled);
|
|
3755
|
+
settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
|
|
3756
|
+
}
|
|
3757
|
+
return {
|
|
3758
|
+
messages: retained,
|
|
3759
|
+
commit: () => {
|
|
3760
|
+
for (const [deliveryRunId, stageId] of endedStages) {
|
|
3761
|
+
const byStage = this.stages.get(deliveryRunId);
|
|
3762
|
+
byStage?.delete(stageId);
|
|
3763
|
+
if (byStage?.size === 0)
|
|
3764
|
+
this.stages.delete(deliveryRunId);
|
|
3765
|
+
}
|
|
3766
|
+
for (const entry of dirty)
|
|
3767
|
+
this.fixed.delete(entry);
|
|
3768
|
+
for (const entry of removed)
|
|
3769
|
+
this.unindex(entry);
|
|
3770
|
+
for (const entry of added)
|
|
3771
|
+
this.index(entry);
|
|
3772
|
+
this.compacted = retained;
|
|
3773
|
+
this.settled = settled;
|
|
3774
|
+
}
|
|
3775
|
+
};
|
|
3776
|
+
}
|
|
3777
|
+
index(entry) {
|
|
3778
|
+
this.fixed.add(entry);
|
|
3779
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3780
|
+
let byStage = this.stages.get(delivery.runId);
|
|
3781
|
+
if (byStage === undefined) {
|
|
3782
|
+
byStage = new Map;
|
|
3783
|
+
this.stages.set(delivery.runId, byStage);
|
|
3784
|
+
}
|
|
3785
|
+
let entries = byStage.get(delivery.stageId);
|
|
3786
|
+
if (entries === undefined) {
|
|
3787
|
+
entries = new Set;
|
|
3788
|
+
byStage.set(delivery.stageId, entries);
|
|
3789
|
+
}
|
|
3790
|
+
entries.add(entry);
|
|
3791
|
+
}
|
|
3792
|
+
}
|
|
3793
|
+
unindex(entry) {
|
|
3794
|
+
this.fixed.delete(entry);
|
|
3795
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3796
|
+
const byStage = this.stages.get(delivery.runId);
|
|
3797
|
+
const entries = byStage?.get(delivery.stageId);
|
|
3798
|
+
if (entries === undefined)
|
|
3799
|
+
continue;
|
|
3800
|
+
entries.delete(entry);
|
|
3801
|
+
if (entries.size > 0)
|
|
3802
|
+
continue;
|
|
3803
|
+
byStage.delete(delivery.stageId);
|
|
3804
|
+
if (byStage.size === 0)
|
|
3805
|
+
this.stages.delete(delivery.runId);
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
}
|
|
3809
|
+
function isCompactingBackend(backend) {
|
|
3810
|
+
return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
|
|
3811
|
+
}
|
|
3812
|
+
function createStageStatusLookup(runs) {
|
|
3813
|
+
let runById;
|
|
3814
|
+
return (runId, stageId) => {
|
|
3815
|
+
if (runById === undefined) {
|
|
3816
|
+
runById = new Map;
|
|
3817
|
+
for (const run of runs)
|
|
3818
|
+
if (!runById.has(run.id))
|
|
3819
|
+
runById.set(run.id, run);
|
|
3820
|
+
}
|
|
3821
|
+
return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
|
|
3822
|
+
};
|
|
3823
|
+
}
|
|
3824
|
+
function isEndedStageStatus(status) {
|
|
3825
|
+
return status === "completed" || status === "skipped" || status === "failed";
|
|
3826
|
+
}
|
|
3827
|
+
async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
|
|
3828
|
+
if (entry.sticky !== true || entry.deliveries === undefined)
|
|
3829
|
+
return entry;
|
|
3830
|
+
const latest = new Map;
|
|
3831
|
+
entry.deliveries.forEach((delivery, index) => {
|
|
3832
|
+
latest.set(deliverySlot(delivery), index);
|
|
3833
|
+
});
|
|
3834
|
+
const deliveries = [];
|
|
3835
|
+
let archivedThrough = -1;
|
|
3836
|
+
for (let index = 0;index < entry.deliveries.length; index++) {
|
|
3837
|
+
const delivery = entry.deliveries[index];
|
|
3838
|
+
if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
|
|
3839
|
+
while (archivedThrough < index) {
|
|
3840
|
+
const prefixIndex = archivedThrough + 1;
|
|
3841
|
+
const receipt = entry.deliveries[prefixIndex];
|
|
3842
|
+
if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
|
|
3843
|
+
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
|
|
3844
|
+
messageRunId: entry.runId,
|
|
3845
|
+
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
3846
|
+
});
|
|
3847
|
+
archivedThrough = prefixIndex;
|
|
3848
|
+
}
|
|
3849
|
+
} else {
|
|
3850
|
+
deliveries.push(delivery);
|
|
3851
|
+
}
|
|
3852
|
+
}
|
|
3853
|
+
return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
|
|
3854
|
+
}
|
|
3855
|
+
function deliverySlot(delivery) {
|
|
3856
|
+
return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
|
|
3857
|
+
}
|
|
3858
|
+
var init_pending_stage_transition_ledger = () => {};
|
|
3859
|
+
|
|
3629
3860
|
// dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
|
|
3630
3861
|
function createPendingStageDeliveryStoreMethods(context) {
|
|
3631
|
-
const
|
|
3632
|
-
const
|
|
3633
|
-
const previous = transitions.get(runId) ?? Promise.resolve();
|
|
3634
|
-
const result = previous.catch(() => {
|
|
3635
|
-
return;
|
|
3636
|
-
}).then(transition);
|
|
3637
|
-
const settled = result.then(() => {
|
|
3638
|
-
return;
|
|
3639
|
-
}, () => {
|
|
3640
|
-
return;
|
|
3641
|
-
});
|
|
3642
|
-
transitions.set(runId, settled);
|
|
3643
|
-
settled.finally(() => {
|
|
3644
|
-
if (transitions.get(runId) === settled)
|
|
3645
|
-
transitions.delete(runId);
|
|
3646
|
-
});
|
|
3647
|
-
return await result;
|
|
3648
|
-
};
|
|
3862
|
+
const ledgers = new PendingStageLedgers;
|
|
3863
|
+
const batches = new PendingStageTransitionBatches(context, ledgers);
|
|
3649
3864
|
return {
|
|
3650
3865
|
async queueStageMessage(input, senderGroup, runGroup, backend) {
|
|
3651
|
-
return await
|
|
3866
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3652
3867
|
const run = context.findRun(input.runId);
|
|
3653
3868
|
if (run === undefined)
|
|
3654
3869
|
return;
|
|
3655
3870
|
const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
|
|
3656
3871
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3657
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3872
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
|
|
3658
3873
|
if (result.ok && !result.deduplicated) {
|
|
3659
|
-
|
|
3660
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3661
|
-
live.pendingStageMessages = [...messages];
|
|
3662
|
-
context.bumpAndNotify();
|
|
3874
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3663
3875
|
}
|
|
3664
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3876
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3665
3877
|
});
|
|
3666
3878
|
},
|
|
3667
3879
|
async validateLiveStageMessage(input) {
|
|
3668
|
-
return await
|
|
3880
|
+
return await batches.run(input.runId, undefined, async (batch) => {
|
|
3669
3881
|
const run = context.findRun(input.runId);
|
|
3670
3882
|
if (run === undefined)
|
|
3671
3883
|
return;
|
|
3672
3884
|
const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
|
|
3673
3885
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
|
|
3674
3886
|
const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3675
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3887
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
|
|
3676
3888
|
if (!result.ok) {
|
|
3677
3889
|
return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
|
|
3678
3890
|
}
|
|
@@ -3694,77 +3906,65 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3694
3906
|
return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
|
|
3695
3907
|
},
|
|
3696
3908
|
async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
|
|
3697
|
-
return await
|
|
3909
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3698
3910
|
const run = context.findRun(runId);
|
|
3699
3911
|
if (run === undefined)
|
|
3700
3912
|
return false;
|
|
3701
|
-
const current = run
|
|
3913
|
+
const current = batch.messages(run);
|
|
3702
3914
|
const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
|
|
3703
3915
|
if (next === current)
|
|
3704
3916
|
return false;
|
|
3705
|
-
|
|
3706
|
-
const live = context.findRun(runId) ?? run;
|
|
3707
|
-
live.pendingStageMessages = [...messages];
|
|
3708
|
-
context.bumpAndNotify();
|
|
3917
|
+
await batch.apply(backend, runId, next);
|
|
3709
3918
|
return true;
|
|
3710
3919
|
});
|
|
3711
3920
|
},
|
|
3712
3921
|
async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
|
|
3713
|
-
return await
|
|
3922
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3714
3923
|
const run = context.findRun(runId);
|
|
3715
3924
|
if (run === undefined)
|
|
3716
3925
|
return false;
|
|
3717
|
-
const current = run
|
|
3926
|
+
const current = batch.messages(run);
|
|
3718
3927
|
const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
|
|
3719
3928
|
if (next === current)
|
|
3720
3929
|
return false;
|
|
3721
|
-
|
|
3722
|
-
const live = context.findRun(runId) ?? run;
|
|
3723
|
-
live.pendingStageMessages = [...messages];
|
|
3724
|
-
context.bumpAndNotify();
|
|
3930
|
+
await batch.apply(backend, runId, next);
|
|
3725
3931
|
return true;
|
|
3726
3932
|
});
|
|
3727
3933
|
},
|
|
3728
3934
|
async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
|
|
3729
|
-
return await
|
|
3935
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3730
3936
|
const run = context.findRun(runId);
|
|
3731
3937
|
if (run === undefined)
|
|
3732
3938
|
return false;
|
|
3733
|
-
const current = run
|
|
3939
|
+
const current = batch.messages(run);
|
|
3734
3940
|
const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
|
|
3735
3941
|
if (next === current)
|
|
3736
3942
|
return false;
|
|
3737
|
-
|
|
3738
|
-
const live = context.findRun(runId) ?? run;
|
|
3739
|
-
live.pendingStageMessages = [...messages];
|
|
3740
|
-
context.bumpAndNotify();
|
|
3943
|
+
await batch.apply(backend, runId, next);
|
|
3741
3944
|
return true;
|
|
3742
3945
|
});
|
|
3743
3946
|
},
|
|
3744
3947
|
async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
|
|
3745
|
-
return await
|
|
3948
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3746
3949
|
const run = context.findRun(input.runId);
|
|
3747
3950
|
if (run === undefined)
|
|
3748
3951
|
return;
|
|
3749
3952
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3750
|
-
const result = queueStickyStageMessage(receipt === undefined ? run
|
|
3953
|
+
const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
|
|
3751
3954
|
if (result.ok && !result.deduplicated) {
|
|
3752
|
-
|
|
3753
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3754
|
-
live.pendingStageMessages = [...messages];
|
|
3755
|
-
context.bumpAndNotify();
|
|
3955
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3756
3956
|
}
|
|
3757
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3957
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3758
3958
|
});
|
|
3759
3959
|
},
|
|
3760
3960
|
async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
|
|
3761
|
-
return await
|
|
3961
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3762
3962
|
const run = context.findRun(runId);
|
|
3763
3963
|
if (run === undefined)
|
|
3764
3964
|
return false;
|
|
3765
3965
|
if (records.length === 0)
|
|
3766
3966
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3767
|
-
let current =
|
|
3967
|
+
let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3768
3968
|
const unconfirmedRecords = [];
|
|
3769
3969
|
for (const record of records) {
|
|
3770
3970
|
const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
|
|
@@ -3772,35 +3972,26 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3772
3972
|
unconfirmedRecords.push(record);
|
|
3773
3973
|
}
|
|
3774
3974
|
}
|
|
3775
|
-
current =
|
|
3776
|
-
|
|
3777
|
-
run.pendingStageMessages = [...current];
|
|
3778
|
-
context.bumpAndNotify();
|
|
3779
|
-
}
|
|
3975
|
+
current = ledgers.for(run).reconcile(backend, runId, current);
|
|
3976
|
+
batch.reconciled(run, current);
|
|
3780
3977
|
const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
|
|
3781
3978
|
if (next === current)
|
|
3782
3979
|
return false;
|
|
3783
|
-
|
|
3784
|
-
const live = context.findRun(runId) ?? run;
|
|
3785
|
-
live.pendingStageMessages = [...messages];
|
|
3786
|
-
context.bumpAndNotify();
|
|
3980
|
+
await batch.apply(backend, runId, next);
|
|
3787
3981
|
return true;
|
|
3788
3982
|
});
|
|
3789
3983
|
},
|
|
3790
3984
|
async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
|
|
3791
|
-
return await
|
|
3985
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3792
3986
|
const run = context.findRun(runId);
|
|
3793
3987
|
if (run === undefined)
|
|
3794
3988
|
return false;
|
|
3795
3989
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3796
|
-
const current =
|
|
3990
|
+
const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3797
3991
|
const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
|
|
3798
3992
|
if (next === current)
|
|
3799
3993
|
return false;
|
|
3800
|
-
|
|
3801
|
-
const live = context.findRun(runId) ?? run;
|
|
3802
|
-
live.pendingStageMessages = [...messages];
|
|
3803
|
-
context.bumpAndNotify();
|
|
3994
|
+
await batch.apply(backend, runId, next);
|
|
3804
3995
|
return true;
|
|
3805
3996
|
});
|
|
3806
3997
|
}
|
|
@@ -3818,78 +4009,186 @@ function resolvePendingStageIdentity(run, stageKey) {
|
|
|
3818
4009
|
aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
|
|
3819
4010
|
};
|
|
3820
4011
|
}
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
4012
|
+
|
|
4013
|
+
class PendingStageTransitionBatches {
|
|
4014
|
+
queues = new Map;
|
|
4015
|
+
context;
|
|
4016
|
+
ledgers;
|
|
4017
|
+
constructor(context, ledgers) {
|
|
4018
|
+
this.context = context;
|
|
4019
|
+
this.ledgers = ledgers;
|
|
4020
|
+
}
|
|
4021
|
+
run(runId, backend, step) {
|
|
4022
|
+
return new Promise((resolve, reject) => {
|
|
4023
|
+
const transition = {
|
|
4024
|
+
backend,
|
|
4025
|
+
step,
|
|
4026
|
+
resolve: (value2) => resolve(value2),
|
|
4027
|
+
reject
|
|
4028
|
+
};
|
|
4029
|
+
const queue = this.queues.get(runId);
|
|
4030
|
+
if (queue !== undefined) {
|
|
4031
|
+
queue.push(transition);
|
|
4032
|
+
return;
|
|
4033
|
+
}
|
|
4034
|
+
const created = [transition];
|
|
4035
|
+
this.queues.set(runId, created);
|
|
4036
|
+
Promise.resolve().then(() => this.drain(runId, created));
|
|
4037
|
+
});
|
|
4038
|
+
}
|
|
4039
|
+
async drain(runId, queue) {
|
|
4040
|
+
while (queue.length > 0)
|
|
4041
|
+
await this.runBatch(runId, takeBatch(queue));
|
|
4042
|
+
this.queues.delete(runId);
|
|
4043
|
+
}
|
|
4044
|
+
async runBatch(runId, transitions) {
|
|
4045
|
+
const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
|
|
4046
|
+
const outcomes = [];
|
|
4047
|
+
for (const transition of transitions) {
|
|
4048
|
+
const changedBefore = batch.changed;
|
|
4049
|
+
try {
|
|
4050
|
+
const value2 = await transition.step(batch);
|
|
4051
|
+
outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
|
|
4052
|
+
} catch (error) {
|
|
4053
|
+
outcomes.push({ ok: false, error });
|
|
3854
4054
|
}
|
|
3855
|
-
retained = compacted;
|
|
3856
4055
|
}
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
4056
|
+
let failure;
|
|
4057
|
+
try {
|
|
4058
|
+
await batch.persist();
|
|
4059
|
+
} catch (error) {
|
|
4060
|
+
failure = { error };
|
|
4061
|
+
}
|
|
4062
|
+
transitions.forEach((transition, index) => {
|
|
4063
|
+
const outcome = outcomes[index];
|
|
4064
|
+
if (!outcome.ok)
|
|
4065
|
+
transition.reject(outcome.error);
|
|
4066
|
+
else if (failure !== undefined && outcome.uncommitted)
|
|
4067
|
+
transition.reject(failure.error);
|
|
4068
|
+
else
|
|
4069
|
+
transition.resolve(outcome.value);
|
|
4070
|
+
});
|
|
4071
|
+
}
|
|
4072
|
+
}
|
|
4073
|
+
function takeBatch(queue) {
|
|
4074
|
+
let count = 1;
|
|
4075
|
+
let backend = queue[0].backend;
|
|
4076
|
+
while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
|
|
4077
|
+
const next = queue[count].backend;
|
|
4078
|
+
if (next !== undefined && backend !== undefined && next !== backend)
|
|
4079
|
+
break;
|
|
4080
|
+
backend ??= next;
|
|
4081
|
+
count++;
|
|
4082
|
+
}
|
|
4083
|
+
return queue.splice(0, count);
|
|
4084
|
+
}
|
|
4085
|
+
|
|
4086
|
+
class PendingStageTransitionBatch {
|
|
4087
|
+
changed = false;
|
|
4088
|
+
list;
|
|
4089
|
+
pending;
|
|
4090
|
+
context;
|
|
4091
|
+
ledgers;
|
|
4092
|
+
runId;
|
|
4093
|
+
constructor(context, ledgers, runId) {
|
|
4094
|
+
this.context = context;
|
|
4095
|
+
this.ledgers = ledgers;
|
|
4096
|
+
this.runId = runId;
|
|
4097
|
+
}
|
|
4098
|
+
messages(run) {
|
|
4099
|
+
return this.list ?? run.pendingStageMessages ?? [];
|
|
4100
|
+
}
|
|
4101
|
+
reconciled(run, messages) {
|
|
4102
|
+
if (this.changed) {
|
|
4103
|
+
this.list = messages;
|
|
4104
|
+
return;
|
|
3860
4105
|
}
|
|
3861
|
-
if (
|
|
3862
|
-
|
|
4106
|
+
if (messages !== run.pendingStageMessages) {
|
|
4107
|
+
run.pendingStageMessages = [...messages];
|
|
4108
|
+
this.context.bumpAndNotify();
|
|
3863
4109
|
}
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
const run = context.findRun(runId);
|
|
3867
|
-
|
|
3868
|
-
const
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
4110
|
+
}
|
|
4111
|
+
async apply(backend, runId, messages) {
|
|
4112
|
+
const run = this.context.findRun(runId);
|
|
4113
|
+
try {
|
|
4114
|
+
const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
|
|
4115
|
+
this.list = compacted.messages;
|
|
4116
|
+
this.changed = true;
|
|
4117
|
+
this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
|
|
4118
|
+
return compacted.messages;
|
|
4119
|
+
} catch (error) {
|
|
4120
|
+
if (this.changed && this.list !== undefined) {
|
|
4121
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
|
|
4122
|
+
this.list = ledger.reconcile(backend, runId, this.list);
|
|
4123
|
+
} else
|
|
4124
|
+
reconcileLive(backend, runId, this.context, this.ledgers);
|
|
4125
|
+
throw error;
|
|
3874
4126
|
}
|
|
3875
4127
|
}
|
|
4128
|
+
async persist() {
|
|
4129
|
+
if (this.pending === undefined)
|
|
4130
|
+
return;
|
|
4131
|
+
const { backend, messages, commit, run } = this.pending;
|
|
4132
|
+
const published = this.list ?? messages;
|
|
4133
|
+
try {
|
|
4134
|
+
if (!await backend.persistPendingStageMessages(this.runId, messages)) {
|
|
4135
|
+
throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
|
|
4136
|
+
}
|
|
4137
|
+
commit?.();
|
|
4138
|
+
} finally {
|
|
4139
|
+
reconcileLive(backend, this.runId, this.context, this.ledgers);
|
|
4140
|
+
}
|
|
4141
|
+
const live = this.context.findRun(this.runId) ?? run;
|
|
4142
|
+
if (live !== undefined)
|
|
4143
|
+
live.pendingStageMessages = [...published];
|
|
4144
|
+
this.context.bumpAndNotify();
|
|
4145
|
+
}
|
|
4146
|
+
}
|
|
4147
|
+
async function compactTransition(backend, runId, messages, context, ledgers, run) {
|
|
4148
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
|
|
4149
|
+
const reconciled = ledger.reconcile(backend, runId, messages);
|
|
4150
|
+
const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
|
|
4151
|
+
if (isCompactingBackend(backend)) {
|
|
4152
|
+
return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
|
|
4153
|
+
}
|
|
4154
|
+
let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
|
|
4155
|
+
if (archiveSettled !== undefined)
|
|
4156
|
+
retained = await retainPendingStageMessages(retained, archiveSettled);
|
|
4157
|
+
if (run !== undefined)
|
|
4158
|
+
ledgers.reset(run);
|
|
4159
|
+
return { messages: retained };
|
|
4160
|
+
}
|
|
4161
|
+
function reconcileLive(backend, runId, context, ledgers) {
|
|
4162
|
+
const current = context.findRun(runId);
|
|
4163
|
+
if (current === undefined)
|
|
4164
|
+
return;
|
|
4165
|
+
const messages = current.pendingStageMessages ?? [];
|
|
4166
|
+
const recovered = ledgers.for(current).reconcile(backend, runId, messages);
|
|
4167
|
+
if (recovered !== messages) {
|
|
4168
|
+
current.pendingStageMessages = [...recovered];
|
|
4169
|
+
context.bumpAndNotify();
|
|
4170
|
+
}
|
|
3876
4171
|
}
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
return
|
|
3887
|
-
}
|
|
3888
|
-
|
|
4172
|
+
|
|
4173
|
+
class PendingStageLedgers {
|
|
4174
|
+
ledgers = new WeakMap;
|
|
4175
|
+
for(run) {
|
|
4176
|
+
let ledger = this.ledgers.get(run);
|
|
4177
|
+
if (ledger === undefined) {
|
|
4178
|
+
ledger = new PendingStageTransitionLedger;
|
|
4179
|
+
this.ledgers.set(run, ledger);
|
|
4180
|
+
}
|
|
4181
|
+
return ledger;
|
|
4182
|
+
}
|
|
4183
|
+
reset(run) {
|
|
4184
|
+
this.ledgers.delete(run);
|
|
4185
|
+
}
|
|
3889
4186
|
}
|
|
4187
|
+
var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
|
|
3890
4188
|
var init_store_pending_stage_delivery_methods = __esm(() => {
|
|
3891
4189
|
init_run_owner_backend();
|
|
3892
4190
|
init_pending_stage_delivery();
|
|
4191
|
+
init_pending_stage_transition_ledger();
|
|
3893
4192
|
});
|
|
3894
4193
|
|
|
3895
4194
|
// dist/builtin/workflows/src/shared/store-prompt-methods.ts
|
|
@@ -5843,8 +6142,59 @@ function firstError(schema, value2) {
|
|
|
5843
6142
|
}
|
|
5844
6143
|
return;
|
|
5845
6144
|
}
|
|
6145
|
+
function certainlySerializable(root, rootMustBeObject) {
|
|
6146
|
+
if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
|
|
6147
|
+
return false;
|
|
6148
|
+
const pending = [root];
|
|
6149
|
+
const ancestors = new Set;
|
|
6150
|
+
const exits = [undefined];
|
|
6151
|
+
while (pending.length > 0) {
|
|
6152
|
+
const exit = exits.pop();
|
|
6153
|
+
const value2 = pending.pop();
|
|
6154
|
+
if (exit !== undefined) {
|
|
6155
|
+
ancestors.delete(exit);
|
|
6156
|
+
continue;
|
|
6157
|
+
}
|
|
6158
|
+
if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
|
|
6159
|
+
continue;
|
|
6160
|
+
if (typeof value2 === "number") {
|
|
6161
|
+
if (!Number.isFinite(value2))
|
|
6162
|
+
return false;
|
|
6163
|
+
continue;
|
|
6164
|
+
}
|
|
6165
|
+
if (typeof value2 !== "object" || ancestors.has(value2))
|
|
6166
|
+
return false;
|
|
6167
|
+
if (Object.getOwnPropertySymbols(value2).length > 0)
|
|
6168
|
+
return false;
|
|
6169
|
+
const names = Object.getOwnPropertyNames(value2);
|
|
6170
|
+
let children;
|
|
6171
|
+
if (Array.isArray(value2)) {
|
|
6172
|
+
if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
|
|
6173
|
+
return false;
|
|
6174
|
+
children = [];
|
|
6175
|
+
for (let index = 0;index < value2.length; index++)
|
|
6176
|
+
children.push(value2[index]);
|
|
6177
|
+
} else {
|
|
6178
|
+
const proto = Object.getPrototypeOf(value2);
|
|
6179
|
+
if (proto !== Object.prototype && proto !== null)
|
|
6180
|
+
return false;
|
|
6181
|
+
const keys = Object.keys(value2);
|
|
6182
|
+
if (keys.length !== names.length)
|
|
6183
|
+
return false;
|
|
6184
|
+
children = keys.map((key) => value2[key]);
|
|
6185
|
+
}
|
|
6186
|
+
ancestors.add(value2);
|
|
6187
|
+
pending.push(value2);
|
|
6188
|
+
exits.push(value2);
|
|
6189
|
+
for (const child of children) {
|
|
6190
|
+
pending.push(child);
|
|
6191
|
+
exits.push(undefined);
|
|
6192
|
+
}
|
|
6193
|
+
}
|
|
6194
|
+
return true;
|
|
6195
|
+
}
|
|
5846
6196
|
function workflowSerializableValidationError(value2, label) {
|
|
5847
|
-
if (Value.Check(workflowSerializableValueSchema, value2))
|
|
6197
|
+
if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
|
|
5848
6198
|
return;
|
|
5849
6199
|
const issue = firstError(workflowSerializableValueSchema, value2);
|
|
5850
6200
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -5853,7 +6203,7 @@ function workflowSerializableValidationError(value2, label) {
|
|
|
5853
6203
|
return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
|
|
5854
6204
|
}
|
|
5855
6205
|
function workflowSerializableObjectValidationError(value2, label) {
|
|
5856
|
-
if (Value.Check(workflowSerializableObjectSchema, value2))
|
|
6206
|
+
if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
|
|
5857
6207
|
return;
|
|
5858
6208
|
const issue = firstError(workflowSerializableObjectSchema, value2);
|
|
5859
6209
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -6882,6 +7232,204 @@ var init_resume_eligibility = () => {};
|
|
|
6882
7232
|
// dist/builtin/workflows/src/durable/types.ts
|
|
6883
7233
|
var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
|
|
6884
7234
|
|
|
7235
|
+
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
7236
|
+
function immutableStageGroupError(stages) {
|
|
7237
|
+
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
7238
|
+
if (occurrenceError !== undefined)
|
|
7239
|
+
return occurrenceError;
|
|
7240
|
+
const byReplayKey = groupByDurableStageKey(stages);
|
|
7241
|
+
for (const group of byReplayKey.values()) {
|
|
7242
|
+
const firstName = group[0]?.name;
|
|
7243
|
+
if (group.some((stage) => stage.name !== firstName))
|
|
7244
|
+
return "stage name drift within one replay-key group";
|
|
7245
|
+
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
7246
|
+
const first = topologies[0];
|
|
7247
|
+
if (first === undefined)
|
|
7248
|
+
continue;
|
|
7249
|
+
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
7250
|
+
return "immutable stage identity drift within one replay-key group";
|
|
7251
|
+
}
|
|
7252
|
+
}
|
|
7253
|
+
return;
|
|
7254
|
+
}
|
|
7255
|
+
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
7256
|
+
return directChildTopologyErrorFromRecords({
|
|
7257
|
+
claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
|
|
7258
|
+
direct: (claiming) => {
|
|
7259
|
+
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
7260
|
+
return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
7261
|
+
}
|
|
7262
|
+
}, replayScope, expectedRun, allowedNodeIds);
|
|
7263
|
+
}
|
|
7264
|
+
function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
7265
|
+
const prefix = `${replayScope}:`;
|
|
7266
|
+
const claiming = records.claiming;
|
|
7267
|
+
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
7268
|
+
return "child run topology exists outside its durable invocation scope";
|
|
7269
|
+
}
|
|
7270
|
+
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
7271
|
+
return "child checkpoint ownership is not reciprocal";
|
|
7272
|
+
}
|
|
7273
|
+
const direct = records.direct(claiming);
|
|
7274
|
+
const immutableError = immutableStageGroupError(direct);
|
|
7275
|
+
if (immutableError !== undefined)
|
|
7276
|
+
return immutableError;
|
|
7277
|
+
const merged = [];
|
|
7278
|
+
for (const group of groupByDurableStageKey(direct).values()) {
|
|
7279
|
+
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
7280
|
+
if (topology === undefined)
|
|
7281
|
+
return "direct child stage group lacks owning-run topology";
|
|
7282
|
+
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
7283
|
+
return "direct child stage topology lacks current source identity";
|
|
7284
|
+
}
|
|
7285
|
+
if (topology.stageId.length === 0)
|
|
7286
|
+
return "direct child stage id is empty";
|
|
7287
|
+
merged.push(topology);
|
|
7288
|
+
}
|
|
7289
|
+
const ids = new Set(allowedNodeIds);
|
|
7290
|
+
const stageIds = new Set;
|
|
7291
|
+
const orders = new Set;
|
|
7292
|
+
for (const topology of merged) {
|
|
7293
|
+
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
7294
|
+
return "direct child stage ids are not unique";
|
|
7295
|
+
stageIds.add(topology.stageId);
|
|
7296
|
+
ids.add(topology.stageId);
|
|
7297
|
+
if (topology.sourceOrder !== undefined) {
|
|
7298
|
+
if (orders.has(topology.sourceOrder))
|
|
7299
|
+
return "direct child source orders are not unique";
|
|
7300
|
+
orders.add(topology.sourceOrder);
|
|
7301
|
+
}
|
|
7302
|
+
}
|
|
7303
|
+
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
7304
|
+
return "direct child stage references an unknown parent";
|
|
7305
|
+
}
|
|
7306
|
+
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
7307
|
+
}
|
|
7308
|
+
function owningBoundaryIdError(stages, owningRunId) {
|
|
7309
|
+
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
7310
|
+
const seen = new Map;
|
|
7311
|
+
for (const start of starts) {
|
|
7312
|
+
const stageId = start.topology.stageId;
|
|
7313
|
+
if (stageId.length === 0)
|
|
7314
|
+
return "owning-run boundary stage id is empty";
|
|
7315
|
+
const prior = seen.get(stageId);
|
|
7316
|
+
if (prior !== undefined && prior !== start.replayKey)
|
|
7317
|
+
return "owning-run boundary stage id is aliased";
|
|
7318
|
+
seen.set(stageId, start.replayKey);
|
|
7319
|
+
}
|
|
7320
|
+
return;
|
|
7321
|
+
}
|
|
7322
|
+
function sameRun(actual, expected) {
|
|
7323
|
+
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
7324
|
+
}
|
|
7325
|
+
function sameStrings(left, right) {
|
|
7326
|
+
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
7327
|
+
}
|
|
7328
|
+
function sameOptionalRun(actual, expected) {
|
|
7329
|
+
if (actual === undefined || expected === undefined)
|
|
7330
|
+
return actual === expected;
|
|
7331
|
+
return sameRun(actual, expected);
|
|
7332
|
+
}
|
|
7333
|
+
function promptOccurrenceIdentityError(stages) {
|
|
7334
|
+
const replayByOccurrence = new Map;
|
|
7335
|
+
const occurrenceByReplayStage = new Map;
|
|
7336
|
+
const stageByReplayOccurrence = new Map;
|
|
7337
|
+
for (const stage of stages) {
|
|
7338
|
+
const topology = stage.topology;
|
|
7339
|
+
const occurrence = topology?.occurrenceKey;
|
|
7340
|
+
if (topology === undefined || occurrence === undefined)
|
|
7341
|
+
continue;
|
|
7342
|
+
const priorReplay = replayByOccurrence.get(occurrence);
|
|
7343
|
+
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
7344
|
+
return "prompt occurrence is bound to multiple replay keys";
|
|
7345
|
+
}
|
|
7346
|
+
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
7347
|
+
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
7348
|
+
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
7349
|
+
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
7350
|
+
return "prompt stage identity is bound to multiple occurrences";
|
|
7351
|
+
}
|
|
7352
|
+
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
7353
|
+
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7354
|
+
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
7355
|
+
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
7356
|
+
return "prompt occurrence is bound to multiple stage identities";
|
|
7357
|
+
}
|
|
7358
|
+
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
7359
|
+
}
|
|
7360
|
+
return;
|
|
7361
|
+
}
|
|
7362
|
+
function durableStageGroupKey(stage) {
|
|
7363
|
+
const occurrence = stage.topology?.occurrenceKey;
|
|
7364
|
+
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7365
|
+
}
|
|
7366
|
+
function groupByDurableStageKey(stages) {
|
|
7367
|
+
const groups = new Map;
|
|
7368
|
+
for (const stage of stages) {
|
|
7369
|
+
const key = durableStageGroupKey(stage);
|
|
7370
|
+
const group = groups.get(key) ?? [];
|
|
7371
|
+
group.push(stage);
|
|
7372
|
+
groups.set(key, group);
|
|
7373
|
+
}
|
|
7374
|
+
return groups;
|
|
7375
|
+
}
|
|
7376
|
+
function compositeKey(domain, ...parts) {
|
|
7377
|
+
return JSON.stringify([domain, ...parts]);
|
|
7378
|
+
}
|
|
7379
|
+
function hasCycle(topologies) {
|
|
7380
|
+
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
7381
|
+
const visiting = new Set;
|
|
7382
|
+
const visited = new Set;
|
|
7383
|
+
const cyclic = (stageId) => {
|
|
7384
|
+
if (visiting.has(stageId))
|
|
7385
|
+
return true;
|
|
7386
|
+
if (visited.has(stageId))
|
|
7387
|
+
return false;
|
|
7388
|
+
visiting.add(stageId);
|
|
7389
|
+
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
7390
|
+
return true;
|
|
7391
|
+
visiting.delete(stageId);
|
|
7392
|
+
visited.add(stageId);
|
|
7393
|
+
return false;
|
|
7394
|
+
};
|
|
7395
|
+
return topologies.some((topology) => cyclic(topology.stageId));
|
|
7396
|
+
}
|
|
7397
|
+
var init_stage_topology_validation = () => {};
|
|
7398
|
+
|
|
7399
|
+
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
7400
|
+
function parseWorkflowChildResult(value2) {
|
|
7401
|
+
if (!isOrdinaryObject(value2))
|
|
7402
|
+
return;
|
|
7403
|
+
const workflow = value2.workflow;
|
|
7404
|
+
const runId = value2.runId;
|
|
7405
|
+
const status = value2.status;
|
|
7406
|
+
const exited = value2.exited;
|
|
7407
|
+
const outputs = value2.outputs;
|
|
7408
|
+
const exitReason = value2.exitReason;
|
|
7409
|
+
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
7410
|
+
return;
|
|
7411
|
+
if (exited === false) {
|
|
7412
|
+
if (status !== "completed" || exitReason !== undefined)
|
|
7413
|
+
return;
|
|
7414
|
+
return value2;
|
|
7415
|
+
}
|
|
7416
|
+
if (exited !== true || !isWorkflowExitStatus2(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
7417
|
+
return;
|
|
7418
|
+
return value2;
|
|
7419
|
+
}
|
|
7420
|
+
function isWorkflowChildResult(value2) {
|
|
7421
|
+
return parseWorkflowChildResult(value2) !== undefined;
|
|
7422
|
+
}
|
|
7423
|
+
function isWorkflowExitStatus2(value2) {
|
|
7424
|
+
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
7425
|
+
}
|
|
7426
|
+
function isOrdinaryObject(value2) {
|
|
7427
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
7428
|
+
return false;
|
|
7429
|
+
const prototype = Object.getPrototypeOf(value2);
|
|
7430
|
+
return prototype === Object.prototype || prototype === null;
|
|
7431
|
+
}
|
|
7432
|
+
|
|
6885
7433
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
6886
7434
|
import { createHash as createHash2 } from "node:crypto";
|
|
6887
7435
|
function durableHash(value2) {
|
|
@@ -6948,6 +7496,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
|
|
|
6948
7496
|
function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
|
|
6949
7497
|
return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
|
|
6950
7498
|
}
|
|
7499
|
+
|
|
7500
|
+
class StageReceipts {
|
|
7501
|
+
exact = new Map;
|
|
7502
|
+
anySession = new Map;
|
|
7503
|
+
has(logicalRunId, sessionId, admission) {
|
|
7504
|
+
return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
|
|
7505
|
+
}
|
|
7506
|
+
hasAnySession(logicalRunId, admission) {
|
|
7507
|
+
return this.anySession.get(logicalRunId)?.has(admission) ?? false;
|
|
7508
|
+
}
|
|
7509
|
+
add(logicalRunId, sessionId, admission) {
|
|
7510
|
+
let bySession = this.exact.get(logicalRunId);
|
|
7511
|
+
if (bySession === undefined) {
|
|
7512
|
+
bySession = new Map;
|
|
7513
|
+
this.exact.set(logicalRunId, bySession);
|
|
7514
|
+
}
|
|
7515
|
+
let admissions = bySession.get(sessionId);
|
|
7516
|
+
if (admissions === undefined) {
|
|
7517
|
+
admissions = new Set;
|
|
7518
|
+
bySession.set(sessionId, admissions);
|
|
7519
|
+
}
|
|
7520
|
+
if (admissions.has(admission))
|
|
7521
|
+
return false;
|
|
7522
|
+
admissions.add(admission);
|
|
7523
|
+
return true;
|
|
7524
|
+
}
|
|
7525
|
+
addAnySession(logicalRunId, admission) {
|
|
7526
|
+
let admissions = this.anySession.get(logicalRunId);
|
|
7527
|
+
if (admissions === undefined) {
|
|
7528
|
+
admissions = new Set;
|
|
7529
|
+
this.anySession.set(logicalRunId, admissions);
|
|
7530
|
+
}
|
|
7531
|
+
if (admissions.has(admission))
|
|
7532
|
+
return false;
|
|
7533
|
+
admissions.add(admission);
|
|
7534
|
+
return true;
|
|
7535
|
+
}
|
|
7536
|
+
}
|
|
7537
|
+
|
|
7538
|
+
class PendingStageReceiptIndex {
|
|
7539
|
+
workflows = new Map;
|
|
7540
|
+
add(workflowId, messageId, delivery, logicalRunId) {
|
|
7541
|
+
let workflow = this.workflows.get(workflowId);
|
|
7542
|
+
if (workflow === undefined) {
|
|
7543
|
+
workflow = { byMessage: new Map, log: [] };
|
|
7544
|
+
this.workflows.set(workflowId, workflow);
|
|
7545
|
+
}
|
|
7546
|
+
let byRun = workflow.byMessage.get(messageId);
|
|
7547
|
+
if (byRun === undefined) {
|
|
7548
|
+
byRun = new Map;
|
|
7549
|
+
workflow.byMessage.set(messageId, byRun);
|
|
7550
|
+
}
|
|
7551
|
+
let byStage = byRun.get(delivery.runId);
|
|
7552
|
+
if (byStage === undefined) {
|
|
7553
|
+
byStage = new Map;
|
|
7554
|
+
byRun.set(delivery.runId, byStage);
|
|
7555
|
+
}
|
|
7556
|
+
let receipts = byStage.get(delivery.stageId);
|
|
7557
|
+
if (receipts === undefined) {
|
|
7558
|
+
receipts = new StageReceipts;
|
|
7559
|
+
byStage.set(delivery.stageId, receipts);
|
|
7560
|
+
}
|
|
7561
|
+
const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
|
|
7562
|
+
let changed = added;
|
|
7563
|
+
if (receipts.add(logicalRunId, undefined, undefined))
|
|
7564
|
+
changed = true;
|
|
7565
|
+
if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
|
|
7566
|
+
changed = true;
|
|
7567
|
+
if (changed)
|
|
7568
|
+
workflow.log.push(messageId);
|
|
7569
|
+
return added;
|
|
7570
|
+
}
|
|
7571
|
+
has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
|
|
7572
|
+
const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
|
|
7573
|
+
if (receipts === undefined)
|
|
7574
|
+
return false;
|
|
7575
|
+
if (anySession && delivery.admission === "context")
|
|
7576
|
+
return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
|
|
7577
|
+
return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
|
|
7578
|
+
}
|
|
7579
|
+
changesSince(workflowId, cursor) {
|
|
7580
|
+
let workflow = this.workflows.get(workflowId);
|
|
7581
|
+
if (workflow === undefined) {
|
|
7582
|
+
workflow = { byMessage: new Map, log: [] };
|
|
7583
|
+
this.workflows.set(workflowId, workflow);
|
|
7584
|
+
}
|
|
7585
|
+
const next = { log: workflow.log, length: workflow.log.length };
|
|
7586
|
+
if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
|
|
7587
|
+
return { cursor: next, messageIds: undefined };
|
|
7588
|
+
return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
|
|
7589
|
+
}
|
|
7590
|
+
delete(workflowId) {
|
|
7591
|
+
this.workflows.delete(workflowId);
|
|
7592
|
+
}
|
|
7593
|
+
clear() {
|
|
7594
|
+
this.workflows.clear();
|
|
7595
|
+
}
|
|
7596
|
+
}
|
|
7597
|
+
function nextVersion() {
|
|
7598
|
+
lastVersion += 1;
|
|
7599
|
+
return lastVersion;
|
|
7600
|
+
}
|
|
7601
|
+
|
|
7602
|
+
class NestedSnapshotVersion {
|
|
7603
|
+
rootWorkflowId;
|
|
7604
|
+
current = nextVersion();
|
|
7605
|
+
latestCompletedAt = Number.NEGATIVE_INFINITY;
|
|
7606
|
+
replayGroups = new Map;
|
|
7607
|
+
boundaryReplayKeys = new Set;
|
|
7608
|
+
constructor(rootWorkflowId) {
|
|
7609
|
+
this.rootWorkflowId = rootWorkflowId;
|
|
7610
|
+
}
|
|
7611
|
+
get version() {
|
|
7612
|
+
return this.current;
|
|
7613
|
+
}
|
|
7614
|
+
record(checkpoint) {
|
|
7615
|
+
if (!this.isInert(checkpoint))
|
|
7616
|
+
this.current = nextVersion();
|
|
7617
|
+
this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
|
|
7618
|
+
if (checkpoint.kind !== "stage")
|
|
7619
|
+
return;
|
|
7620
|
+
if (checkpoint.topology?.boundary !== undefined)
|
|
7621
|
+
this.boundaryReplayKeys.add(checkpoint.replayKey);
|
|
7622
|
+
if (checkpoint.topology?.occurrenceKey !== undefined)
|
|
7623
|
+
return;
|
|
7624
|
+
const identity = stageIdentity(checkpoint);
|
|
7625
|
+
const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
|
|
7626
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
7627
|
+
if (summary === undefined) {
|
|
7628
|
+
this.replayGroups.set(checkpoint.replayKey, {
|
|
7629
|
+
name: checkpoint.name,
|
|
7630
|
+
identity,
|
|
7631
|
+
consistent: true,
|
|
7632
|
+
hasOutput: checkpoint.output !== undefined,
|
|
7633
|
+
hasExitedChildOutput: exited
|
|
7634
|
+
});
|
|
7635
|
+
return;
|
|
7636
|
+
}
|
|
7637
|
+
summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
|
|
7638
|
+
summary.hasOutput ||= checkpoint.output !== undefined;
|
|
7639
|
+
summary.hasExitedChildOutput ||= exited;
|
|
7640
|
+
}
|
|
7641
|
+
isInert(checkpoint) {
|
|
7642
|
+
if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
|
|
7643
|
+
return false;
|
|
7644
|
+
}
|
|
7645
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
7646
|
+
if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
|
|
7647
|
+
return false;
|
|
7648
|
+
}
|
|
7649
|
+
const id = checkpoint.checkpointId;
|
|
7650
|
+
for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
|
|
7651
|
+
if (this.boundaryReplayKeys.has(id.slice(0, colon)))
|
|
7652
|
+
return false;
|
|
7653
|
+
}
|
|
7654
|
+
return true;
|
|
7655
|
+
}
|
|
7656
|
+
isRootOwned(checkpoint) {
|
|
7657
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
7658
|
+
return runId === undefined || runId === this.rootWorkflowId;
|
|
7659
|
+
}
|
|
7660
|
+
}
|
|
7661
|
+
function stageIdentity(checkpoint) {
|
|
7662
|
+
const topology = checkpoint.topology;
|
|
7663
|
+
return JSON.stringify([
|
|
7664
|
+
durableStageGroupKey(checkpoint),
|
|
7665
|
+
topology?.version,
|
|
7666
|
+
topology?.stageId,
|
|
7667
|
+
topology?.parentIds,
|
|
7668
|
+
topology?.sourceOrder,
|
|
7669
|
+
topology?.run === undefined ? undefined : [
|
|
7670
|
+
topology.run.runId,
|
|
7671
|
+
topology.run.runName,
|
|
7672
|
+
topology.run.parentRunId,
|
|
7673
|
+
topology.run.parentStageId,
|
|
7674
|
+
topology.run.rootRunId
|
|
7675
|
+
]
|
|
7676
|
+
]);
|
|
7677
|
+
}
|
|
6951
7678
|
function checkpointKey(c) {
|
|
6952
7679
|
return `${c.kind}:${c.checkpointId}`;
|
|
6953
7680
|
}
|
|
@@ -6959,7 +7686,7 @@ class JournalProjection {
|
|
|
6959
7686
|
workflows = new Map;
|
|
6960
7687
|
deletedWorkflowIds = new Set;
|
|
6961
7688
|
pendingReceipts = new Map;
|
|
6962
|
-
stickyReceipts = new
|
|
7689
|
+
stickyReceipts = new PendingStageReceiptIndex;
|
|
6963
7690
|
registerWorkflow(handle, authoritativeOwner = false) {
|
|
6964
7691
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
6965
7692
|
const existing = this.workflows.get(handle.workflowId);
|
|
@@ -7007,7 +7734,8 @@ class JournalProjection {
|
|
|
7007
7734
|
toolByHash: new Map,
|
|
7008
7735
|
uiByHash: new Map,
|
|
7009
7736
|
stageOutputByReplayKey: new Map,
|
|
7010
|
-
stageSessionByReplayKey: new Map
|
|
7737
|
+
stageSessionByReplayKey: new Map,
|
|
7738
|
+
nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
|
|
7011
7739
|
});
|
|
7012
7740
|
}
|
|
7013
7741
|
applyWorkflowSnapshot(handle) {
|
|
@@ -7033,39 +7761,43 @@ class JournalProjection {
|
|
|
7033
7761
|
return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
|
|
7034
7762
|
}
|
|
7035
7763
|
stickyDeliveryCounts = new Map;
|
|
7764
|
+
stickyDeliveryCountVersions = new Map;
|
|
7036
7765
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
7037
|
-
return this.stickyDeliveryCounts.get(workflowId)?.get(
|
|
7766
|
+
return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
|
|
7767
|
+
}
|
|
7768
|
+
pendingStageDeliveryCountVersion(workflowId) {
|
|
7769
|
+
return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
|
|
7770
|
+
}
|
|
7771
|
+
bumpDeliveryCountVersion(workflowId) {
|
|
7772
|
+
this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
|
|
7038
7773
|
}
|
|
7039
7774
|
async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
|
|
7040
|
-
|
|
7041
|
-
if (
|
|
7042
|
-
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7046
|
-
|
|
7047
|
-
|
|
7048
|
-
receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
|
|
7049
|
-
if (delivery.admission === "context" && delivery.sessionId !== undefined)
|
|
7050
|
-
receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
|
|
7051
|
-
if (accounting !== undefined && !existing) {
|
|
7052
|
-
let counts = this.stickyDeliveryCounts.get(workflowId);
|
|
7775
|
+
const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
|
|
7776
|
+
if (accounting !== undefined && added) {
|
|
7777
|
+
let byRun = this.stickyDeliveryCounts.get(workflowId);
|
|
7778
|
+
if (byRun === undefined) {
|
|
7779
|
+
byRun = new Map;
|
|
7780
|
+
this.stickyDeliveryCounts.set(workflowId, byRun);
|
|
7781
|
+
}
|
|
7782
|
+
let counts = byRun.get(accounting.messageRunId);
|
|
7053
7783
|
if (counts === undefined) {
|
|
7054
7784
|
counts = new Map;
|
|
7055
|
-
|
|
7785
|
+
byRun.set(accounting.messageRunId, counts);
|
|
7056
7786
|
}
|
|
7057
|
-
const
|
|
7058
|
-
counts.set(
|
|
7787
|
+
const previous = counts.get(messageId) ?? 0;
|
|
7788
|
+
counts.set(messageId, Math.max(previous, accounting.deliveryCount));
|
|
7789
|
+
if (accounting.deliveryCount > previous)
|
|
7790
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
7059
7791
|
}
|
|
7060
7792
|
}
|
|
7061
7793
|
async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
|
|
7062
|
-
return
|
|
7794
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
|
|
7063
7795
|
}
|
|
7064
7796
|
hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
|
|
7065
|
-
return
|
|
7066
|
-
|
|
7067
|
-
|
|
7068
|
-
|
|
7797
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
|
|
7798
|
+
}
|
|
7799
|
+
pendingStageDeliveryReceiptChanges(workflowId, cursor) {
|
|
7800
|
+
return this.stickyReceipts.changesSince(workflowId, cursor);
|
|
7069
7801
|
}
|
|
7070
7802
|
recordCheckpoint(recorded) {
|
|
7071
7803
|
const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
|
|
@@ -7084,6 +7816,7 @@ class JournalProjection {
|
|
|
7084
7816
|
if (rec.checkpoints.has(key))
|
|
7085
7817
|
return;
|
|
7086
7818
|
rec.checkpoints.set(key, currentCheckpoint);
|
|
7819
|
+
rec.nestedSnapshotVersion.record(currentCheckpoint);
|
|
7087
7820
|
if (currentCheckpoint.kind === "tool") {
|
|
7088
7821
|
const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
|
|
7089
7822
|
if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
|
|
@@ -7141,6 +7874,9 @@ class JournalProjection {
|
|
|
7141
7874
|
getWorkflow(workflowId) {
|
|
7142
7875
|
return this.workflows.get(workflowId)?.handle;
|
|
7143
7876
|
}
|
|
7877
|
+
nestedSnapshotVersion(workflowId) {
|
|
7878
|
+
return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
|
|
7879
|
+
}
|
|
7144
7880
|
getLoadableWorkflow(workflowId) {
|
|
7145
7881
|
return this.getWorkflow(workflowId);
|
|
7146
7882
|
}
|
|
@@ -7218,6 +7954,7 @@ class JournalProjection {
|
|
|
7218
7954
|
this.pendingReceipts.delete(workflowId);
|
|
7219
7955
|
this.stickyReceipts.delete(workflowId);
|
|
7220
7956
|
this.stickyDeliveryCounts.delete(workflowId);
|
|
7957
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
7221
7958
|
this.deletedWorkflowIds.add(workflowId);
|
|
7222
7959
|
}
|
|
7223
7960
|
isWorkflowLoadable(workflowId) {
|
|
@@ -7228,6 +7965,8 @@ class JournalProjection {
|
|
|
7228
7965
|
this.pendingReceipts.clear();
|
|
7229
7966
|
this.stickyReceipts.clear();
|
|
7230
7967
|
this.stickyDeliveryCounts.clear();
|
|
7968
|
+
for (const workflowId of this.stickyDeliveryCountVersions.keys())
|
|
7969
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
7231
7970
|
this.deletedWorkflowIds.clear();
|
|
7232
7971
|
}
|
|
7233
7972
|
}
|
|
@@ -7281,13 +8020,10 @@ function toResumableEntry(handle) {
|
|
|
7281
8020
|
updatedAt: handle.updatedAt
|
|
7282
8021
|
};
|
|
7283
8022
|
}
|
|
7284
|
-
|
|
7285
|
-
const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
|
|
7286
|
-
return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
|
|
7287
|
-
}
|
|
7288
|
-
var SESSION_HEADER_READ_LIMIT_BYTES;
|
|
8023
|
+
var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
|
|
7289
8024
|
var init_backend = __esm(() => {
|
|
7290
8025
|
init_resume_eligibility();
|
|
8026
|
+
init_stage_topology_validation();
|
|
7291
8027
|
init_durable_hash();
|
|
7292
8028
|
SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
|
|
7293
8029
|
});
|
|
@@ -7348,205 +8084,24 @@ var init_executor_child_helpers = __esm(() => {
|
|
|
7348
8084
|
init_workflow();
|
|
7349
8085
|
});
|
|
7350
8086
|
|
|
7351
|
-
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
7352
|
-
function immutableStageGroupError(stages) {
|
|
7353
|
-
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
7354
|
-
if (occurrenceError !== undefined)
|
|
7355
|
-
return occurrenceError;
|
|
7356
|
-
const byReplayKey = groupByDurableStageKey(stages);
|
|
7357
|
-
for (const group of byReplayKey.values()) {
|
|
7358
|
-
const firstName = group[0]?.name;
|
|
7359
|
-
if (group.some((stage) => stage.name !== firstName))
|
|
7360
|
-
return "stage name drift within one replay-key group";
|
|
7361
|
-
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
7362
|
-
const first = topologies[0];
|
|
7363
|
-
if (first === undefined)
|
|
7364
|
-
continue;
|
|
7365
|
-
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
7366
|
-
return "immutable stage identity drift within one replay-key group";
|
|
7367
|
-
}
|
|
7368
|
-
}
|
|
7369
|
-
return;
|
|
7370
|
-
}
|
|
7371
|
-
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
7372
|
-
const prefix = `${replayScope}:`;
|
|
7373
|
-
const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
|
|
7374
|
-
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
7375
|
-
return "child run topology exists outside its durable invocation scope";
|
|
7376
|
-
}
|
|
7377
|
-
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
7378
|
-
return "child checkpoint ownership is not reciprocal";
|
|
7379
|
-
}
|
|
7380
|
-
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
7381
|
-
const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
7382
|
-
const immutableError = immutableStageGroupError(direct);
|
|
7383
|
-
if (immutableError !== undefined)
|
|
7384
|
-
return immutableError;
|
|
7385
|
-
const merged = [];
|
|
7386
|
-
for (const group of groupByDurableStageKey(direct).values()) {
|
|
7387
|
-
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
7388
|
-
if (topology === undefined)
|
|
7389
|
-
return "direct child stage group lacks owning-run topology";
|
|
7390
|
-
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
7391
|
-
return "direct child stage topology lacks current source identity";
|
|
7392
|
-
}
|
|
7393
|
-
if (topology.stageId.length === 0)
|
|
7394
|
-
return "direct child stage id is empty";
|
|
7395
|
-
merged.push(topology);
|
|
7396
|
-
}
|
|
7397
|
-
const ids = new Set(allowedNodeIds);
|
|
7398
|
-
const stageIds = new Set;
|
|
7399
|
-
const orders = new Set;
|
|
7400
|
-
for (const topology of merged) {
|
|
7401
|
-
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
7402
|
-
return "direct child stage ids are not unique";
|
|
7403
|
-
stageIds.add(topology.stageId);
|
|
7404
|
-
ids.add(topology.stageId);
|
|
7405
|
-
if (topology.sourceOrder !== undefined) {
|
|
7406
|
-
if (orders.has(topology.sourceOrder))
|
|
7407
|
-
return "direct child source orders are not unique";
|
|
7408
|
-
orders.add(topology.sourceOrder);
|
|
7409
|
-
}
|
|
7410
|
-
}
|
|
7411
|
-
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
7412
|
-
return "direct child stage references an unknown parent";
|
|
7413
|
-
}
|
|
7414
|
-
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
7415
|
-
}
|
|
7416
|
-
function owningBoundaryIdError(stages, owningRunId) {
|
|
7417
|
-
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
7418
|
-
const seen = new Map;
|
|
7419
|
-
for (const start of starts) {
|
|
7420
|
-
const stageId = start.topology.stageId;
|
|
7421
|
-
if (stageId.length === 0)
|
|
7422
|
-
return "owning-run boundary stage id is empty";
|
|
7423
|
-
const prior = seen.get(stageId);
|
|
7424
|
-
if (prior !== undefined && prior !== start.replayKey)
|
|
7425
|
-
return "owning-run boundary stage id is aliased";
|
|
7426
|
-
seen.set(stageId, start.replayKey);
|
|
7427
|
-
}
|
|
7428
|
-
return;
|
|
7429
|
-
}
|
|
7430
|
-
function sameRun(actual, expected) {
|
|
7431
|
-
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
7432
|
-
}
|
|
7433
|
-
function sameStrings(left, right) {
|
|
7434
|
-
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
7435
|
-
}
|
|
7436
|
-
function sameOptionalRun(actual, expected) {
|
|
7437
|
-
if (actual === undefined || expected === undefined)
|
|
7438
|
-
return actual === expected;
|
|
7439
|
-
return sameRun(actual, expected);
|
|
7440
|
-
}
|
|
7441
|
-
function promptOccurrenceIdentityError(stages) {
|
|
7442
|
-
const replayByOccurrence = new Map;
|
|
7443
|
-
const occurrenceByReplayStage = new Map;
|
|
7444
|
-
const stageByReplayOccurrence = new Map;
|
|
7445
|
-
for (const stage of stages) {
|
|
7446
|
-
const topology = stage.topology;
|
|
7447
|
-
const occurrence = topology?.occurrenceKey;
|
|
7448
|
-
if (topology === undefined || occurrence === undefined)
|
|
7449
|
-
continue;
|
|
7450
|
-
const priorReplay = replayByOccurrence.get(occurrence);
|
|
7451
|
-
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
7452
|
-
return "prompt occurrence is bound to multiple replay keys";
|
|
7453
|
-
}
|
|
7454
|
-
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
7455
|
-
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
7456
|
-
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
7457
|
-
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
7458
|
-
return "prompt stage identity is bound to multiple occurrences";
|
|
7459
|
-
}
|
|
7460
|
-
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
7461
|
-
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7462
|
-
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
7463
|
-
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
7464
|
-
return "prompt occurrence is bound to multiple stage identities";
|
|
7465
|
-
}
|
|
7466
|
-
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
7467
|
-
}
|
|
7468
|
-
return;
|
|
7469
|
-
}
|
|
7470
|
-
function durableStageGroupKey(stage) {
|
|
7471
|
-
const occurrence = stage.topology?.occurrenceKey;
|
|
7472
|
-
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
7473
|
-
}
|
|
7474
|
-
function groupByDurableStageKey(stages) {
|
|
7475
|
-
const groups = new Map;
|
|
7476
|
-
for (const stage of stages) {
|
|
7477
|
-
const key = durableStageGroupKey(stage);
|
|
7478
|
-
const group = groups.get(key) ?? [];
|
|
7479
|
-
group.push(stage);
|
|
7480
|
-
groups.set(key, group);
|
|
7481
|
-
}
|
|
7482
|
-
return groups;
|
|
7483
|
-
}
|
|
7484
|
-
function compositeKey(domain, ...parts) {
|
|
7485
|
-
return JSON.stringify([domain, ...parts]);
|
|
7486
|
-
}
|
|
7487
|
-
function hasCycle(topologies) {
|
|
7488
|
-
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
7489
|
-
const visiting = new Set;
|
|
7490
|
-
const visited = new Set;
|
|
7491
|
-
const cyclic = (stageId) => {
|
|
7492
|
-
if (visiting.has(stageId))
|
|
7493
|
-
return true;
|
|
7494
|
-
if (visited.has(stageId))
|
|
7495
|
-
return false;
|
|
7496
|
-
visiting.add(stageId);
|
|
7497
|
-
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
7498
|
-
return true;
|
|
7499
|
-
visiting.delete(stageId);
|
|
7500
|
-
visited.add(stageId);
|
|
7501
|
-
return false;
|
|
7502
|
-
};
|
|
7503
|
-
return topologies.some((topology) => cyclic(topology.stageId));
|
|
7504
|
-
}
|
|
7505
|
-
var init_stage_topology_validation = () => {};
|
|
7506
|
-
|
|
7507
|
-
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
7508
|
-
function parseWorkflowChildResult(value2) {
|
|
7509
|
-
if (!isOrdinaryObject(value2))
|
|
7510
|
-
return;
|
|
7511
|
-
const workflow = value2.workflow;
|
|
7512
|
-
const runId = value2.runId;
|
|
7513
|
-
const status = value2.status;
|
|
7514
|
-
const exited = value2.exited;
|
|
7515
|
-
const outputs = value2.outputs;
|
|
7516
|
-
const exitReason = value2.exitReason;
|
|
7517
|
-
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
7518
|
-
return;
|
|
7519
|
-
if (exited === false) {
|
|
7520
|
-
if (status !== "completed" || exitReason !== undefined)
|
|
7521
|
-
return;
|
|
7522
|
-
return value2;
|
|
7523
|
-
}
|
|
7524
|
-
if (exited !== true || !isWorkflowExitStatus2(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
7525
|
-
return;
|
|
7526
|
-
return value2;
|
|
7527
|
-
}
|
|
7528
|
-
function isWorkflowChildResult(value2) {
|
|
7529
|
-
return parseWorkflowChildResult(value2) !== undefined;
|
|
7530
|
-
}
|
|
7531
|
-
function isWorkflowExitStatus2(value2) {
|
|
7532
|
-
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
7533
|
-
}
|
|
7534
|
-
function isOrdinaryObject(value2) {
|
|
7535
|
-
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
7536
|
-
return false;
|
|
7537
|
-
const prototype = Object.getPrototypeOf(value2);
|
|
7538
|
-
return prototype === Object.prototype || prototype === null;
|
|
7539
|
-
}
|
|
7540
|
-
|
|
7541
8087
|
// dist/builtin/workflows/src/durable/boundary-lifecycle.ts
|
|
7542
8088
|
function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
7543
8089
|
const startTopology = start.topology;
|
|
7544
8090
|
const startBoundary = startTopology?.boundary;
|
|
7545
|
-
const startRun = startTopology?.run;
|
|
7546
8091
|
const related = stages.filter((stage) => {
|
|
7547
8092
|
const boundary = stage.topology?.boundary;
|
|
7548
8093
|
return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
|
|
7549
8094
|
});
|
|
8095
|
+
return resolveBoundaryLifecycleFromRecords(start, {
|
|
8096
|
+
related,
|
|
8097
|
+
outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
|
|
8098
|
+
}, requireTerminal);
|
|
8099
|
+
}
|
|
8100
|
+
function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
|
|
8101
|
+
const startTopology = start.topology;
|
|
8102
|
+
const startBoundary = startTopology?.boundary;
|
|
8103
|
+
const startRun = startTopology?.run;
|
|
8104
|
+
const related = records.related;
|
|
7550
8105
|
const starts = related.filter((stage) => stage.topology.boundary.event === "start");
|
|
7551
8106
|
if (starts.length !== 1 || starts[0] !== start)
|
|
7552
8107
|
return failure("boundary has no unique start record");
|
|
@@ -7583,7 +8138,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
|
7583
8138
|
return failure("boundary has an illegal terminal transition");
|
|
7584
8139
|
}
|
|
7585
8140
|
const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
|
|
7586
|
-
const outputRecords =
|
|
8141
|
+
const outputRecords = records.outputRecords();
|
|
7587
8142
|
if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
|
|
7588
8143
|
return failure("boundary replay key has conflicting output records");
|
|
7589
8144
|
}
|
|
@@ -9892,9 +10447,19 @@ function* invoke(action) {
|
|
|
9892
10447
|
} catch (error) {
|
|
9893
10448
|
if (error instanceof JournalError || error instanceof StorageError)
|
|
9894
10449
|
throw error;
|
|
9895
|
-
throw
|
|
10450
|
+
throw customJournalUnavailable(error);
|
|
9896
10451
|
}
|
|
9897
10452
|
}
|
|
10453
|
+
function customJournalUnavailable(failure) {
|
|
10454
|
+
const cause = failure instanceof Error ? failure : new Error(String(failure));
|
|
10455
|
+
const code = hasErrorCode(cause) ? cause.code : undefined;
|
|
10456
|
+
const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
|
|
10457
|
+
return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
|
|
10458
|
+
}
|
|
10459
|
+
function hasErrorCode(error) {
|
|
10460
|
+
const code = error.code;
|
|
10461
|
+
return typeof code === "string" || typeof code === "number";
|
|
10462
|
+
}
|
|
9898
10463
|
async function execute(operation) {
|
|
9899
10464
|
let step = operation.next();
|
|
9900
10465
|
while (!step.done) {
|
|
@@ -9922,6 +10487,8 @@ class JournalDurableBackend {
|
|
|
9922
10487
|
executions = new Map;
|
|
9923
10488
|
executionContext = new AsyncLocalStorage3;
|
|
9924
10489
|
pending = new Map;
|
|
10490
|
+
reconciledPendingStages = new Map;
|
|
10491
|
+
appendBatches = new Map;
|
|
9925
10492
|
catalogQueue = Symbol("catalog");
|
|
9926
10493
|
lost = new Map;
|
|
9927
10494
|
prompts = new Map;
|
|
@@ -9954,7 +10521,7 @@ class JournalDurableBackend {
|
|
|
9954
10521
|
await backend.close();
|
|
9955
10522
|
if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
|
|
9956
10523
|
throw error;
|
|
9957
|
-
throw
|
|
10524
|
+
throw customJournalUnavailable(error);
|
|
9958
10525
|
}
|
|
9959
10526
|
}
|
|
9960
10527
|
serialize(runId, action, requiresActiveExecution = true) {
|
|
@@ -9969,6 +10536,8 @@ class JournalDurableBackend {
|
|
|
9969
10536
|
return this.enqueue(runId, action, admitted);
|
|
9970
10537
|
}
|
|
9971
10538
|
enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
|
|
10539
|
+
if (typeof runId === "string")
|
|
10540
|
+
this.appendBatches.delete(runId);
|
|
9972
10541
|
const prior = this.pending.get(runId) ?? Promise.resolve();
|
|
9973
10542
|
const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
|
|
9974
10543
|
const settled = result.then(() => {}, () => {});
|
|
@@ -9983,6 +10552,7 @@ class JournalDurableBackend {
|
|
|
9983
10552
|
if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
|
|
9984
10553
|
return;
|
|
9985
10554
|
const event = entry.event;
|
|
10555
|
+
let raisedCount;
|
|
9986
10556
|
if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
|
|
9987
10557
|
const checkpoint = event.payload;
|
|
9988
10558
|
validateCheckpointInvocationFingerprint(checkpoint);
|
|
@@ -9991,7 +10561,10 @@ class JournalDurableBackend {
|
|
|
9991
10561
|
this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
|
|
9992
10562
|
} else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
|
|
9993
10563
|
const payload = event.payload;
|
|
10564
|
+
const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
|
|
9994
10565
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
10566
|
+
if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
|
|
10567
|
+
raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
|
|
9995
10568
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
9996
10569
|
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
9997
10570
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
@@ -10002,20 +10575,80 @@ class JournalDurableBackend {
|
|
|
10002
10575
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
10003
10576
|
this.prompts.delete(entry.runId);
|
|
10004
10577
|
}
|
|
10005
|
-
|
|
10006
|
-
if (handle?.pendingStageMessages !== undefined) {
|
|
10007
|
-
const messages = handle.pendingStageMessages.map((message) => {
|
|
10008
|
-
if (message.sticky !== true)
|
|
10009
|
-
return message;
|
|
10010
|
-
const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
|
|
10011
|
-
return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
|
|
10012
|
-
});
|
|
10013
|
-
if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
|
|
10014
|
-
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
|
|
10015
|
-
}
|
|
10578
|
+
this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
|
|
10016
10579
|
this.sequences.set(entry.runId, entry.seq);
|
|
10017
10580
|
this.foldedAt.set(entry.runId, entry.at);
|
|
10018
10581
|
}
|
|
10582
|
+
reconcileFoldedDeliveryCounts(runId, raisedCount) {
|
|
10583
|
+
const handle = this.projection.getWorkflow(runId);
|
|
10584
|
+
const list = handle?.pendingStageMessages;
|
|
10585
|
+
if (handle === undefined || list === undefined) {
|
|
10586
|
+
this.reconciledPendingStages.delete(runId);
|
|
10587
|
+
return;
|
|
10588
|
+
}
|
|
10589
|
+
let state = this.reconciledPendingStages.get(runId);
|
|
10590
|
+
let next;
|
|
10591
|
+
const raise = (index, count) => {
|
|
10592
|
+
const message = (next ?? list)[index];
|
|
10593
|
+
if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
|
|
10594
|
+
return;
|
|
10595
|
+
next ??= [...list];
|
|
10596
|
+
next[index] = { ...message, deliveryCount: count };
|
|
10597
|
+
};
|
|
10598
|
+
const reconcile = (index) => {
|
|
10599
|
+
const message = list[index];
|
|
10600
|
+
if (message.sticky !== true)
|
|
10601
|
+
return;
|
|
10602
|
+
raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
|
|
10603
|
+
};
|
|
10604
|
+
if (state?.list !== list) {
|
|
10605
|
+
const previous = state?.list ?? [];
|
|
10606
|
+
let aligned = list.length >= previous.length;
|
|
10607
|
+
const changed = [];
|
|
10608
|
+
for (let index = 0;index < list.length; index++) {
|
|
10609
|
+
const message = list[index];
|
|
10610
|
+
if (previous[index] === message)
|
|
10611
|
+
continue;
|
|
10612
|
+
if (index < previous.length && previous[index].id !== message.id)
|
|
10613
|
+
aligned = false;
|
|
10614
|
+
changed.push(index);
|
|
10615
|
+
}
|
|
10616
|
+
const reconciled = aligned ? undefined : new Set(previous);
|
|
10617
|
+
for (const index of changed)
|
|
10618
|
+
if (!reconciled?.has(list[index]))
|
|
10619
|
+
reconcile(index);
|
|
10620
|
+
const positions = aligned ? state?.positions : undefined;
|
|
10621
|
+
if (positions !== undefined) {
|
|
10622
|
+
for (let index = previous.length;index < list.length; index++) {
|
|
10623
|
+
const indexes = positions.get(list[index].id);
|
|
10624
|
+
if (indexes === undefined)
|
|
10625
|
+
positions.set(list[index].id, [index]);
|
|
10626
|
+
else
|
|
10627
|
+
indexes.push(index);
|
|
10628
|
+
}
|
|
10629
|
+
}
|
|
10630
|
+
state = { list, positions };
|
|
10631
|
+
}
|
|
10632
|
+
if (raisedCount !== undefined) {
|
|
10633
|
+
const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
|
|
10634
|
+
state.positions ??= messagePositions(list);
|
|
10635
|
+
for (const index of state.positions.get(raisedCount.messageId) ?? []) {
|
|
10636
|
+
const message = list[index];
|
|
10637
|
+
if (message.sticky === true && message.runId === raisedCount.messageRunId)
|
|
10638
|
+
raise(index, count);
|
|
10639
|
+
}
|
|
10640
|
+
}
|
|
10641
|
+
if (next === undefined) {
|
|
10642
|
+
this.reconciledPendingStages.set(runId, state);
|
|
10643
|
+
return;
|
|
10644
|
+
}
|
|
10645
|
+
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
|
|
10646
|
+
const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
|
|
10647
|
+
this.reconciledPendingStages.set(runId, {
|
|
10648
|
+
list: applied,
|
|
10649
|
+
positions: applied.length === list.length ? state.positions : undefined
|
|
10650
|
+
});
|
|
10651
|
+
}
|
|
10019
10652
|
summarize(runId, tail) {
|
|
10020
10653
|
const event = tail.snapshot.event;
|
|
10021
10654
|
const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
|
|
@@ -10039,6 +10672,7 @@ class JournalDurableBackend {
|
|
|
10039
10672
|
*rebuild(runId) {
|
|
10040
10673
|
const entries = yield* this.entries(runId);
|
|
10041
10674
|
this.projection.deleteWorkflow(runId);
|
|
10675
|
+
this.reconciledPendingStages.delete(runId);
|
|
10042
10676
|
this.sequences.delete(runId);
|
|
10043
10677
|
this.foldedAt.delete(runId);
|
|
10044
10678
|
this.summarized.delete(runId);
|
|
@@ -10134,9 +10768,9 @@ class JournalDurableBackend {
|
|
|
10134
10768
|
}
|
|
10135
10769
|
yield* this.write(runId, lease, event, options);
|
|
10136
10770
|
}
|
|
10137
|
-
*write(runId, lease, event, options) {
|
|
10771
|
+
*write(runId, lease, event, options, context = (action) => action()) {
|
|
10138
10772
|
try {
|
|
10139
|
-
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
10773
|
+
const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
|
|
10140
10774
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
10141
10775
|
if (outcome.entry.seq > nextSeq) {
|
|
10142
10776
|
for (const entry of yield* this.entries(runId, nextSeq))
|
|
@@ -10320,13 +10954,80 @@ class JournalDurableBackend {
|
|
|
10320
10954
|
return true;
|
|
10321
10955
|
}
|
|
10322
10956
|
async archivePendingStageMessage(runId, message) {
|
|
10323
|
-
await this.
|
|
10957
|
+
await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
|
|
10324
10958
|
}
|
|
10325
10959
|
async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
|
|
10326
|
-
await this.
|
|
10960
|
+
await this.queueAppend(runId, {
|
|
10327
10961
|
type: RUN_EVENT_TYPES.messageDelivered,
|
|
10328
10962
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
10329
|
-
})
|
|
10963
|
+
});
|
|
10964
|
+
}
|
|
10965
|
+
queueAppend(runId, event, key) {
|
|
10966
|
+
try {
|
|
10967
|
+
this.ensureOpen();
|
|
10968
|
+
this.assertExecutionActive();
|
|
10969
|
+
} catch (error) {
|
|
10970
|
+
return Promise.reject(error);
|
|
10971
|
+
}
|
|
10972
|
+
const open = this.appendBatches.get(runId);
|
|
10973
|
+
const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
|
|
10974
|
+
const queued = new Promise((resolve, reject) => {
|
|
10975
|
+
batch.push({
|
|
10976
|
+
event,
|
|
10977
|
+
...key === undefined ? {} : { key },
|
|
10978
|
+
context: AsyncLocalStorage3.snapshot(),
|
|
10979
|
+
resolve,
|
|
10980
|
+
reject
|
|
10981
|
+
});
|
|
10982
|
+
});
|
|
10983
|
+
if (batch !== open) {
|
|
10984
|
+
this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
|
|
10985
|
+
for (const append of batch)
|
|
10986
|
+
append.reject(error);
|
|
10987
|
+
});
|
|
10988
|
+
this.appendBatches.set(runId, batch);
|
|
10989
|
+
}
|
|
10990
|
+
return queued;
|
|
10991
|
+
}
|
|
10992
|
+
*appendBatch(runId, batch) {
|
|
10993
|
+
if (this.appendBatches.get(runId) === batch)
|
|
10994
|
+
this.appendBatches.delete(runId);
|
|
10995
|
+
const appendMany = runJournalAppendMany(this.journal);
|
|
10996
|
+
if (appendMany === undefined || batch.length === 1) {
|
|
10997
|
+
for (const append of batch) {
|
|
10998
|
+
try {
|
|
10999
|
+
const lease = yield* this.ownership(runId);
|
|
11000
|
+
yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
|
|
11001
|
+
append.resolve();
|
|
11002
|
+
} catch (error) {
|
|
11003
|
+
append.reject(error);
|
|
11004
|
+
}
|
|
11005
|
+
}
|
|
11006
|
+
return;
|
|
11007
|
+
}
|
|
11008
|
+
let settled = 0;
|
|
11009
|
+
try {
|
|
11010
|
+
const lease = yield* this.ownership(runId);
|
|
11011
|
+
const results = yield* invoke(() => appendMany(lease, batch));
|
|
11012
|
+
for (const result of results) {
|
|
11013
|
+
const append = batch[settled];
|
|
11014
|
+
if (result.status === "rejected") {
|
|
11015
|
+
append.reject(yield* this.mapOwnership(result.reason, runId));
|
|
11016
|
+
} else {
|
|
11017
|
+
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
11018
|
+
if (result.value.entry.seq > nextSeq)
|
|
11019
|
+
for (const entry of yield* this.entries(runId, nextSeq))
|
|
11020
|
+
this.fold(entry);
|
|
11021
|
+
this.fold(result.value.entry);
|
|
11022
|
+
append.resolve();
|
|
11023
|
+
}
|
|
11024
|
+
settled++;
|
|
11025
|
+
}
|
|
11026
|
+
} catch (error) {
|
|
11027
|
+
const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
|
|
11028
|
+
for (const append of batch.slice(settled))
|
|
11029
|
+
append.reject(failure);
|
|
11030
|
+
}
|
|
10330
11031
|
}
|
|
10331
11032
|
getToolOutput(...args) {
|
|
10332
11033
|
return this.projection.getToolOutput(...args);
|
|
@@ -10346,6 +11047,9 @@ class JournalDurableBackend {
|
|
|
10346
11047
|
listCheckpoints(runId) {
|
|
10347
11048
|
return this.projection.listCheckpoints(runId);
|
|
10348
11049
|
}
|
|
11050
|
+
nestedSnapshotVersion(runId) {
|
|
11051
|
+
return this.projection.nestedSnapshotVersion(runId);
|
|
11052
|
+
}
|
|
10349
11053
|
getWorkflow(runId) {
|
|
10350
11054
|
return this.projection.getWorkflow(runId);
|
|
10351
11055
|
}
|
|
@@ -10370,6 +11074,12 @@ class JournalDurableBackend {
|
|
|
10370
11074
|
getPendingStageMessageDeliveryCount(...args) {
|
|
10371
11075
|
return this.projection.getPendingStageMessageDeliveryCount(...args);
|
|
10372
11076
|
}
|
|
11077
|
+
pendingStageDeliveryReceiptChanges(...args) {
|
|
11078
|
+
return this.projection.pendingStageDeliveryReceiptChanges(...args);
|
|
11079
|
+
}
|
|
11080
|
+
pendingStageDeliveryCountVersion(runId) {
|
|
11081
|
+
return this.projection.pendingStageDeliveryCountVersion(runId);
|
|
11082
|
+
}
|
|
10373
11083
|
hasPendingStageDeliveryReceipt(...args) {
|
|
10374
11084
|
return this.projection.hasPendingStageDeliveryReceipt(...args);
|
|
10375
11085
|
}
|
|
@@ -10454,6 +11164,7 @@ class JournalDurableBackend {
|
|
|
10454
11164
|
this.foldedAt.delete(runId);
|
|
10455
11165
|
this.summarized.delete(runId);
|
|
10456
11166
|
this.prompts.delete(runId);
|
|
11167
|
+
this.reconciledPendingStages.delete(runId);
|
|
10457
11168
|
this.projection.deleteWorkflow(runId);
|
|
10458
11169
|
}
|
|
10459
11170
|
async deleteWorkflow(runId) {
|
|
@@ -10636,8 +11347,24 @@ class JournalDurableBackend {
|
|
|
10636
11347
|
}
|
|
10637
11348
|
}
|
|
10638
11349
|
}
|
|
11350
|
+
function messagePositions(list) {
|
|
11351
|
+
const positions = new Map;
|
|
11352
|
+
list.forEach((message, index) => {
|
|
11353
|
+
const indexes = positions.get(message.id);
|
|
11354
|
+
if (indexes === undefined)
|
|
11355
|
+
positions.set(message.id, [index]);
|
|
11356
|
+
else
|
|
11357
|
+
indexes.push(index);
|
|
11358
|
+
});
|
|
11359
|
+
return positions;
|
|
11360
|
+
}
|
|
10639
11361
|
function identity(message) {
|
|
10640
|
-
|
|
11362
|
+
let key = entryIdentity.get(message);
|
|
11363
|
+
if (key === undefined) {
|
|
11364
|
+
key = JSON.stringify([message.runId, message.id]);
|
|
11365
|
+
entryIdentity.set(message, key);
|
|
11366
|
+
}
|
|
11367
|
+
return key;
|
|
10641
11368
|
}
|
|
10642
11369
|
function uniqueByIdentity(messages) {
|
|
10643
11370
|
const result = new Map;
|
|
@@ -10811,7 +11538,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
10811
11538
|
function hasPendingStageDeliveryDelta(messages) {
|
|
10812
11539
|
return messages?.some(isDeltaEncoded) === true;
|
|
10813
11540
|
}
|
|
10814
|
-
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
11541
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
|
|
10815
11542
|
var init_journal_backend = __esm(() => {
|
|
10816
11543
|
init_workflow_instance_ownership_error();
|
|
10817
11544
|
init_backend();
|
|
@@ -10823,6 +11550,7 @@ var init_journal_backend = __esm(() => {
|
|
|
10823
11550
|
init_types();
|
|
10824
11551
|
PROCESS_HOLDER_ID = randomUUID4();
|
|
10825
11552
|
processExitCleanups = new Set;
|
|
11553
|
+
CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
|
|
10826
11554
|
JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
|
|
10827
11555
|
runId;
|
|
10828
11556
|
holderId;
|
|
@@ -10869,6 +11597,7 @@ var init_journal_backend = __esm(() => {
|
|
|
10869
11597
|
this.name = "JournalMessageScopeError";
|
|
10870
11598
|
}
|
|
10871
11599
|
};
|
|
11600
|
+
entryIdentity = new WeakMap;
|
|
10872
11601
|
entryJson = new WeakMap;
|
|
10873
11602
|
});
|
|
10874
11603
|
|
|
@@ -57517,9 +58246,146 @@ var init_definitions = __esm(() => {
|
|
|
57517
58246
|
workerDefinitionSources = new WeakMap;
|
|
57518
58247
|
});
|
|
57519
58248
|
|
|
58249
|
+
// dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
|
|
58250
|
+
function push(map, key, value2) {
|
|
58251
|
+
if (key === undefined)
|
|
58252
|
+
return;
|
|
58253
|
+
const list = map.get(key);
|
|
58254
|
+
if (list)
|
|
58255
|
+
list.push(value2);
|
|
58256
|
+
else
|
|
58257
|
+
map.set(key, [value2]);
|
|
58258
|
+
}
|
|
58259
|
+
|
|
58260
|
+
class Partitions {
|
|
58261
|
+
byRun = new Map;
|
|
58262
|
+
startsByChild = new Map;
|
|
58263
|
+
stagesByGroup = new Map;
|
|
58264
|
+
stagesByReplayKey = new Map;
|
|
58265
|
+
boundariesByStageId = new Map;
|
|
58266
|
+
boundariesByChild = new Map;
|
|
58267
|
+
add(entry) {
|
|
58268
|
+
const checkpoint = entry.checkpoint;
|
|
58269
|
+
push(this.byRun, checkpoint.topology?.run?.runId, entry);
|
|
58270
|
+
if (checkpoint.kind !== "stage")
|
|
58271
|
+
return;
|
|
58272
|
+
const stage = entry;
|
|
58273
|
+
push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
|
|
58274
|
+
push(this.stagesByReplayKey, checkpoint.replayKey, stage);
|
|
58275
|
+
const boundary = checkpoint.topology?.boundary;
|
|
58276
|
+
if (!boundary)
|
|
58277
|
+
return;
|
|
58278
|
+
push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
|
|
58279
|
+
push(this.boundariesByChild, boundary.child?.runId, stage);
|
|
58280
|
+
if (boundary.event === "start")
|
|
58281
|
+
push(this.startsByChild, boundary.child?.runId, stage);
|
|
58282
|
+
}
|
|
58283
|
+
}
|
|
58284
|
+
function inKeyOrder(entries) {
|
|
58285
|
+
return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
|
|
58286
|
+
}
|
|
58287
|
+
|
|
58288
|
+
class PoolCheckpointView {
|
|
58289
|
+
base;
|
|
58290
|
+
from;
|
|
58291
|
+
delta;
|
|
58292
|
+
constructor(base, from, delta) {
|
|
58293
|
+
this.base = base;
|
|
58294
|
+
this.from = from;
|
|
58295
|
+
this.delta = delta;
|
|
58296
|
+
}
|
|
58297
|
+
entries(partition, key) {
|
|
58298
|
+
const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
|
|
58299
|
+
return [...committed, ...this.delta[partition].get(key) ?? []];
|
|
58300
|
+
}
|
|
58301
|
+
}
|
|
58302
|
+
|
|
58303
|
+
class PoolCheckpointRead {
|
|
58304
|
+
index;
|
|
58305
|
+
root;
|
|
58306
|
+
pending;
|
|
58307
|
+
constructor(index, root) {
|
|
58308
|
+
this.index = index;
|
|
58309
|
+
this.root = root;
|
|
58310
|
+
}
|
|
58311
|
+
read(tx, events) {
|
|
58312
|
+
return this.index.read(tx, events, this.root, (apply) => {
|
|
58313
|
+
this.pending = apply;
|
|
58314
|
+
});
|
|
58315
|
+
}
|
|
58316
|
+
commit() {
|
|
58317
|
+
this.pending?.();
|
|
58318
|
+
this.pending = undefined;
|
|
58319
|
+
}
|
|
58320
|
+
}
|
|
58321
|
+
|
|
58322
|
+
class PoolCheckpointIndex {
|
|
58323
|
+
roots = new Map;
|
|
58324
|
+
begin(root) {
|
|
58325
|
+
return new PoolCheckpointRead(this, root);
|
|
58326
|
+
}
|
|
58327
|
+
async read(tx, events, root, onCommit) {
|
|
58328
|
+
const current = this.roots.get(root);
|
|
58329
|
+
let base = current;
|
|
58330
|
+
const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
|
|
58331
|
+
if (base && base.seq > 0) {
|
|
58332
|
+
const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
|
|
58333
|
+
if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
|
|
58334
|
+
base = undefined;
|
|
58335
|
+
}
|
|
58336
|
+
const from = base?.seq ?? 0;
|
|
58337
|
+
const to = head ? Number(head.seq) : 0;
|
|
58338
|
+
const delta = new Partitions;
|
|
58339
|
+
const added = [];
|
|
58340
|
+
if (to > from) {
|
|
58341
|
+
const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
|
|
58342
|
+
for (const row of rows) {
|
|
58343
|
+
const checkpoint = JSON.parse(String(row.payload));
|
|
58344
|
+
if (checkpoint.kind === "ui")
|
|
58345
|
+
continue;
|
|
58346
|
+
const entry = {
|
|
58347
|
+
seq: Number(row.seq),
|
|
58348
|
+
order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
|
|
58349
|
+
checkpoint
|
|
58350
|
+
};
|
|
58351
|
+
delta.add(entry);
|
|
58352
|
+
added.push(entry);
|
|
58353
|
+
}
|
|
58354
|
+
}
|
|
58355
|
+
onCommit(() => {
|
|
58356
|
+
const latest = this.roots.get(root);
|
|
58357
|
+
if (!base) {
|
|
58358
|
+
if (latest !== current)
|
|
58359
|
+
return;
|
|
58360
|
+
const fresh = {
|
|
58361
|
+
seq: to,
|
|
58362
|
+
key: head?.key ?? null,
|
|
58363
|
+
at: Number(head?.at ?? 0),
|
|
58364
|
+
partitions: delta
|
|
58365
|
+
};
|
|
58366
|
+
this.roots.set(root, fresh);
|
|
58367
|
+
return;
|
|
58368
|
+
}
|
|
58369
|
+
if (latest !== base || to <= base.seq)
|
|
58370
|
+
return;
|
|
58371
|
+
for (const entry of added)
|
|
58372
|
+
if (entry.seq > base.seq)
|
|
58373
|
+
base.partitions.add(entry);
|
|
58374
|
+
base.seq = to;
|
|
58375
|
+
base.key = head?.key ?? null;
|
|
58376
|
+
base.at = Number(head?.at ?? 0);
|
|
58377
|
+
});
|
|
58378
|
+
return new PoolCheckpointView(base, from, delta);
|
|
58379
|
+
}
|
|
58380
|
+
}
|
|
58381
|
+
var init_pool_checkpoint_index = __esm(() => {
|
|
58382
|
+
init_stage_topology_validation();
|
|
58383
|
+
init_types();
|
|
58384
|
+
});
|
|
58385
|
+
|
|
57520
58386
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
57521
58387
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
57522
|
-
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
58388
|
+
async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
|
|
57523
58389
|
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
57524
58390
|
throw new Error("Invalid pool mutation");
|
|
57525
58391
|
const checkpoint = mutation.checkpoint;
|
|
@@ -57539,42 +58405,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
|
57539
58405
|
}
|
|
57540
58406
|
if (checkpoint.kind === "ui")
|
|
57541
58407
|
return;
|
|
57542
|
-
const
|
|
57543
|
-
const records = rows.map((row) => JSON.parse(String(row.payload)));
|
|
57544
|
-
const stages = records.filter((entry) => entry.kind === "stage");
|
|
58408
|
+
const view = await checkpoints.read(tx, journal.events);
|
|
57545
58409
|
const topology = checkpoint.topology;
|
|
57546
58410
|
const run = topology?.run;
|
|
57547
58411
|
if (!topology || !run || run.rootRunId !== root || run.runId === root)
|
|
57548
58412
|
throw new Error("Pool topology scope mismatch");
|
|
57549
|
-
const owner =
|
|
58413
|
+
const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
|
|
57550
58414
|
if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
|
|
57551
58415
|
throw new Error("Pool topology scope mismatch");
|
|
57552
58416
|
if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
|
|
57553
58417
|
throw new Error("Pool session scope mismatch");
|
|
58418
|
+
const runEntries = view.entries("byRun", run.runId);
|
|
57554
58419
|
if (checkpoint.kind === "tool") {
|
|
57555
58420
|
const tool = checkpoint.topology;
|
|
57556
|
-
const peers =
|
|
58421
|
+
const peers = runEntries.map((entry) => entry.checkpoint);
|
|
57557
58422
|
const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
|
|
57558
58423
|
if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
|
|
57559
58424
|
throw new Error("Pool tool topology scope mismatch");
|
|
57560
58425
|
return;
|
|
57561
58426
|
}
|
|
57562
|
-
const
|
|
57563
|
-
const
|
|
58427
|
+
const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
|
|
58428
|
+
const relevant = [...runStages];
|
|
58429
|
+
const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
|
|
58430
|
+
for (const group of groups)
|
|
58431
|
+
relevant.push(...view.entries("stagesByGroup", group));
|
|
58432
|
+
const boundary = checkpoint.topology?.boundary;
|
|
58433
|
+
let start;
|
|
58434
|
+
if (boundary) {
|
|
58435
|
+
const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
|
|
58436
|
+
start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
58437
|
+
relevant.push(...replayed);
|
|
58438
|
+
if (start) {
|
|
58439
|
+
const startTopology = start.topology;
|
|
58440
|
+
relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
|
|
58441
|
+
relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
|
|
58442
|
+
}
|
|
58443
|
+
}
|
|
58444
|
+
const next = [...stagesExcept(relevant, checkpoint), checkpoint];
|
|
58445
|
+
const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
|
|
57564
58446
|
const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
|
|
57565
58447
|
if (stageError)
|
|
57566
58448
|
throw new Error(`Pool topology scope: ${stageError}`);
|
|
57567
|
-
const boundary = checkpoint.topology?.boundary;
|
|
57568
58449
|
if (!boundary) {
|
|
57569
58450
|
if (parseWorkflowChildResult(checkpoint.output))
|
|
57570
58451
|
throw new Error("Pool boundary output without ownership");
|
|
57571
58452
|
return;
|
|
57572
58453
|
}
|
|
57573
|
-
const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
57574
58454
|
const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
|
|
57575
58455
|
if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
|
|
57576
58456
|
throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
|
|
57577
58457
|
}
|
|
58458
|
+
function stagesExcept(entries, checkpoint) {
|
|
58459
|
+
return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
|
|
58460
|
+
}
|
|
57578
58461
|
async function sessionStore(tx, prefix) {
|
|
57579
58462
|
const database = {
|
|
57580
58463
|
descriptor: { adapter: "pool-transaction", location: prefix },
|
|
@@ -57623,20 +58506,52 @@ var init_pool_mutations = __esm(() => {
|
|
|
57623
58506
|
init_boundary_lifecycle();
|
|
57624
58507
|
init_boundary_topology();
|
|
57625
58508
|
init_stage_topology_validation();
|
|
57626
|
-
|
|
58509
|
+
init_pool_checkpoint_index();
|
|
57627
58510
|
});
|
|
57628
58511
|
|
|
57629
58512
|
// dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
|
|
57630
58513
|
function poolUsageTable(runs) {
|
|
57631
58514
|
return `${runs}_pool_usage`;
|
|
57632
58515
|
}
|
|
58516
|
+
function poolUsageTotalsTable(runs) {
|
|
58517
|
+
return `${runs}_pool_usage_totals`;
|
|
58518
|
+
}
|
|
58519
|
+
function addToTotals(totals, runs, work, sign, row) {
|
|
58520
|
+
const assignments = POOL_USAGE_FIELDS.map((field) => {
|
|
58521
|
+
const total = field.toLowerCase();
|
|
58522
|
+
const value2 = `(${sign}${row}.${total})`;
|
|
58523
|
+
return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
|
|
58524
|
+
});
|
|
58525
|
+
return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
|
|
58526
|
+
}
|
|
57633
58527
|
function poolUsageSchema(runs) {
|
|
57634
|
-
|
|
58528
|
+
const usage = poolUsageTable(runs);
|
|
58529
|
+
const totals = poolUsageTotalsTable(runs);
|
|
58530
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
58531
|
+
return `CREATE TABLE IF NOT EXISTS ${usage}(
|
|
57635
58532
|
work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
|
|
57636
58533
|
generation INTEGER NOT NULL, key TEXT NOT NULL,
|
|
57637
58534
|
input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
|
|
57638
58535
|
PRIMARY KEY(work, generation, key)
|
|
57639
|
-
)
|
|
58536
|
+
);
|
|
58537
|
+
CREATE TABLE IF NOT EXISTS ${totals}(
|
|
58538
|
+
root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
|
|
58539
|
+
${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
|
|
58540
|
+
);
|
|
58541
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
|
|
58542
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
58543
|
+
END;
|
|
58544
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
|
|
58545
|
+
WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
|
|
58546
|
+
${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
|
|
58547
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
58548
|
+
END;
|
|
58549
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
|
|
58550
|
+
DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
|
|
58551
|
+
END;
|
|
58552
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
|
|
58553
|
+
DELETE FROM ${totals} WHERE root = OLD.root;
|
|
58554
|
+
END;`;
|
|
57640
58555
|
}
|
|
57641
58556
|
function poolStageUsageKey(runId, stageId) {
|
|
57642
58557
|
return `stage:${runId}:${stageId}`;
|
|
@@ -57663,7 +58578,7 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
57663
58578
|
|
|
57664
58579
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
57665
58580
|
import { createHash as createHash4, randomUUID as randomUUID10 } from "node:crypto";
|
|
57666
|
-
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
58581
|
+
import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
|
|
57667
58582
|
function requestIdentity(request) {
|
|
57668
58583
|
return createHash4("sha256").update(request).digest("hex");
|
|
57669
58584
|
}
|
|
@@ -57702,6 +58617,7 @@ class SqlPoolQueue {
|
|
|
57702
58617
|
operations;
|
|
57703
58618
|
transaction;
|
|
57704
58619
|
journal;
|
|
58620
|
+
checkpoints = new PoolCheckpointIndex;
|
|
57705
58621
|
constructor(storage) {
|
|
57706
58622
|
this.storage = storage;
|
|
57707
58623
|
const { transaction, tables: journal } = storage;
|
|
@@ -57813,17 +58729,23 @@ class SqlPoolQueue {
|
|
|
57813
58729
|
lease = snapshotLease(lease);
|
|
57814
58730
|
token = { ...token };
|
|
57815
58731
|
return this.storage.withAppendGuard(async (tx, runId) => {
|
|
57816
|
-
|
|
57817
|
-
|
|
57818
|
-
|
|
57819
|
-
|
|
57820
|
-
|
|
57821
|
-
|
|
57822
|
-
|
|
57823
|
-
|
|
57824
|
-
|
|
57825
|
-
|
|
57826
|
-
|
|
58732
|
+
try {
|
|
58733
|
+
if (runId !== lease.runId)
|
|
58734
|
+
throw new Error("Pool delivery cannot write another root");
|
|
58735
|
+
await this.authority(tx, lease).catch((error) => {
|
|
58736
|
+
if (error instanceof StaleLeaseError)
|
|
58737
|
+
throw new Error("Stale pool coordinator authority", { cause: error });
|
|
58738
|
+
throw error;
|
|
58739
|
+
});
|
|
58740
|
+
const row = await this.current(tx, lease, token);
|
|
58741
|
+
if (row.epoch !== lease.epoch)
|
|
58742
|
+
throw new Error(STALE_WORK_LEASE);
|
|
58743
|
+
assertSession();
|
|
58744
|
+
} catch (error) {
|
|
58745
|
+
if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
|
|
58746
|
+
throw error;
|
|
58747
|
+
throw new PoolAppendRefusedError(error.message, { cause: error });
|
|
58748
|
+
}
|
|
57827
58749
|
}, action);
|
|
57828
58750
|
}
|
|
57829
58751
|
async assertCurrent(lease, token) {
|
|
@@ -57936,7 +58858,8 @@ class SqlPoolQueue {
|
|
|
57936
58858
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
57937
58859
|
const identity = requestIdentity(request);
|
|
57938
58860
|
mutation = JSON.parse(request).mutation;
|
|
57939
|
-
|
|
58861
|
+
const checkpoints = this.checkpoints.begin(lease.runId);
|
|
58862
|
+
const receipt = await this.transaction(async (tx) => {
|
|
57940
58863
|
const { now, status } = await this.authority(tx, lease);
|
|
57941
58864
|
const row = await this.current(tx, lease, token);
|
|
57942
58865
|
if (row.epoch !== lease.epoch)
|
|
@@ -57951,24 +58874,26 @@ class SqlPoolQueue {
|
|
|
57951
58874
|
throw new Error(STALE_WORK_LEASE);
|
|
57952
58875
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
57953
58876
|
throw new Error("Pool root is not running");
|
|
57954
|
-
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
58877
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
|
|
57955
58878
|
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
57956
58879
|
return receipt;
|
|
57957
58880
|
});
|
|
58881
|
+
checkpoints.commit();
|
|
58882
|
+
return receipt;
|
|
57958
58883
|
}
|
|
57959
58884
|
async appendSession(tx, row, mutation, now) {
|
|
57960
58885
|
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
57961
58886
|
}
|
|
57962
|
-
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
58887
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
|
|
57963
58888
|
const checkpoint = mutation.checkpoint;
|
|
57964
|
-
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
58889
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
|
|
57965
58890
|
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
57966
58891
|
throw new Error("Pool checkpoint timestamp is in the future");
|
|
57967
58892
|
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
57968
58893
|
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
57969
58894
|
const payload = serializeJson(checkpoint);
|
|
57970
58895
|
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
57971
|
-
if (existing && existing.payload
|
|
58896
|
+
if (existing && !isRecordedCheckpoint(existing.payload, payload))
|
|
57972
58897
|
throw new Error("Pool checkpoint identity conflict");
|
|
57973
58898
|
if (existing && mutation.kind !== "checkpoint")
|
|
57974
58899
|
throw new Error("Pool session checkpoint identity conflict");
|
|
@@ -58066,24 +58991,60 @@ class SqlPoolQueue {
|
|
|
58066
58991
|
return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
|
|
58067
58992
|
}
|
|
58068
58993
|
}
|
|
58994
|
+
function isRecordedCheckpoint(stored, payload) {
|
|
58995
|
+
if (stored === payload)
|
|
58996
|
+
return true;
|
|
58997
|
+
const { completedAt } = JSON.parse(stored);
|
|
58998
|
+
if (typeof completedAt !== "number")
|
|
58999
|
+
return false;
|
|
59000
|
+
return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
|
|
59001
|
+
}
|
|
58069
59002
|
function openSqlPoolQueue(journal) {
|
|
58070
59003
|
const storage = sqlPoolStorage(journal);
|
|
58071
59004
|
if (!storage)
|
|
58072
59005
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
58073
59006
|
return SqlPoolQueue.open(storage);
|
|
58074
59007
|
}
|
|
58075
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
59008
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
|
|
58076
59009
|
var init_sql_pool_queue = __esm(() => {
|
|
58077
59010
|
init_serializable();
|
|
58078
59011
|
init_boundary_topology();
|
|
58079
59012
|
init_durable_hash();
|
|
58080
59013
|
init_internal();
|
|
59014
|
+
init_pool_checkpoint_index();
|
|
58081
59015
|
init_pool_mutations();
|
|
58082
59016
|
init_pool_stage_usage();
|
|
58083
59017
|
init_types();
|
|
58084
59018
|
init_values();
|
|
59019
|
+
PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
|
|
59020
|
+
constructor(message, options) {
|
|
59021
|
+
super(message, options);
|
|
59022
|
+
this.name = "PoolAppendRefusedError";
|
|
59023
|
+
}
|
|
59024
|
+
};
|
|
58085
59025
|
});
|
|
58086
59026
|
|
|
59027
|
+
// dist/builtin/workflows/src/engine/session-closed-error.ts
|
|
59028
|
+
import { isStaleExtensionContextError as isStaleExtensionContextError2 } from "@bastani/atomic";
|
|
59029
|
+
function reportsClosedSession(error) {
|
|
59030
|
+
return "code" in error && error.code === "SessionClosed" || isStaleExtensionContextError2(error);
|
|
59031
|
+
}
|
|
59032
|
+
function isSessionClosedError(error) {
|
|
59033
|
+
let current = error;
|
|
59034
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined; depth++) {
|
|
59035
|
+
try {
|
|
59036
|
+
if (reportsClosedSession(current))
|
|
59037
|
+
return true;
|
|
59038
|
+
current = current.cause instanceof Error ? current.cause : undefined;
|
|
59039
|
+
} catch {
|
|
59040
|
+
return false;
|
|
59041
|
+
}
|
|
59042
|
+
}
|
|
59043
|
+
return false;
|
|
59044
|
+
}
|
|
59045
|
+
var MAX_CAUSE_DEPTH = 8;
|
|
59046
|
+
var init_session_closed_error = () => {};
|
|
59047
|
+
|
|
58087
59048
|
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
58088
59049
|
function claimExternalRunRoute(store, runId, owner) {
|
|
58089
59050
|
let owners = registry.owners.get(store);
|
|
@@ -58409,39 +59370,399 @@ var init_pending_stage_status = __esm(() => {
|
|
|
58409
59370
|
init_workflow_stage_target();
|
|
58410
59371
|
});
|
|
58411
59372
|
|
|
58412
|
-
// dist/builtin/workflows/src/shared/sticky-
|
|
58413
|
-
function
|
|
58414
|
-
|
|
58415
|
-
|
|
58416
|
-
const
|
|
58417
|
-
|
|
58418
|
-
const
|
|
58419
|
-
|
|
58420
|
-
const
|
|
58421
|
-
|
|
59373
|
+
// dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
|
|
59374
|
+
function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
|
|
59375
|
+
if (states.length === 0)
|
|
59376
|
+
return { deliveryStates: states };
|
|
59377
|
+
const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
|
|
59378
|
+
counts.set("delivered", 0);
|
|
59379
|
+
for (const delivery of states)
|
|
59380
|
+
counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
|
|
59381
|
+
const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
|
|
59382
|
+
if (states.length <= limit)
|
|
59383
|
+
return { deliveryStates: states, deliveryStateCounts };
|
|
59384
|
+
const exemplars = [];
|
|
59385
|
+
for (const state of UNRESOLVED_STATES) {
|
|
59386
|
+
for (const delivery of states) {
|
|
59387
|
+
if (exemplars.length === limit)
|
|
59388
|
+
break;
|
|
59389
|
+
if (delivery.state === state)
|
|
59390
|
+
exemplars.push(delivery);
|
|
59391
|
+
}
|
|
59392
|
+
}
|
|
59393
|
+
for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
|
|
59394
|
+
const delivery = states[index];
|
|
59395
|
+
if (delivery.state === "delivered")
|
|
59396
|
+
exemplars.push(delivery);
|
|
59397
|
+
}
|
|
59398
|
+
return {
|
|
59399
|
+
deliveryStates: exemplars,
|
|
59400
|
+
deliveryStateCounts,
|
|
59401
|
+
omittedDeliveryStates: states.length - exemplars.length
|
|
59402
|
+
};
|
|
59403
|
+
}
|
|
59404
|
+
var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
|
|
59405
|
+
var init_sticky_delivery_state_bound = __esm(() => {
|
|
59406
|
+
UNRESOLVED_STATES = [
|
|
59407
|
+
"queued",
|
|
59408
|
+
"receipt-unverified",
|
|
59409
|
+
"session-replaced",
|
|
59410
|
+
"delivery-unavailable",
|
|
59411
|
+
"root-unresolved",
|
|
59412
|
+
"stage-terminal"
|
|
59413
|
+
];
|
|
59414
|
+
});
|
|
59415
|
+
|
|
59416
|
+
// dist/builtin/workflows/src/shared/possible-stage-rows.ts
|
|
59417
|
+
function boundPossibleStageDeliveryStates(states) {
|
|
59418
|
+
const bounded = boundStickyDeliveryStates(states);
|
|
59419
|
+
return {
|
|
59420
|
+
...bounded,
|
|
59421
|
+
deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
|
|
59422
|
+
};
|
|
59423
|
+
}
|
|
59424
|
+
|
|
59425
|
+
class WorkflowTopologyIndex {
|
|
59426
|
+
lineage = [];
|
|
59427
|
+
runById = new Map;
|
|
59428
|
+
roots = new Map;
|
|
59429
|
+
hops = new Map;
|
|
59430
|
+
refresh(runs) {
|
|
59431
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
59432
|
+
const lineage = [];
|
|
59433
|
+
for (const run of runs) {
|
|
59434
|
+
lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
|
|
59435
|
+
if (run.parentRunId === undefined)
|
|
59436
|
+
continue;
|
|
59437
|
+
const parent = runById.get(run.parentRunId);
|
|
59438
|
+
const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
|
|
59439
|
+
lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
|
|
59440
|
+
}
|
|
59441
|
+
if (sameValues(lineage, this.lineage))
|
|
59442
|
+
return;
|
|
59443
|
+
this.lineage = lineage;
|
|
59444
|
+
this.runById = runById;
|
|
59445
|
+
this.roots.clear();
|
|
59446
|
+
this.hops.clear();
|
|
59447
|
+
}
|
|
59448
|
+
rootOf(runId) {
|
|
59449
|
+
if (this.roots.has(runId))
|
|
59450
|
+
return this.roots.get(runId);
|
|
59451
|
+
const root = reciprocalWorkflowRootRunId(this.runById, runId);
|
|
59452
|
+
this.roots.set(runId, root);
|
|
59453
|
+
return root;
|
|
59454
|
+
}
|
|
59455
|
+
hopsOf(runId) {
|
|
59456
|
+
if (this.hops.has(runId))
|
|
59457
|
+
return this.hops.get(runId);
|
|
59458
|
+
const hops = boundaryHops(this.runById, runId);
|
|
59459
|
+
this.hops.set(runId, hops);
|
|
59460
|
+
return hops;
|
|
59461
|
+
}
|
|
59462
|
+
}
|
|
59463
|
+
function boundaryHops(runById, runId) {
|
|
59464
|
+
const hops = [];
|
|
59465
|
+
let current = runById.get(runId);
|
|
59466
|
+
while (current !== undefined && current.parentRunId !== undefined) {
|
|
59467
|
+
const parent = runById.get(current.parentRunId);
|
|
59468
|
+
const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
|
|
59469
|
+
if (parent === undefined || boundary === undefined)
|
|
59470
|
+
return;
|
|
59471
|
+
hops.unshift({ name: boundary.name, runId: current.id });
|
|
59472
|
+
current = parent;
|
|
59473
|
+
}
|
|
59474
|
+
return current === undefined ? undefined : hops;
|
|
59475
|
+
}
|
|
59476
|
+
function sameValues(left, right) {
|
|
59477
|
+
if (left.length !== right.length)
|
|
59478
|
+
return false;
|
|
59479
|
+
for (let index = 0;index < left.length; index++)
|
|
59480
|
+
if (left[index] !== right[index])
|
|
59481
|
+
return false;
|
|
59482
|
+
return true;
|
|
59483
|
+
}
|
|
59484
|
+
|
|
59485
|
+
class PossibleStageRowsIndex {
|
|
59486
|
+
rootRunId;
|
|
59487
|
+
topology;
|
|
59488
|
+
targets = new Map;
|
|
59489
|
+
patterns = [];
|
|
59490
|
+
targetOf = new Map;
|
|
59491
|
+
targetsByMessage = new Map;
|
|
59492
|
+
list = [];
|
|
59493
|
+
stageFields = new WeakMap;
|
|
59494
|
+
backend;
|
|
59495
|
+
receiptCursor;
|
|
59496
|
+
scanMatches = new Map;
|
|
59497
|
+
rowCache = new Map;
|
|
59498
|
+
boundedStates = new WeakMap;
|
|
59499
|
+
deliveriesByEntry = new WeakMap;
|
|
59500
|
+
statusCache = new WeakMap;
|
|
59501
|
+
receiptGeneration = 0;
|
|
59502
|
+
receiptEpochs = new Map;
|
|
59503
|
+
constructor(rootRunId, topology) {
|
|
59504
|
+
this.rootRunId = rootRunId;
|
|
59505
|
+
this.topology = topology;
|
|
59506
|
+
}
|
|
59507
|
+
rows(runs, rootRun, availableBackend) {
|
|
59508
|
+
if (isTerminalRunStatus(rootRun.status))
|
|
59509
|
+
return [];
|
|
59510
|
+
this.refreshEntries(rootRun.pendingStageMessages ?? []);
|
|
59511
|
+
const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
|
|
59512
|
+
const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
|
|
59513
|
+
this.refreshReceipts(backend);
|
|
59514
|
+
this.refreshStages(runs);
|
|
59515
|
+
for (const state of this.patterns)
|
|
59516
|
+
if (state.dirty)
|
|
59517
|
+
this.recompute(state, backend);
|
|
59518
|
+
return this.assemble(runs, rootRun);
|
|
59519
|
+
}
|
|
59520
|
+
targetFor(entry) {
|
|
58422
59521
|
const target = entry.targetPath ?? entry.stageKey;
|
|
58423
|
-
let
|
|
58424
|
-
if (
|
|
59522
|
+
let state = this.targets.get(target);
|
|
59523
|
+
if (state === undefined) {
|
|
58425
59524
|
const parsed = parseWorkflowStageTarget(target);
|
|
58426
|
-
|
|
58427
|
-
|
|
58428
|
-
|
|
58429
|
-
|
|
59525
|
+
const owned = parsed?.rootRunId === this.rootRunId;
|
|
59526
|
+
state = {
|
|
59527
|
+
target,
|
|
59528
|
+
segments: owned ? parsed.segments : undefined,
|
|
59529
|
+
pattern: owned ? this.patterns.length : -1,
|
|
59530
|
+
entries: [],
|
|
59531
|
+
positions: [],
|
|
59532
|
+
queued: 0,
|
|
59533
|
+
stages: [],
|
|
59534
|
+
statuses: [],
|
|
59535
|
+
dirty: true
|
|
59536
|
+
};
|
|
59537
|
+
this.targets.set(target, state);
|
|
59538
|
+
if (owned)
|
|
59539
|
+
this.patterns.push(state);
|
|
58430
59540
|
}
|
|
58431
|
-
|
|
59541
|
+
return state;
|
|
59542
|
+
}
|
|
59543
|
+
refreshEntries(list) {
|
|
59544
|
+
const previous = this.list;
|
|
59545
|
+
if (list === previous)
|
|
58432
59546
|
return;
|
|
58433
|
-
|
|
58434
|
-
|
|
58435
|
-
|
|
58436
|
-
|
|
58437
|
-
|
|
58438
|
-
|
|
58439
|
-
|
|
58440
|
-
|
|
58441
|
-
|
|
59547
|
+
this.list = list;
|
|
59548
|
+
let incremental = list.length >= previous.length;
|
|
59549
|
+
const replaced = [];
|
|
59550
|
+
for (let index = 0;incremental && index < previous.length; index++) {
|
|
59551
|
+
if (list[index] === previous[index])
|
|
59552
|
+
continue;
|
|
59553
|
+
const before = previous[index];
|
|
59554
|
+
const after = list[index];
|
|
59555
|
+
if (before.sticky !== true && after.sticky !== true)
|
|
59556
|
+
continue;
|
|
59557
|
+
if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
|
|
59558
|
+
incremental = false;
|
|
59559
|
+
break;
|
|
59560
|
+
}
|
|
59561
|
+
replaced.push(index);
|
|
59562
|
+
}
|
|
59563
|
+
if (!incremental) {
|
|
59564
|
+
this.rebuildEntries(list);
|
|
59565
|
+
return;
|
|
59566
|
+
}
|
|
59567
|
+
for (const index of replaced) {
|
|
59568
|
+
const before = previous[index];
|
|
59569
|
+
const after = list[index];
|
|
59570
|
+
const state = this.targetOf.get(before);
|
|
59571
|
+
const slot = state.entries.indexOf(before);
|
|
59572
|
+
state.entries[slot] = after;
|
|
59573
|
+
this.targetOf.delete(before);
|
|
59574
|
+
this.targetOf.set(after, state);
|
|
59575
|
+
state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
|
|
59576
|
+
if (before.id !== after.id)
|
|
59577
|
+
this.rebuildMessageIndex();
|
|
59578
|
+
state.dirty = true;
|
|
59579
|
+
}
|
|
59580
|
+
for (let index = previous.length;index < list.length; index++)
|
|
59581
|
+
this.addEntry(list[index], index);
|
|
59582
|
+
}
|
|
59583
|
+
addEntry(entry, position) {
|
|
59584
|
+
if (entry.sticky !== true)
|
|
59585
|
+
return;
|
|
59586
|
+
const state = this.targetFor(entry);
|
|
59587
|
+
state.entries.push(entry);
|
|
59588
|
+
state.positions.push(position);
|
|
59589
|
+
if (entry.status === "queued")
|
|
59590
|
+
state.queued++;
|
|
59591
|
+
state.dirty = true;
|
|
59592
|
+
this.targetOf.set(entry, state);
|
|
59593
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
59594
|
+
if (targets === undefined) {
|
|
59595
|
+
targets = new Set;
|
|
59596
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
59597
|
+
}
|
|
59598
|
+
targets.add(state);
|
|
59599
|
+
}
|
|
59600
|
+
rebuildEntries(list) {
|
|
59601
|
+
const before = new Map;
|
|
59602
|
+
for (const state of this.targets.values()) {
|
|
59603
|
+
before.set(state, state.entries);
|
|
59604
|
+
state.entries = [];
|
|
59605
|
+
state.positions = [];
|
|
59606
|
+
state.queued = 0;
|
|
59607
|
+
}
|
|
59608
|
+
this.targetOf.clear();
|
|
59609
|
+
this.targetsByMessage.clear();
|
|
59610
|
+
list.forEach((entry, position) => {
|
|
59611
|
+
this.addEntry(entry, position);
|
|
59612
|
+
});
|
|
59613
|
+
for (const state of this.targets.values()) {
|
|
59614
|
+
const previous = before.get(state);
|
|
59615
|
+
state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
|
|
59616
|
+
}
|
|
59617
|
+
}
|
|
59618
|
+
rebuildMessageIndex() {
|
|
59619
|
+
this.targetsByMessage.clear();
|
|
59620
|
+
for (const [entry, state] of this.targetOf) {
|
|
59621
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
59622
|
+
if (targets === undefined) {
|
|
59623
|
+
targets = new Set;
|
|
59624
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
59625
|
+
}
|
|
59626
|
+
targets.add(state);
|
|
59627
|
+
}
|
|
59628
|
+
}
|
|
59629
|
+
refreshReceipts(backend) {
|
|
59630
|
+
if (backend !== this.backend) {
|
|
59631
|
+
this.backend = backend;
|
|
59632
|
+
this.receiptCursor = undefined;
|
|
59633
|
+
this.invalidateReceipts();
|
|
59634
|
+
}
|
|
59635
|
+
if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
|
|
59636
|
+
return;
|
|
59637
|
+
const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
|
|
59638
|
+
if (changes === undefined) {
|
|
59639
|
+
this.invalidateReceipts();
|
|
59640
|
+
return;
|
|
59641
|
+
}
|
|
59642
|
+
const firstRead = this.receiptCursor === undefined;
|
|
59643
|
+
this.receiptCursor = changes.cursor;
|
|
59644
|
+
if (changes.messageIds === undefined) {
|
|
59645
|
+
if (!firstRead)
|
|
59646
|
+
this.invalidateReceipts();
|
|
59647
|
+
return;
|
|
59648
|
+
}
|
|
59649
|
+
for (const messageId of changes.messageIds) {
|
|
59650
|
+
this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
|
|
59651
|
+
for (const state of this.targetsByMessage.get(messageId) ?? [])
|
|
59652
|
+
state.dirty = true;
|
|
59653
|
+
}
|
|
59654
|
+
}
|
|
59655
|
+
invalidateReceipts() {
|
|
59656
|
+
this.receiptGeneration++;
|
|
59657
|
+
this.receiptEpochs.clear();
|
|
59658
|
+
for (const state of this.patterns)
|
|
59659
|
+
state.dirty = true;
|
|
59660
|
+
}
|
|
59661
|
+
refreshStages(runs) {
|
|
59662
|
+
const current = new Map;
|
|
59663
|
+
let ordinal = 0;
|
|
59664
|
+
for (const run of runs) {
|
|
59665
|
+
const root = this.topology.rootOf(run.id);
|
|
59666
|
+
if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
|
|
59667
|
+
continue;
|
|
59668
|
+
const resolved = root === this.rootRunId;
|
|
59669
|
+
const hops = this.topology.hopsOf(run.id);
|
|
59670
|
+
for (const stage of run.stages) {
|
|
59671
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
59672
|
+
continue;
|
|
59673
|
+
const fields = this.fieldsOf(run, stage, resolved, hops);
|
|
59674
|
+
const status = { run, stage, resolved, hops, ordinal: ordinal++ };
|
|
59675
|
+
for (const state of fields.matched) {
|
|
59676
|
+
const stages = current.get(state);
|
|
59677
|
+
if (stages === undefined)
|
|
59678
|
+
current.set(state, [status]);
|
|
59679
|
+
else
|
|
59680
|
+
stages.push(status);
|
|
59681
|
+
}
|
|
59682
|
+
}
|
|
59683
|
+
}
|
|
59684
|
+
for (const state of this.patterns) {
|
|
59685
|
+
const stages = current.get(state) ?? [];
|
|
59686
|
+
if (stages.length !== state.stages.length || stages.some((status, index) => {
|
|
59687
|
+
const previous = state.stages[index];
|
|
59688
|
+
return previous.stage !== status.stage || previous.run !== status.run;
|
|
59689
|
+
}))
|
|
59690
|
+
state.dirty = true;
|
|
59691
|
+
state.stages = stages;
|
|
59692
|
+
}
|
|
59693
|
+
}
|
|
59694
|
+
fieldsOf(run, stage, resolved, hops) {
|
|
59695
|
+
let fields = this.stageFields.get(stage);
|
|
59696
|
+
const matchable = resolved && hops !== undefined;
|
|
59697
|
+
let changed = false;
|
|
59698
|
+
if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
|
|
59699
|
+
const previous = fields;
|
|
59700
|
+
const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
|
|
59701
|
+
fields = {
|
|
59702
|
+
run,
|
|
59703
|
+
status: stage.status,
|
|
59704
|
+
error: stage.error,
|
|
59705
|
+
sessionId: stage.sessionId,
|
|
59706
|
+
pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
|
|
59707
|
+
resolved,
|
|
59708
|
+
hops,
|
|
59709
|
+
id: stage.id,
|
|
59710
|
+
name: stage.name,
|
|
59711
|
+
tested: sameMatching ? previous.tested : 0,
|
|
59712
|
+
matched: sameMatching ? previous.matched : []
|
|
59713
|
+
};
|
|
59714
|
+
this.stageFields.set(stage, fields);
|
|
59715
|
+
changed = true;
|
|
59716
|
+
}
|
|
59717
|
+
if (fields.tested < this.patterns.length) {
|
|
59718
|
+
const added = this.patterns.slice(fields.tested);
|
|
59719
|
+
if (!matchable)
|
|
59720
|
+
fields.matched.push(...added);
|
|
59721
|
+
else {
|
|
59722
|
+
const index = createStagePathPatternIndex(added.map((state) => state.segments));
|
|
59723
|
+
for (const position of index.matching(hops, [stage.id, stage.name]))
|
|
59724
|
+
fields.matched.push(added[position]);
|
|
59725
|
+
}
|
|
59726
|
+
fields.tested = this.patterns.length;
|
|
59727
|
+
}
|
|
59728
|
+
if (changed)
|
|
59729
|
+
for (const state of fields.matched)
|
|
59730
|
+
state.dirty = true;
|
|
59731
|
+
return fields;
|
|
59732
|
+
}
|
|
59733
|
+
recompute(state, backend) {
|
|
59734
|
+
state.dirty = false;
|
|
59735
|
+
const statuses = [];
|
|
59736
|
+
for (const { run, stage, resolved, hops } of state.stages) {
|
|
59737
|
+
const fields = this.stageFields.get(stage);
|
|
59738
|
+
for (const entry of state.entries) {
|
|
59739
|
+
let byStage = this.statusCache.get(entry);
|
|
59740
|
+
if (byStage === undefined) {
|
|
59741
|
+
byStage = new Map;
|
|
59742
|
+
this.statusCache.set(entry, byStage);
|
|
59743
|
+
}
|
|
59744
|
+
const epoch = this.receiptEpochs.get(entry.id) ?? 0;
|
|
59745
|
+
let cached = byStage.get(stage);
|
|
59746
|
+
if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
|
|
59747
|
+
cached = {
|
|
59748
|
+
fields,
|
|
59749
|
+
generation: this.receiptGeneration,
|
|
59750
|
+
epoch,
|
|
59751
|
+
status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
|
|
59752
|
+
};
|
|
59753
|
+
byStage.set(stage, cached);
|
|
59754
|
+
}
|
|
59755
|
+
statuses.push(cached.status);
|
|
59756
|
+
}
|
|
59757
|
+
}
|
|
59758
|
+
if (!sameStatuses(statuses, state.statuses))
|
|
59759
|
+
state.statuses = statuses;
|
|
59760
|
+
}
|
|
59761
|
+
receiptsOf(entry, runId, stageId) {
|
|
59762
|
+
let byStage = this.deliveriesByEntry.get(entry);
|
|
58442
59763
|
if (byStage === undefined) {
|
|
58443
59764
|
byStage = new Map;
|
|
58444
|
-
for (const delivery of
|
|
59765
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
58445
59766
|
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
58446
59767
|
const list = byStage.get(key);
|
|
58447
59768
|
if (list === undefined)
|
|
@@ -58449,75 +59770,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
58449
59770
|
else
|
|
58450
59771
|
list.push(delivery);
|
|
58451
59772
|
}
|
|
58452
|
-
deliveriesByEntry.set(
|
|
59773
|
+
this.deliveriesByEntry.set(entry, byStage);
|
|
58453
59774
|
}
|
|
58454
59775
|
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
58455
|
-
}
|
|
58456
|
-
|
|
58457
|
-
|
|
58458
|
-
|
|
58459
|
-
|
|
58460
|
-
|
|
58461
|
-
|
|
58462
|
-
if (
|
|
58463
|
-
|
|
58464
|
-
|
|
58465
|
-
|
|
58466
|
-
|
|
58467
|
-
|
|
59776
|
+
}
|
|
59777
|
+
status(entry, run, stage, matchable, backend) {
|
|
59778
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
59779
|
+
const receipts = this.receiptsOf(entry, run.id, stage.id);
|
|
59780
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
59781
|
+
let state;
|
|
59782
|
+
let reason;
|
|
59783
|
+
if (!matchable) {
|
|
59784
|
+
state = "root-unresolved";
|
|
59785
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
59786
|
+
} else if (delivered) {
|
|
59787
|
+
state = "delivered";
|
|
59788
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
59789
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
59790
|
+
state = "stage-terminal";
|
|
59791
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
59792
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
59793
|
+
state = "receipt-unverified";
|
|
59794
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
59795
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
59796
|
+
state = "delivery-unavailable";
|
|
59797
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
59798
|
+
} else if (receipts.length > 0) {
|
|
59799
|
+
state = "session-replaced";
|
|
59800
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
59801
|
+
} else {
|
|
59802
|
+
state = "queued";
|
|
59803
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
59804
|
+
}
|
|
59805
|
+
return {
|
|
59806
|
+
messageId: entry.message.id,
|
|
59807
|
+
target,
|
|
59808
|
+
runId: run.id,
|
|
59809
|
+
stageId: stage.id,
|
|
59810
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
59811
|
+
state,
|
|
59812
|
+
reason
|
|
59813
|
+
};
|
|
59814
|
+
}
|
|
59815
|
+
assemble(runs, rootRun) {
|
|
59816
|
+
const rows = [];
|
|
59817
|
+
const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
|
|
59818
|
+
let liveStageMatches;
|
|
59819
|
+
const rowTargets = new Set;
|
|
59820
|
+
for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
|
|
59821
|
+
const rowSegments = splitStagePathSegments(scanEntry);
|
|
59822
|
+
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
58468
59823
|
continue;
|
|
58469
|
-
|
|
58470
|
-
|
|
58471
|
-
|
|
58472
|
-
|
|
58473
|
-
const receipts = entryReceipts(index, run.id, stage.id);
|
|
58474
|
-
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
58475
|
-
runId: run.id,
|
|
58476
|
-
stageId: stage.id,
|
|
58477
|
-
sessionId: stage.sessionId,
|
|
58478
|
-
admission: "context"
|
|
58479
|
-
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
58480
|
-
let state;
|
|
58481
|
-
let reason;
|
|
58482
|
-
if (!resolved || !hops) {
|
|
58483
|
-
state = "root-unresolved";
|
|
58484
|
-
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
58485
|
-
} else if (delivered) {
|
|
58486
|
-
state = "delivered";
|
|
58487
|
-
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
58488
|
-
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
58489
|
-
state = "stage-terminal";
|
|
58490
|
-
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
58491
|
-
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
58492
|
-
state = "receipt-unverified";
|
|
58493
|
-
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
58494
|
-
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
58495
|
-
state = "delivery-unavailable";
|
|
58496
|
-
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
58497
|
-
} else if (receipts.length > 0) {
|
|
58498
|
-
state = "session-replaced";
|
|
58499
|
-
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
58500
|
-
} else {
|
|
58501
|
-
state = "queued";
|
|
58502
|
-
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
58503
|
-
}
|
|
58504
|
-
result.push({
|
|
58505
|
-
messageId: entry.message.id,
|
|
58506
|
-
target,
|
|
58507
|
-
runId: run.id,
|
|
58508
|
-
stageId: stage.id,
|
|
58509
|
-
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
58510
|
-
state,
|
|
58511
|
-
reason
|
|
58512
|
-
});
|
|
59824
|
+
if (!rowSegments.some((segment) => segment.includes("*"))) {
|
|
59825
|
+
liveStageMatches ??= this.materializedStageMatcher(runs);
|
|
59826
|
+
if (liveStageMatches(rowSegments))
|
|
59827
|
+
continue;
|
|
58513
59828
|
}
|
|
59829
|
+
let matches = this.scanMatches.get(scanEntry);
|
|
59830
|
+
if (matches === undefined) {
|
|
59831
|
+
matches = new Map;
|
|
59832
|
+
this.scanMatches.set(scanEntry, matches);
|
|
59833
|
+
}
|
|
59834
|
+
let queuedCount = 0;
|
|
59835
|
+
for (const state of scanTargets) {
|
|
59836
|
+
let match = matches.get(state);
|
|
59837
|
+
if (match === undefined) {
|
|
59838
|
+
const segments = state.segments;
|
|
59839
|
+
match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
59840
|
+
matches.set(state, match);
|
|
59841
|
+
}
|
|
59842
|
+
if (match)
|
|
59843
|
+
queuedCount += state.queued;
|
|
59844
|
+
}
|
|
59845
|
+
const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
|
|
59846
|
+
rows.push({ slot: `s${position}`, target, queuedCount });
|
|
59847
|
+
rowTargets.add(target);
|
|
59848
|
+
}
|
|
59849
|
+
let broadcastCount = 0;
|
|
59850
|
+
for (const state of this.targets.values()) {
|
|
59851
|
+
if (state.segments?.length === 1 && state.segments[0] === "**")
|
|
59852
|
+
broadcastCount += state.queued;
|
|
59853
|
+
}
|
|
59854
|
+
const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
|
|
59855
|
+
rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
|
|
59856
|
+
rowTargets.add(broadcast);
|
|
59857
|
+
const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
|
|
59858
|
+
const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
|
|
59859
|
+
for (const state of statusTargets) {
|
|
59860
|
+
if (rowTargets.has(state.target))
|
|
59861
|
+
continue;
|
|
59862
|
+
rowTargets.add(state.target);
|
|
59863
|
+
rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
|
|
58514
59864
|
}
|
|
59865
|
+
const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
|
|
59866
|
+
this.pruneRows(result);
|
|
59867
|
+
return result;
|
|
58515
59868
|
}
|
|
58516
|
-
|
|
59869
|
+
row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
|
|
59870
|
+
const cached = this.rowCache.get(slot);
|
|
59871
|
+
const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
|
|
59872
|
+
if (same && cached.statuses === deliveryStates)
|
|
59873
|
+
return cached.row;
|
|
59874
|
+
const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
|
|
59875
|
+
if (same && sameBoundedRow(cached.row, bounded)) {
|
|
59876
|
+
this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
|
|
59877
|
+
return cached.row;
|
|
59878
|
+
}
|
|
59879
|
+
const row = {
|
|
59880
|
+
target,
|
|
59881
|
+
queuedCount,
|
|
59882
|
+
...deliveryOnly === true ? { deliveryOnly } : {},
|
|
59883
|
+
...bounded
|
|
59884
|
+
};
|
|
59885
|
+
this.rowCache.set(slot, { row, statuses: deliveryStates });
|
|
59886
|
+
return row;
|
|
59887
|
+
}
|
|
59888
|
+
bounded(states) {
|
|
59889
|
+
let bounded = this.boundedStates.get(states);
|
|
59890
|
+
if (bounded === undefined) {
|
|
59891
|
+
bounded = boundPossibleStageDeliveryStates(states);
|
|
59892
|
+
this.boundedStates.set(states, bounded);
|
|
59893
|
+
}
|
|
59894
|
+
return bounded;
|
|
59895
|
+
}
|
|
59896
|
+
pruneRows(rows) {
|
|
59897
|
+
if (this.rowCache.size <= rows.length * 4 + 16)
|
|
59898
|
+
return;
|
|
59899
|
+
const kept = new Set(rows);
|
|
59900
|
+
for (const [key, cached] of this.rowCache)
|
|
59901
|
+
if (!kept.has(cached.row))
|
|
59902
|
+
this.rowCache.delete(key);
|
|
59903
|
+
}
|
|
59904
|
+
materializedStageMatcher(runs) {
|
|
59905
|
+
const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
|
|
59906
|
+
return (entrySegments) => {
|
|
59907
|
+
for (const run of invocationRuns) {
|
|
59908
|
+
const hops = this.topology.hopsOf(run.id);
|
|
59909
|
+
if (hops === undefined)
|
|
59910
|
+
continue;
|
|
59911
|
+
for (const stage of run.stages) {
|
|
59912
|
+
if (stage.pendingStageDeliveryAvailable !== true)
|
|
59913
|
+
continue;
|
|
59914
|
+
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
59915
|
+
continue;
|
|
59916
|
+
}
|
|
59917
|
+
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
59918
|
+
return true;
|
|
59919
|
+
}
|
|
59920
|
+
}
|
|
59921
|
+
return false;
|
|
59922
|
+
};
|
|
59923
|
+
}
|
|
59924
|
+
}
|
|
59925
|
+
function sameBoundedRow(row, bounded) {
|
|
59926
|
+
if (bounded === undefined)
|
|
59927
|
+
return row.deliveryStates === undefined;
|
|
59928
|
+
if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
|
|
59929
|
+
return false;
|
|
59930
|
+
const counts = Object.entries(row.deliveryStateCounts ?? {});
|
|
59931
|
+
const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
|
|
59932
|
+
if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
|
|
59933
|
+
return false;
|
|
59934
|
+
return sameStatuses(row.deliveryStates, bounded.deliveryStates);
|
|
59935
|
+
}
|
|
59936
|
+
function sameStatuses(left, right) {
|
|
59937
|
+
if (left.length !== right.length)
|
|
59938
|
+
return false;
|
|
59939
|
+
for (let index = 0;index < left.length; index++) {
|
|
59940
|
+
const a = left[index];
|
|
59941
|
+
const b = right[index];
|
|
59942
|
+
if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
|
|
59943
|
+
return false;
|
|
59944
|
+
}
|
|
59945
|
+
return true;
|
|
58517
59946
|
}
|
|
58518
|
-
var
|
|
59947
|
+
var init_possible_stage_rows = __esm(() => {
|
|
58519
59948
|
init_run_owner_backend();
|
|
58520
59949
|
init_pending_stage_status();
|
|
59950
|
+
init_sticky_delivery_state_bound();
|
|
59951
|
+
init_store_internal();
|
|
58521
59952
|
init_workflow_stage_target();
|
|
58522
59953
|
});
|
|
58523
59954
|
|
|
@@ -58578,12 +60009,36 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58578
60009
|
};
|
|
58579
60010
|
const announcedRoutes = new Map;
|
|
58580
60011
|
const routeCompletions = new Map;
|
|
58581
|
-
const disposeReadiness = registerWorkflowPendingStageRouteReadiness(activeStore, (runId) =>
|
|
60012
|
+
const disposeReadiness = registerWorkflowPendingStageRouteReadiness(activeStore, (runId) => {
|
|
60013
|
+
announceIfPending();
|
|
60014
|
+
return routeCompletions.get(runId);
|
|
60015
|
+
});
|
|
60016
|
+
const topology = new WorkflowTopologyIndex;
|
|
60017
|
+
const possibleStageRowIndexes = new Map;
|
|
60018
|
+
const rowIdentities = new WeakMap;
|
|
60019
|
+
let nextRowIdentity = 0;
|
|
60020
|
+
const rowIdentity = (row) => {
|
|
60021
|
+
let identity = rowIdentities.get(row);
|
|
60022
|
+
if (identity === undefined) {
|
|
60023
|
+
identity = nextRowIdentity++;
|
|
60024
|
+
rowIdentities.set(row, identity);
|
|
60025
|
+
}
|
|
60026
|
+
return identity;
|
|
60027
|
+
};
|
|
60028
|
+
const possibleStageRowsIndex = (rootRunId) => {
|
|
60029
|
+
let index = possibleStageRowIndexes.get(rootRunId);
|
|
60030
|
+
if (index === undefined) {
|
|
60031
|
+
index = new PossibleStageRowsIndex(rootRunId, topology);
|
|
60032
|
+
possibleStageRowIndexes.set(rootRunId, index);
|
|
60033
|
+
}
|
|
60034
|
+
return index;
|
|
60035
|
+
};
|
|
58582
60036
|
const announceRoutes = () => {
|
|
58583
60037
|
if (disposed)
|
|
58584
60038
|
return;
|
|
58585
60039
|
const ownedRunIds = new Set;
|
|
58586
60040
|
const runs = activeStore.runs();
|
|
60041
|
+
topology.refresh(runs);
|
|
58587
60042
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58588
60043
|
const childrenByBoundary = new Map;
|
|
58589
60044
|
for (const run of runs) {
|
|
@@ -58598,7 +60053,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58598
60053
|
for (const run of runs) {
|
|
58599
60054
|
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
58600
60055
|
continue;
|
|
58601
|
-
const rootRunId =
|
|
60056
|
+
const rootRunId = topology.rootOf(run.id);
|
|
58602
60057
|
if (rootRunId === undefined)
|
|
58603
60058
|
continue;
|
|
58604
60059
|
ownedRunIds.add(run.id);
|
|
@@ -58608,6 +60063,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58608
60063
|
runId: parentRun.id,
|
|
58609
60064
|
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
58610
60065
|
};
|
|
60066
|
+
let possibleStages;
|
|
60067
|
+
if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
|
|
60068
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
60069
|
+
possibleStages = [];
|
|
60070
|
+
} else if (run.id === rootRunId) {
|
|
60071
|
+
possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
|
|
60072
|
+
}
|
|
58611
60073
|
const announcement = {
|
|
58612
60074
|
runId: run.id,
|
|
58613
60075
|
group: workflowInvocationIntercomGroup(rootRunId),
|
|
@@ -58632,11 +60094,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58632
60094
|
group: workflowInvocationIntercomGroup(rootRunId)
|
|
58633
60095
|
}))
|
|
58634
60096
|
],
|
|
58635
|
-
...
|
|
58636
|
-
possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
|
|
58637
|
-
} : {}
|
|
60097
|
+
...possibleStages === undefined ? {} : { possibleStages }
|
|
58638
60098
|
};
|
|
58639
|
-
const signature = JSON.stringify(announcement)
|
|
60099
|
+
const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
|
|
58640
60100
|
if (announcedRoutes.get(run.id)?.signature === signature)
|
|
58641
60101
|
continue;
|
|
58642
60102
|
const entry = { signature };
|
|
@@ -58671,15 +60131,61 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58671
60131
|
if (!ownedRunIds.has(runId))
|
|
58672
60132
|
routeCompletions.delete(runId);
|
|
58673
60133
|
}
|
|
60134
|
+
for (const rootRunId of possibleStageRowIndexes.keys()) {
|
|
60135
|
+
if (!ownedRunIds.has(rootRunId))
|
|
60136
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
60137
|
+
}
|
|
58674
60138
|
sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
|
|
58675
60139
|
return;
|
|
58676
60140
|
}).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
|
|
58677
60141
|
};
|
|
58678
|
-
|
|
58679
|
-
|
|
60142
|
+
let announcementPending = false;
|
|
60143
|
+
let announcementTimer;
|
|
60144
|
+
let announcedAt = Number.NEGATIVE_INFINITY;
|
|
60145
|
+
let announcementCost = 0;
|
|
60146
|
+
const announceNow = () => {
|
|
60147
|
+
if (announcementTimer !== undefined)
|
|
60148
|
+
clearTimeout(announcementTimer);
|
|
60149
|
+
announcementTimer = undefined;
|
|
60150
|
+
announcementPending = false;
|
|
60151
|
+
const started = performance.now();
|
|
60152
|
+
announceRoutes();
|
|
60153
|
+
announcedAt = performance.now();
|
|
60154
|
+
announcementCost = announcedAt - started;
|
|
60155
|
+
};
|
|
60156
|
+
function announceIfPending() {
|
|
60157
|
+
if (announcementPending)
|
|
60158
|
+
announceNow();
|
|
60159
|
+
}
|
|
60160
|
+
const announceBatch = () => {
|
|
60161
|
+
try {
|
|
60162
|
+
announceNow();
|
|
60163
|
+
} catch (error) {
|
|
60164
|
+
if (error instanceof Error && isSessionClosedError(error)) {
|
|
60165
|
+
dispose();
|
|
60166
|
+
return;
|
|
60167
|
+
}
|
|
60168
|
+
try {
|
|
60169
|
+
reportWarning(`atomic-workflows: pending stage route announcement failed: ${String(error)}`);
|
|
60170
|
+
} catch {
|
|
60171
|
+
dispose();
|
|
60172
|
+
}
|
|
60173
|
+
}
|
|
60174
|
+
};
|
|
60175
|
+
const scheduleAnnouncement = () => {
|
|
60176
|
+
if (disposed || announcementTimer !== undefined)
|
|
60177
|
+
return;
|
|
60178
|
+
const idle = announcementCost / ROUTE_ANNOUNCEMENT_EVENT_LOOP_SHARE - announcementCost;
|
|
60179
|
+
const wait = Math.max(ROUTE_ANNOUNCEMENT_BATCH_MS, announcedAt + idle - performance.now());
|
|
60180
|
+
announcementPending = true;
|
|
60181
|
+
announcementTimer = setTimeout(announceBatch, wait);
|
|
60182
|
+
announcementTimer.unref?.();
|
|
60183
|
+
};
|
|
60184
|
+
const unsubscribeStore = activeStore.subscribeInvalidation(scheduleAnnouncement);
|
|
60185
|
+
announceNow();
|
|
58680
60186
|
pi.on?.("session_start", () => {
|
|
58681
60187
|
announcedRoutes.clear();
|
|
58682
|
-
|
|
60188
|
+
announceNow();
|
|
58683
60189
|
});
|
|
58684
60190
|
const routePendingStageMessage = (payload) => {
|
|
58685
60191
|
if (disposed || !isPendingStageMessageEvent(payload) || payload.handled)
|
|
@@ -58761,6 +60267,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58761
60267
|
});
|
|
58762
60268
|
const dispose = () => {
|
|
58763
60269
|
disposed = true;
|
|
60270
|
+
if (announcementTimer !== undefined)
|
|
60271
|
+
clearTimeout(announcementTimer);
|
|
60272
|
+
announcementTimer = undefined;
|
|
60273
|
+
announcementPending = false;
|
|
58764
60274
|
disposeReadiness();
|
|
58765
60275
|
disposeRouter();
|
|
58766
60276
|
routeCompletions.clear();
|
|
@@ -58774,112 +60284,17 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
58774
60284
|
pi.on?.("session_shutdown", dispose);
|
|
58775
60285
|
return Object.assign(dispose, {
|
|
58776
60286
|
async flush() {
|
|
60287
|
+
announceIfPending();
|
|
58777
60288
|
await Promise.allSettled(routeCompletions.values());
|
|
58778
60289
|
await sweepPromise;
|
|
58779
60290
|
},
|
|
58780
|
-
refresh:
|
|
60291
|
+
refresh: announceNow
|
|
58781
60292
|
});
|
|
58782
60293
|
}
|
|
58783
60294
|
function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegments = (id) => workflowBoundarySegments(runs, id)) {
|
|
58784
60295
|
const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
|
|
58785
60296
|
return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
|
|
58786
60297
|
}
|
|
58787
|
-
function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
58788
|
-
if (isTerminalRunStatus(rootRun.status))
|
|
58789
|
-
return [];
|
|
58790
|
-
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
58791
|
-
const segmentsByTarget = new Map;
|
|
58792
|
-
const stickySegments = (entry) => {
|
|
58793
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58794
|
-
if (segmentsByTarget.has(target))
|
|
58795
|
-
return segmentsByTarget.get(target);
|
|
58796
|
-
const parsed = parseWorkflowStageTarget(target);
|
|
58797
|
-
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
58798
|
-
segmentsByTarget.set(target, segments);
|
|
58799
|
-
return segments;
|
|
58800
|
-
};
|
|
58801
|
-
const scanEntries = stickyQueued.filter((entry) => {
|
|
58802
|
-
const segments = stickySegments(entry);
|
|
58803
|
-
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
58804
|
-
});
|
|
58805
|
-
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
58806
|
-
const rows = [];
|
|
58807
|
-
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
58808
|
-
const rowSegments = splitStagePathSegments(scanEntry);
|
|
58809
|
-
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
58810
|
-
continue;
|
|
58811
|
-
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
58812
|
-
continue;
|
|
58813
|
-
}
|
|
58814
|
-
rows.push({
|
|
58815
|
-
target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
|
|
58816
|
-
queuedCount: scanEntries.filter((entry) => {
|
|
58817
|
-
const segments = stickySegments(entry);
|
|
58818
|
-
if (segments === undefined)
|
|
58819
|
-
return false;
|
|
58820
|
-
return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
58821
|
-
}).length
|
|
58822
|
-
});
|
|
58823
|
-
}
|
|
58824
|
-
rows.push({
|
|
58825
|
-
target: formatWorkflowStageTarget(rootRunId, "**"),
|
|
58826
|
-
queuedCount: stickyQueued.filter((entry) => {
|
|
58827
|
-
const segments = stickySegments(entry);
|
|
58828
|
-
return segments !== undefined && segments.length === 1 && segments[0] === "**";
|
|
58829
|
-
}).length
|
|
58830
|
-
});
|
|
58831
|
-
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
58832
|
-
const statusesByTarget = new Map;
|
|
58833
|
-
for (const status of statuses) {
|
|
58834
|
-
const forTarget = statusesByTarget.get(status.target);
|
|
58835
|
-
if (forTarget === undefined)
|
|
58836
|
-
statusesByTarget.set(status.target, [status]);
|
|
58837
|
-
else
|
|
58838
|
-
forTarget.push(status);
|
|
58839
|
-
}
|
|
58840
|
-
const queuedByTarget = new Map;
|
|
58841
|
-
for (const entry of stickyQueued) {
|
|
58842
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
58843
|
-
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
58844
|
-
}
|
|
58845
|
-
const rowTargets = new Set(rows.map((row) => row.target));
|
|
58846
|
-
for (const target of statusesByTarget.keys()) {
|
|
58847
|
-
if (rowTargets.has(target))
|
|
58848
|
-
continue;
|
|
58849
|
-
rowTargets.add(target);
|
|
58850
|
-
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
58851
|
-
}
|
|
58852
|
-
for (let index = 0;index < rows.length; index++) {
|
|
58853
|
-
const row = rows[index];
|
|
58854
|
-
const deliveryStates = statusesByTarget.get(row.target);
|
|
58855
|
-
if (deliveryStates !== undefined)
|
|
58856
|
-
rows[index] = { ...row, deliveryStates };
|
|
58857
|
-
}
|
|
58858
|
-
return rows;
|
|
58859
|
-
}
|
|
58860
|
-
function materializedStageMatcher(runs, rootRunId) {
|
|
58861
|
-
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
58862
|
-
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
58863
|
-
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
58864
|
-
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
58865
|
-
}
|
|
58866
|
-
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
58867
|
-
for (const run of invocationRuns) {
|
|
58868
|
-
const hops = hopsFor(run.id);
|
|
58869
|
-
if (hops === undefined)
|
|
58870
|
-
continue;
|
|
58871
|
-
for (const stage of run.stages) {
|
|
58872
|
-
if (stage.pendingStageDeliveryAvailable !== true)
|
|
58873
|
-
continue;
|
|
58874
|
-
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
58875
|
-
continue;
|
|
58876
|
-
}
|
|
58877
|
-
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
58878
|
-
return true;
|
|
58879
|
-
}
|
|
58880
|
-
}
|
|
58881
|
-
return false;
|
|
58882
|
-
}
|
|
58883
60298
|
function isPendingStageMessageEvent(value2) {
|
|
58884
60299
|
if (typeof value2 !== "object" || value2 === null)
|
|
58885
60300
|
return false;
|
|
@@ -59252,17 +60667,18 @@ async function settleUndeliverablePendingStageMessages(activeStore, notify, owns
|
|
|
59252
60667
|
}
|
|
59253
60668
|
return settled;
|
|
59254
60669
|
}
|
|
59255
|
-
var PENDING_STAGE_ROUTE_EVENT = "atomic:workflow-pending-stage-route", PENDING_STAGE_MESSAGE_EVENT = "atomic:workflow-pending-stage-message", PENDING_STAGE_UNDELIVERABLE_EVENT = "atomic:workflow-pending-stage-undeliverable", STICKY_LIVE_DELIVERED_EVENT = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_ASK_REFUSAL = "Cannot ask a workflow stage whose session has not initialized. Use send; Atomic will queue the message until the stage session initializes.";
|
|
60670
|
+
var PENDING_STAGE_ROUTE_EVENT = "atomic:workflow-pending-stage-route", PENDING_STAGE_MESSAGE_EVENT = "atomic:workflow-pending-stage-message", PENDING_STAGE_UNDELIVERABLE_EVENT = "atomic:workflow-pending-stage-undeliverable", STICKY_LIVE_DELIVERED_EVENT = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_ASK_REFUSAL = "Cannot ask a workflow stage whose session has not initialized. Use send; Atomic will queue the message until the stage session initializes.", ROUTE_ANNOUNCEMENT_BATCH_MS = 25, ROUTE_ANNOUNCEMENT_EVENT_LOOP_SHARE = 0.2;
|
|
59256
60671
|
var init_pending_stage_intercom = __esm(() => {
|
|
59257
60672
|
init_execution_context_boundary();
|
|
59258
60673
|
init_run_owner_backend();
|
|
59259
60674
|
init_store_backend();
|
|
60675
|
+
init_session_closed_error();
|
|
59260
60676
|
init_external_run_routes();
|
|
59261
60677
|
init_intercom_group();
|
|
59262
60678
|
init_pending_stage_route_capability();
|
|
59263
60679
|
init_pending_stage_route_readiness();
|
|
59264
60680
|
init_pending_stage_status();
|
|
59265
|
-
|
|
60681
|
+
init_possible_stage_rows();
|
|
59266
60682
|
init_store_internal();
|
|
59267
60683
|
init_workflow_stage_target();
|
|
59268
60684
|
});
|
|
@@ -61749,7 +63165,7 @@ function isInfrastructureError(value2) {
|
|
|
61749
63165
|
}
|
|
61750
63166
|
function isRetryableWorkflowError(error) {
|
|
61751
63167
|
let current = error;
|
|
61752
|
-
for (let depth = 0;depth <
|
|
63168
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined && current !== null; depth++) {
|
|
61753
63169
|
if (isInfrastructureError(current))
|
|
61754
63170
|
return true;
|
|
61755
63171
|
current = current instanceof Error ? current.cause : undefined;
|
|
@@ -61767,7 +63183,7 @@ function asWorkflowInfrastructureError(error) {
|
|
|
61767
63183
|
const code = workflowInfrastructureFault(error.message);
|
|
61768
63184
|
return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
|
|
61769
63185
|
}
|
|
61770
|
-
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError",
|
|
63186
|
+
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH2 = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
|
|
61771
63187
|
var init_workflow_infrastructure_error = __esm(() => {
|
|
61772
63188
|
FAULT_PATTERNS = [
|
|
61773
63189
|
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
@@ -61809,15 +63225,22 @@ function validate(report) {
|
|
|
61809
63225
|
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
61810
63226
|
}
|
|
61811
63227
|
}
|
|
63228
|
+
function addCompensated(total, value2) {
|
|
63229
|
+
const sum = total.sum + value2;
|
|
63230
|
+
total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
|
|
63231
|
+
total.sum = sum;
|
|
63232
|
+
}
|
|
61812
63233
|
|
|
61813
63234
|
class SqlPoolUsage {
|
|
61814
63235
|
transaction;
|
|
61815
63236
|
journal;
|
|
61816
63237
|
table;
|
|
63238
|
+
totals;
|
|
61817
63239
|
constructor(transaction, journal) {
|
|
61818
63240
|
this.transaction = transaction;
|
|
61819
63241
|
this.journal = journal;
|
|
61820
63242
|
this.table = poolUsageTable(journal.runs);
|
|
63243
|
+
this.totals = poolUsageTotalsTable(journal.runs);
|
|
61821
63244
|
}
|
|
61822
63245
|
static async open(transaction, journal) {
|
|
61823
63246
|
const usage = new SqlPoolUsage(transaction, journal);
|
|
@@ -61825,9 +63248,27 @@ class SqlPoolUsage {
|
|
|
61825
63248
|
return usage;
|
|
61826
63249
|
}
|
|
61827
63250
|
async totalIn(tx, root) {
|
|
63251
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
63252
|
+
const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
|
|
63253
|
+
if (kept)
|
|
63254
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
|
|
63255
|
+
field,
|
|
63256
|
+
Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
|
|
63257
|
+
]));
|
|
61828
63258
|
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
61829
63259
|
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
61830
|
-
|
|
63260
|
+
const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
63261
|
+
if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
|
|
63262
|
+
return total;
|
|
63263
|
+
const sums = columns.map(() => ({ sum: 0, error: 0 }));
|
|
63264
|
+
const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
|
|
63265
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
63266
|
+
for (const usage of rows)
|
|
63267
|
+
for (const [index, column] of columns.entries())
|
|
63268
|
+
addCompensated(sums[index], Number(usage[column]));
|
|
63269
|
+
await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
|
|
63270
|
+
VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
|
|
63271
|
+
return total;
|
|
61831
63272
|
}
|
|
61832
63273
|
async report(lease, token, reports) {
|
|
61833
63274
|
validateText("runId", lease.runId);
|
|
@@ -64956,7 +66397,7 @@ function describeErrorWithCauses(error) {
|
|
|
64956
66397
|
const messages = [];
|
|
64957
66398
|
const visited = new Set;
|
|
64958
66399
|
let current = error;
|
|
64959
|
-
for (let depth = 0;current !== undefined && depth <
|
|
66400
|
+
for (let depth = 0;current !== undefined && depth < MAX_CAUSE_DEPTH3 && !visited.has(current); depth++) {
|
|
64960
66401
|
visited.add(current);
|
|
64961
66402
|
const message = current instanceof Error ? current.message : String(current);
|
|
64962
66403
|
if (!messages.includes(message))
|
|
@@ -64965,7 +66406,7 @@ function describeErrorWithCauses(error) {
|
|
|
64965
66406
|
}
|
|
64966
66407
|
return messages.join(": ");
|
|
64967
66408
|
}
|
|
64968
|
-
var
|
|
66409
|
+
var MAX_CAUSE_DEPTH3 = 5;
|
|
64969
66410
|
|
|
64970
66411
|
// dist/builtin/workflows/src/shared/returned-run-status.ts
|
|
64971
66412
|
function normalizeReturnedWorkflowStatus(status) {
|
|
@@ -65345,10 +66786,120 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
|
65345
66786
|
}
|
|
65346
66787
|
var init_stage_session_store = () => {};
|
|
65347
66788
|
|
|
66789
|
+
// dist/builtin/workflows/src/durable/completed-catalog-index.ts
|
|
66790
|
+
class CheckpointCatalogIndex {
|
|
66791
|
+
checkpoints;
|
|
66792
|
+
stages;
|
|
66793
|
+
positions = new Map;
|
|
66794
|
+
groupKeys = new Map;
|
|
66795
|
+
groups = new Map;
|
|
66796
|
+
stagesByRunId = new Map;
|
|
66797
|
+
boundariesByReplayKey = new Map;
|
|
66798
|
+
boundariesByStageId = new Map;
|
|
66799
|
+
boundariesByChildRunId = new Map;
|
|
66800
|
+
outputsByReplayKey = new Map;
|
|
66801
|
+
evidence;
|
|
66802
|
+
constructor(checkpoints) {
|
|
66803
|
+
this.checkpoints = checkpoints;
|
|
66804
|
+
const stages = [];
|
|
66805
|
+
checkpoints.forEach((checkpoint, index) => {
|
|
66806
|
+
if (!this.positions.has(checkpoint))
|
|
66807
|
+
this.positions.set(checkpoint, index);
|
|
66808
|
+
if (checkpoint.kind !== "stage")
|
|
66809
|
+
return;
|
|
66810
|
+
stages.push(checkpoint);
|
|
66811
|
+
const key = durableStageGroupKey(checkpoint);
|
|
66812
|
+
this.groupKeys.set(checkpoint, key);
|
|
66813
|
+
append(this.groups, key, checkpoint);
|
|
66814
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
66815
|
+
if (runId !== undefined)
|
|
66816
|
+
append(this.stagesByRunId, runId, checkpoint);
|
|
66817
|
+
if (checkpoint.output !== undefined)
|
|
66818
|
+
append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
|
|
66819
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66820
|
+
if (boundary === undefined)
|
|
66821
|
+
return;
|
|
66822
|
+
append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
|
|
66823
|
+
append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
|
|
66824
|
+
append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
|
|
66825
|
+
});
|
|
66826
|
+
this.stages = stages;
|
|
66827
|
+
}
|
|
66828
|
+
sequence(checkpoint) {
|
|
66829
|
+
return (this.positions.get(checkpoint) ?? -1) + 1;
|
|
66830
|
+
}
|
|
66831
|
+
stageGroups() {
|
|
66832
|
+
return this.groups.values();
|
|
66833
|
+
}
|
|
66834
|
+
boundaryLifecycle(start, requireTerminal) {
|
|
66835
|
+
const topology = start.topology;
|
|
66836
|
+
const childRunId = topology?.boundary?.child.runId;
|
|
66837
|
+
const related = this.inCheckpointOrder([
|
|
66838
|
+
this.boundariesByReplayKey.get(start.replayKey),
|
|
66839
|
+
topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
|
|
66840
|
+
childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
|
|
66841
|
+
]);
|
|
66842
|
+
return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
|
|
66843
|
+
}
|
|
66844
|
+
directChildRecords(childRunId) {
|
|
66845
|
+
return {
|
|
66846
|
+
claiming: this.stagesByRunId.get(childRunId) ?? [],
|
|
66847
|
+
direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
|
|
66848
|
+
};
|
|
66849
|
+
}
|
|
66850
|
+
childScopedEvidenceExists(boundaryReplayKey, childRunId) {
|
|
66851
|
+
this.evidence ??= scopedEvidenceIndex(this.checkpoints);
|
|
66852
|
+
return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
|
|
66853
|
+
}
|
|
66854
|
+
inCheckpointOrder(lists) {
|
|
66855
|
+
const present = lists.filter((list) => list !== undefined);
|
|
66856
|
+
if (present.length === 1)
|
|
66857
|
+
return present[0];
|
|
66858
|
+
return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
|
|
66859
|
+
}
|
|
66860
|
+
}
|
|
66861
|
+
function scopedEvidenceIndex(checkpoints) {
|
|
66862
|
+
const runIds = new Set;
|
|
66863
|
+
const identities = [];
|
|
66864
|
+
for (const checkpoint of checkpoints) {
|
|
66865
|
+
identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
|
|
66866
|
+
if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
|
|
66867
|
+
runIds.add(checkpoint.topology.run.runId);
|
|
66868
|
+
}
|
|
66869
|
+
}
|
|
66870
|
+
identities.sort(compareCodeUnits);
|
|
66871
|
+
return { runIds, identities };
|
|
66872
|
+
}
|
|
66873
|
+
function hasPrefix(sorted, prefix) {
|
|
66874
|
+
let low = 0;
|
|
66875
|
+
let high = sorted.length;
|
|
66876
|
+
while (low < high) {
|
|
66877
|
+
const middle = low + high >>> 1;
|
|
66878
|
+
if (compareCodeUnits(sorted[middle], prefix) < 0)
|
|
66879
|
+
low = middle + 1;
|
|
66880
|
+
else
|
|
66881
|
+
high = middle;
|
|
66882
|
+
}
|
|
66883
|
+
return low < sorted.length && sorted[low].startsWith(prefix);
|
|
66884
|
+
}
|
|
66885
|
+
function compareCodeUnits(left, right) {
|
|
66886
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
66887
|
+
}
|
|
66888
|
+
function append(map, key, value2) {
|
|
66889
|
+
const list = map.get(key);
|
|
66890
|
+
if (list === undefined)
|
|
66891
|
+
map.set(key, [value2]);
|
|
66892
|
+
else
|
|
66893
|
+
list.push(value2);
|
|
66894
|
+
}
|
|
66895
|
+
var init_completed_catalog_index = __esm(() => {
|
|
66896
|
+
init_boundary_lifecycle();
|
|
66897
|
+
init_stage_topology_validation();
|
|
66898
|
+
});
|
|
66899
|
+
|
|
65348
66900
|
// dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
|
|
65349
|
-
function validBoundaryRecordSet(
|
|
65350
|
-
const
|
|
65351
|
-
const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
|
|
66901
|
+
function validBoundaryRecordSet(index, strict) {
|
|
66902
|
+
const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
|
|
65352
66903
|
const startsByReplay = new Map;
|
|
65353
66904
|
for (const checkpoint of boundaries) {
|
|
65354
66905
|
if (checkpoint.topology.boundary.event !== "start")
|
|
@@ -65363,7 +66914,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
|
|
|
65363
66914
|
return false;
|
|
65364
66915
|
return [...startsByReplay.values()].every((starts) => {
|
|
65365
66916
|
const start = starts[0];
|
|
65366
|
-
if (
|
|
66917
|
+
if ("error" in index.boundaryLifecycle(start, strict))
|
|
65367
66918
|
return false;
|
|
65368
66919
|
return start.topology.boundary.invocationFingerprint !== undefined;
|
|
65369
66920
|
});
|
|
@@ -65427,8 +66978,8 @@ function compareDraftSourceOrder(left, right) {
|
|
|
65427
66978
|
function runTopologyFor(drafts, tools) {
|
|
65428
66979
|
return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
|
|
65429
66980
|
}
|
|
65430
|
-
function validateRunGroups(grouped, rootRunId,
|
|
65431
|
-
const toolsFor = (runId) =>
|
|
66981
|
+
function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
|
|
66982
|
+
const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
|
|
65432
66983
|
const owners = new Map;
|
|
65433
66984
|
const syntheticChildIds = new Set;
|
|
65434
66985
|
for (const [parentRunId, drafts] of grouped) {
|
|
@@ -65439,12 +66990,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
|
|
|
65439
66990
|
if (childRunId === parentRunId || owners.has(childRunId))
|
|
65440
66991
|
return false;
|
|
65441
66992
|
if (!grouped.has(childRunId)) {
|
|
65442
|
-
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(
|
|
66993
|
+
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
|
|
65443
66994
|
syntheticChildIds.add(childRunId);
|
|
65444
66995
|
owners.set(childRunId, draft);
|
|
65445
66996
|
continue;
|
|
65446
66997
|
}
|
|
65447
|
-
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(
|
|
66998
|
+
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
|
|
65448
66999
|
return false;
|
|
65449
67000
|
syntheticChildIds.add(childRunId);
|
|
65450
67001
|
owners.set(childRunId, draft);
|
|
@@ -65485,13 +67036,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
|
|
|
65485
67036
|
const boundary = draft.topology?.boundary;
|
|
65486
67037
|
return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
|
|
65487
67038
|
}
|
|
65488
|
-
function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
65489
|
-
const prefix = `${boundaryReplayKey}:`;
|
|
65490
|
-
return checkpoints.some((checkpoint) => {
|
|
65491
|
-
const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
|
|
65492
|
-
return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
|
|
65493
|
-
});
|
|
65494
|
-
}
|
|
65495
67039
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
65496
67040
|
const reachable = new Set([rootRunId]);
|
|
65497
67041
|
const settled = new Set;
|
|
@@ -65532,14 +67076,6 @@ function childRunIdFromDraft(draft) {
|
|
|
65532
67076
|
return;
|
|
65533
67077
|
return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
|
|
65534
67078
|
}
|
|
65535
|
-
function boundaryOwner(grouped, childRunId) {
|
|
65536
|
-
for (const drafts of grouped.values()) {
|
|
65537
|
-
const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
|
|
65538
|
-
if (owner !== undefined)
|
|
65539
|
-
return owner;
|
|
65540
|
-
}
|
|
65541
|
-
return;
|
|
65542
|
-
}
|
|
65543
67079
|
function childRunStatus(owner) {
|
|
65544
67080
|
const child = workflowChildFromDraft(owner);
|
|
65545
67081
|
if (child !== undefined)
|
|
@@ -65547,15 +67083,15 @@ function childRunStatus(owner) {
|
|
|
65547
67083
|
const status = owner.topology?.boundary?.status;
|
|
65548
67084
|
return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
|
|
65549
67085
|
}
|
|
65550
|
-
function mergeStageGroup(records,
|
|
67086
|
+
function mergeStageGroup(records, index) {
|
|
65551
67087
|
let merged;
|
|
65552
67088
|
for (const record of records)
|
|
65553
|
-
merged = mergeStageDraft(merged, record,
|
|
67089
|
+
merged = mergeStageDraft(merged, record, index.sequence(record));
|
|
65554
67090
|
const draft = merged;
|
|
65555
67091
|
const start = records.find((record) => record.topology?.boundary?.event === "start");
|
|
65556
67092
|
if (start === undefined)
|
|
65557
67093
|
return draft;
|
|
65558
|
-
const resolved =
|
|
67094
|
+
const resolved = index.boundaryLifecycle(start, false);
|
|
65559
67095
|
if ("error" in resolved)
|
|
65560
67096
|
return draft;
|
|
65561
67097
|
const terminal = resolved.lifecycle.latestTerminal;
|
|
@@ -65618,9 +67154,7 @@ function workflowChildFromDraft(draft) {
|
|
|
65618
67154
|
...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
|
|
65619
67155
|
};
|
|
65620
67156
|
}
|
|
65621
|
-
var init_completed_catalog_stage_groups =
|
|
65622
|
-
init_boundary_lifecycle();
|
|
65623
|
-
});
|
|
67157
|
+
var init_completed_catalog_stage_groups = () => {};
|
|
65624
67158
|
|
|
65625
67159
|
// dist/builtin/workflows/src/engine/run-tool-control-registry.ts
|
|
65626
67160
|
function createToolControlRegistry() {
|
|
@@ -65825,11 +67359,68 @@ var init_resume_runtime = __esm(() => {
|
|
|
65825
67359
|
function isReservedRunCheckpointHash(argsHash) {
|
|
65826
67360
|
return [RUN_TIMING_CHECKPOINT_NAME, RUN_USAGE_CHECKPOINT_NAME, RUN_RESULT_CHECKPOINT_NAME].includes(argsHash) || argsHash === WORKFLOW_HEARTBEAT_ANCHOR_CHECKPOINT_NAME;
|
|
65827
67361
|
}
|
|
65828
|
-
function
|
|
67362
|
+
function durableNestedRunCatalog(backend, rootWorkflowId) {
|
|
65829
67363
|
const handle = backend.getWorkflow(rootWorkflowId);
|
|
65830
67364
|
if (handle === undefined)
|
|
65831
|
-
return
|
|
65832
|
-
|
|
67365
|
+
return EMPTY_NESTED_RUN_CATALOG;
|
|
67366
|
+
const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
|
|
67367
|
+
const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
|
|
67368
|
+
if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
|
|
67369
|
+
return cached.catalog;
|
|
67370
|
+
}
|
|
67371
|
+
const checkpoints = backend.listCheckpoints(rootWorkflowId);
|
|
67372
|
+
const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
|
|
67373
|
+
const catalog = nestedRunCatalog(runs, checkpoints);
|
|
67374
|
+
if (version !== undefined) {
|
|
67375
|
+
const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
|
|
67376
|
+
nestedRunCatalogs.set(backend, byRoot);
|
|
67377
|
+
byRoot.delete(rootWorkflowId);
|
|
67378
|
+
byRoot.set(rootWorkflowId, {
|
|
67379
|
+
version,
|
|
67380
|
+
name: handle.name,
|
|
67381
|
+
updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
|
|
67382
|
+
catalog
|
|
67383
|
+
});
|
|
67384
|
+
if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
|
|
67385
|
+
byRoot.delete(byRoot.keys().next().value);
|
|
67386
|
+
}
|
|
67387
|
+
return catalog;
|
|
67388
|
+
}
|
|
67389
|
+
function nestedRunCatalog(runs, checkpoints) {
|
|
67390
|
+
const positionsById = new Map;
|
|
67391
|
+
const childIdsByParent = new Map;
|
|
67392
|
+
runs.forEach((run, position) => {
|
|
67393
|
+
const positions = positionsById.get(run.id);
|
|
67394
|
+
if (positions === undefined)
|
|
67395
|
+
positionsById.set(run.id, [position]);
|
|
67396
|
+
else
|
|
67397
|
+
positions.push(position);
|
|
67398
|
+
if (run.parentRunId === undefined)
|
|
67399
|
+
return;
|
|
67400
|
+
const children = childIdsByParent.get(run.parentRunId);
|
|
67401
|
+
if (children === undefined)
|
|
67402
|
+
childIdsByParent.set(run.parentRunId, [run.id]);
|
|
67403
|
+
else
|
|
67404
|
+
children.push(run.id);
|
|
67405
|
+
});
|
|
67406
|
+
const intentionalExitStatuses = new Map;
|
|
67407
|
+
for (const checkpoint of checkpoints) {
|
|
67408
|
+
if (checkpoint.kind !== "stage")
|
|
67409
|
+
continue;
|
|
67410
|
+
const child = parseWorkflowChildResult(checkpoint.output);
|
|
67411
|
+
if (child?.exited === true)
|
|
67412
|
+
intentionalExitStatuses.set(child.runId, child.status);
|
|
67413
|
+
}
|
|
67414
|
+
return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
|
|
67415
|
+
}
|
|
67416
|
+
function readsFallbackCompletedAt(run) {
|
|
67417
|
+
return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
|
|
67418
|
+
}
|
|
67419
|
+
function cloneRunSnapshot(run) {
|
|
67420
|
+
const copy = structuredClone(run);
|
|
67421
|
+
for (const tool of copy.toolNodes)
|
|
67422
|
+
Object.freeze(tool.parentIds);
|
|
67423
|
+
return copy;
|
|
65833
67424
|
}
|
|
65834
67425
|
function checkpointDrafts(checkpoints) {
|
|
65835
67426
|
const stageByReplayKey = new Map;
|
|
@@ -65853,11 +67444,8 @@ function checkpointDrafts(checkpoints) {
|
|
|
65853
67444
|
tools: [...toolByHash.values()]
|
|
65854
67445
|
};
|
|
65855
67446
|
}
|
|
65856
|
-
function completedToolNodes(
|
|
65857
|
-
return
|
|
65858
|
-
const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
65859
|
-
if (topologyRunId !== runId)
|
|
65860
|
-
return [];
|
|
67447
|
+
function completedToolNodes(ownedCheckpoints, sourceIds) {
|
|
67448
|
+
return ownedCheckpoints.flatMap((checkpoint) => {
|
|
65861
67449
|
const topology = checkpoint.topology;
|
|
65862
67450
|
if (topology === undefined)
|
|
65863
67451
|
return [];
|
|
@@ -65894,18 +67482,34 @@ function summarizeCompletedToolResult(value2) {
|
|
|
65894
67482
|
return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
|
|
65895
67483
|
}
|
|
65896
67484
|
function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
|
|
65897
|
-
|
|
67485
|
+
const index = new CheckpointCatalogIndex(checkpoints);
|
|
67486
|
+
if (!validBoundaryRecordSet(index, policy.requireTerminal))
|
|
65898
67487
|
return [];
|
|
65899
|
-
const stageRecords =
|
|
67488
|
+
const stageRecords = index.stages;
|
|
65900
67489
|
const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
|
|
65901
67490
|
if (immutableStageGroupError(identityRecords) !== undefined)
|
|
65902
67491
|
return [];
|
|
65903
|
-
const
|
|
65904
|
-
const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
67492
|
+
const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
65905
67493
|
const drafts = checkpointDrafts(checkpoints);
|
|
65906
67494
|
const toolCheckpoints = drafts.tools;
|
|
65907
67495
|
if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
|
|
65908
67496
|
return [];
|
|
67497
|
+
const toolNodeIdsByClaimedRun = new Map;
|
|
67498
|
+
const toolsByRun = new Map;
|
|
67499
|
+
for (const checkpoint of toolCheckpoints) {
|
|
67500
|
+
const claimedRunId = checkpoint.topology?.run?.runId;
|
|
67501
|
+
if (claimedRunId !== undefined) {
|
|
67502
|
+
const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
|
|
67503
|
+
ids.add(checkpoint.topology.nodeId);
|
|
67504
|
+
toolNodeIdsByClaimedRun.set(claimedRunId, ids);
|
|
67505
|
+
}
|
|
67506
|
+
const ownerRunId = claimedRunId ?? rootRunId;
|
|
67507
|
+
const owned = toolsByRun.get(ownerRunId);
|
|
67508
|
+
if (owned === undefined)
|
|
67509
|
+
toolsByRun.set(ownerRunId, [checkpoint]);
|
|
67510
|
+
else
|
|
67511
|
+
owned.push(checkpoint);
|
|
67512
|
+
}
|
|
65909
67513
|
for (const draft of candidates) {
|
|
65910
67514
|
const boundary = draft.topology?.boundary;
|
|
65911
67515
|
if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
|
|
@@ -65919,8 +67523,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
65919
67523
|
parentStageId: boundary.child.parentStageId,
|
|
65920
67524
|
rootRunId: boundary.child.rootRunId
|
|
65921
67525
|
};
|
|
65922
|
-
const childToolIds =
|
|
65923
|
-
if (
|
|
67526
|
+
const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
|
|
67527
|
+
if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
|
|
65924
67528
|
return [];
|
|
65925
67529
|
}
|
|
65926
67530
|
if (candidates.length === 0 && toolCheckpoints.length === 0) {
|
|
@@ -65948,7 +67552,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
65948
67552
|
group.sort(compareDraftSourceOrder);
|
|
65949
67553
|
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
65950
67554
|
return [];
|
|
65951
|
-
if (!validateRunGroups(grouped, rootRunId,
|
|
67555
|
+
if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
|
|
65952
67556
|
return [];
|
|
65953
67557
|
const syntheticChildren = [];
|
|
65954
67558
|
const pendingChildren = [];
|
|
@@ -65962,7 +67566,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
65962
67566
|
const child = workflowChildFromDraft(draft);
|
|
65963
67567
|
if (child?.exited !== true || grouped.has(child.runId))
|
|
65964
67568
|
continue;
|
|
65965
|
-
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(
|
|
67569
|
+
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
|
|
65966
67570
|
return [];
|
|
65967
67571
|
}
|
|
65968
67572
|
syntheticChildren.push({ parentRunId, owner: draft, child });
|
|
@@ -65970,7 +67574,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
65970
67574
|
}
|
|
65971
67575
|
const idMaps = new Map;
|
|
65972
67576
|
for (const [runId, runDrafts] of grouped) {
|
|
65973
|
-
const ownedTools =
|
|
67577
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
65974
67578
|
if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
|
|
65975
67579
|
return [];
|
|
65976
67580
|
const ids = new Map;
|
|
@@ -65988,32 +67592,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
65988
67592
|
});
|
|
65989
67593
|
if (hasIdentityConflict)
|
|
65990
67594
|
return [];
|
|
65991
|
-
for (const checkpoint of
|
|
65992
|
-
const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
67595
|
+
for (const checkpoint of ownedTools) {
|
|
65993
67596
|
const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
|
|
65994
|
-
|
|
65995
|
-
ids.set(sourceId, sourceId);
|
|
67597
|
+
ids.set(sourceId, sourceId);
|
|
65996
67598
|
}
|
|
65997
67599
|
idMaps.set(runId, ids);
|
|
65998
67600
|
}
|
|
67601
|
+
const boundaryOwners = new Map;
|
|
67602
|
+
const boundarySourcesByParent = new Map;
|
|
67603
|
+
for (const [parentRunId, runDrafts] of grouped) {
|
|
67604
|
+
const sources = new Map;
|
|
67605
|
+
for (const draft of runDrafts) {
|
|
67606
|
+
const childRunId = childRunIdFromDraft(draft);
|
|
67607
|
+
if (childRunId === undefined)
|
|
67608
|
+
continue;
|
|
67609
|
+
if (!boundaryOwners.has(childRunId))
|
|
67610
|
+
boundaryOwners.set(childRunId, draft);
|
|
67611
|
+
if (!sources.has(childRunId))
|
|
67612
|
+
sources.set(childRunId, draft);
|
|
67613
|
+
}
|
|
67614
|
+
boundarySourcesByParent.set(parentRunId, sources);
|
|
67615
|
+
}
|
|
65999
67616
|
const runs = [];
|
|
66000
67617
|
const emittedRunIds = new Set;
|
|
66001
67618
|
for (const [runId, runDrafts] of grouped) {
|
|
66002
67619
|
const ids = idMaps.get(runId);
|
|
66003
|
-
const ownedTools =
|
|
67620
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
66004
67621
|
const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
|
|
66005
67622
|
if (hasUnknownParents)
|
|
66006
67623
|
return [];
|
|
66007
67624
|
const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
|
|
66008
|
-
const toolNodes = completedToolNodes(
|
|
67625
|
+
const toolNodes = completedToolNodes(ownedTools, ids);
|
|
66009
67626
|
const run = runTopologyFor(runDrafts, ownedTools);
|
|
66010
67627
|
const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
|
|
66011
67628
|
const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
|
|
66012
|
-
const owner = runId === rootRunId ? undefined :
|
|
67629
|
+
const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
|
|
66013
67630
|
const status = owner === undefined ? "completed" : childRunStatus(owner);
|
|
66014
67631
|
const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
|
|
66015
67632
|
const parentRunId = run?.parentRunId ?? rootRunId;
|
|
66016
|
-
const boundarySourceId =
|
|
67633
|
+
const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
|
|
66017
67634
|
const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
|
|
66018
67635
|
const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
|
|
66019
67636
|
const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
|
|
@@ -66149,10 +67766,11 @@ function emptyGraphRun(runId, runName, completedAt) {
|
|
|
66149
67766
|
resumable: false
|
|
66150
67767
|
};
|
|
66151
67768
|
}
|
|
66152
|
-
var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
67769
|
+
var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
66153
67770
|
var init_completed_catalog = __esm(() => {
|
|
66154
67771
|
init_stage_session_store();
|
|
66155
67772
|
init_tool_payload_bounds();
|
|
67773
|
+
init_completed_catalog_index();
|
|
66156
67774
|
init_completed_catalog_stage_groups();
|
|
66157
67775
|
init_resume_eligibility();
|
|
66158
67776
|
init_resume_runtime();
|
|
@@ -66161,6 +67779,13 @@ var init_completed_catalog = __esm(() => {
|
|
|
66161
67779
|
init_workflow_heartbeat_anchor();
|
|
66162
67780
|
init_stage_topology_validation();
|
|
66163
67781
|
init_tool_outcome();
|
|
67782
|
+
nestedRunCatalogs = new WeakMap;
|
|
67783
|
+
EMPTY_NESTED_RUN_CATALOG = {
|
|
67784
|
+
runs: [],
|
|
67785
|
+
positionsById: new Map,
|
|
67786
|
+
childIdsByParent: new Map,
|
|
67787
|
+
intentionalExitStatuses: new Map
|
|
67788
|
+
};
|
|
66164
67789
|
COMPLETED_RECONSTRUCTION = {
|
|
66165
67790
|
requireTerminal: true,
|
|
66166
67791
|
retainActive: false,
|
|
@@ -66175,29 +67800,23 @@ var init_completed_catalog = __esm(() => {
|
|
|
66175
67800
|
|
|
66176
67801
|
// dist/builtin/workflows/src/durable/completed-subtree.ts
|
|
66177
67802
|
function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
|
|
66178
|
-
const
|
|
66179
|
-
if (!
|
|
67803
|
+
const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
|
|
67804
|
+
if (!catalog.positionsById.has(childRunId))
|
|
66180
67805
|
return;
|
|
66181
|
-
const intentionalExitStatuses = new Map;
|
|
66182
|
-
for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
|
|
66183
|
-
if (checkpoint.kind !== "stage")
|
|
66184
|
-
continue;
|
|
66185
|
-
const child = parseWorkflowChildResult(checkpoint.output);
|
|
66186
|
-
if (child?.exited === true)
|
|
66187
|
-
intentionalExitStatuses.set(child.runId, child.status);
|
|
66188
|
-
}
|
|
66189
67806
|
const included = new Set([childRunId]);
|
|
66190
|
-
|
|
66191
|
-
|
|
66192
|
-
|
|
66193
|
-
|
|
66194
|
-
|
|
66195
|
-
|
|
67807
|
+
const pending = [childRunId];
|
|
67808
|
+
for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
|
|
67809
|
+
for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
|
|
67810
|
+
if (included.has(childId))
|
|
67811
|
+
continue;
|
|
67812
|
+
included.add(childId);
|
|
67813
|
+
pending.push(childId);
|
|
66196
67814
|
}
|
|
66197
67815
|
}
|
|
66198
|
-
const
|
|
66199
|
-
const
|
|
66200
|
-
|
|
67816
|
+
const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
|
|
67817
|
+
const subtree = positions.map((position) => catalog.runs[position]);
|
|
67818
|
+
const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
|
|
67819
|
+
return complete ? subtree.map(cloneRunSnapshot) : undefined;
|
|
66201
67820
|
}
|
|
66202
67821
|
function isCompleteCachedRun(run, intentionalExitStatuses) {
|
|
66203
67822
|
if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
|
|
@@ -66318,7 +67937,7 @@ function createDurableCachedStageRecorder(input) {
|
|
|
66318
67937
|
const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
|
|
66319
67938
|
kind: "stage",
|
|
66320
67939
|
workflowId: input.run.id,
|
|
66321
|
-
checkpointId:
|
|
67940
|
+
checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
|
|
66322
67941
|
name,
|
|
66323
67942
|
replayKey,
|
|
66324
67943
|
completedAt: Date.now(),
|
|
@@ -66490,7 +68109,7 @@ function recordedOwners(boundary, reason) {
|
|
|
66490
68109
|
const pending = [{ value: reason, depth: 0 }];
|
|
66491
68110
|
while (pending.length > 0) {
|
|
66492
68111
|
const { value: value2, depth } = pending.pop();
|
|
66493
|
-
if (depth >
|
|
68112
|
+
if (depth > MAX_CAUSE_DEPTH4)
|
|
66494
68113
|
continue;
|
|
66495
68114
|
if (isPrimitive(value2)) {
|
|
66496
68115
|
if (depth === 0 || value2 !== undefined && value2 !== null) {
|
|
@@ -66812,7 +68431,7 @@ function containUiRejections(ui, scope) {
|
|
|
66812
68431
|
custom: (factory, options) => scope.watch(ui.custom(factory, options))
|
|
66813
68432
|
};
|
|
66814
68433
|
}
|
|
66815
|
-
var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections",
|
|
68434
|
+
var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections", MAX_CAUSE_DEPTH4 = 8, MAX_PRIMITIVES_PER_RUN = 64, MAX_STOP_STRINGS = 64, ENDED_RUN_GRACE_MS = 30000, NATIVE_REPORTING_MODES, WATCHED_STAGE_METHODS;
|
|
66816
68435
|
var init_run_rejection_boundary = __esm(() => {
|
|
66817
68436
|
init_prompt_callsite();
|
|
66818
68437
|
init_workflow_source();
|
|
@@ -72703,27 +74322,6 @@ var init_runtime = __esm(() => {
|
|
|
72703
74322
|
init_intercom_group();
|
|
72704
74323
|
});
|
|
72705
74324
|
|
|
72706
|
-
// dist/builtin/workflows/src/engine/session-closed-error.ts
|
|
72707
|
-
import { isStaleExtensionContextError as isStaleExtensionContextError2 } from "@bastani/atomic";
|
|
72708
|
-
function reportsClosedSession(error) {
|
|
72709
|
-
return "code" in error && error.code === "SessionClosed" || isStaleExtensionContextError2(error);
|
|
72710
|
-
}
|
|
72711
|
-
function isSessionClosedError(error) {
|
|
72712
|
-
let current = error;
|
|
72713
|
-
for (let depth = 0;depth < MAX_CAUSE_DEPTH4 && current !== undefined; depth++) {
|
|
72714
|
-
try {
|
|
72715
|
-
if (reportsClosedSession(current))
|
|
72716
|
-
return true;
|
|
72717
|
-
current = current.cause instanceof Error ? current.cause : undefined;
|
|
72718
|
-
} catch {
|
|
72719
|
-
return false;
|
|
72720
|
-
}
|
|
72721
|
-
}
|
|
72722
|
-
return false;
|
|
72723
|
-
}
|
|
72724
|
-
var MAX_CAUSE_DEPTH4 = 8;
|
|
72725
|
-
var init_session_closed_error = () => {};
|
|
72726
|
-
|
|
72727
74325
|
// dist/builtin/workflows/src/engine/workflow-activity.ts
|
|
72728
74326
|
import { runCallback as runCallback4 } from "@bastani/atomic";
|
|
72729
74327
|
function nextEventLoopTurn() {
|