@dudousxd/nestjs-durable-core 0.85.6 → 0.85.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +619 -148
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +125 -19
- package/dist/index.d.ts +125 -19
- package/dist/index.js +619 -148
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -937,7 +937,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
|
|
|
937
937
|
"failed",
|
|
938
938
|
"dead"
|
|
939
939
|
]);
|
|
940
|
-
var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
|
|
941
940
|
function canonicalJson(value) {
|
|
942
941
|
const plain = JSON.stringify(value);
|
|
943
942
|
return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
|
|
@@ -952,77 +951,235 @@ function sortedJson(value) {
|
|
|
952
951
|
return JSON.stringify(value);
|
|
953
952
|
}
|
|
954
953
|
__name(sortedJson, "sortedJson");
|
|
955
|
-
|
|
954
|
+
var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
|
|
955
|
+
var SEQ_CHILD = /\.child\.(\d+)$/;
|
|
956
|
+
function slotOf(childId) {
|
|
956
957
|
const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
|
|
957
|
-
const fan =
|
|
958
|
-
if (
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
958
|
+
const fan = FAN_CHILD_PARTS.exec(childId);
|
|
959
|
+
if (fan) {
|
|
960
|
+
const root = fan[1];
|
|
961
|
+
const wave = fan[3];
|
|
962
|
+
const firstSeq = Number(fan[2]);
|
|
963
|
+
const i = Number(fan[4]);
|
|
964
|
+
if (wave === void 0) {
|
|
965
|
+
return {
|
|
966
|
+
signal,
|
|
967
|
+
signalSeq: firstSeq + i,
|
|
968
|
+
fan: {
|
|
969
|
+
fold: `${FOLDED_PREFIX}${root}`,
|
|
970
|
+
firstSeq,
|
|
971
|
+
index: i
|
|
972
|
+
}
|
|
973
|
+
};
|
|
966
974
|
}
|
|
975
|
+
return {
|
|
976
|
+
signal,
|
|
977
|
+
signalSeq: firstSeq + i,
|
|
978
|
+
fan: {
|
|
979
|
+
fold: `${FOLDED_PREFIX}${root}w${wave}.`,
|
|
980
|
+
firstSeq,
|
|
981
|
+
index: i,
|
|
982
|
+
wave: Number(wave),
|
|
983
|
+
waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
|
|
984
|
+
}
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
const seq = SEQ_CHILD.exec(childId);
|
|
988
|
+
return seq ? {
|
|
989
|
+
signal,
|
|
990
|
+
signalSeq: Number(seq[1])
|
|
991
|
+
} : {
|
|
992
|
+
signal
|
|
967
993
|
};
|
|
968
994
|
}
|
|
969
|
-
__name(
|
|
970
|
-
function
|
|
971
|
-
|
|
972
|
-
const
|
|
973
|
-
return
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
995
|
+
__name(slotOf, "slotOf");
|
|
996
|
+
function foldOf(cp, name) {
|
|
997
|
+
if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
|
|
998
|
+
const { from, values } = cp.output ?? {};
|
|
999
|
+
return typeof from === "number" && Array.isArray(values) ? {
|
|
1000
|
+
from,
|
|
1001
|
+
values
|
|
1002
|
+
} : void 0;
|
|
1003
|
+
}
|
|
1004
|
+
__name(foldOf, "foldOf");
|
|
1005
|
+
function covers(fold, index) {
|
|
1006
|
+
return !!fold && index >= fold.from && index < fold.from + fold.values.length;
|
|
1007
|
+
}
|
|
1008
|
+
__name(covers, "covers");
|
|
1009
|
+
function holdsSignal(cp, signal, expected) {
|
|
1010
|
+
return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
|
|
1011
|
+
}
|
|
1012
|
+
__name(holdsSignal, "holdsSignal");
|
|
1013
|
+
async function checkpointsAt(store, runId, seqs) {
|
|
1014
|
+
const unique = [
|
|
1015
|
+
...new Set(seqs)
|
|
1016
|
+
];
|
|
1017
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
1018
|
+
if (unique.length === 0) return bySeq;
|
|
1019
|
+
const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
|
|
1020
|
+
for (const cp of rows) bySeq.set(cp.seq, cp);
|
|
1021
|
+
return bySeq;
|
|
1022
|
+
}
|
|
1023
|
+
__name(checkpointsAt, "checkpointsAt");
|
|
1024
|
+
async function checkpointsNamed(store, runId, names) {
|
|
1025
|
+
const wanted = new Set(names);
|
|
1026
|
+
const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
|
|
1027
|
+
...wanted
|
|
1028
|
+
]) : await store.listCheckpoints(runId);
|
|
1029
|
+
return rows.filter((cp) => wanted.has(cp.name));
|
|
1030
|
+
}
|
|
1031
|
+
__name(checkpointsNamed, "checkpointsNamed");
|
|
1032
|
+
async function parentSettles(store, parentId, children) {
|
|
1033
|
+
const entries = children.map(({ childId, completion }) => ({
|
|
1034
|
+
slot: slotOf(childId),
|
|
1035
|
+
expected: canonicalJson(completion)
|
|
1036
|
+
}));
|
|
1037
|
+
const held = entries.map(() => void 0);
|
|
1038
|
+
const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
|
|
1039
|
+
...slot.signalSeq === void 0 ? [] : [
|
|
1040
|
+
slot.signalSeq
|
|
1041
|
+
],
|
|
1042
|
+
...slot.fan ? [
|
|
1043
|
+
slot.fan.firstSeq
|
|
1044
|
+
] : []
|
|
1045
|
+
]));
|
|
1046
|
+
const waveFolds = [];
|
|
1047
|
+
const byName = [];
|
|
1048
|
+
entries.forEach(({ slot, expected }, at) => {
|
|
1049
|
+
const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
|
|
1050
|
+
if (own?.name === slot.signal) {
|
|
1051
|
+
held[at] = holdsSignal(own, slot.signal, expected);
|
|
1052
|
+
return;
|
|
979
1053
|
}
|
|
980
|
-
|
|
1054
|
+
const fan = slot.fan;
|
|
1055
|
+
if (fan) {
|
|
1056
|
+
if (covers(foldOf(own, fan.fold), fan.index)) {
|
|
1057
|
+
held[at] = true;
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
const atFirst = first.get(fan.firstSeq);
|
|
1061
|
+
if (fan.wave === void 0 || fan.wave === 0) {
|
|
1062
|
+
held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
const waveZero = foldOf(atFirst, fan.waveZeroFold);
|
|
1066
|
+
if (waveZero && waveZero.values.length > 0) {
|
|
1067
|
+
waveFolds.push({
|
|
1068
|
+
at,
|
|
1069
|
+
seq: fan.firstSeq + fan.wave * waveZero.values.length
|
|
1070
|
+
});
|
|
1071
|
+
return;
|
|
1072
|
+
}
|
|
1073
|
+
byName.push(at);
|
|
1074
|
+
return;
|
|
1075
|
+
}
|
|
1076
|
+
if (slot.signalSeq !== void 0 && !own) {
|
|
1077
|
+
held[at] = false;
|
|
1078
|
+
return;
|
|
1079
|
+
}
|
|
1080
|
+
byName.push(at);
|
|
981
1081
|
});
|
|
1082
|
+
if (waveFolds.length > 0) {
|
|
1083
|
+
const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
|
|
1084
|
+
for (const { at, seq } of waveFolds) {
|
|
1085
|
+
const fan = entries[at].slot.fan;
|
|
1086
|
+
held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
if (byName.length > 0) {
|
|
1090
|
+
const names = byName.flatMap((at) => {
|
|
1091
|
+
const { slot } = entries[at];
|
|
1092
|
+
return slot.fan ? [
|
|
1093
|
+
slot.signal,
|
|
1094
|
+
slot.fan.fold
|
|
1095
|
+
] : [
|
|
1096
|
+
slot.signal
|
|
1097
|
+
];
|
|
1098
|
+
});
|
|
1099
|
+
const named = await checkpointsNamed(store, parentId, names);
|
|
1100
|
+
for (const at of byName) {
|
|
1101
|
+
const { slot, expected } = entries[at];
|
|
1102
|
+
held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
return held.map((h) => h === true);
|
|
982
1106
|
}
|
|
983
|
-
__name(
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
const prefixes = fold ? [
|
|
987
|
-
signal,
|
|
988
|
-
fold.name
|
|
989
|
-
] : [
|
|
990
|
-
signal
|
|
991
|
-
];
|
|
992
|
-
return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
|
|
1107
|
+
__name(parentSettles, "parentSettles");
|
|
1108
|
+
function parentClosed(run) {
|
|
1109
|
+
return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
|
|
993
1110
|
}
|
|
994
|
-
__name(
|
|
1111
|
+
__name(parentClosed, "parentClosed");
|
|
995
1112
|
async function childCompletionNeedsBuffer(store, childId, completion, parent) {
|
|
996
1113
|
const parentId = parentRunIdOf(childId);
|
|
997
1114
|
if (!parentId) return false;
|
|
998
1115
|
const run = parent === void 0 ? await store.getRun(parentId) : parent;
|
|
999
|
-
if (
|
|
1000
|
-
|
|
1001
|
-
|
|
1116
|
+
if (parentClosed(run)) return false;
|
|
1117
|
+
const [held] = await parentSettles(store, parentId, [
|
|
1118
|
+
{
|
|
1119
|
+
childId,
|
|
1120
|
+
completion
|
|
1121
|
+
}
|
|
1122
|
+
]);
|
|
1123
|
+
return !held;
|
|
1002
1124
|
}
|
|
1003
1125
|
__name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
|
|
1004
|
-
async function unneededChildSignals(store, rows) {
|
|
1126
|
+
async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
|
|
1127
|
+
const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
|
|
1005
1128
|
const parentIds = rows.flatMap((row) => {
|
|
1006
|
-
const
|
|
1129
|
+
const childId = childOf(row);
|
|
1130
|
+
const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
|
|
1007
1131
|
return parentId ? [
|
|
1008
1132
|
parentId
|
|
1009
1133
|
] : [];
|
|
1010
1134
|
});
|
|
1011
1135
|
const { runs, unread } = await readRunsByIds(store, parentIds);
|
|
1012
1136
|
const unreadable = new Set(unread);
|
|
1013
|
-
const doomed =
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
const childId = row
|
|
1137
|
+
const doomed = /* @__PURE__ */ new Set();
|
|
1138
|
+
const open = /* @__PURE__ */ new Map();
|
|
1139
|
+
rows.forEach((row, at) => {
|
|
1140
|
+
const childId = childOf(row);
|
|
1141
|
+
if (childId === void 0) return;
|
|
1017
1142
|
const parentId = parentRunIdOf(childId);
|
|
1018
|
-
if (parentId
|
|
1019
|
-
|
|
1143
|
+
if (!parentId) {
|
|
1144
|
+
doomed.add(row);
|
|
1145
|
+
return;
|
|
1146
|
+
}
|
|
1147
|
+
if (unreadable.has(parentId)) return;
|
|
1148
|
+
if (parentClosed(runs.get(parentId) ?? null)) {
|
|
1149
|
+
doomed.add(row);
|
|
1150
|
+
return;
|
|
1151
|
+
}
|
|
1152
|
+
const group = open.get(parentId) ?? [];
|
|
1153
|
+
group.push({
|
|
1154
|
+
at,
|
|
1155
|
+
row,
|
|
1156
|
+
childId
|
|
1157
|
+
});
|
|
1158
|
+
open.set(parentId, group);
|
|
1159
|
+
});
|
|
1160
|
+
let decided = rows.length;
|
|
1161
|
+
let checked = 0;
|
|
1162
|
+
for (const [parentId, group] of open) {
|
|
1163
|
+
if (checked > 0 && Date.now() >= deadline) {
|
|
1164
|
+
decided = group[0].at;
|
|
1165
|
+
break;
|
|
1166
|
+
}
|
|
1167
|
+
checked += 1;
|
|
1020
1168
|
try {
|
|
1021
|
-
|
|
1169
|
+
const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
|
|
1170
|
+
childId,
|
|
1171
|
+
completion: row.payload
|
|
1172
|
+
})));
|
|
1173
|
+
group.forEach(({ row }, k) => {
|
|
1174
|
+
if (held[k]) doomed.add(row);
|
|
1175
|
+
});
|
|
1022
1176
|
} catch {
|
|
1023
1177
|
}
|
|
1024
1178
|
}
|
|
1025
|
-
return
|
|
1179
|
+
return {
|
|
1180
|
+
doomed: rows.filter((row) => doomed.has(row)),
|
|
1181
|
+
decided
|
|
1182
|
+
};
|
|
1026
1183
|
}
|
|
1027
1184
|
__name(unneededChildSignals, "unneededChildSignals");
|
|
1028
1185
|
|
|
@@ -3347,6 +3504,7 @@ function constantTimeEquals(a, b) {
|
|
|
3347
3504
|
__name(constantTimeEquals, "constantTimeEquals");
|
|
3348
3505
|
|
|
3349
3506
|
// src/transport-pool.ts
|
|
3507
|
+
var DISPATCH_CHUNK = 200;
|
|
3350
3508
|
var TransportPool = class {
|
|
3351
3509
|
static {
|
|
3352
3510
|
__name(this, "TransportPool");
|
|
@@ -3406,6 +3564,28 @@ var TransportPool = class {
|
|
|
3406
3564
|
}
|
|
3407
3565
|
throw lastErr ?? new Error("no transport accepted the dispatch");
|
|
3408
3566
|
}
|
|
3567
|
+
/**
|
|
3568
|
+
* {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
|
|
3569
|
+
* first) transport when it has one, else (or when that call rejects) each task through
|
|
3570
|
+
* {@link dispatch}, a bounded number at a time.
|
|
3571
|
+
*/
|
|
3572
|
+
async dispatchMany(tasks, preferId) {
|
|
3573
|
+
if (tasks.length === 0) return;
|
|
3574
|
+
const first = this.ordered(preferId)[0];
|
|
3575
|
+
if (first?.transport.dispatchMany) {
|
|
3576
|
+
try {
|
|
3577
|
+
await first.transport.dispatchMany(tasks.map((task) => ({
|
|
3578
|
+
...task,
|
|
3579
|
+
transport: first.id
|
|
3580
|
+
})));
|
|
3581
|
+
return;
|
|
3582
|
+
} catch {
|
|
3583
|
+
}
|
|
3584
|
+
}
|
|
3585
|
+
for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
|
|
3586
|
+
await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
|
|
3587
|
+
}
|
|
3588
|
+
}
|
|
3409
3589
|
/** Worker-health for `group`, merged across every transport that can report it (a group's queue
|
|
3410
3590
|
* and its workers may live on any pinned transport). Undefined when no transport implements
|
|
3411
3591
|
* `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
|
|
@@ -4625,6 +4805,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
|
4625
4805
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
4626
4806
|
var LEASE_RECHECK_SLACK_MS = 100;
|
|
4627
4807
|
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
4808
|
+
function wakeSeen(request, settledFans, served) {
|
|
4809
|
+
if (request.id !== void 0 && served.has(request.id)) return true;
|
|
4810
|
+
return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
|
|
4811
|
+
}
|
|
4812
|
+
__name(wakeSeen, "wakeSeen");
|
|
4813
|
+
function isFanWakeRequest(payload) {
|
|
4814
|
+
return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
|
|
4815
|
+
}
|
|
4816
|
+
__name(isFanWakeRequest, "isFanWakeRequest");
|
|
4628
4817
|
function deferred() {
|
|
4629
4818
|
let resolve;
|
|
4630
4819
|
let reject;
|
|
@@ -4643,12 +4832,52 @@ __name(deferred, "deferred");
|
|
|
4643
4832
|
var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
|
|
4644
4833
|
var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
|
|
4645
4834
|
var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
|
|
4835
|
+
var WAKE_REQUEST_SCAN = 100;
|
|
4646
4836
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4647
4837
|
var BUFFERED_SWEEP_PAGE = 500;
|
|
4648
4838
|
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
4839
|
+
var BUFFERED_SWEEP_BUDGET_MS = 250;
|
|
4649
4840
|
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4650
4841
|
var DELIVER_CHUNK = 50;
|
|
4651
4842
|
var RESULT_BATCH_MAX = 256;
|
|
4843
|
+
function isResultError(outcome) {
|
|
4844
|
+
return typeof outcome === "object" && "error" in outcome;
|
|
4845
|
+
}
|
|
4846
|
+
__name(isResultError, "isResultError");
|
|
4847
|
+
var FanWakeCollector = class FanWakeCollector2 {
|
|
4848
|
+
static {
|
|
4849
|
+
__name(this, "FanWakeCollector");
|
|
4850
|
+
}
|
|
4851
|
+
carrier;
|
|
4852
|
+
generic = false;
|
|
4853
|
+
fan;
|
|
4854
|
+
/** The highest seq the batch settled, per fan. */
|
|
4855
|
+
highest = /* @__PURE__ */ new Map();
|
|
4856
|
+
/** A checkpoint the batch settled. */
|
|
4857
|
+
note(cp) {
|
|
4858
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4859
|
+
if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
|
|
4860
|
+
}
|
|
4861
|
+
/** A settled checkpoint that resumes the run. */
|
|
4862
|
+
add(index, cp, status) {
|
|
4863
|
+
this.carrier = index;
|
|
4864
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4865
|
+
if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
|
|
4866
|
+
this.generic = true;
|
|
4867
|
+
return;
|
|
4868
|
+
}
|
|
4869
|
+
this.fan = fan.firstSeq;
|
|
4870
|
+
}
|
|
4871
|
+
applyTo(outcomes) {
|
|
4872
|
+
if (this.carrier === void 0) return outcomes;
|
|
4873
|
+
const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
|
|
4874
|
+
outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
|
|
4875
|
+
first: this.fan,
|
|
4876
|
+
seq
|
|
4877
|
+
};
|
|
4878
|
+
return outcomes;
|
|
4879
|
+
}
|
|
4880
|
+
};
|
|
4652
4881
|
function shuffle(items) {
|
|
4653
4882
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
4654
4883
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -4837,6 +5066,23 @@ var WorkflowEngine = class {
|
|
|
4837
5066
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
4838
5067
|
leaseRetries = /* @__PURE__ */ new Map();
|
|
4839
5068
|
/**
|
|
5069
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
5070
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
5071
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
5072
|
+
*/
|
|
5073
|
+
unconditionalRechecks = /* @__PURE__ */ new Set();
|
|
5074
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
5075
|
+
genericWakeRequested = /* @__PURE__ */ new Set();
|
|
5076
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
5077
|
+
wakeReasons = /* @__PURE__ */ new Map();
|
|
5078
|
+
/**
|
|
5079
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
5080
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
5081
|
+
*/
|
|
5082
|
+
turnSettledFans = /* @__PURE__ */ new Map();
|
|
5083
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
5084
|
+
turnServedWakes = /* @__PURE__ */ new Map();
|
|
5085
|
+
/**
|
|
4840
5086
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
4841
5087
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
4842
5088
|
* enqueued pickups and the runs it takes off the transport's run queue.
|
|
@@ -4960,8 +5206,9 @@ var WorkflowEngine = class {
|
|
|
4960
5206
|
}
|
|
4961
5207
|
return;
|
|
4962
5208
|
}
|
|
4963
|
-
|
|
4964
|
-
|
|
5209
|
+
const wake = await this.completeRemoteResult(result);
|
|
5210
|
+
if (!wake) return;
|
|
5211
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
|
|
4965
5212
|
if (err instanceof NamespaceMismatch) return;
|
|
4966
5213
|
throw err;
|
|
4967
5214
|
}));
|
|
@@ -5178,7 +5425,10 @@ var WorkflowEngine = class {
|
|
|
5178
5425
|
const resumed = await this.resumeLeased([
|
|
5179
5426
|
run
|
|
5180
5427
|
], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
|
|
5181
|
-
if (resumed.length === 0)
|
|
5428
|
+
if (resumed.length === 0) {
|
|
5429
|
+
this.unconditionalRechecks.add(runId);
|
|
5430
|
+
this.recheckAfterLease(runId, run.lockedUntil);
|
|
5431
|
+
}
|
|
5182
5432
|
return;
|
|
5183
5433
|
}
|
|
5184
5434
|
await this.resumeImplicit(runId).catch((err) => {
|
|
@@ -5479,13 +5729,17 @@ var WorkflowEngine = class {
|
|
|
5479
5729
|
*
|
|
5480
5730
|
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
5481
5731
|
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
5482
|
-
* few timer ticks without a long query or a big delete.
|
|
5483
|
-
*
|
|
5484
|
-
*
|
|
5485
|
-
*
|
|
5486
|
-
*
|
|
5487
|
-
|
|
5488
|
-
|
|
5732
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
5733
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
5734
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
5735
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
5736
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
5737
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
5738
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
5739
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
5740
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5741
|
+
*/
|
|
5742
|
+
async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
|
|
5489
5743
|
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
5490
5744
|
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
5491
5745
|
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
@@ -5493,15 +5747,21 @@ var WorkflowEngine = class {
|
|
|
5493
5747
|
return 0;
|
|
5494
5748
|
}
|
|
5495
5749
|
this.bufferedSweepActive = true;
|
|
5750
|
+
const deadline = Date.now() + budgetMs;
|
|
5496
5751
|
let deleted = 0;
|
|
5497
5752
|
try {
|
|
5498
5753
|
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
5754
|
+
if (page > 0 && (Date.now() >= deadline || this.draining)) break;
|
|
5499
5755
|
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
5500
|
-
const doomed = await unneededChildSignals(this.store, rows);
|
|
5756
|
+
const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
|
|
5501
5757
|
if (doomed.length > 0) {
|
|
5502
5758
|
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
5503
5759
|
deleted += doomed.length;
|
|
5504
5760
|
}
|
|
5761
|
+
if (decided < rows.length) {
|
|
5762
|
+
if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
|
|
5763
|
+
break;
|
|
5764
|
+
}
|
|
5505
5765
|
const last = rows.at(-1);
|
|
5506
5766
|
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
5507
5767
|
this.bufferedSweepCursor = void 0;
|
|
@@ -5745,15 +6005,27 @@ var WorkflowEngine = class {
|
|
|
5745
6005
|
* terminal too.
|
|
5746
6006
|
*/
|
|
5747
6007
|
resume(runId) {
|
|
6008
|
+
this.noteWakeReason(runId, void 0);
|
|
5748
6009
|
return this.resumeRun(runId, true);
|
|
5749
6010
|
}
|
|
6011
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
6012
|
+
noteWakeReason(runId, fan) {
|
|
6013
|
+
const reason = this.wakeReasons.get(runId) ?? {
|
|
6014
|
+
generic: false,
|
|
6015
|
+
fans: /* @__PURE__ */ new Map()
|
|
6016
|
+
};
|
|
6017
|
+
if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
|
|
6018
|
+
else reason.generic = true;
|
|
6019
|
+
this.wakeReasons.set(runId, reason);
|
|
6020
|
+
}
|
|
5750
6021
|
/**
|
|
5751
6022
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5752
6023
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
5753
6024
|
* these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
|
|
5754
6025
|
* stale waiter or a late result left behind by a failed run silently re-executes its body.
|
|
5755
6026
|
*/
|
|
5756
|
-
resumeImplicit(runId) {
|
|
6027
|
+
resumeImplicit(runId, fan) {
|
|
6028
|
+
if (fan !== "coalesced") this.noteWakeReason(runId, fan);
|
|
5757
6029
|
const turn = this.wakeTurns.get(runId);
|
|
5758
6030
|
if (turn) {
|
|
5759
6031
|
turn.again ??= deferred();
|
|
@@ -5778,7 +6050,7 @@ var WorkflowEngine = class {
|
|
|
5778
6050
|
});
|
|
5779
6051
|
return;
|
|
5780
6052
|
}
|
|
5781
|
-
const next = this.resumeImplicit(runId);
|
|
6053
|
+
const next = this.resumeImplicit(runId, "coalesced");
|
|
5782
6054
|
next.then(again.resolve, again.reject);
|
|
5783
6055
|
this.inBackground(next);
|
|
5784
6056
|
}, "finish");
|
|
@@ -7843,9 +8115,26 @@ var WorkflowEngine = class {
|
|
|
7843
8115
|
async execute(run, fn) {
|
|
7844
8116
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
7845
8117
|
const lockNow = this.clock();
|
|
8118
|
+
const reason = this.wakeReasons.get(run.id) ?? {
|
|
8119
|
+
generic: true,
|
|
8120
|
+
fans: /* @__PURE__ */ new Map()
|
|
8121
|
+
};
|
|
8122
|
+
this.wakeReasons.delete(run.id);
|
|
7846
8123
|
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
7847
8124
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
7848
|
-
if (
|
|
8125
|
+
if (reason.generic) {
|
|
8126
|
+
if (!this.genericWakeRequested.has(run.id)) {
|
|
8127
|
+
this.genericWakeRequested.add(run.id);
|
|
8128
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, null));
|
|
8129
|
+
}
|
|
8130
|
+
} else {
|
|
8131
|
+
const fans = {
|
|
8132
|
+
fans: [
|
|
8133
|
+
...reason.fans
|
|
8134
|
+
]
|
|
8135
|
+
};
|
|
8136
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
8137
|
+
}
|
|
7849
8138
|
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
7850
8139
|
return {
|
|
7851
8140
|
runId: run.id,
|
|
@@ -7853,56 +8142,142 @@ var WorkflowEngine = class {
|
|
|
7853
8142
|
};
|
|
7854
8143
|
}
|
|
7855
8144
|
}
|
|
8145
|
+
const settledFans = /* @__PURE__ */ new Map();
|
|
8146
|
+
this.turnSettledFans.set(run.id, settledFans);
|
|
7856
8147
|
const held = this.holdExecutionLease(run.id);
|
|
7857
8148
|
try {
|
|
8149
|
+
const served = /* @__PURE__ */ new Set();
|
|
8150
|
+
this.turnServedWakes.set(run.id, served);
|
|
8151
|
+
if (registered?.remote) await this.noteServedWakes(run.id);
|
|
7858
8152
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
7859
8153
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
7860
8154
|
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
7861
8155
|
}
|
|
7862
|
-
if (result.status === "suspended")
|
|
7863
|
-
|
|
7864
|
-
|
|
8156
|
+
if (result.status === "suspended") {
|
|
8157
|
+
this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
|
|
8158
|
+
} else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
|
|
8159
|
+
this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
|
|
7865
8160
|
}
|
|
7866
8161
|
return result;
|
|
7867
8162
|
} finally {
|
|
7868
8163
|
held.stop();
|
|
8164
|
+
if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
|
|
8165
|
+
this.turnServedWakes.delete(run.id);
|
|
7869
8166
|
}
|
|
7870
8167
|
}
|
|
7871
8168
|
/**
|
|
7872
8169
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
7873
8170
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
7874
|
-
* (a signal, a child's completion). The request is a buffered
|
|
7875
|
-
* on
|
|
8171
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
8172
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
8173
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
7876
8174
|
*
|
|
7877
8175
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
7878
8176
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
7879
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
7880
|
-
*
|
|
8177
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
8178
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
8179
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
8180
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
7881
8181
|
*/
|
|
7882
|
-
async requestWakeAfterTurn(runId) {
|
|
7883
|
-
|
|
8182
|
+
async requestWakeAfterTurn(runId, request) {
|
|
8183
|
+
const id = globalThis.crypto.randomUUID();
|
|
8184
|
+
await this.store.bufferEvent({
|
|
8185
|
+
name: wakeRequestToken(runId),
|
|
8186
|
+
payload: request,
|
|
8187
|
+
id,
|
|
8188
|
+
publishedAt: this.clock()
|
|
8189
|
+
});
|
|
7884
8190
|
const run = await this.store.getRun(runId);
|
|
7885
|
-
if (run
|
|
7886
|
-
|
|
7887
|
-
|
|
8191
|
+
if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
8192
|
+
await this.store.removeBufferedEvent(id);
|
|
8193
|
+
return;
|
|
8194
|
+
}
|
|
8195
|
+
if (run.status === "running") return;
|
|
8196
|
+
if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
|
|
8197
|
+
await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
|
|
7888
8198
|
return;
|
|
7889
8199
|
}
|
|
7890
|
-
if (!await this.takeWakeRequests(runId)) return;
|
|
7891
|
-
if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
7892
8200
|
this.recheckAfterLease(runId, run.lockedUntil);
|
|
7893
8201
|
}
|
|
7894
|
-
/**
|
|
7895
|
-
|
|
7896
|
-
|
|
8202
|
+
/**
|
|
8203
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
8204
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
8205
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
8206
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
8207
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
8208
|
+
*/
|
|
8209
|
+
async honorWakeRequests(runId, settledFans, served) {
|
|
8210
|
+
const kept = await this.listWakeRequests(runId, true);
|
|
8211
|
+
const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
|
|
8212
|
+
const needed = kept.filter((request) => !seen(request));
|
|
8213
|
+
await this.dropWakeRequests(kept.filter(seen));
|
|
8214
|
+
if (needed.length === 0) return;
|
|
8215
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
7897
8216
|
await this.wake(runId).catch((err) => {
|
|
7898
8217
|
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
7899
8218
|
});
|
|
7900
8219
|
}
|
|
7901
|
-
/**
|
|
7902
|
-
|
|
7903
|
-
|
|
7904
|
-
|
|
7905
|
-
|
|
8220
|
+
/**
|
|
8221
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
8222
|
+
* signals, which are taken as they are read.
|
|
8223
|
+
*/
|
|
8224
|
+
async listWakeRequests(runId, legacy = false) {
|
|
8225
|
+
const token = wakeRequestToken(runId);
|
|
8226
|
+
const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
|
|
8227
|
+
id: request.id,
|
|
8228
|
+
payload: request.payload
|
|
8229
|
+
}));
|
|
8230
|
+
if (!legacy) return kept;
|
|
8231
|
+
for (; ; ) {
|
|
8232
|
+
const signal = await this.store.takeBufferedSignal(token);
|
|
8233
|
+
if (!signal) return kept;
|
|
8234
|
+
kept.push({
|
|
8235
|
+
payload: signal.payload
|
|
8236
|
+
});
|
|
8237
|
+
}
|
|
8238
|
+
}
|
|
8239
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
8240
|
+
async dropWakeRequests(requests) {
|
|
8241
|
+
await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
|
|
8242
|
+
this.store.removeBufferedEvent(request.id)
|
|
8243
|
+
]));
|
|
8244
|
+
}
|
|
8245
|
+
/**
|
|
8246
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
8247
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
8248
|
+
* until then stay for whoever runs the run next.
|
|
8249
|
+
*/
|
|
8250
|
+
async noteServedWakes(runId) {
|
|
8251
|
+
const served = this.turnServedWakes.get(runId);
|
|
8252
|
+
if (!served) return;
|
|
8253
|
+
for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
|
|
8254
|
+
served.add(request.id);
|
|
8255
|
+
}
|
|
8256
|
+
}
|
|
8257
|
+
/**
|
|
8258
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
8259
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
8260
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
8261
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
8262
|
+
*/
|
|
8263
|
+
async keepDueForKeptWakes(runId) {
|
|
8264
|
+
const served = this.turnServedWakes.get(runId);
|
|
8265
|
+
if (!served) return;
|
|
8266
|
+
const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
|
|
8267
|
+
const kept = await this.listWakeRequests(runId);
|
|
8268
|
+
if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
|
|
8269
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8270
|
+
}
|
|
8271
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
8272
|
+
async makeDueBy(runId, at) {
|
|
8273
|
+
const run = await this.store.getRun(runId);
|
|
8274
|
+
if (run?.status !== "suspended") return;
|
|
8275
|
+
if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
|
|
8276
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
|
|
8277
|
+
"suspended"
|
|
8278
|
+
], {
|
|
8279
|
+
wakeAt: at
|
|
8280
|
+
});
|
|
7906
8281
|
}
|
|
7907
8282
|
/**
|
|
7908
8283
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
@@ -7921,6 +8296,7 @@ var WorkflowEngine = class {
|
|
|
7921
8296
|
const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
|
|
7922
8297
|
const timer = setTimeout(() => {
|
|
7923
8298
|
this.leaseRetries.delete(runId);
|
|
8299
|
+
this.genericWakeRequested.delete(runId);
|
|
7924
8300
|
this.inBackground(this.retryDroppedWake(runId, attempt));
|
|
7925
8301
|
}, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
|
|
7926
8302
|
timer.unref?.();
|
|
@@ -7929,12 +8305,15 @@ var WorkflowEngine = class {
|
|
|
7929
8305
|
async retryDroppedWake(runId, attempt) {
|
|
7930
8306
|
if (this.draining) return;
|
|
7931
8307
|
const run = await this.store.getRun(runId);
|
|
7932
|
-
if (run?.status !== "suspended") return;
|
|
7933
8308
|
const nowMs = this.clock();
|
|
7934
|
-
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
|
|
7935
|
-
|
|
8309
|
+
if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
|
|
8310
|
+
this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
|
|
7936
8311
|
return;
|
|
7937
8312
|
}
|
|
8313
|
+
const unconditional = this.unconditionalRechecks.delete(runId);
|
|
8314
|
+
if (run?.status !== "suspended") return;
|
|
8315
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
|
|
8316
|
+
if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
|
|
7938
8317
|
if (this.queuesRunsOverTransport) {
|
|
7939
8318
|
await this.wake(runId);
|
|
7940
8319
|
return;
|
|
@@ -8273,6 +8652,7 @@ ${run.workflowVersion}`;
|
|
|
8273
8652
|
updatedAt
|
|
8274
8653
|
});
|
|
8275
8654
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
8655
|
+
await this.keepDueForKeptWakes(run.id);
|
|
8276
8656
|
this.emit({
|
|
8277
8657
|
type: "run.suspended",
|
|
8278
8658
|
runId: run.id,
|
|
@@ -8966,6 +9346,7 @@ ${run.workflowVersion}`;
|
|
|
8966
9346
|
namespace: run.namespace
|
|
8967
9347
|
});
|
|
8968
9348
|
}
|
|
9349
|
+
await this.noteServedWakes(run.id);
|
|
8969
9350
|
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
8970
9351
|
return this.singletonWaitResult(run);
|
|
8971
9352
|
}
|
|
@@ -9639,6 +10020,14 @@ ${run.workflowVersion}`;
|
|
|
9639
10020
|
throw new WorkflowSuspended();
|
|
9640
10021
|
}
|
|
9641
10022
|
/**
|
|
10023
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
10024
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
10025
|
+
*/
|
|
10026
|
+
stepIsWaiting(cp) {
|
|
10027
|
+
if (this.remoteRedispatchMs == null) return true;
|
|
10028
|
+
return cp.wakeAt != null && this.clock() < cp.wakeAt;
|
|
10029
|
+
}
|
|
10030
|
+
/**
|
|
9642
10031
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
9643
10032
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
9644
10033
|
*
|
|
@@ -9668,6 +10057,7 @@ ${run.workflowVersion}`;
|
|
|
9668
10057
|
const failures = [];
|
|
9669
10058
|
const unsent = [];
|
|
9670
10059
|
let inFlight = 0;
|
|
10060
|
+
let highest = -1;
|
|
9671
10061
|
let wakeAt;
|
|
9672
10062
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
9673
10063
|
const seq = positions[i];
|
|
@@ -9677,10 +10067,19 @@ ${run.workflowVersion}`;
|
|
|
9677
10067
|
continue;
|
|
9678
10068
|
}
|
|
9679
10069
|
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
10070
|
+
if (seq > highest) highest = seq;
|
|
9680
10071
|
if (cp.status === "completed") {
|
|
9681
10072
|
outputs[i] = cp.output;
|
|
9682
10073
|
continue;
|
|
9683
10074
|
}
|
|
10075
|
+
if (cp.status === "pending" && this.stepIsWaiting(cp)) {
|
|
10076
|
+
inFlight += 1;
|
|
10077
|
+
if (cp.wakeAt != null) {
|
|
10078
|
+
const at = this.stepLeaseWakeAt(cp.wakeAt);
|
|
10079
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
10080
|
+
}
|
|
10081
|
+
continue;
|
|
10082
|
+
}
|
|
9684
10083
|
try {
|
|
9685
10084
|
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
9686
10085
|
...routing,
|
|
@@ -9708,6 +10107,7 @@ ${run.workflowVersion}`;
|
|
|
9708
10107
|
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
9709
10108
|
}
|
|
9710
10109
|
}
|
|
10110
|
+
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
9711
10111
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
9712
10112
|
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
9713
10113
|
if (inFlight + batch.length > 0) {
|
|
@@ -9738,53 +10138,62 @@ ${run.workflowVersion}`;
|
|
|
9738
10138
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
9739
10139
|
return outputs;
|
|
9740
10140
|
}
|
|
9741
|
-
/**
|
|
10141
|
+
/**
|
|
10142
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
10143
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
10144
|
+
* batch, where the store and transport have them.
|
|
10145
|
+
*/
|
|
9742
10146
|
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
9743
10147
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
9744
10148
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
9745
10149
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
9746
|
-
|
|
9747
|
-
|
|
9748
|
-
|
|
9749
|
-
|
|
9750
|
-
|
|
9751
|
-
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9756
|
-
|
|
9757
|
-
|
|
9758
|
-
|
|
9759
|
-
|
|
9760
|
-
|
|
9761
|
-
|
|
9762
|
-
|
|
9763
|
-
|
|
9764
|
-
|
|
10150
|
+
const checkpoints = batch.map((i) => {
|
|
10151
|
+
const seq = positions[i];
|
|
10152
|
+
return {
|
|
10153
|
+
runId,
|
|
10154
|
+
seq,
|
|
10155
|
+
name: step.name,
|
|
10156
|
+
kind: "remote",
|
|
10157
|
+
stepId: stepId(runId, seq),
|
|
10158
|
+
status: "pending",
|
|
10159
|
+
input: inputs[i],
|
|
10160
|
+
attempts: 1,
|
|
10161
|
+
workerGroup,
|
|
10162
|
+
parallelGroup: group,
|
|
10163
|
+
enqueuedAt,
|
|
10164
|
+
startedAt: enqueuedAt,
|
|
10165
|
+
finishedAt: enqueuedAt
|
|
10166
|
+
};
|
|
10167
|
+
});
|
|
10168
|
+
if (this.store.saveCheckpoints) {
|
|
10169
|
+
await this.store.saveCheckpoints(checkpoints);
|
|
10170
|
+
} else {
|
|
10171
|
+
for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
|
|
10172
|
+
await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
|
|
10173
|
+
}
|
|
9765
10174
|
}
|
|
9766
|
-
|
|
9767
|
-
|
|
9768
|
-
|
|
9769
|
-
|
|
9770
|
-
|
|
9771
|
-
|
|
9772
|
-
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
9778
|
-
|
|
9779
|
-
|
|
9780
|
-
|
|
9781
|
-
|
|
9782
|
-
|
|
9783
|
-
|
|
9784
|
-
|
|
9785
|
-
|
|
9786
|
-
|
|
9787
|
-
})
|
|
10175
|
+
const traceparent = this.traceparent?.();
|
|
10176
|
+
const context = this.context?.();
|
|
10177
|
+
await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
|
|
10178
|
+
runId,
|
|
10179
|
+
seq: cp.seq,
|
|
10180
|
+
name: step.name,
|
|
10181
|
+
stepId: cp.stepId,
|
|
10182
|
+
group: workerGroup,
|
|
10183
|
+
input: inputs[batch[k]],
|
|
10184
|
+
traceparent,
|
|
10185
|
+
context,
|
|
10186
|
+
priority: opts.admission?.priority,
|
|
10187
|
+
attempt: 1
|
|
10188
|
+
})), opts.transport);
|
|
10189
|
+
for (const cp of checkpoints) {
|
|
10190
|
+
this.emit({
|
|
10191
|
+
type: "step.started",
|
|
10192
|
+
runId,
|
|
10193
|
+
seq: cp.seq,
|
|
10194
|
+
name: step.name,
|
|
10195
|
+
kind: "remote"
|
|
10196
|
+
});
|
|
9788
10197
|
}
|
|
9789
10198
|
}
|
|
9790
10199
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
@@ -9825,6 +10234,27 @@ ${run.workflowVersion}`;
|
|
|
9825
10234
|
error: s.reason
|
|
9826
10235
|
});
|
|
9827
10236
|
}
|
|
10237
|
+
/**
|
|
10238
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10239
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10240
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10241
|
+
*/
|
|
10242
|
+
async readCheckpoints(runId, seqs) {
|
|
10243
|
+
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10244
|
+
try {
|
|
10245
|
+
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
10246
|
+
cp.seq,
|
|
10247
|
+
cp
|
|
10248
|
+
]));
|
|
10249
|
+
return seqs.map((seq) => ({
|
|
10250
|
+
status: "fulfilled",
|
|
10251
|
+
value: bySeq.get(seq) ?? null
|
|
10252
|
+
}));
|
|
10253
|
+
} catch {
|
|
10254
|
+
}
|
|
10255
|
+
}
|
|
10256
|
+
return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
|
|
10257
|
+
}
|
|
9828
10258
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
9829
10259
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
9830
10260
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -9866,9 +10296,10 @@ ${run.workflowVersion}`;
|
|
|
9866
10296
|
*
|
|
9867
10297
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
9868
10298
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
9869
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
9870
|
-
*
|
|
9871
|
-
* transport acknowledges nothing that is not durable yet.
|
|
10299
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
10300
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
10301
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
10302
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
9872
10303
|
*/
|
|
9873
10304
|
completeRemoteResult(result) {
|
|
9874
10305
|
return new Promise((resolve, reject) => {
|
|
@@ -9896,7 +10327,7 @@ ${run.workflowVersion}`;
|
|
|
9896
10327
|
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
9897
10328
|
let outcomes;
|
|
9898
10329
|
try {
|
|
9899
|
-
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
10330
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
|
|
9900
10331
|
} catch (err) {
|
|
9901
10332
|
outcomes = batch.map(() => ({
|
|
9902
10333
|
error: err
|
|
@@ -9904,8 +10335,8 @@ ${run.workflowVersion}`;
|
|
|
9904
10335
|
}
|
|
9905
10336
|
batch.forEach((q, i) => {
|
|
9906
10337
|
const outcome = outcomes[i] ?? false;
|
|
9907
|
-
if (
|
|
9908
|
-
else q.
|
|
10338
|
+
if (isResultError(outcome)) q.reject(outcome.error);
|
|
10339
|
+
else q.resolve(outcome);
|
|
9909
10340
|
});
|
|
9910
10341
|
}
|
|
9911
10342
|
this.resultQueues.delete(runId);
|
|
@@ -9914,8 +10345,13 @@ ${run.workflowVersion}`;
|
|
|
9914
10345
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
9915
10346
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
9916
10347
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
10348
|
+
*
|
|
10349
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
10350
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
10351
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
10352
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
9917
10353
|
*/
|
|
9918
|
-
async completeRemoteResults(runId, results) {
|
|
10354
|
+
async completeRemoteResults(runId, results, queuedIn = () => false) {
|
|
9919
10355
|
const outcomes = results.map(() => false);
|
|
9920
10356
|
const seen = /* @__PURE__ */ new Set();
|
|
9921
10357
|
const live = results.flatMap((r, i) => {
|
|
@@ -9925,7 +10361,7 @@ ${run.workflowVersion}`;
|
|
|
9925
10361
|
i
|
|
9926
10362
|
];
|
|
9927
10363
|
});
|
|
9928
|
-
const reads = await
|
|
10364
|
+
const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
|
|
9929
10365
|
const pending = [];
|
|
9930
10366
|
const resettled = [];
|
|
9931
10367
|
reads.forEach((read, k) => {
|
|
@@ -9946,7 +10382,7 @@ ${run.workflowVersion}`;
|
|
|
9946
10382
|
});
|
|
9947
10383
|
if (pending.length === 0) {
|
|
9948
10384
|
if (resettled.length === 0) return outcomes;
|
|
9949
|
-
return this.resumeForRedelivered(runId, resettled, outcomes);
|
|
10385
|
+
return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
|
|
9950
10386
|
}
|
|
9951
10387
|
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
9952
10388
|
const run = await this.store.getRun(runId);
|
|
@@ -9973,7 +10409,9 @@ ${run.workflowVersion}`;
|
|
|
9973
10409
|
cp,
|
|
9974
10410
|
result: results[index]
|
|
9975
10411
|
});
|
|
9976
|
-
else if (claim !== false) outcomes[index] =
|
|
10412
|
+
else if (claim !== false) outcomes[index] = {
|
|
10413
|
+
error: claim.error
|
|
10414
|
+
};
|
|
9977
10415
|
});
|
|
9978
10416
|
for (const { cp, result } of settled) {
|
|
9979
10417
|
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
@@ -9991,19 +10429,21 @@ ${run.workflowVersion}`;
|
|
|
9991
10429
|
});
|
|
9992
10430
|
}
|
|
9993
10431
|
const barriers = /* @__PURE__ */ new Map();
|
|
9994
|
-
|
|
10432
|
+
const wake = new FanWakeCollector();
|
|
10433
|
+
for (const { cp } of settled) wake.note(cp);
|
|
9995
10434
|
for (const { index, cp, result } of settled) {
|
|
9996
|
-
if (!await this.stepFanStillRunning(cp, result, barriers))
|
|
10435
|
+
if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
|
|
10436
|
+
wake.add(index, cp, result.status);
|
|
10437
|
+
}
|
|
9997
10438
|
}
|
|
9998
|
-
|
|
9999
|
-
return outcomes;
|
|
10439
|
+
return wake.applyTo(outcomes);
|
|
10000
10440
|
}
|
|
10001
10441
|
/**
|
|
10002
10442
|
* The outcomes for a batch of redelivered results (their steps already settled): one of them
|
|
10003
10443
|
* resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
|
|
10004
10444
|
* resume of a run that had in fact moved on replays to its current wait and re-parks.
|
|
10005
10445
|
*/
|
|
10006
|
-
async resumeForRedelivered(runId, resettled, outcomes) {
|
|
10446
|
+
async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
|
|
10007
10447
|
const run = await this.store.getRun(runId);
|
|
10008
10448
|
if (run?.status !== "suspended") return outcomes;
|
|
10009
10449
|
const barriers = /* @__PURE__ */ new Map();
|
|
@@ -10014,9 +10454,11 @@ ${run.workflowVersion}`;
|
|
|
10014
10454
|
stepId: cp.stepId ?? stepId(runId, cp.seq),
|
|
10015
10455
|
status: cp.status === "failed" ? "failed" : "completed"
|
|
10016
10456
|
};
|
|
10017
|
-
if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
|
|
10018
|
-
|
|
10019
|
-
|
|
10457
|
+
if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
|
|
10458
|
+
const wake = new FanWakeCollector();
|
|
10459
|
+
for (const settled of resettled) wake.note(settled.cp);
|
|
10460
|
+
wake.add(index, cp, asResult.status);
|
|
10461
|
+
return wake.applyTo(outcomes);
|
|
10020
10462
|
}
|
|
10021
10463
|
}
|
|
10022
10464
|
return outcomes;
|
|
@@ -10027,15 +10469,17 @@ ${run.workflowVersion}`;
|
|
|
10027
10469
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
10028
10470
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
10029
10471
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
10030
|
-
* re-suspends, a lost one strands the run.
|
|
10472
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
10473
|
+
* that result (see {@link completeRemoteResults}).
|
|
10031
10474
|
*/
|
|
10032
|
-
async stepFanStillRunning(cp, result, barriers) {
|
|
10475
|
+
async stepFanStillRunning(cp, result, barriers, queuedIn) {
|
|
10033
10476
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
10034
10477
|
if (!fan) return false;
|
|
10035
10478
|
if (result.status === "failed") return false;
|
|
10036
10479
|
let closed = barriers.get(fan.firstSeq);
|
|
10037
10480
|
if (!closed) {
|
|
10038
|
-
|
|
10481
|
+
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10482
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
|
|
10039
10483
|
barriers.set(fan.firstSeq, closed);
|
|
10040
10484
|
}
|
|
10041
10485
|
if (await closed) return true;
|
|
@@ -10869,6 +11313,10 @@ var CodecStateStore = class {
|
|
|
10869
11313
|
saveCheckpointIfPending;
|
|
10870
11314
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
10871
11315
|
saveCheckpointsIfPending;
|
|
11316
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11317
|
+
saveCheckpoints;
|
|
11318
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11319
|
+
getCheckpoints;
|
|
10872
11320
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
10873
11321
|
countCheckpoints;
|
|
10874
11322
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -10919,6 +11367,10 @@ var CodecStateStore = class {
|
|
|
10919
11367
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
10920
11368
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
10921
11369
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11370
|
+
const saveMany = inner.saveCheckpoints;
|
|
11371
|
+
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11372
|
+
const getMany = inner.getCheckpoints;
|
|
11373
|
+
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
10922
11374
|
const countCps = inner.countCheckpoints;
|
|
10923
11375
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
10924
11376
|
const stats = inner.checkpointStats;
|
|
@@ -11555,11 +12007,30 @@ var InMemoryStateStore = class {
|
|
|
11555
12007
|
...cp
|
|
11556
12008
|
} : null;
|
|
11557
12009
|
}
|
|
12010
|
+
async getCheckpoints(runId, seqs) {
|
|
12011
|
+
return [
|
|
12012
|
+
...new Set(seqs)
|
|
12013
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
12014
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
12015
|
+
return cp ? [
|
|
12016
|
+
{
|
|
12017
|
+
...cp
|
|
12018
|
+
}
|
|
12019
|
+
] : [];
|
|
12020
|
+
});
|
|
12021
|
+
}
|
|
11558
12022
|
async saveCheckpoint(checkpoint) {
|
|
11559
12023
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11560
12024
|
...checkpoint
|
|
11561
12025
|
});
|
|
11562
12026
|
}
|
|
12027
|
+
async saveCheckpoints(checkpoints) {
|
|
12028
|
+
for (const checkpoint of checkpoints) {
|
|
12029
|
+
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
12030
|
+
...checkpoint
|
|
12031
|
+
});
|
|
12032
|
+
}
|
|
12033
|
+
}
|
|
11563
12034
|
async saveCheckpointIfPending(checkpoint) {
|
|
11564
12035
|
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
11565
12036
|
if (this.checkpoints.get(key)?.status !== "pending") return false;
|