@dudousxd/nestjs-durable-core 0.85.6 → 0.85.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +619 -148
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +125 -19
- package/dist/index.d.ts +125 -19
- package/dist/index.js +619 -148
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1151,7 +1151,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
|
|
|
1151
1151
|
"failed",
|
|
1152
1152
|
"dead"
|
|
1153
1153
|
]);
|
|
1154
|
-
var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
|
|
1155
1154
|
function canonicalJson(value) {
|
|
1156
1155
|
const plain = JSON.stringify(value);
|
|
1157
1156
|
return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
|
|
@@ -1166,77 +1165,235 @@ function sortedJson(value) {
|
|
|
1166
1165
|
return JSON.stringify(value);
|
|
1167
1166
|
}
|
|
1168
1167
|
__name(sortedJson, "sortedJson");
|
|
1169
|
-
|
|
1168
|
+
var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
|
|
1169
|
+
var SEQ_CHILD = /\.child\.(\d+)$/;
|
|
1170
|
+
function slotOf(childId) {
|
|
1170
1171
|
const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
|
|
1171
|
-
const fan =
|
|
1172
|
-
if (
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1172
|
+
const fan = FAN_CHILD_PARTS.exec(childId);
|
|
1173
|
+
if (fan) {
|
|
1174
|
+
const root = fan[1];
|
|
1175
|
+
const wave = fan[3];
|
|
1176
|
+
const firstSeq = Number(fan[2]);
|
|
1177
|
+
const i = Number(fan[4]);
|
|
1178
|
+
if (wave === void 0) {
|
|
1179
|
+
return {
|
|
1180
|
+
signal,
|
|
1181
|
+
signalSeq: firstSeq + i,
|
|
1182
|
+
fan: {
|
|
1183
|
+
fold: `${FOLDED_PREFIX}${root}`,
|
|
1184
|
+
firstSeq,
|
|
1185
|
+
index: i
|
|
1186
|
+
}
|
|
1187
|
+
};
|
|
1180
1188
|
}
|
|
1189
|
+
return {
|
|
1190
|
+
signal,
|
|
1191
|
+
signalSeq: firstSeq + i,
|
|
1192
|
+
fan: {
|
|
1193
|
+
fold: `${FOLDED_PREFIX}${root}w${wave}.`,
|
|
1194
|
+
firstSeq,
|
|
1195
|
+
index: i,
|
|
1196
|
+
wave: Number(wave),
|
|
1197
|
+
waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
|
|
1198
|
+
}
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
const seq = SEQ_CHILD.exec(childId);
|
|
1202
|
+
return seq ? {
|
|
1203
|
+
signal,
|
|
1204
|
+
signalSeq: Number(seq[1])
|
|
1205
|
+
} : {
|
|
1206
|
+
signal
|
|
1181
1207
|
};
|
|
1182
1208
|
}
|
|
1183
|
-
__name(
|
|
1184
|
-
function
|
|
1185
|
-
|
|
1186
|
-
const
|
|
1187
|
-
return
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1209
|
+
__name(slotOf, "slotOf");
|
|
1210
|
+
function foldOf(cp, name) {
|
|
1211
|
+
if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
|
|
1212
|
+
const { from, values } = cp.output ?? {};
|
|
1213
|
+
return typeof from === "number" && Array.isArray(values) ? {
|
|
1214
|
+
from,
|
|
1215
|
+
values
|
|
1216
|
+
} : void 0;
|
|
1217
|
+
}
|
|
1218
|
+
__name(foldOf, "foldOf");
|
|
1219
|
+
function covers(fold, index) {
|
|
1220
|
+
return !!fold && index >= fold.from && index < fold.from + fold.values.length;
|
|
1221
|
+
}
|
|
1222
|
+
__name(covers, "covers");
|
|
1223
|
+
function holdsSignal(cp, signal, expected) {
|
|
1224
|
+
return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
|
|
1225
|
+
}
|
|
1226
|
+
__name(holdsSignal, "holdsSignal");
|
|
1227
|
+
async function checkpointsAt(store, runId, seqs) {
|
|
1228
|
+
const unique = [
|
|
1229
|
+
...new Set(seqs)
|
|
1230
|
+
];
|
|
1231
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
1232
|
+
if (unique.length === 0) return bySeq;
|
|
1233
|
+
const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
|
|
1234
|
+
for (const cp of rows) bySeq.set(cp.seq, cp);
|
|
1235
|
+
return bySeq;
|
|
1236
|
+
}
|
|
1237
|
+
__name(checkpointsAt, "checkpointsAt");
|
|
1238
|
+
async function checkpointsNamed(store, runId, names) {
|
|
1239
|
+
const wanted = new Set(names);
|
|
1240
|
+
const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
|
|
1241
|
+
...wanted
|
|
1242
|
+
]) : await store.listCheckpoints(runId);
|
|
1243
|
+
return rows.filter((cp) => wanted.has(cp.name));
|
|
1244
|
+
}
|
|
1245
|
+
__name(checkpointsNamed, "checkpointsNamed");
|
|
1246
|
+
async function parentSettles(store, parentId, children) {
|
|
1247
|
+
const entries = children.map(({ childId, completion }) => ({
|
|
1248
|
+
slot: slotOf(childId),
|
|
1249
|
+
expected: canonicalJson(completion)
|
|
1250
|
+
}));
|
|
1251
|
+
const held = entries.map(() => void 0);
|
|
1252
|
+
const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
|
|
1253
|
+
...slot.signalSeq === void 0 ? [] : [
|
|
1254
|
+
slot.signalSeq
|
|
1255
|
+
],
|
|
1256
|
+
...slot.fan ? [
|
|
1257
|
+
slot.fan.firstSeq
|
|
1258
|
+
] : []
|
|
1259
|
+
]));
|
|
1260
|
+
const waveFolds = [];
|
|
1261
|
+
const byName = [];
|
|
1262
|
+
entries.forEach(({ slot, expected }, at) => {
|
|
1263
|
+
const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
|
|
1264
|
+
if (own?.name === slot.signal) {
|
|
1265
|
+
held[at] = holdsSignal(own, slot.signal, expected);
|
|
1266
|
+
return;
|
|
1193
1267
|
}
|
|
1194
|
-
|
|
1268
|
+
const fan = slot.fan;
|
|
1269
|
+
if (fan) {
|
|
1270
|
+
if (covers(foldOf(own, fan.fold), fan.index)) {
|
|
1271
|
+
held[at] = true;
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
const atFirst = first.get(fan.firstSeq);
|
|
1275
|
+
if (fan.wave === void 0 || fan.wave === 0) {
|
|
1276
|
+
held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
1279
|
+
const waveZero = foldOf(atFirst, fan.waveZeroFold);
|
|
1280
|
+
if (waveZero && waveZero.values.length > 0) {
|
|
1281
|
+
waveFolds.push({
|
|
1282
|
+
at,
|
|
1283
|
+
seq: fan.firstSeq + fan.wave * waveZero.values.length
|
|
1284
|
+
});
|
|
1285
|
+
return;
|
|
1286
|
+
}
|
|
1287
|
+
byName.push(at);
|
|
1288
|
+
return;
|
|
1289
|
+
}
|
|
1290
|
+
if (slot.signalSeq !== void 0 && !own) {
|
|
1291
|
+
held[at] = false;
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
byName.push(at);
|
|
1195
1295
|
});
|
|
1296
|
+
if (waveFolds.length > 0) {
|
|
1297
|
+
const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
|
|
1298
|
+
for (const { at, seq } of waveFolds) {
|
|
1299
|
+
const fan = entries[at].slot.fan;
|
|
1300
|
+
held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
if (byName.length > 0) {
|
|
1304
|
+
const names = byName.flatMap((at) => {
|
|
1305
|
+
const { slot } = entries[at];
|
|
1306
|
+
return slot.fan ? [
|
|
1307
|
+
slot.signal,
|
|
1308
|
+
slot.fan.fold
|
|
1309
|
+
] : [
|
|
1310
|
+
slot.signal
|
|
1311
|
+
];
|
|
1312
|
+
});
|
|
1313
|
+
const named = await checkpointsNamed(store, parentId, names);
|
|
1314
|
+
for (const at of byName) {
|
|
1315
|
+
const { slot, expected } = entries[at];
|
|
1316
|
+
held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
return held.map((h) => h === true);
|
|
1196
1320
|
}
|
|
1197
|
-
__name(
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
const prefixes = fold ? [
|
|
1201
|
-
signal,
|
|
1202
|
-
fold.name
|
|
1203
|
-
] : [
|
|
1204
|
-
signal
|
|
1205
|
-
];
|
|
1206
|
-
return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
|
|
1321
|
+
__name(parentSettles, "parentSettles");
|
|
1322
|
+
function parentClosed(run) {
|
|
1323
|
+
return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
|
|
1207
1324
|
}
|
|
1208
|
-
__name(
|
|
1325
|
+
__name(parentClosed, "parentClosed");
|
|
1209
1326
|
async function childCompletionNeedsBuffer(store, childId, completion, parent) {
|
|
1210
1327
|
const parentId = parentRunIdOf(childId);
|
|
1211
1328
|
if (!parentId) return false;
|
|
1212
1329
|
const run = parent === void 0 ? await store.getRun(parentId) : parent;
|
|
1213
|
-
if (
|
|
1214
|
-
|
|
1215
|
-
|
|
1330
|
+
if (parentClosed(run)) return false;
|
|
1331
|
+
const [held] = await parentSettles(store, parentId, [
|
|
1332
|
+
{
|
|
1333
|
+
childId,
|
|
1334
|
+
completion
|
|
1335
|
+
}
|
|
1336
|
+
]);
|
|
1337
|
+
return !held;
|
|
1216
1338
|
}
|
|
1217
1339
|
__name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
|
|
1218
|
-
async function unneededChildSignals(store, rows) {
|
|
1340
|
+
async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
|
|
1341
|
+
const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
|
|
1219
1342
|
const parentIds = rows.flatMap((row) => {
|
|
1220
|
-
const
|
|
1343
|
+
const childId = childOf(row);
|
|
1344
|
+
const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
|
|
1221
1345
|
return parentId ? [
|
|
1222
1346
|
parentId
|
|
1223
1347
|
] : [];
|
|
1224
1348
|
});
|
|
1225
1349
|
const { runs, unread } = await readRunsByIds(store, parentIds);
|
|
1226
1350
|
const unreadable = new Set(unread);
|
|
1227
|
-
const doomed =
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
const childId = row
|
|
1351
|
+
const doomed = /* @__PURE__ */ new Set();
|
|
1352
|
+
const open = /* @__PURE__ */ new Map();
|
|
1353
|
+
rows.forEach((row, at) => {
|
|
1354
|
+
const childId = childOf(row);
|
|
1355
|
+
if (childId === void 0) return;
|
|
1231
1356
|
const parentId = parentRunIdOf(childId);
|
|
1232
|
-
if (parentId
|
|
1233
|
-
|
|
1357
|
+
if (!parentId) {
|
|
1358
|
+
doomed.add(row);
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
if (unreadable.has(parentId)) return;
|
|
1362
|
+
if (parentClosed(runs.get(parentId) ?? null)) {
|
|
1363
|
+
doomed.add(row);
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
const group = open.get(parentId) ?? [];
|
|
1367
|
+
group.push({
|
|
1368
|
+
at,
|
|
1369
|
+
row,
|
|
1370
|
+
childId
|
|
1371
|
+
});
|
|
1372
|
+
open.set(parentId, group);
|
|
1373
|
+
});
|
|
1374
|
+
let decided = rows.length;
|
|
1375
|
+
let checked = 0;
|
|
1376
|
+
for (const [parentId, group] of open) {
|
|
1377
|
+
if (checked > 0 && Date.now() >= deadline) {
|
|
1378
|
+
decided = group[0].at;
|
|
1379
|
+
break;
|
|
1380
|
+
}
|
|
1381
|
+
checked += 1;
|
|
1234
1382
|
try {
|
|
1235
|
-
|
|
1383
|
+
const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
|
|
1384
|
+
childId,
|
|
1385
|
+
completion: row.payload
|
|
1386
|
+
})));
|
|
1387
|
+
group.forEach(({ row }, k) => {
|
|
1388
|
+
if (held[k]) doomed.add(row);
|
|
1389
|
+
});
|
|
1236
1390
|
} catch {
|
|
1237
1391
|
}
|
|
1238
1392
|
}
|
|
1239
|
-
return
|
|
1393
|
+
return {
|
|
1394
|
+
doomed: rows.filter((row) => doomed.has(row)),
|
|
1395
|
+
decided
|
|
1396
|
+
};
|
|
1240
1397
|
}
|
|
1241
1398
|
__name(unneededChildSignals, "unneededChildSignals");
|
|
1242
1399
|
|
|
@@ -3561,6 +3718,7 @@ function constantTimeEquals(a, b) {
|
|
|
3561
3718
|
__name(constantTimeEquals, "constantTimeEquals");
|
|
3562
3719
|
|
|
3563
3720
|
// src/transport-pool.ts
|
|
3721
|
+
var DISPATCH_CHUNK = 200;
|
|
3564
3722
|
var TransportPool = class {
|
|
3565
3723
|
static {
|
|
3566
3724
|
__name(this, "TransportPool");
|
|
@@ -3620,6 +3778,28 @@ var TransportPool = class {
|
|
|
3620
3778
|
}
|
|
3621
3779
|
throw lastErr ?? new Error("no transport accepted the dispatch");
|
|
3622
3780
|
}
|
|
3781
|
+
/**
|
|
3782
|
+
* {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
|
|
3783
|
+
* first) transport when it has one, else (or when that call rejects) each task through
|
|
3784
|
+
* {@link dispatch}, a bounded number at a time.
|
|
3785
|
+
*/
|
|
3786
|
+
async dispatchMany(tasks, preferId) {
|
|
3787
|
+
if (tasks.length === 0) return;
|
|
3788
|
+
const first = this.ordered(preferId)[0];
|
|
3789
|
+
if (first?.transport.dispatchMany) {
|
|
3790
|
+
try {
|
|
3791
|
+
await first.transport.dispatchMany(tasks.map((task) => ({
|
|
3792
|
+
...task,
|
|
3793
|
+
transport: first.id
|
|
3794
|
+
})));
|
|
3795
|
+
return;
|
|
3796
|
+
} catch {
|
|
3797
|
+
}
|
|
3798
|
+
}
|
|
3799
|
+
for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
|
|
3800
|
+
await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
|
|
3801
|
+
}
|
|
3802
|
+
}
|
|
3623
3803
|
/** Worker-health for `group`, merged across every transport that can report it (a group's queue
|
|
3624
3804
|
* and its workers may live on any pinned transport). Undefined when no transport implements
|
|
3625
3805
|
* `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
|
|
@@ -4839,6 +5019,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
|
4839
5019
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
4840
5020
|
var LEASE_RECHECK_SLACK_MS = 100;
|
|
4841
5021
|
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
5022
|
+
function wakeSeen(request, settledFans, served) {
|
|
5023
|
+
if (request.id !== void 0 && served.has(request.id)) return true;
|
|
5024
|
+
return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
|
|
5025
|
+
}
|
|
5026
|
+
__name(wakeSeen, "wakeSeen");
|
|
5027
|
+
function isFanWakeRequest(payload) {
|
|
5028
|
+
return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
|
|
5029
|
+
}
|
|
5030
|
+
__name(isFanWakeRequest, "isFanWakeRequest");
|
|
4842
5031
|
function deferred() {
|
|
4843
5032
|
let resolve;
|
|
4844
5033
|
let reject;
|
|
@@ -4857,12 +5046,52 @@ __name(deferred, "deferred");
|
|
|
4857
5046
|
var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
|
|
4858
5047
|
var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
|
|
4859
5048
|
var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
|
|
5049
|
+
var WAKE_REQUEST_SCAN = 100;
|
|
4860
5050
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4861
5051
|
var BUFFERED_SWEEP_PAGE = 500;
|
|
4862
5052
|
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
5053
|
+
var BUFFERED_SWEEP_BUDGET_MS = 250;
|
|
4863
5054
|
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4864
5055
|
var DELIVER_CHUNK = 50;
|
|
4865
5056
|
var RESULT_BATCH_MAX = 256;
|
|
5057
|
+
function isResultError(outcome) {
|
|
5058
|
+
return typeof outcome === "object" && "error" in outcome;
|
|
5059
|
+
}
|
|
5060
|
+
__name(isResultError, "isResultError");
|
|
5061
|
+
var FanWakeCollector = class FanWakeCollector2 {
|
|
5062
|
+
static {
|
|
5063
|
+
__name(this, "FanWakeCollector");
|
|
5064
|
+
}
|
|
5065
|
+
carrier;
|
|
5066
|
+
generic = false;
|
|
5067
|
+
fan;
|
|
5068
|
+
/** The highest seq the batch settled, per fan. */
|
|
5069
|
+
highest = /* @__PURE__ */ new Map();
|
|
5070
|
+
/** A checkpoint the batch settled. */
|
|
5071
|
+
note(cp) {
|
|
5072
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
5073
|
+
if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
|
|
5074
|
+
}
|
|
5075
|
+
/** A settled checkpoint that resumes the run. */
|
|
5076
|
+
add(index, cp, status) {
|
|
5077
|
+
this.carrier = index;
|
|
5078
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
5079
|
+
if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
|
|
5080
|
+
this.generic = true;
|
|
5081
|
+
return;
|
|
5082
|
+
}
|
|
5083
|
+
this.fan = fan.firstSeq;
|
|
5084
|
+
}
|
|
5085
|
+
applyTo(outcomes) {
|
|
5086
|
+
if (this.carrier === void 0) return outcomes;
|
|
5087
|
+
const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
|
|
5088
|
+
outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
|
|
5089
|
+
first: this.fan,
|
|
5090
|
+
seq
|
|
5091
|
+
};
|
|
5092
|
+
return outcomes;
|
|
5093
|
+
}
|
|
5094
|
+
};
|
|
4866
5095
|
function shuffle(items) {
|
|
4867
5096
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
4868
5097
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -5051,6 +5280,23 @@ var WorkflowEngine = class {
|
|
|
5051
5280
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
5052
5281
|
leaseRetries = /* @__PURE__ */ new Map();
|
|
5053
5282
|
/**
|
|
5283
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
5284
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
5285
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
5286
|
+
*/
|
|
5287
|
+
unconditionalRechecks = /* @__PURE__ */ new Set();
|
|
5288
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
5289
|
+
genericWakeRequested = /* @__PURE__ */ new Set();
|
|
5290
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
5291
|
+
wakeReasons = /* @__PURE__ */ new Map();
|
|
5292
|
+
/**
|
|
5293
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
5294
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
5295
|
+
*/
|
|
5296
|
+
turnSettledFans = /* @__PURE__ */ new Map();
|
|
5297
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
5298
|
+
turnServedWakes = /* @__PURE__ */ new Map();
|
|
5299
|
+
/**
|
|
5054
5300
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
5055
5301
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
5056
5302
|
* enqueued pickups and the runs it takes off the transport's run queue.
|
|
@@ -5174,8 +5420,9 @@ var WorkflowEngine = class {
|
|
|
5174
5420
|
}
|
|
5175
5421
|
return;
|
|
5176
5422
|
}
|
|
5177
|
-
|
|
5178
|
-
|
|
5423
|
+
const wake = await this.completeRemoteResult(result);
|
|
5424
|
+
if (!wake) return;
|
|
5425
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
|
|
5179
5426
|
if (err instanceof NamespaceMismatch) return;
|
|
5180
5427
|
throw err;
|
|
5181
5428
|
}));
|
|
@@ -5392,7 +5639,10 @@ var WorkflowEngine = class {
|
|
|
5392
5639
|
const resumed = await this.resumeLeased([
|
|
5393
5640
|
run
|
|
5394
5641
|
], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
|
|
5395
|
-
if (resumed.length === 0)
|
|
5642
|
+
if (resumed.length === 0) {
|
|
5643
|
+
this.unconditionalRechecks.add(runId);
|
|
5644
|
+
this.recheckAfterLease(runId, run.lockedUntil);
|
|
5645
|
+
}
|
|
5396
5646
|
return;
|
|
5397
5647
|
}
|
|
5398
5648
|
await this.resumeImplicit(runId).catch((err) => {
|
|
@@ -5693,13 +5943,17 @@ var WorkflowEngine = class {
|
|
|
5693
5943
|
*
|
|
5694
5944
|
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
5695
5945
|
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
5696
|
-
* few timer ticks without a long query or a big delete.
|
|
5697
|
-
*
|
|
5698
|
-
*
|
|
5699
|
-
*
|
|
5700
|
-
*
|
|
5701
|
-
|
|
5702
|
-
|
|
5946
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
5947
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
5948
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
5949
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
5950
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
5951
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
5952
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
5953
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
5954
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5955
|
+
*/
|
|
5956
|
+
async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
|
|
5703
5957
|
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
5704
5958
|
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
5705
5959
|
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
@@ -5707,15 +5961,21 @@ var WorkflowEngine = class {
|
|
|
5707
5961
|
return 0;
|
|
5708
5962
|
}
|
|
5709
5963
|
this.bufferedSweepActive = true;
|
|
5964
|
+
const deadline = Date.now() + budgetMs;
|
|
5710
5965
|
let deleted = 0;
|
|
5711
5966
|
try {
|
|
5712
5967
|
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
5968
|
+
if (page > 0 && (Date.now() >= deadline || this.draining)) break;
|
|
5713
5969
|
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
5714
|
-
const doomed = await unneededChildSignals(this.store, rows);
|
|
5970
|
+
const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
|
|
5715
5971
|
if (doomed.length > 0) {
|
|
5716
5972
|
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
5717
5973
|
deleted += doomed.length;
|
|
5718
5974
|
}
|
|
5975
|
+
if (decided < rows.length) {
|
|
5976
|
+
if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
|
|
5977
|
+
break;
|
|
5978
|
+
}
|
|
5719
5979
|
const last = rows.at(-1);
|
|
5720
5980
|
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
5721
5981
|
this.bufferedSweepCursor = void 0;
|
|
@@ -5959,15 +6219,27 @@ var WorkflowEngine = class {
|
|
|
5959
6219
|
* terminal too.
|
|
5960
6220
|
*/
|
|
5961
6221
|
resume(runId) {
|
|
6222
|
+
this.noteWakeReason(runId, void 0);
|
|
5962
6223
|
return this.resumeRun(runId, true);
|
|
5963
6224
|
}
|
|
6225
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
6226
|
+
noteWakeReason(runId, fan) {
|
|
6227
|
+
const reason = this.wakeReasons.get(runId) ?? {
|
|
6228
|
+
generic: false,
|
|
6229
|
+
fans: /* @__PURE__ */ new Map()
|
|
6230
|
+
};
|
|
6231
|
+
if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
|
|
6232
|
+
else reason.generic = true;
|
|
6233
|
+
this.wakeReasons.set(runId, reason);
|
|
6234
|
+
}
|
|
5964
6235
|
/**
|
|
5965
6236
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5966
6237
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
5967
6238
|
* these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
|
|
5968
6239
|
* stale waiter or a late result left behind by a failed run silently re-executes its body.
|
|
5969
6240
|
*/
|
|
5970
|
-
resumeImplicit(runId) {
|
|
6241
|
+
resumeImplicit(runId, fan) {
|
|
6242
|
+
if (fan !== "coalesced") this.noteWakeReason(runId, fan);
|
|
5971
6243
|
const turn = this.wakeTurns.get(runId);
|
|
5972
6244
|
if (turn) {
|
|
5973
6245
|
turn.again ??= deferred();
|
|
@@ -5992,7 +6264,7 @@ var WorkflowEngine = class {
|
|
|
5992
6264
|
});
|
|
5993
6265
|
return;
|
|
5994
6266
|
}
|
|
5995
|
-
const next = this.resumeImplicit(runId);
|
|
6267
|
+
const next = this.resumeImplicit(runId, "coalesced");
|
|
5996
6268
|
next.then(again.resolve, again.reject);
|
|
5997
6269
|
this.inBackground(next);
|
|
5998
6270
|
}, "finish");
|
|
@@ -8057,9 +8329,26 @@ var WorkflowEngine = class {
|
|
|
8057
8329
|
async execute(run, fn) {
|
|
8058
8330
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
8059
8331
|
const lockNow = this.clock();
|
|
8332
|
+
const reason = this.wakeReasons.get(run.id) ?? {
|
|
8333
|
+
generic: true,
|
|
8334
|
+
fans: /* @__PURE__ */ new Map()
|
|
8335
|
+
};
|
|
8336
|
+
this.wakeReasons.delete(run.id);
|
|
8060
8337
|
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
8061
8338
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
8062
|
-
if (
|
|
8339
|
+
if (reason.generic) {
|
|
8340
|
+
if (!this.genericWakeRequested.has(run.id)) {
|
|
8341
|
+
this.genericWakeRequested.add(run.id);
|
|
8342
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, null));
|
|
8343
|
+
}
|
|
8344
|
+
} else {
|
|
8345
|
+
const fans = {
|
|
8346
|
+
fans: [
|
|
8347
|
+
...reason.fans
|
|
8348
|
+
]
|
|
8349
|
+
};
|
|
8350
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
8351
|
+
}
|
|
8063
8352
|
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
8064
8353
|
return {
|
|
8065
8354
|
runId: run.id,
|
|
@@ -8067,56 +8356,142 @@ var WorkflowEngine = class {
|
|
|
8067
8356
|
};
|
|
8068
8357
|
}
|
|
8069
8358
|
}
|
|
8359
|
+
const settledFans = /* @__PURE__ */ new Map();
|
|
8360
|
+
this.turnSettledFans.set(run.id, settledFans);
|
|
8070
8361
|
const held = this.holdExecutionLease(run.id);
|
|
8071
8362
|
try {
|
|
8363
|
+
const served = /* @__PURE__ */ new Set();
|
|
8364
|
+
this.turnServedWakes.set(run.id, served);
|
|
8365
|
+
if (registered?.remote) await this.noteServedWakes(run.id);
|
|
8072
8366
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
8073
8367
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
8074
8368
|
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
8075
8369
|
}
|
|
8076
|
-
if (result.status === "suspended")
|
|
8077
|
-
|
|
8078
|
-
|
|
8370
|
+
if (result.status === "suspended") {
|
|
8371
|
+
this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
|
|
8372
|
+
} else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
|
|
8373
|
+
this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
|
|
8079
8374
|
}
|
|
8080
8375
|
return result;
|
|
8081
8376
|
} finally {
|
|
8082
8377
|
held.stop();
|
|
8378
|
+
if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
|
|
8379
|
+
this.turnServedWakes.delete(run.id);
|
|
8083
8380
|
}
|
|
8084
8381
|
}
|
|
8085
8382
|
/**
|
|
8086
8383
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
8087
8384
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
8088
|
-
* (a signal, a child's completion). The request is a buffered
|
|
8089
|
-
* on
|
|
8385
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
8386
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
8387
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
8090
8388
|
*
|
|
8091
8389
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
8092
8390
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
8093
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
8094
|
-
*
|
|
8391
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
8392
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
8393
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
8394
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
8095
8395
|
*/
|
|
8096
|
-
async requestWakeAfterTurn(runId) {
|
|
8097
|
-
|
|
8396
|
+
async requestWakeAfterTurn(runId, request) {
|
|
8397
|
+
const id = globalThis.crypto.randomUUID();
|
|
8398
|
+
await this.store.bufferEvent({
|
|
8399
|
+
name: wakeRequestToken(runId),
|
|
8400
|
+
payload: request,
|
|
8401
|
+
id,
|
|
8402
|
+
publishedAt: this.clock()
|
|
8403
|
+
});
|
|
8098
8404
|
const run = await this.store.getRun(runId);
|
|
8099
|
-
if (run
|
|
8100
|
-
|
|
8101
|
-
|
|
8405
|
+
if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
8406
|
+
await this.store.removeBufferedEvent(id);
|
|
8407
|
+
return;
|
|
8408
|
+
}
|
|
8409
|
+
if (run.status === "running") return;
|
|
8410
|
+
if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
|
|
8411
|
+
await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
|
|
8102
8412
|
return;
|
|
8103
8413
|
}
|
|
8104
|
-
if (!await this.takeWakeRequests(runId)) return;
|
|
8105
|
-
if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
8106
8414
|
this.recheckAfterLease(runId, run.lockedUntil);
|
|
8107
8415
|
}
|
|
8108
|
-
/**
|
|
8109
|
-
|
|
8110
|
-
|
|
8416
|
+
/**
|
|
8417
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
8418
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
8419
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
8420
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
8421
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
8422
|
+
*/
|
|
8423
|
+
async honorWakeRequests(runId, settledFans, served) {
|
|
8424
|
+
const kept = await this.listWakeRequests(runId, true);
|
|
8425
|
+
const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
|
|
8426
|
+
const needed = kept.filter((request) => !seen(request));
|
|
8427
|
+
await this.dropWakeRequests(kept.filter(seen));
|
|
8428
|
+
if (needed.length === 0) return;
|
|
8429
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8111
8430
|
await this.wake(runId).catch((err) => {
|
|
8112
8431
|
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
8113
8432
|
});
|
|
8114
8433
|
}
|
|
8115
|
-
/**
|
|
8116
|
-
|
|
8117
|
-
|
|
8118
|
-
|
|
8119
|
-
|
|
8434
|
+
/**
|
|
8435
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
8436
|
+
* signals, which are taken as they are read.
|
|
8437
|
+
*/
|
|
8438
|
+
async listWakeRequests(runId, legacy = false) {
|
|
8439
|
+
const token = wakeRequestToken(runId);
|
|
8440
|
+
const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
|
|
8441
|
+
id: request.id,
|
|
8442
|
+
payload: request.payload
|
|
8443
|
+
}));
|
|
8444
|
+
if (!legacy) return kept;
|
|
8445
|
+
for (; ; ) {
|
|
8446
|
+
const signal = await this.store.takeBufferedSignal(token);
|
|
8447
|
+
if (!signal) return kept;
|
|
8448
|
+
kept.push({
|
|
8449
|
+
payload: signal.payload
|
|
8450
|
+
});
|
|
8451
|
+
}
|
|
8452
|
+
}
|
|
8453
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
8454
|
+
async dropWakeRequests(requests) {
|
|
8455
|
+
await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
|
|
8456
|
+
this.store.removeBufferedEvent(request.id)
|
|
8457
|
+
]));
|
|
8458
|
+
}
|
|
8459
|
+
/**
|
|
8460
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
8461
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
8462
|
+
* until then stay for whoever runs the run next.
|
|
8463
|
+
*/
|
|
8464
|
+
async noteServedWakes(runId) {
|
|
8465
|
+
const served = this.turnServedWakes.get(runId);
|
|
8466
|
+
if (!served) return;
|
|
8467
|
+
for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
|
|
8468
|
+
served.add(request.id);
|
|
8469
|
+
}
|
|
8470
|
+
}
|
|
8471
|
+
/**
|
|
8472
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
8473
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
8474
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
8475
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
8476
|
+
*/
|
|
8477
|
+
async keepDueForKeptWakes(runId) {
|
|
8478
|
+
const served = this.turnServedWakes.get(runId);
|
|
8479
|
+
if (!served) return;
|
|
8480
|
+
const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
|
|
8481
|
+
const kept = await this.listWakeRequests(runId);
|
|
8482
|
+
if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
|
|
8483
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8484
|
+
}
|
|
8485
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
8486
|
+
async makeDueBy(runId, at) {
|
|
8487
|
+
const run = await this.store.getRun(runId);
|
|
8488
|
+
if (run?.status !== "suspended") return;
|
|
8489
|
+
if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
|
|
8490
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
|
|
8491
|
+
"suspended"
|
|
8492
|
+
], {
|
|
8493
|
+
wakeAt: at
|
|
8494
|
+
});
|
|
8120
8495
|
}
|
|
8121
8496
|
/**
|
|
8122
8497
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
@@ -8135,6 +8510,7 @@ var WorkflowEngine = class {
|
|
|
8135
8510
|
const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
|
|
8136
8511
|
const timer = setTimeout(() => {
|
|
8137
8512
|
this.leaseRetries.delete(runId);
|
|
8513
|
+
this.genericWakeRequested.delete(runId);
|
|
8138
8514
|
this.inBackground(this.retryDroppedWake(runId, attempt));
|
|
8139
8515
|
}, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
|
|
8140
8516
|
timer.unref?.();
|
|
@@ -8143,12 +8519,15 @@ var WorkflowEngine = class {
|
|
|
8143
8519
|
async retryDroppedWake(runId, attempt) {
|
|
8144
8520
|
if (this.draining) return;
|
|
8145
8521
|
const run = await this.store.getRun(runId);
|
|
8146
|
-
if (run?.status !== "suspended") return;
|
|
8147
8522
|
const nowMs = this.clock();
|
|
8148
|
-
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
|
|
8149
|
-
|
|
8523
|
+
if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
|
|
8524
|
+
this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
|
|
8150
8525
|
return;
|
|
8151
8526
|
}
|
|
8527
|
+
const unconditional = this.unconditionalRechecks.delete(runId);
|
|
8528
|
+
if (run?.status !== "suspended") return;
|
|
8529
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
|
|
8530
|
+
if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
|
|
8152
8531
|
if (this.queuesRunsOverTransport) {
|
|
8153
8532
|
await this.wake(runId);
|
|
8154
8533
|
return;
|
|
@@ -8487,6 +8866,7 @@ ${run.workflowVersion}`;
|
|
|
8487
8866
|
updatedAt
|
|
8488
8867
|
});
|
|
8489
8868
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
8869
|
+
await this.keepDueForKeptWakes(run.id);
|
|
8490
8870
|
this.emit({
|
|
8491
8871
|
type: "run.suspended",
|
|
8492
8872
|
runId: run.id,
|
|
@@ -9180,6 +9560,7 @@ ${run.workflowVersion}`;
|
|
|
9180
9560
|
namespace: run.namespace
|
|
9181
9561
|
});
|
|
9182
9562
|
}
|
|
9563
|
+
await this.noteServedWakes(run.id);
|
|
9183
9564
|
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
9184
9565
|
return this.singletonWaitResult(run);
|
|
9185
9566
|
}
|
|
@@ -9853,6 +10234,14 @@ ${run.workflowVersion}`;
|
|
|
9853
10234
|
throw new WorkflowSuspended();
|
|
9854
10235
|
}
|
|
9855
10236
|
/**
|
|
10237
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
10238
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
10239
|
+
*/
|
|
10240
|
+
stepIsWaiting(cp) {
|
|
10241
|
+
if (this.remoteRedispatchMs == null) return true;
|
|
10242
|
+
return cp.wakeAt != null && this.clock() < cp.wakeAt;
|
|
10243
|
+
}
|
|
10244
|
+
/**
|
|
9856
10245
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
9857
10246
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
9858
10247
|
*
|
|
@@ -9882,6 +10271,7 @@ ${run.workflowVersion}`;
|
|
|
9882
10271
|
const failures = [];
|
|
9883
10272
|
const unsent = [];
|
|
9884
10273
|
let inFlight = 0;
|
|
10274
|
+
let highest = -1;
|
|
9885
10275
|
let wakeAt;
|
|
9886
10276
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
9887
10277
|
const seq = positions[i];
|
|
@@ -9891,10 +10281,19 @@ ${run.workflowVersion}`;
|
|
|
9891
10281
|
continue;
|
|
9892
10282
|
}
|
|
9893
10283
|
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
10284
|
+
if (seq > highest) highest = seq;
|
|
9894
10285
|
if (cp.status === "completed") {
|
|
9895
10286
|
outputs[i] = cp.output;
|
|
9896
10287
|
continue;
|
|
9897
10288
|
}
|
|
10289
|
+
if (cp.status === "pending" && this.stepIsWaiting(cp)) {
|
|
10290
|
+
inFlight += 1;
|
|
10291
|
+
if (cp.wakeAt != null) {
|
|
10292
|
+
const at = this.stepLeaseWakeAt(cp.wakeAt);
|
|
10293
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
10294
|
+
}
|
|
10295
|
+
continue;
|
|
10296
|
+
}
|
|
9898
10297
|
try {
|
|
9899
10298
|
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
9900
10299
|
...routing,
|
|
@@ -9922,6 +10321,7 @@ ${run.workflowVersion}`;
|
|
|
9922
10321
|
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
9923
10322
|
}
|
|
9924
10323
|
}
|
|
10324
|
+
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
9925
10325
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
9926
10326
|
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
9927
10327
|
if (inFlight + batch.length > 0) {
|
|
@@ -9952,53 +10352,62 @@ ${run.workflowVersion}`;
|
|
|
9952
10352
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
9953
10353
|
return outputs;
|
|
9954
10354
|
}
|
|
9955
|
-
/**
|
|
10355
|
+
/**
|
|
10356
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
10357
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
10358
|
+
* batch, where the store and transport have them.
|
|
10359
|
+
*/
|
|
9956
10360
|
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
9957
10361
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
9958
10362
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
9959
10363
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
9960
|
-
|
|
9961
|
-
|
|
9962
|
-
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
|
|
9973
|
-
|
|
9974
|
-
|
|
9975
|
-
|
|
9976
|
-
|
|
9977
|
-
|
|
9978
|
-
|
|
10364
|
+
const checkpoints = batch.map((i) => {
|
|
10365
|
+
const seq = positions[i];
|
|
10366
|
+
return {
|
|
10367
|
+
runId,
|
|
10368
|
+
seq,
|
|
10369
|
+
name: step.name,
|
|
10370
|
+
kind: "remote",
|
|
10371
|
+
stepId: stepId(runId, seq),
|
|
10372
|
+
status: "pending",
|
|
10373
|
+
input: inputs[i],
|
|
10374
|
+
attempts: 1,
|
|
10375
|
+
workerGroup,
|
|
10376
|
+
parallelGroup: group,
|
|
10377
|
+
enqueuedAt,
|
|
10378
|
+
startedAt: enqueuedAt,
|
|
10379
|
+
finishedAt: enqueuedAt
|
|
10380
|
+
};
|
|
10381
|
+
});
|
|
10382
|
+
if (this.store.saveCheckpoints) {
|
|
10383
|
+
await this.store.saveCheckpoints(checkpoints);
|
|
10384
|
+
} else {
|
|
10385
|
+
for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
|
|
10386
|
+
await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
|
|
10387
|
+
}
|
|
9979
10388
|
}
|
|
9980
|
-
|
|
9981
|
-
|
|
9982
|
-
|
|
9983
|
-
|
|
9984
|
-
|
|
9985
|
-
|
|
9986
|
-
|
|
9987
|
-
|
|
9988
|
-
|
|
9989
|
-
|
|
9990
|
-
|
|
9991
|
-
|
|
9992
|
-
|
|
9993
|
-
|
|
9994
|
-
|
|
9995
|
-
|
|
9996
|
-
|
|
9997
|
-
|
|
9998
|
-
|
|
9999
|
-
|
|
10000
|
-
|
|
10001
|
-
})
|
|
10389
|
+
const traceparent = this.traceparent?.();
|
|
10390
|
+
const context = this.context?.();
|
|
10391
|
+
await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
|
|
10392
|
+
runId,
|
|
10393
|
+
seq: cp.seq,
|
|
10394
|
+
name: step.name,
|
|
10395
|
+
stepId: cp.stepId,
|
|
10396
|
+
group: workerGroup,
|
|
10397
|
+
input: inputs[batch[k]],
|
|
10398
|
+
traceparent,
|
|
10399
|
+
context,
|
|
10400
|
+
priority: opts.admission?.priority,
|
|
10401
|
+
attempt: 1
|
|
10402
|
+
})), opts.transport);
|
|
10403
|
+
for (const cp of checkpoints) {
|
|
10404
|
+
this.emit({
|
|
10405
|
+
type: "step.started",
|
|
10406
|
+
runId,
|
|
10407
|
+
seq: cp.seq,
|
|
10408
|
+
name: step.name,
|
|
10409
|
+
kind: "remote"
|
|
10410
|
+
});
|
|
10002
10411
|
}
|
|
10003
10412
|
}
|
|
10004
10413
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
@@ -10039,6 +10448,27 @@ ${run.workflowVersion}`;
|
|
|
10039
10448
|
error: s.reason
|
|
10040
10449
|
});
|
|
10041
10450
|
}
|
|
10451
|
+
/**
|
|
10452
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10453
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10454
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10455
|
+
*/
|
|
10456
|
+
async readCheckpoints(runId, seqs) {
|
|
10457
|
+
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10458
|
+
try {
|
|
10459
|
+
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
10460
|
+
cp.seq,
|
|
10461
|
+
cp
|
|
10462
|
+
]));
|
|
10463
|
+
return seqs.map((seq) => ({
|
|
10464
|
+
status: "fulfilled",
|
|
10465
|
+
value: bySeq.get(seq) ?? null
|
|
10466
|
+
}));
|
|
10467
|
+
} catch {
|
|
10468
|
+
}
|
|
10469
|
+
}
|
|
10470
|
+
return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
|
|
10471
|
+
}
|
|
10042
10472
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
10043
10473
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
10044
10474
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -10080,9 +10510,10 @@ ${run.workflowVersion}`;
|
|
|
10080
10510
|
*
|
|
10081
10511
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
10082
10512
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
10083
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
10084
|
-
*
|
|
10085
|
-
* transport acknowledges nothing that is not durable yet.
|
|
10513
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
10514
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
10515
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
10516
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
10086
10517
|
*/
|
|
10087
10518
|
completeRemoteResult(result) {
|
|
10088
10519
|
return new Promise((resolve, reject) => {
|
|
@@ -10110,7 +10541,7 @@ ${run.workflowVersion}`;
|
|
|
10110
10541
|
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
10111
10542
|
let outcomes;
|
|
10112
10543
|
try {
|
|
10113
|
-
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
10544
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
|
|
10114
10545
|
} catch (err) {
|
|
10115
10546
|
outcomes = batch.map(() => ({
|
|
10116
10547
|
error: err
|
|
@@ -10118,8 +10549,8 @@ ${run.workflowVersion}`;
|
|
|
10118
10549
|
}
|
|
10119
10550
|
batch.forEach((q, i) => {
|
|
10120
10551
|
const outcome = outcomes[i] ?? false;
|
|
10121
|
-
if (
|
|
10122
|
-
else q.
|
|
10552
|
+
if (isResultError(outcome)) q.reject(outcome.error);
|
|
10553
|
+
else q.resolve(outcome);
|
|
10123
10554
|
});
|
|
10124
10555
|
}
|
|
10125
10556
|
this.resultQueues.delete(runId);
|
|
@@ -10128,8 +10559,13 @@ ${run.workflowVersion}`;
|
|
|
10128
10559
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
10129
10560
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
10130
10561
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
10562
|
+
*
|
|
10563
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
10564
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
10565
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
10566
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
10131
10567
|
*/
|
|
10132
|
-
async completeRemoteResults(runId, results) {
|
|
10568
|
+
async completeRemoteResults(runId, results, queuedIn = () => false) {
|
|
10133
10569
|
const outcomes = results.map(() => false);
|
|
10134
10570
|
const seen = /* @__PURE__ */ new Set();
|
|
10135
10571
|
const live = results.flatMap((r, i) => {
|
|
@@ -10139,7 +10575,7 @@ ${run.workflowVersion}`;
|
|
|
10139
10575
|
i
|
|
10140
10576
|
];
|
|
10141
10577
|
});
|
|
10142
|
-
const reads = await
|
|
10578
|
+
const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
|
|
10143
10579
|
const pending = [];
|
|
10144
10580
|
const resettled = [];
|
|
10145
10581
|
reads.forEach((read, k) => {
|
|
@@ -10160,7 +10596,7 @@ ${run.workflowVersion}`;
|
|
|
10160
10596
|
});
|
|
10161
10597
|
if (pending.length === 0) {
|
|
10162
10598
|
if (resettled.length === 0) return outcomes;
|
|
10163
|
-
return this.resumeForRedelivered(runId, resettled, outcomes);
|
|
10599
|
+
return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
|
|
10164
10600
|
}
|
|
10165
10601
|
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
10166
10602
|
const run = await this.store.getRun(runId);
|
|
@@ -10187,7 +10623,9 @@ ${run.workflowVersion}`;
|
|
|
10187
10623
|
cp,
|
|
10188
10624
|
result: results[index]
|
|
10189
10625
|
});
|
|
10190
|
-
else if (claim !== false) outcomes[index] =
|
|
10626
|
+
else if (claim !== false) outcomes[index] = {
|
|
10627
|
+
error: claim.error
|
|
10628
|
+
};
|
|
10191
10629
|
});
|
|
10192
10630
|
for (const { cp, result } of settled) {
|
|
10193
10631
|
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
@@ -10205,19 +10643,21 @@ ${run.workflowVersion}`;
|
|
|
10205
10643
|
});
|
|
10206
10644
|
}
|
|
10207
10645
|
const barriers = /* @__PURE__ */ new Map();
|
|
10208
|
-
|
|
10646
|
+
const wake = new FanWakeCollector();
|
|
10647
|
+
for (const { cp } of settled) wake.note(cp);
|
|
10209
10648
|
for (const { index, cp, result } of settled) {
|
|
10210
|
-
if (!await this.stepFanStillRunning(cp, result, barriers))
|
|
10649
|
+
if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
|
|
10650
|
+
wake.add(index, cp, result.status);
|
|
10651
|
+
}
|
|
10211
10652
|
}
|
|
10212
|
-
|
|
10213
|
-
return outcomes;
|
|
10653
|
+
return wake.applyTo(outcomes);
|
|
10214
10654
|
}
|
|
10215
10655
|
/**
|
|
10216
10656
|
* The outcomes for a batch of redelivered results (their steps already settled): one of them
|
|
10217
10657
|
* resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
|
|
10218
10658
|
* resume of a run that had in fact moved on replays to its current wait and re-parks.
|
|
10219
10659
|
*/
|
|
10220
|
-
async resumeForRedelivered(runId, resettled, outcomes) {
|
|
10660
|
+
async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
|
|
10221
10661
|
const run = await this.store.getRun(runId);
|
|
10222
10662
|
if (run?.status !== "suspended") return outcomes;
|
|
10223
10663
|
const barriers = /* @__PURE__ */ new Map();
|
|
@@ -10228,9 +10668,11 @@ ${run.workflowVersion}`;
|
|
|
10228
10668
|
stepId: cp.stepId ?? stepId(runId, cp.seq),
|
|
10229
10669
|
status: cp.status === "failed" ? "failed" : "completed"
|
|
10230
10670
|
};
|
|
10231
|
-
if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
|
|
10232
|
-
|
|
10233
|
-
|
|
10671
|
+
if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
|
|
10672
|
+
const wake = new FanWakeCollector();
|
|
10673
|
+
for (const settled of resettled) wake.note(settled.cp);
|
|
10674
|
+
wake.add(index, cp, asResult.status);
|
|
10675
|
+
return wake.applyTo(outcomes);
|
|
10234
10676
|
}
|
|
10235
10677
|
}
|
|
10236
10678
|
return outcomes;
|
|
@@ -10241,15 +10683,17 @@ ${run.workflowVersion}`;
|
|
|
10241
10683
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
10242
10684
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
10243
10685
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
10244
|
-
* re-suspends, a lost one strands the run.
|
|
10686
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
10687
|
+
* that result (see {@link completeRemoteResults}).
|
|
10245
10688
|
*/
|
|
10246
|
-
async stepFanStillRunning(cp, result, barriers) {
|
|
10689
|
+
async stepFanStillRunning(cp, result, barriers, queuedIn) {
|
|
10247
10690
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
10248
10691
|
if (!fan) return false;
|
|
10249
10692
|
if (result.status === "failed") return false;
|
|
10250
10693
|
let closed = barriers.get(fan.firstSeq);
|
|
10251
10694
|
if (!closed) {
|
|
10252
|
-
|
|
10695
|
+
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10696
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
|
|
10253
10697
|
barriers.set(fan.firstSeq, closed);
|
|
10254
10698
|
}
|
|
10255
10699
|
if (await closed) return true;
|
|
@@ -11083,6 +11527,10 @@ var CodecStateStore = class {
|
|
|
11083
11527
|
saveCheckpointIfPending;
|
|
11084
11528
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
11085
11529
|
saveCheckpointsIfPending;
|
|
11530
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11531
|
+
saveCheckpoints;
|
|
11532
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11533
|
+
getCheckpoints;
|
|
11086
11534
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
11087
11535
|
countCheckpoints;
|
|
11088
11536
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -11133,6 +11581,10 @@ var CodecStateStore = class {
|
|
|
11133
11581
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
11134
11582
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
11135
11583
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11584
|
+
const saveMany = inner.saveCheckpoints;
|
|
11585
|
+
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11586
|
+
const getMany = inner.getCheckpoints;
|
|
11587
|
+
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11136
11588
|
const countCps = inner.countCheckpoints;
|
|
11137
11589
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
11138
11590
|
const stats = inner.checkpointStats;
|
|
@@ -11769,11 +12221,30 @@ var InMemoryStateStore = class {
|
|
|
11769
12221
|
...cp
|
|
11770
12222
|
} : null;
|
|
11771
12223
|
}
|
|
12224
|
+
async getCheckpoints(runId, seqs) {
|
|
12225
|
+
return [
|
|
12226
|
+
...new Set(seqs)
|
|
12227
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
12228
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
12229
|
+
return cp ? [
|
|
12230
|
+
{
|
|
12231
|
+
...cp
|
|
12232
|
+
}
|
|
12233
|
+
] : [];
|
|
12234
|
+
});
|
|
12235
|
+
}
|
|
11772
12236
|
async saveCheckpoint(checkpoint) {
|
|
11773
12237
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11774
12238
|
...checkpoint
|
|
11775
12239
|
});
|
|
11776
12240
|
}
|
|
12241
|
+
async saveCheckpoints(checkpoints) {
|
|
12242
|
+
for (const checkpoint of checkpoints) {
|
|
12243
|
+
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
12244
|
+
...checkpoint
|
|
12245
|
+
});
|
|
12246
|
+
}
|
|
12247
|
+
}
|
|
11777
12248
|
async saveCheckpointIfPending(checkpoint) {
|
|
11778
12249
|
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
11779
12250
|
if (this.checkpoints.get(key)?.status !== "pending") return false;
|