@dudousxd/nestjs-durable-core 0.85.7 → 0.85.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +444 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +106 -9
- package/dist/index.d.ts +106 -9
- package/dist/index.js +444 -74
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1151,7 +1151,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
|
|
|
1151
1151
|
"failed",
|
|
1152
1152
|
"dead"
|
|
1153
1153
|
]);
|
|
1154
|
-
var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
|
|
1155
1154
|
function canonicalJson(value) {
|
|
1156
1155
|
const plain = JSON.stringify(value);
|
|
1157
1156
|
return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
|
|
@@ -1166,77 +1165,235 @@ function sortedJson(value) {
|
|
|
1166
1165
|
return JSON.stringify(value);
|
|
1167
1166
|
}
|
|
1168
1167
|
__name(sortedJson, "sortedJson");
|
|
1169
|
-
|
|
1168
|
+
var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
|
|
1169
|
+
var SEQ_CHILD = /\.child\.(\d+)$/;
|
|
1170
|
+
function slotOf(childId) {
|
|
1170
1171
|
const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
|
|
1171
|
-
const fan =
|
|
1172
|
-
if (
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1172
|
+
const fan = FAN_CHILD_PARTS.exec(childId);
|
|
1173
|
+
if (fan) {
|
|
1174
|
+
const root = fan[1];
|
|
1175
|
+
const wave = fan[3];
|
|
1176
|
+
const firstSeq = Number(fan[2]);
|
|
1177
|
+
const i = Number(fan[4]);
|
|
1178
|
+
if (wave === void 0) {
|
|
1179
|
+
return {
|
|
1180
|
+
signal,
|
|
1181
|
+
signalSeq: firstSeq + i,
|
|
1182
|
+
fan: {
|
|
1183
|
+
fold: `${FOLDED_PREFIX}${root}`,
|
|
1184
|
+
firstSeq,
|
|
1185
|
+
index: i
|
|
1186
|
+
}
|
|
1187
|
+
};
|
|
1180
1188
|
}
|
|
1189
|
+
return {
|
|
1190
|
+
signal,
|
|
1191
|
+
signalSeq: firstSeq + i,
|
|
1192
|
+
fan: {
|
|
1193
|
+
fold: `${FOLDED_PREFIX}${root}w${wave}.`,
|
|
1194
|
+
firstSeq,
|
|
1195
|
+
index: i,
|
|
1196
|
+
wave: Number(wave),
|
|
1197
|
+
waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
|
|
1198
|
+
}
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
const seq = SEQ_CHILD.exec(childId);
|
|
1202
|
+
return seq ? {
|
|
1203
|
+
signal,
|
|
1204
|
+
signalSeq: Number(seq[1])
|
|
1205
|
+
} : {
|
|
1206
|
+
signal
|
|
1181
1207
|
};
|
|
1182
1208
|
}
|
|
1183
|
-
__name(
|
|
1184
|
-
function
|
|
1185
|
-
|
|
1186
|
-
const
|
|
1187
|
-
return
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1209
|
+
__name(slotOf, "slotOf");
|
|
1210
|
+
function foldOf(cp, name) {
|
|
1211
|
+
if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
|
|
1212
|
+
const { from, values } = cp.output ?? {};
|
|
1213
|
+
return typeof from === "number" && Array.isArray(values) ? {
|
|
1214
|
+
from,
|
|
1215
|
+
values
|
|
1216
|
+
} : void 0;
|
|
1217
|
+
}
|
|
1218
|
+
__name(foldOf, "foldOf");
|
|
1219
|
+
function covers(fold, index) {
|
|
1220
|
+
return !!fold && index >= fold.from && index < fold.from + fold.values.length;
|
|
1221
|
+
}
|
|
1222
|
+
__name(covers, "covers");
|
|
1223
|
+
function holdsSignal(cp, signal, expected) {
|
|
1224
|
+
return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
|
|
1225
|
+
}
|
|
1226
|
+
__name(holdsSignal, "holdsSignal");
|
|
1227
|
+
async function checkpointsAt(store, runId, seqs) {
|
|
1228
|
+
const unique = [
|
|
1229
|
+
...new Set(seqs)
|
|
1230
|
+
];
|
|
1231
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
1232
|
+
if (unique.length === 0) return bySeq;
|
|
1233
|
+
const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
|
|
1234
|
+
for (const cp of rows) bySeq.set(cp.seq, cp);
|
|
1235
|
+
return bySeq;
|
|
1236
|
+
}
|
|
1237
|
+
__name(checkpointsAt, "checkpointsAt");
|
|
1238
|
+
async function checkpointsNamed(store, runId, names) {
|
|
1239
|
+
const wanted = new Set(names);
|
|
1240
|
+
const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
|
|
1241
|
+
...wanted
|
|
1242
|
+
]) : await store.listCheckpoints(runId);
|
|
1243
|
+
return rows.filter((cp) => wanted.has(cp.name));
|
|
1244
|
+
}
|
|
1245
|
+
__name(checkpointsNamed, "checkpointsNamed");
|
|
1246
|
+
async function parentSettles(store, parentId, children) {
|
|
1247
|
+
const entries = children.map(({ childId, completion }) => ({
|
|
1248
|
+
slot: slotOf(childId),
|
|
1249
|
+
expected: canonicalJson(completion)
|
|
1250
|
+
}));
|
|
1251
|
+
const held = entries.map(() => void 0);
|
|
1252
|
+
const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
|
|
1253
|
+
...slot.signalSeq === void 0 ? [] : [
|
|
1254
|
+
slot.signalSeq
|
|
1255
|
+
],
|
|
1256
|
+
...slot.fan ? [
|
|
1257
|
+
slot.fan.firstSeq
|
|
1258
|
+
] : []
|
|
1259
|
+
]));
|
|
1260
|
+
const waveFolds = [];
|
|
1261
|
+
const byName = [];
|
|
1262
|
+
entries.forEach(({ slot, expected }, at) => {
|
|
1263
|
+
const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
|
|
1264
|
+
if (own?.name === slot.signal) {
|
|
1265
|
+
held[at] = holdsSignal(own, slot.signal, expected);
|
|
1266
|
+
return;
|
|
1193
1267
|
}
|
|
1194
|
-
|
|
1268
|
+
const fan = slot.fan;
|
|
1269
|
+
if (fan) {
|
|
1270
|
+
if (covers(foldOf(own, fan.fold), fan.index)) {
|
|
1271
|
+
held[at] = true;
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
const atFirst = first.get(fan.firstSeq);
|
|
1275
|
+
if (fan.wave === void 0 || fan.wave === 0) {
|
|
1276
|
+
held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
1279
|
+
const waveZero = foldOf(atFirst, fan.waveZeroFold);
|
|
1280
|
+
if (waveZero && waveZero.values.length > 0) {
|
|
1281
|
+
waveFolds.push({
|
|
1282
|
+
at,
|
|
1283
|
+
seq: fan.firstSeq + fan.wave * waveZero.values.length
|
|
1284
|
+
});
|
|
1285
|
+
return;
|
|
1286
|
+
}
|
|
1287
|
+
byName.push(at);
|
|
1288
|
+
return;
|
|
1289
|
+
}
|
|
1290
|
+
if (slot.signalSeq !== void 0 && !own) {
|
|
1291
|
+
held[at] = false;
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
byName.push(at);
|
|
1195
1295
|
});
|
|
1296
|
+
if (waveFolds.length > 0) {
|
|
1297
|
+
const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
|
|
1298
|
+
for (const { at, seq } of waveFolds) {
|
|
1299
|
+
const fan = entries[at].slot.fan;
|
|
1300
|
+
held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
if (byName.length > 0) {
|
|
1304
|
+
const names = byName.flatMap((at) => {
|
|
1305
|
+
const { slot } = entries[at];
|
|
1306
|
+
return slot.fan ? [
|
|
1307
|
+
slot.signal,
|
|
1308
|
+
slot.fan.fold
|
|
1309
|
+
] : [
|
|
1310
|
+
slot.signal
|
|
1311
|
+
];
|
|
1312
|
+
});
|
|
1313
|
+
const named = await checkpointsNamed(store, parentId, names);
|
|
1314
|
+
for (const at of byName) {
|
|
1315
|
+
const { slot, expected } = entries[at];
|
|
1316
|
+
held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
return held.map((h) => h === true);
|
|
1196
1320
|
}
|
|
1197
|
-
__name(
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
const prefixes = fold ? [
|
|
1201
|
-
signal,
|
|
1202
|
-
fold.name
|
|
1203
|
-
] : [
|
|
1204
|
-
signal
|
|
1205
|
-
];
|
|
1206
|
-
return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
|
|
1321
|
+
__name(parentSettles, "parentSettles");
|
|
1322
|
+
function parentClosed(run) {
|
|
1323
|
+
return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
|
|
1207
1324
|
}
|
|
1208
|
-
__name(
|
|
1325
|
+
__name(parentClosed, "parentClosed");
|
|
1209
1326
|
async function childCompletionNeedsBuffer(store, childId, completion, parent) {
|
|
1210
1327
|
const parentId = parentRunIdOf(childId);
|
|
1211
1328
|
if (!parentId) return false;
|
|
1212
1329
|
const run = parent === void 0 ? await store.getRun(parentId) : parent;
|
|
1213
|
-
if (
|
|
1214
|
-
|
|
1215
|
-
|
|
1330
|
+
if (parentClosed(run)) return false;
|
|
1331
|
+
const [held] = await parentSettles(store, parentId, [
|
|
1332
|
+
{
|
|
1333
|
+
childId,
|
|
1334
|
+
completion
|
|
1335
|
+
}
|
|
1336
|
+
]);
|
|
1337
|
+
return !held;
|
|
1216
1338
|
}
|
|
1217
1339
|
__name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
|
|
1218
|
-
async function unneededChildSignals(store, rows) {
|
|
1340
|
+
async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
|
|
1341
|
+
const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
|
|
1219
1342
|
const parentIds = rows.flatMap((row) => {
|
|
1220
|
-
const
|
|
1343
|
+
const childId = childOf(row);
|
|
1344
|
+
const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
|
|
1221
1345
|
return parentId ? [
|
|
1222
1346
|
parentId
|
|
1223
1347
|
] : [];
|
|
1224
1348
|
});
|
|
1225
1349
|
const { runs, unread } = await readRunsByIds(store, parentIds);
|
|
1226
1350
|
const unreadable = new Set(unread);
|
|
1227
|
-
const doomed =
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
const childId = row
|
|
1351
|
+
const doomed = /* @__PURE__ */ new Set();
|
|
1352
|
+
const open = /* @__PURE__ */ new Map();
|
|
1353
|
+
rows.forEach((row, at) => {
|
|
1354
|
+
const childId = childOf(row);
|
|
1355
|
+
if (childId === void 0) return;
|
|
1231
1356
|
const parentId = parentRunIdOf(childId);
|
|
1232
|
-
if (parentId
|
|
1233
|
-
|
|
1357
|
+
if (!parentId) {
|
|
1358
|
+
doomed.add(row);
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
if (unreadable.has(parentId)) return;
|
|
1362
|
+
if (parentClosed(runs.get(parentId) ?? null)) {
|
|
1363
|
+
doomed.add(row);
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
const group = open.get(parentId) ?? [];
|
|
1367
|
+
group.push({
|
|
1368
|
+
at,
|
|
1369
|
+
row,
|
|
1370
|
+
childId
|
|
1371
|
+
});
|
|
1372
|
+
open.set(parentId, group);
|
|
1373
|
+
});
|
|
1374
|
+
let decided = rows.length;
|
|
1375
|
+
let checked = 0;
|
|
1376
|
+
for (const [parentId, group] of open) {
|
|
1377
|
+
if (checked > 0 && Date.now() >= deadline) {
|
|
1378
|
+
decided = group[0].at;
|
|
1379
|
+
break;
|
|
1380
|
+
}
|
|
1381
|
+
checked += 1;
|
|
1234
1382
|
try {
|
|
1235
|
-
|
|
1383
|
+
const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
|
|
1384
|
+
childId,
|
|
1385
|
+
completion: row.payload
|
|
1386
|
+
})));
|
|
1387
|
+
group.forEach(({ row }, k) => {
|
|
1388
|
+
if (held[k]) doomed.add(row);
|
|
1389
|
+
});
|
|
1236
1390
|
} catch {
|
|
1237
1391
|
}
|
|
1238
1392
|
}
|
|
1239
|
-
return
|
|
1393
|
+
return {
|
|
1394
|
+
doomed: rows.filter((row) => doomed.has(row)),
|
|
1395
|
+
decided
|
|
1396
|
+
};
|
|
1240
1397
|
}
|
|
1241
1398
|
__name(unneededChildSignals, "unneededChildSignals");
|
|
1242
1399
|
|
|
@@ -4857,6 +5014,7 @@ var ANNOUNCED_ROUTES_TTL_MS = 5e3;
|
|
|
4857
5014
|
var VERSION_UNDECLARED_TAG = "version:undeclared";
|
|
4858
5015
|
var versionKey = /* @__PURE__ */ __name((name, version) => `${name}@${version}`, "versionKey");
|
|
4859
5016
|
var STEP_FAN_CHUNK = 200;
|
|
5017
|
+
var FAN_REREAD_CHUNK = 1e3;
|
|
4860
5018
|
var PARENT_NOTIFY_ATTEMPTS = 4;
|
|
4861
5019
|
var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
4862
5020
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
@@ -4893,6 +5051,7 @@ var WAKE_REQUEST_SCAN = 100;
|
|
|
4893
5051
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4894
5052
|
var BUFFERED_SWEEP_PAGE = 500;
|
|
4895
5053
|
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
5054
|
+
var BUFFERED_SWEEP_BUDGET_MS = 250;
|
|
4896
5055
|
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4897
5056
|
var DELIVER_CHUNK = 50;
|
|
4898
5057
|
var RESULT_BATCH_MAX = 256;
|
|
@@ -5785,13 +5944,17 @@ var WorkflowEngine = class {
|
|
|
5785
5944
|
*
|
|
5786
5945
|
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
5787
5946
|
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
5788
|
-
* few timer ticks without a long query or a big delete.
|
|
5789
|
-
*
|
|
5790
|
-
*
|
|
5791
|
-
*
|
|
5792
|
-
*
|
|
5793
|
-
|
|
5794
|
-
|
|
5947
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
5948
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
5949
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
5950
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
5951
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
5952
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
5953
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
5954
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
5955
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5956
|
+
*/
|
|
5957
|
+
async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
|
|
5795
5958
|
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
5796
5959
|
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
5797
5960
|
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
@@ -5799,15 +5962,21 @@ var WorkflowEngine = class {
|
|
|
5799
5962
|
return 0;
|
|
5800
5963
|
}
|
|
5801
5964
|
this.bufferedSweepActive = true;
|
|
5965
|
+
const deadline = Date.now() + budgetMs;
|
|
5802
5966
|
let deleted = 0;
|
|
5803
5967
|
try {
|
|
5804
5968
|
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
5969
|
+
if (page > 0 && (Date.now() >= deadline || this.draining)) break;
|
|
5805
5970
|
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
5806
|
-
const doomed = await unneededChildSignals(this.store, rows);
|
|
5971
|
+
const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
|
|
5807
5972
|
if (doomed.length > 0) {
|
|
5808
5973
|
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
5809
5974
|
deleted += doomed.length;
|
|
5810
5975
|
}
|
|
5976
|
+
if (decided < rows.length) {
|
|
5977
|
+
if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
|
|
5978
|
+
break;
|
|
5979
|
+
}
|
|
5811
5980
|
const last = rows.at(-1);
|
|
5812
5981
|
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
5813
5982
|
this.bufferedSweepCursor = void 0;
|
|
@@ -7282,8 +7451,7 @@ var WorkflowEngine = class {
|
|
|
7282
7451
|
*/
|
|
7283
7452
|
async redriveSettledSteps(runId, seqs) {
|
|
7284
7453
|
for (const seq of seqs) {
|
|
7285
|
-
|
|
7286
|
-
if (cp && (cp.status === "completed" || cp.status === "failed")) {
|
|
7454
|
+
if (await this.stepSettled(runId, seq)) {
|
|
7287
7455
|
if (this.settledHere.has(`${runId}:${seq}`)) return;
|
|
7288
7456
|
const nowMs = this.clock();
|
|
7289
7457
|
if (!await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs)) {
|
|
@@ -7299,6 +7467,17 @@ var WorkflowEngine = class {
|
|
|
7299
7467
|
}
|
|
7300
7468
|
}
|
|
7301
7469
|
}
|
|
7470
|
+
/** Whether the step at `seq` is settled: a `COUNT` when the store has one, else its checkpoint. */
|
|
7471
|
+
async stepSettled(runId, seq) {
|
|
7472
|
+
if (this.store.countCheckpoints) {
|
|
7473
|
+
return await this.store.countCheckpoints(runId, seq, seq, [
|
|
7474
|
+
"completed",
|
|
7475
|
+
"failed"
|
|
7476
|
+
]) > 0;
|
|
7477
|
+
}
|
|
7478
|
+
const cp = await this.store.getCheckpoint(runId, seq);
|
|
7479
|
+
return cp?.status === "completed" || cp?.status === "failed";
|
|
7480
|
+
}
|
|
7302
7481
|
/**
|
|
7303
7482
|
* {@link redriveSettledSteps} for a step fan: once the turn that suspended on it released its lease,
|
|
7304
7483
|
* re-drive the run if no step of the fan is still `pending` — the last result landed on another
|
|
@@ -8770,7 +8949,7 @@ ${run.workflowVersion}`;
|
|
|
8770
8949
|
}
|
|
8771
8950
|
const wfPlan = await this.planRoute(tenantGroup(sanitizeQueueToken(remote.group), void 0), registered.requires);
|
|
8772
8951
|
if (wfPlan?.status === "blocked") return this.parkBlocked(run, wfPlan);
|
|
8773
|
-
const checkpoints = await this.
|
|
8952
|
+
const checkpoints = await this.replayCheckpoints(run.id);
|
|
8774
8953
|
await this.deliverFinishedChildren(run.id, checkpoints);
|
|
8775
8954
|
await this.deliverFinishedChildWaiters(run.id, checkpoints);
|
|
8776
8955
|
const handoff = liveHandoff(run.id, checkpoints);
|
|
@@ -9046,6 +9225,37 @@ ${run.workflowVersion}`;
|
|
|
9046
9225
|
* throttle for {@link rearmStepLease}, the step-scoped sibling of {@link decisionDeadlineRearms}. */
|
|
9047
9226
|
stepLeaseRearms = /* @__PURE__ */ new Map();
|
|
9048
9227
|
/**
|
|
9228
|
+
* {@link stepLeaseRearms} for the steps of a fan this instance sent, by run: one entry per sent
|
|
9229
|
+
* block of consecutive seqs instead of one per step, since they share the lease stamped at send.
|
|
9230
|
+
*/
|
|
9231
|
+
fanLeaseRearms = /* @__PURE__ */ new Map();
|
|
9232
|
+
noteFanLease(runId, seqs, lease) {
|
|
9233
|
+
if (this.remoteRedispatchMs == null || seqs.length === 0) return;
|
|
9234
|
+
let from = seqs[0];
|
|
9235
|
+
let to = from;
|
|
9236
|
+
for (const seq of seqs) {
|
|
9237
|
+
if (seq < from) from = seq;
|
|
9238
|
+
if (seq > to) to = seq;
|
|
9239
|
+
}
|
|
9240
|
+
if (to - from + 1 !== seqs.length) return;
|
|
9241
|
+
if (this.fanLeaseRearms.size > 1024) this.fanLeaseRearms.clear();
|
|
9242
|
+
const blocks = this.fanLeaseRearms.get(runId) ?? [];
|
|
9243
|
+
blocks.push({
|
|
9244
|
+
from,
|
|
9245
|
+
to,
|
|
9246
|
+
lookAgainAt: lease - Math.ceil(this.remoteRedispatchMs / 2)
|
|
9247
|
+
});
|
|
9248
|
+
this.fanLeaseRearms.set(runId, blocks);
|
|
9249
|
+
}
|
|
9250
|
+
fanLeaseLookAgainAt(runId, seq, now) {
|
|
9251
|
+
const blocks = this.fanLeaseRearms.get(runId);
|
|
9252
|
+
if (!blocks) return void 0;
|
|
9253
|
+
const live = blocks.filter((b) => now < b.lookAgainAt);
|
|
9254
|
+
if (live.length === 0) this.fanLeaseRearms.delete(runId);
|
|
9255
|
+
else if (live.length < blocks.length) this.fanLeaseRearms.set(runId, live);
|
|
9256
|
+
return live.find((b) => seq >= b.from && seq <= b.to)?.lookAgainAt;
|
|
9257
|
+
}
|
|
9258
|
+
/**
|
|
9049
9259
|
* A step-scoped heartbeat for a durably-suspended remote step: push its LEASE forward so a worker
|
|
9050
9260
|
* that is still working on it is never presumed dead and double-dispatched. The durable counterpart
|
|
9051
9261
|
* of {@link callRemoteInMemory}'s in-memory `heartbeatResets` rearm (which only exists on the
|
|
@@ -9074,7 +9284,7 @@ ${run.workflowVersion}`;
|
|
|
9074
9284
|
if (this.remoteRedispatchMs == null) return;
|
|
9075
9285
|
const id = stepId(runId, seq);
|
|
9076
9286
|
const now = this.clock();
|
|
9077
|
-
const lookAgainAt = this.stepLeaseRearms.get(id);
|
|
9287
|
+
const lookAgainAt = this.stepLeaseRearms.get(id) ?? this.fanLeaseLookAgainAt(runId, seq, now);
|
|
9078
9288
|
if (lookAgainAt !== void 0 && now < lookAgainAt) return;
|
|
9079
9289
|
try {
|
|
9080
9290
|
const cp = await this.store.getCheckpoint(runId, seq);
|
|
@@ -9397,7 +9607,7 @@ ${run.workflowVersion}`;
|
|
|
9397
9607
|
return this.singletonWaitResult(run);
|
|
9398
9608
|
}
|
|
9399
9609
|
const compensations = [];
|
|
9400
|
-
const snapshot = await this.
|
|
9610
|
+
const snapshot = await this.replayCheckpoints(run.id);
|
|
9401
9611
|
await this.deliverFinishedChildren(run.id, snapshot);
|
|
9402
9612
|
const replay = /* @__PURE__ */ new Map();
|
|
9403
9613
|
for (const cp of snapshot) replay.set(cp.seq, cp);
|
|
@@ -9727,7 +9937,7 @@ ${run.workflowVersion}`;
|
|
|
9727
9937
|
}), "callRemote"),
|
|
9728
9938
|
callRemoteAll: /* @__PURE__ */ __name((runId, positions, step, inputs, opts) => this.callRemoteAll(runId, positions, step, inputs, opts, {
|
|
9729
9939
|
namespace: parentNamespace
|
|
9730
|
-
}), "callRemoteAll"),
|
|
9940
|
+
}, replay), "callRemoteAll"),
|
|
9731
9941
|
// Defer so a fast child can't reentrantly resume a still-running parent. Stamp the child with
|
|
9732
9942
|
// the PARENT run's namespace (not this engine's) so a child belongs to the tenant/partition of
|
|
9733
9943
|
// the run it was spawned from — even when an operator (namespace: undefined, drives every
|
|
@@ -10084,7 +10294,7 @@ ${run.workflowVersion}`;
|
|
|
10084
10294
|
* concurrently with this one finds them already sent instead of sending them again. The join
|
|
10085
10295
|
* barrier ({@link stepFanStillRunning}) then wakes the parent once per wave, not once per result.
|
|
10086
10296
|
*/
|
|
10087
|
-
async callRemoteAll(runId, positions, step, inputs, opts, routing) {
|
|
10297
|
+
async callRemoteAll(runId, positions, step, inputs, opts, routing, replay) {
|
|
10088
10298
|
if (step.timeoutMs) {
|
|
10089
10299
|
throw new Error(`ctx.all: step "${step.name}" has a liveness timeoutMs; a step fan needs durable steps`);
|
|
10090
10300
|
}
|
|
@@ -10095,10 +10305,7 @@ ${run.workflowVersion}`;
|
|
|
10095
10305
|
const first = positions[0];
|
|
10096
10306
|
const last = positions[positions.length - 1];
|
|
10097
10307
|
const group = stepFanGroup(first, positions.length, opts.mode);
|
|
10098
|
-
const bySeq =
|
|
10099
|
-
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
10100
|
-
if (cp.seq >= first && cp.seq <= last) bySeq.set(cp.seq, cp);
|
|
10101
|
-
}
|
|
10308
|
+
const bySeq = await this.fanCheckpoints(runId, positions, replay);
|
|
10102
10309
|
const outputs = new Array(inputs.length);
|
|
10103
10310
|
const failures = [];
|
|
10104
10311
|
const unsent = [];
|
|
@@ -10155,7 +10362,13 @@ ${run.workflowVersion}`;
|
|
|
10155
10362
|
}
|
|
10156
10363
|
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
10157
10364
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
10158
|
-
if (batch.length > 0)
|
|
10365
|
+
if (batch.length > 0) {
|
|
10366
|
+
const lease = await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
10367
|
+
if (lease !== void 0) {
|
|
10368
|
+
const at = this.stepLeaseWakeAt(lease);
|
|
10369
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
10370
|
+
}
|
|
10371
|
+
}
|
|
10159
10372
|
if (inFlight + batch.length > 0) {
|
|
10160
10373
|
this.awaitedStepFans.get(runId)?.push({
|
|
10161
10374
|
from: first,
|
|
@@ -10185,6 +10398,34 @@ ${run.workflowVersion}`;
|
|
|
10185
10398
|
return outputs;
|
|
10186
10399
|
}
|
|
10187
10400
|
/**
|
|
10401
|
+
* The checkpoints of a step fan's seqs, by seq. A settled one is taken from the turn's snapshot
|
|
10402
|
+
* (`replay`), since nothing but a turn of the run changes it once settled; one the snapshot holds
|
|
10403
|
+
* `pending`, or does not hold, is read again, as {@link callRemote} does for a single step: its
|
|
10404
|
+
* result may have landed since. The fan's last turn so reads nothing it already read.
|
|
10405
|
+
*/
|
|
10406
|
+
async fanCheckpoints(runId, positions, replay) {
|
|
10407
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
10408
|
+
const reread = [];
|
|
10409
|
+
for (const seq of positions) {
|
|
10410
|
+
const cp = replay?.get(seq);
|
|
10411
|
+
if (cp && (cp.status === "completed" || cp.status === "failed")) bySeq.set(seq, cp);
|
|
10412
|
+
else reread.push(seq);
|
|
10413
|
+
}
|
|
10414
|
+
if (reread.length === 0) return bySeq;
|
|
10415
|
+
if (reread.length === positions.length || !this.store.getCheckpoints) {
|
|
10416
|
+
const wanted = new Set(reread);
|
|
10417
|
+
for (const cp of await this.store.listCheckpoints(runId)) {
|
|
10418
|
+
if (wanted.has(cp.seq)) bySeq.set(cp.seq, cp);
|
|
10419
|
+
}
|
|
10420
|
+
return bySeq;
|
|
10421
|
+
}
|
|
10422
|
+
for (let at = 0; at < reread.length; at += FAN_REREAD_CHUNK) {
|
|
10423
|
+
const read = await this.store.getCheckpoints(runId, reread.slice(at, at + FAN_REREAD_CHUNK));
|
|
10424
|
+
for (const cp of read) bySeq.set(cp.seq, cp);
|
|
10425
|
+
}
|
|
10426
|
+
return bySeq;
|
|
10427
|
+
}
|
|
10428
|
+
/**
|
|
10188
10429
|
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
10189
10430
|
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
10190
10431
|
* batch, where the store and transport have them.
|
|
@@ -10193,6 +10434,7 @@ ${run.workflowVersion}`;
|
|
|
10193
10434
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
10194
10435
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
10195
10436
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
10437
|
+
const lease = this.remoteRedispatchMs == null ? void 0 : this.clock() + this.remoteRedispatchMs;
|
|
10196
10438
|
const checkpoints = batch.map((i) => {
|
|
10197
10439
|
const seq = positions[i];
|
|
10198
10440
|
return {
|
|
@@ -10208,7 +10450,10 @@ ${run.workflowVersion}`;
|
|
|
10208
10450
|
parallelGroup: group,
|
|
10209
10451
|
enqueuedAt,
|
|
10210
10452
|
startedAt: enqueuedAt,
|
|
10211
|
-
finishedAt: enqueuedAt
|
|
10453
|
+
finishedAt: enqueuedAt,
|
|
10454
|
+
...lease !== void 0 ? {
|
|
10455
|
+
wakeAt: lease
|
|
10456
|
+
} : {}
|
|
10212
10457
|
};
|
|
10213
10458
|
});
|
|
10214
10459
|
if (this.store.saveCheckpoints) {
|
|
@@ -10241,20 +10486,36 @@ ${run.workflowVersion}`;
|
|
|
10241
10486
|
kind: "remote"
|
|
10242
10487
|
});
|
|
10243
10488
|
}
|
|
10489
|
+
if (lease !== void 0) {
|
|
10490
|
+
this.noteFanLease(runId, checkpoints.map((cp) => cp.seq), lease);
|
|
10491
|
+
}
|
|
10492
|
+
return lease;
|
|
10244
10493
|
}
|
|
10245
10494
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
10246
10495
|
noteSettledHere(key) {
|
|
10247
10496
|
this.noteRecent(this.settledHere, key);
|
|
10248
10497
|
}
|
|
10249
|
-
/**
|
|
10498
|
+
/**
|
|
10499
|
+
* Record `key` in a "happened here recently" map and drop the entries older than a lease. A key is
|
|
10500
|
+
* re-inserted at the end, so the map stays in the order its keys were noted and the stale ones are
|
|
10501
|
+
* all at its front: pruning stops at the first fresh entry instead of walking the whole map.
|
|
10502
|
+
*/
|
|
10250
10503
|
noteRecent(map, key) {
|
|
10251
10504
|
const now = this.clock();
|
|
10505
|
+
map.delete(key);
|
|
10252
10506
|
map.set(key, now);
|
|
10253
|
-
if (map.size < 1024) return;
|
|
10254
10507
|
for (const [k, at] of map) {
|
|
10255
|
-
if (now - at
|
|
10508
|
+
if (now - at <= this.leaseMs) break;
|
|
10509
|
+
map.delete(k);
|
|
10256
10510
|
}
|
|
10257
10511
|
}
|
|
10512
|
+
/**
|
|
10513
|
+
* The run's checkpoints as a turn replays them: {@link StateStore.listCheckpointsForReplay} when
|
|
10514
|
+
* the store has it (no input on a completed row), else {@link StateStore.listCheckpoints}.
|
|
10515
|
+
*/
|
|
10516
|
+
replayCheckpoints(runId) {
|
|
10517
|
+
return this.store.listCheckpointsForReplay ? this.store.listCheckpointsForReplay(runId) : this.store.listCheckpoints(runId);
|
|
10518
|
+
}
|
|
10258
10519
|
/** {@link StateStore.saveCheckpointIfPending}, or a plain save on a store without it. */
|
|
10259
10520
|
async savePendingCheckpoint(checkpoint) {
|
|
10260
10521
|
if (this.store.saveCheckpointIfPending) return this.store.saveCheckpointIfPending(checkpoint);
|
|
@@ -10281,11 +10542,59 @@ ${run.workflowVersion}`;
|
|
|
10281
10542
|
});
|
|
10282
10543
|
}
|
|
10283
10544
|
/**
|
|
10545
|
+
* Settle steps of one run at distinct seqs from their results ({@link StepSettlement}s, `settled[i]`
|
|
10546
|
+
* being the `pending` row `settlements[i]` settles): one
|
|
10547
|
+
* {@link StateStore.settleCheckpointsIfPending} call when the store has it, which writes only what
|
|
10548
|
+
* a result changes, else the whole rows through {@link savePendingCheckpoints}. Never rejects as a
|
|
10549
|
+
* whole, the same way.
|
|
10550
|
+
*/
|
|
10551
|
+
async settlePendingCheckpoints(settlements, settled) {
|
|
10552
|
+
const settle = this.store.settleCheckpointsIfPending?.bind(this.store);
|
|
10553
|
+
if (!settle) {
|
|
10554
|
+
return this.savePendingCheckpoints(settlements.map((s, i) => ({
|
|
10555
|
+
...settled[i],
|
|
10556
|
+
...s
|
|
10557
|
+
})));
|
|
10558
|
+
}
|
|
10559
|
+
if (settlements.length > 1) {
|
|
10560
|
+
try {
|
|
10561
|
+
const wrote = await settle(settlements);
|
|
10562
|
+
return settlements.map((_, i) => wrote[i] === true);
|
|
10563
|
+
} catch {
|
|
10564
|
+
}
|
|
10565
|
+
}
|
|
10566
|
+
const one = await Promise.allSettled(settlements.map((s) => settle([
|
|
10567
|
+
s
|
|
10568
|
+
])));
|
|
10569
|
+
return one.map((s) => s.status === "fulfilled" ? s.value[0] === true : {
|
|
10570
|
+
error: s.reason
|
|
10571
|
+
});
|
|
10572
|
+
}
|
|
10573
|
+
/**
|
|
10284
10574
|
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10285
10575
|
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10286
10576
|
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10287
10577
|
*/
|
|
10288
10578
|
async readCheckpoints(runId, seqs) {
|
|
10579
|
+
const forSettle = this.store.settleCheckpointsIfPending ? this.store.getCheckpointsForSettle?.bind(this.store) : void 0;
|
|
10580
|
+
if (forSettle) {
|
|
10581
|
+
if (seqs.length > 1) {
|
|
10582
|
+
try {
|
|
10583
|
+
const bySeq = new Map((await forSettle(runId, seqs)).map((cp) => [
|
|
10584
|
+
cp.seq,
|
|
10585
|
+
cp
|
|
10586
|
+
]));
|
|
10587
|
+
return seqs.map((seq) => ({
|
|
10588
|
+
status: "fulfilled",
|
|
10589
|
+
value: bySeq.get(seq) ?? null
|
|
10590
|
+
}));
|
|
10591
|
+
} catch {
|
|
10592
|
+
}
|
|
10593
|
+
}
|
|
10594
|
+
return Promise.allSettled(seqs.map(async (seq) => (await forSettle(runId, [
|
|
10595
|
+
seq
|
|
10596
|
+
]))[0] ?? null));
|
|
10597
|
+
}
|
|
10289
10598
|
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10290
10599
|
try {
|
|
10291
10600
|
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
@@ -10434,10 +10743,11 @@ ${run.workflowVersion}`;
|
|
|
10434
10743
|
const run = await this.store.getRun(runId);
|
|
10435
10744
|
if (run && (run.status === "cancelled" || run.status === "completed")) return outcomes;
|
|
10436
10745
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
10437
|
-
const
|
|
10746
|
+
const settlements = pending.map(({ index, cp }) => {
|
|
10438
10747
|
const result = results[index];
|
|
10439
10748
|
return {
|
|
10440
|
-
|
|
10749
|
+
runId,
|
|
10750
|
+
seq: cp.seq,
|
|
10441
10751
|
status: result.status,
|
|
10442
10752
|
output: result.status === "completed" ? result.output : cp.output,
|
|
10443
10753
|
error: result.error,
|
|
@@ -10446,7 +10756,7 @@ ${run.workflowVersion}`;
|
|
|
10446
10756
|
finishedAt
|
|
10447
10757
|
};
|
|
10448
10758
|
});
|
|
10449
|
-
const claims = await this.
|
|
10759
|
+
const claims = await this.settlePendingCheckpoints(settlements, pending.map(({ cp }) => cp));
|
|
10450
10760
|
const settled = [];
|
|
10451
10761
|
claims.forEach((claim, k) => {
|
|
10452
10762
|
const { index, cp } = pending[k];
|
|
@@ -10460,7 +10770,7 @@ ${run.workflowVersion}`;
|
|
|
10460
10770
|
};
|
|
10461
10771
|
});
|
|
10462
10772
|
for (const { cp, result } of settled) {
|
|
10463
|
-
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
10773
|
+
if (!parseStepFanGroup(cp.parallelGroup)) this.noteSettledHere(`${runId}:${result.seq}`);
|
|
10464
10774
|
const startedAt = result.startedAt ? new Date(result.startedAt) : cp.startedAt;
|
|
10465
10775
|
this.emit({
|
|
10466
10776
|
type: result.status === "completed" ? "step.completed" : "step.failed",
|
|
@@ -11359,8 +11669,16 @@ var CodecStateStore = class {
|
|
|
11359
11669
|
saveCheckpointIfPending;
|
|
11360
11670
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
11361
11671
|
saveCheckpointsIfPending;
|
|
11672
|
+
/** Encodes each settlement's payloads like {@link saveCheckpoint}. Bound only when the inner store
|
|
11673
|
+
* has it, so the engine's whole-row fallback still kicks in through the wrapper. */
|
|
11674
|
+
settleCheckpointsIfPending;
|
|
11362
11675
|
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11363
11676
|
saveCheckpoints;
|
|
11677
|
+
/** Decodes each checkpoint read like {@link listCheckpoints}. Bound only when the inner store has
|
|
11678
|
+
* it, so the engine's full-read fallback still kicks in through the wrapper. */
|
|
11679
|
+
listCheckpointsForReplay;
|
|
11680
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11681
|
+
getCheckpointsForSettle;
|
|
11364
11682
|
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11365
11683
|
getCheckpoints;
|
|
11366
11684
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
@@ -11413,8 +11731,14 @@ var CodecStateStore = class {
|
|
|
11413
11731
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
11414
11732
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
11415
11733
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11734
|
+
const settleMany = inner.settleCheckpointsIfPending;
|
|
11735
|
+
if (settleMany) this.settleCheckpointsIfPending = (settlements) => settleMany.call(inner, settlements.map((s) => this.encSettlement(s)));
|
|
11416
11736
|
const saveMany = inner.saveCheckpoints;
|
|
11417
11737
|
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11738
|
+
const listForReplay = inner.listCheckpointsForReplay;
|
|
11739
|
+
if (listForReplay) this.listCheckpointsForReplay = async (runId) => (await listForReplay.call(inner, runId)).map((cp) => this.decCp(cp));
|
|
11740
|
+
const getForSettle = inner.getCheckpointsForSettle;
|
|
11741
|
+
if (getForSettle) this.getCheckpointsForSettle = async (runId, seqs) => (await getForSettle.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11418
11742
|
const getMany = inner.getCheckpoints;
|
|
11419
11743
|
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
11420
11744
|
const countCps = inner.countCheckpoints;
|
|
@@ -11506,6 +11830,21 @@ var CodecStateStore = class {
|
|
|
11506
11830
|
}
|
|
11507
11831
|
return next;
|
|
11508
11832
|
}
|
|
11833
|
+
encSettlement(settlement) {
|
|
11834
|
+
const next = {
|
|
11835
|
+
...settlement,
|
|
11836
|
+
output: this.enc(settlement.output)
|
|
11837
|
+
};
|
|
11838
|
+
if (this.extended) {
|
|
11839
|
+
if (settlement.events !== void 0) {
|
|
11840
|
+
next.events = this.enc(settlement.events);
|
|
11841
|
+
}
|
|
11842
|
+
if (settlement.error !== void 0) {
|
|
11843
|
+
next.error = this.enc(settlement.error);
|
|
11844
|
+
}
|
|
11845
|
+
}
|
|
11846
|
+
return next;
|
|
11847
|
+
}
|
|
11509
11848
|
decCp(cp) {
|
|
11510
11849
|
if (!cp) return cp;
|
|
11511
11850
|
const next = {
|
|
@@ -12065,6 +12404,18 @@ var InMemoryStateStore = class {
|
|
|
12065
12404
|
] : [];
|
|
12066
12405
|
});
|
|
12067
12406
|
}
|
|
12407
|
+
async getCheckpointsForSettle(runId, seqs) {
|
|
12408
|
+
return [
|
|
12409
|
+
...new Set(seqs)
|
|
12410
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
12411
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
12412
|
+
if (!cp) return [];
|
|
12413
|
+
const { input: _input, ...rest } = cp;
|
|
12414
|
+
return [
|
|
12415
|
+
rest
|
|
12416
|
+
];
|
|
12417
|
+
});
|
|
12418
|
+
}
|
|
12068
12419
|
async saveCheckpoint(checkpoint) {
|
|
12069
12420
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
12070
12421
|
...checkpoint
|
|
@@ -12095,6 +12446,18 @@ var InMemoryStateStore = class {
|
|
|
12095
12446
|
return true;
|
|
12096
12447
|
});
|
|
12097
12448
|
}
|
|
12449
|
+
async settleCheckpointsIfPending(settlements) {
|
|
12450
|
+
return settlements.map((settlement) => {
|
|
12451
|
+
const key = this.key(settlement.runId, settlement.seq);
|
|
12452
|
+
const cp = this.checkpoints.get(key);
|
|
12453
|
+
if (cp?.status !== "pending") return false;
|
|
12454
|
+
this.checkpoints.set(key, {
|
|
12455
|
+
...cp,
|
|
12456
|
+
...settlement
|
|
12457
|
+
});
|
|
12458
|
+
return true;
|
|
12459
|
+
});
|
|
12460
|
+
}
|
|
12098
12461
|
async listOrphanedRuns(nowMs, limit, namespace, ownedBy) {
|
|
12099
12462
|
return [
|
|
12100
12463
|
...this.runs.values()
|
|
@@ -12411,6 +12774,13 @@ var InMemoryStateStore = class {
|
|
|
12411
12774
|
this.checkpointRowsRead += rows.length;
|
|
12412
12775
|
return rows;
|
|
12413
12776
|
}
|
|
12777
|
+
async listCheckpointsForReplay(runId) {
|
|
12778
|
+
return (await this.listCheckpoints(runId)).map((cp) => {
|
|
12779
|
+
if (cp.status !== "completed") return cp;
|
|
12780
|
+
const { input: _input, ...rest } = cp;
|
|
12781
|
+
return rest;
|
|
12782
|
+
});
|
|
12783
|
+
}
|
|
12414
12784
|
runCheckpoints(runId) {
|
|
12415
12785
|
return [
|
|
12416
12786
|
...this.checkpoints.values()
|