@dudousxd/nestjs-durable-core 0.85.7 → 0.85.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +222 -54
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +11 -7
- package/dist/index.d.ts +11 -7
- package/dist/index.js +222 -54
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1151,7 +1151,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
|
|
|
1151
1151
|
"failed",
|
|
1152
1152
|
"dead"
|
|
1153
1153
|
]);
|
|
1154
|
-
var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
|
|
1155
1154
|
function canonicalJson(value) {
|
|
1156
1155
|
const plain = JSON.stringify(value);
|
|
1157
1156
|
return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
|
|
@@ -1166,77 +1165,235 @@ function sortedJson(value) {
|
|
|
1166
1165
|
return JSON.stringify(value);
|
|
1167
1166
|
}
|
|
1168
1167
|
__name(sortedJson, "sortedJson");
|
|
1169
|
-
|
|
1168
|
+
var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
|
|
1169
|
+
var SEQ_CHILD = /\.child\.(\d+)$/;
|
|
1170
|
+
function slotOf(childId) {
|
|
1170
1171
|
const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
|
|
1171
|
-
const fan =
|
|
1172
|
-
if (
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1172
|
+
const fan = FAN_CHILD_PARTS.exec(childId);
|
|
1173
|
+
if (fan) {
|
|
1174
|
+
const root = fan[1];
|
|
1175
|
+
const wave = fan[3];
|
|
1176
|
+
const firstSeq = Number(fan[2]);
|
|
1177
|
+
const i = Number(fan[4]);
|
|
1178
|
+
if (wave === void 0) {
|
|
1179
|
+
return {
|
|
1180
|
+
signal,
|
|
1181
|
+
signalSeq: firstSeq + i,
|
|
1182
|
+
fan: {
|
|
1183
|
+
fold: `${FOLDED_PREFIX}${root}`,
|
|
1184
|
+
firstSeq,
|
|
1185
|
+
index: i
|
|
1186
|
+
}
|
|
1187
|
+
};
|
|
1180
1188
|
}
|
|
1189
|
+
return {
|
|
1190
|
+
signal,
|
|
1191
|
+
signalSeq: firstSeq + i,
|
|
1192
|
+
fan: {
|
|
1193
|
+
fold: `${FOLDED_PREFIX}${root}w${wave}.`,
|
|
1194
|
+
firstSeq,
|
|
1195
|
+
index: i,
|
|
1196
|
+
wave: Number(wave),
|
|
1197
|
+
waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
|
|
1198
|
+
}
|
|
1199
|
+
};
|
|
1200
|
+
}
|
|
1201
|
+
const seq = SEQ_CHILD.exec(childId);
|
|
1202
|
+
return seq ? {
|
|
1203
|
+
signal,
|
|
1204
|
+
signalSeq: Number(seq[1])
|
|
1205
|
+
} : {
|
|
1206
|
+
signal
|
|
1181
1207
|
};
|
|
1182
1208
|
}
|
|
1183
|
-
__name(
|
|
1184
|
-
function
|
|
1185
|
-
|
|
1186
|
-
const
|
|
1187
|
-
return
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1209
|
+
__name(slotOf, "slotOf");
|
|
1210
|
+
function foldOf(cp, name) {
|
|
1211
|
+
if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
|
|
1212
|
+
const { from, values } = cp.output ?? {};
|
|
1213
|
+
return typeof from === "number" && Array.isArray(values) ? {
|
|
1214
|
+
from,
|
|
1215
|
+
values
|
|
1216
|
+
} : void 0;
|
|
1217
|
+
}
|
|
1218
|
+
__name(foldOf, "foldOf");
|
|
1219
|
+
function covers(fold, index) {
|
|
1220
|
+
return !!fold && index >= fold.from && index < fold.from + fold.values.length;
|
|
1221
|
+
}
|
|
1222
|
+
__name(covers, "covers");
|
|
1223
|
+
function holdsSignal(cp, signal, expected) {
|
|
1224
|
+
return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
|
|
1225
|
+
}
|
|
1226
|
+
__name(holdsSignal, "holdsSignal");
|
|
1227
|
+
async function checkpointsAt(store, runId, seqs) {
|
|
1228
|
+
const unique = [
|
|
1229
|
+
...new Set(seqs)
|
|
1230
|
+
];
|
|
1231
|
+
const bySeq = /* @__PURE__ */ new Map();
|
|
1232
|
+
if (unique.length === 0) return bySeq;
|
|
1233
|
+
const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
|
|
1234
|
+
for (const cp of rows) bySeq.set(cp.seq, cp);
|
|
1235
|
+
return bySeq;
|
|
1236
|
+
}
|
|
1237
|
+
__name(checkpointsAt, "checkpointsAt");
|
|
1238
|
+
async function checkpointsNamed(store, runId, names) {
|
|
1239
|
+
const wanted = new Set(names);
|
|
1240
|
+
const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
|
|
1241
|
+
...wanted
|
|
1242
|
+
]) : await store.listCheckpoints(runId);
|
|
1243
|
+
return rows.filter((cp) => wanted.has(cp.name));
|
|
1244
|
+
}
|
|
1245
|
+
__name(checkpointsNamed, "checkpointsNamed");
|
|
1246
|
+
async function parentSettles(store, parentId, children) {
|
|
1247
|
+
const entries = children.map(({ childId, completion }) => ({
|
|
1248
|
+
slot: slotOf(childId),
|
|
1249
|
+
expected: canonicalJson(completion)
|
|
1250
|
+
}));
|
|
1251
|
+
const held = entries.map(() => void 0);
|
|
1252
|
+
const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
|
|
1253
|
+
...slot.signalSeq === void 0 ? [] : [
|
|
1254
|
+
slot.signalSeq
|
|
1255
|
+
],
|
|
1256
|
+
...slot.fan ? [
|
|
1257
|
+
slot.fan.firstSeq
|
|
1258
|
+
] : []
|
|
1259
|
+
]));
|
|
1260
|
+
const waveFolds = [];
|
|
1261
|
+
const byName = [];
|
|
1262
|
+
entries.forEach(({ slot, expected }, at) => {
|
|
1263
|
+
const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
|
|
1264
|
+
if (own?.name === slot.signal) {
|
|
1265
|
+
held[at] = holdsSignal(own, slot.signal, expected);
|
|
1266
|
+
return;
|
|
1193
1267
|
}
|
|
1194
|
-
|
|
1268
|
+
const fan = slot.fan;
|
|
1269
|
+
if (fan) {
|
|
1270
|
+
if (covers(foldOf(own, fan.fold), fan.index)) {
|
|
1271
|
+
held[at] = true;
|
|
1272
|
+
return;
|
|
1273
|
+
}
|
|
1274
|
+
const atFirst = first.get(fan.firstSeq);
|
|
1275
|
+
if (fan.wave === void 0 || fan.wave === 0) {
|
|
1276
|
+
held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
|
|
1277
|
+
return;
|
|
1278
|
+
}
|
|
1279
|
+
const waveZero = foldOf(atFirst, fan.waveZeroFold);
|
|
1280
|
+
if (waveZero && waveZero.values.length > 0) {
|
|
1281
|
+
waveFolds.push({
|
|
1282
|
+
at,
|
|
1283
|
+
seq: fan.firstSeq + fan.wave * waveZero.values.length
|
|
1284
|
+
});
|
|
1285
|
+
return;
|
|
1286
|
+
}
|
|
1287
|
+
byName.push(at);
|
|
1288
|
+
return;
|
|
1289
|
+
}
|
|
1290
|
+
if (slot.signalSeq !== void 0 && !own) {
|
|
1291
|
+
held[at] = false;
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
byName.push(at);
|
|
1195
1295
|
});
|
|
1296
|
+
if (waveFolds.length > 0) {
|
|
1297
|
+
const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
|
|
1298
|
+
for (const { at, seq } of waveFolds) {
|
|
1299
|
+
const fan = entries[at].slot.fan;
|
|
1300
|
+
held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
if (byName.length > 0) {
|
|
1304
|
+
const names = byName.flatMap((at) => {
|
|
1305
|
+
const { slot } = entries[at];
|
|
1306
|
+
return slot.fan ? [
|
|
1307
|
+
slot.signal,
|
|
1308
|
+
slot.fan.fold
|
|
1309
|
+
] : [
|
|
1310
|
+
slot.signal
|
|
1311
|
+
];
|
|
1312
|
+
});
|
|
1313
|
+
const named = await checkpointsNamed(store, parentId, names);
|
|
1314
|
+
for (const at of byName) {
|
|
1315
|
+
const { slot, expected } = entries[at];
|
|
1316
|
+
held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
return held.map((h) => h === true);
|
|
1196
1320
|
}
|
|
1197
|
-
__name(
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
const prefixes = fold ? [
|
|
1201
|
-
signal,
|
|
1202
|
-
fold.name
|
|
1203
|
-
] : [
|
|
1204
|
-
signal
|
|
1205
|
-
];
|
|
1206
|
-
return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
|
|
1321
|
+
__name(parentSettles, "parentSettles");
|
|
1322
|
+
function parentClosed(run) {
|
|
1323
|
+
return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
|
|
1207
1324
|
}
|
|
1208
|
-
__name(
|
|
1325
|
+
__name(parentClosed, "parentClosed");
|
|
1209
1326
|
async function childCompletionNeedsBuffer(store, childId, completion, parent) {
|
|
1210
1327
|
const parentId = parentRunIdOf(childId);
|
|
1211
1328
|
if (!parentId) return false;
|
|
1212
1329
|
const run = parent === void 0 ? await store.getRun(parentId) : parent;
|
|
1213
|
-
if (
|
|
1214
|
-
|
|
1215
|
-
|
|
1330
|
+
if (parentClosed(run)) return false;
|
|
1331
|
+
const [held] = await parentSettles(store, parentId, [
|
|
1332
|
+
{
|
|
1333
|
+
childId,
|
|
1334
|
+
completion
|
|
1335
|
+
}
|
|
1336
|
+
]);
|
|
1337
|
+
return !held;
|
|
1216
1338
|
}
|
|
1217
1339
|
__name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
|
|
1218
|
-
async function unneededChildSignals(store, rows) {
|
|
1340
|
+
async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
|
|
1341
|
+
const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
|
|
1219
1342
|
const parentIds = rows.flatMap((row) => {
|
|
1220
|
-
const
|
|
1343
|
+
const childId = childOf(row);
|
|
1344
|
+
const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
|
|
1221
1345
|
return parentId ? [
|
|
1222
1346
|
parentId
|
|
1223
1347
|
] : [];
|
|
1224
1348
|
});
|
|
1225
1349
|
const { runs, unread } = await readRunsByIds(store, parentIds);
|
|
1226
1350
|
const unreadable = new Set(unread);
|
|
1227
|
-
const doomed =
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
const childId = row
|
|
1351
|
+
const doomed = /* @__PURE__ */ new Set();
|
|
1352
|
+
const open = /* @__PURE__ */ new Map();
|
|
1353
|
+
rows.forEach((row, at) => {
|
|
1354
|
+
const childId = childOf(row);
|
|
1355
|
+
if (childId === void 0) return;
|
|
1231
1356
|
const parentId = parentRunIdOf(childId);
|
|
1232
|
-
if (parentId
|
|
1233
|
-
|
|
1357
|
+
if (!parentId) {
|
|
1358
|
+
doomed.add(row);
|
|
1359
|
+
return;
|
|
1360
|
+
}
|
|
1361
|
+
if (unreadable.has(parentId)) return;
|
|
1362
|
+
if (parentClosed(runs.get(parentId) ?? null)) {
|
|
1363
|
+
doomed.add(row);
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
const group = open.get(parentId) ?? [];
|
|
1367
|
+
group.push({
|
|
1368
|
+
at,
|
|
1369
|
+
row,
|
|
1370
|
+
childId
|
|
1371
|
+
});
|
|
1372
|
+
open.set(parentId, group);
|
|
1373
|
+
});
|
|
1374
|
+
let decided = rows.length;
|
|
1375
|
+
let checked = 0;
|
|
1376
|
+
for (const [parentId, group] of open) {
|
|
1377
|
+
if (checked > 0 && Date.now() >= deadline) {
|
|
1378
|
+
decided = group[0].at;
|
|
1379
|
+
break;
|
|
1380
|
+
}
|
|
1381
|
+
checked += 1;
|
|
1234
1382
|
try {
|
|
1235
|
-
|
|
1383
|
+
const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
|
|
1384
|
+
childId,
|
|
1385
|
+
completion: row.payload
|
|
1386
|
+
})));
|
|
1387
|
+
group.forEach(({ row }, k) => {
|
|
1388
|
+
if (held[k]) doomed.add(row);
|
|
1389
|
+
});
|
|
1236
1390
|
} catch {
|
|
1237
1391
|
}
|
|
1238
1392
|
}
|
|
1239
|
-
return
|
|
1393
|
+
return {
|
|
1394
|
+
doomed: rows.filter((row) => doomed.has(row)),
|
|
1395
|
+
decided
|
|
1396
|
+
};
|
|
1240
1397
|
}
|
|
1241
1398
|
__name(unneededChildSignals, "unneededChildSignals");
|
|
1242
1399
|
|
|
@@ -4893,6 +5050,7 @@ var WAKE_REQUEST_SCAN = 100;
|
|
|
4893
5050
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4894
5051
|
var BUFFERED_SWEEP_PAGE = 500;
|
|
4895
5052
|
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
5053
|
+
var BUFFERED_SWEEP_BUDGET_MS = 250;
|
|
4896
5054
|
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4897
5055
|
var DELIVER_CHUNK = 50;
|
|
4898
5056
|
var RESULT_BATCH_MAX = 256;
|
|
@@ -5785,13 +5943,17 @@ var WorkflowEngine = class {
|
|
|
5785
5943
|
*
|
|
5786
5944
|
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
5787
5945
|
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
5788
|
-
* few timer ticks without a long query or a big delete.
|
|
5789
|
-
*
|
|
5790
|
-
*
|
|
5791
|
-
*
|
|
5792
|
-
*
|
|
5793
|
-
|
|
5794
|
-
|
|
5946
|
+
* few timer ticks without a long query or a big delete. Each page is decided set-based (see
|
|
5947
|
+
* `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
|
|
5948
|
+
* read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
|
|
5949
|
+
* {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
|
|
5950
|
+
* call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
|
|
5951
|
+
* before the next one. Call it every tick (the NestJS timer poller does); a call made while
|
|
5952
|
+
* another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
|
|
5953
|
+
* carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
|
|
5954
|
+
* store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5955
|
+
*/
|
|
5956
|
+
async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
|
|
5795
5957
|
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
5796
5958
|
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
5797
5959
|
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
@@ -5799,15 +5961,21 @@ var WorkflowEngine = class {
|
|
|
5799
5961
|
return 0;
|
|
5800
5962
|
}
|
|
5801
5963
|
this.bufferedSweepActive = true;
|
|
5964
|
+
const deadline = Date.now() + budgetMs;
|
|
5802
5965
|
let deleted = 0;
|
|
5803
5966
|
try {
|
|
5804
5967
|
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
5968
|
+
if (page > 0 && (Date.now() >= deadline || this.draining)) break;
|
|
5805
5969
|
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
5806
|
-
const doomed = await unneededChildSignals(this.store, rows);
|
|
5970
|
+
const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
|
|
5807
5971
|
if (doomed.length > 0) {
|
|
5808
5972
|
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
5809
5973
|
deleted += doomed.length;
|
|
5810
5974
|
}
|
|
5975
|
+
if (decided < rows.length) {
|
|
5976
|
+
if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
|
|
5977
|
+
break;
|
|
5978
|
+
}
|
|
5811
5979
|
const last = rows.at(-1);
|
|
5812
5980
|
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
5813
5981
|
this.bufferedSweepCursor = void 0;
|