@dudousxd/nestjs-durable-store-prisma 0.24.2 → 0.24.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +322 -37
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +6 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +322 -37
- package/dist/index.js.map +1 -1
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -254,6 +254,44 @@ var PrismaStateStore = class {
|
|
|
254
254
|
});
|
|
255
255
|
return row ? fromCheckpointRow(row) : null;
|
|
256
256
|
}
|
|
257
|
+
async getCheckpoints(runId, seqs) {
|
|
258
|
+
const unique = [
|
|
259
|
+
...new Set(seqs)
|
|
260
|
+
];
|
|
261
|
+
const rows = [];
|
|
262
|
+
for (let at = 0; at < unique.length; at += 500) {
|
|
263
|
+
rows.push(...await this.db.durableStepCheckpoint.findMany({
|
|
264
|
+
where: {
|
|
265
|
+
runId,
|
|
266
|
+
seq: {
|
|
267
|
+
in: unique.slice(at, at + 500)
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}));
|
|
271
|
+
}
|
|
272
|
+
return rows.sort((a, b) => a.seq - b.seq).map(fromCheckpointRow);
|
|
273
|
+
}
|
|
274
|
+
async getCheckpointsForSettle(runId, seqs) {
|
|
275
|
+
const unique = [
|
|
276
|
+
...new Set(seqs)
|
|
277
|
+
];
|
|
278
|
+
const rows = [];
|
|
279
|
+
for (let at = 0; at < unique.length; at += 500) {
|
|
280
|
+
rows.push(...await this.db.durableStepCheckpoint.findMany({
|
|
281
|
+
where: {
|
|
282
|
+
runId,
|
|
283
|
+
seq: {
|
|
284
|
+
in: unique.slice(at, at + 500)
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
select: CHECKPOINT_SELECT_WITHOUT_INPUT
|
|
288
|
+
}));
|
|
289
|
+
}
|
|
290
|
+
return rows.sort((a, b) => a.seq - b.seq).map((row) => fromCheckpointRow({
|
|
291
|
+
...row,
|
|
292
|
+
input: null
|
|
293
|
+
}));
|
|
294
|
+
}
|
|
257
295
|
async saveCheckpoint(checkpoint) {
|
|
258
296
|
const data = toCheckpointData(checkpoint);
|
|
259
297
|
await this.db.durableStepCheckpoint.upsert({
|
|
@@ -282,22 +320,106 @@ var PrismaStateStore = class {
|
|
|
282
320
|
async saveCheckpointsIfPending(checkpoints) {
|
|
283
321
|
if (checkpoints.length === 0) return [];
|
|
284
322
|
return this.db.$transaction(async (tx) => {
|
|
285
|
-
const
|
|
286
|
-
for (const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
323
|
+
const pending = /* @__PURE__ */ new Set();
|
|
324
|
+
for (const [runId, seqs] of seqsByRun(checkpoints)) {
|
|
325
|
+
for (let at = 0; at < seqs.length; at += 500) {
|
|
326
|
+
const rows = await tx.durableStepCheckpoint.findMany({
|
|
327
|
+
where: {
|
|
328
|
+
runId,
|
|
329
|
+
seq: {
|
|
330
|
+
in: seqs.slice(at, at + 500)
|
|
331
|
+
},
|
|
332
|
+
status: "pending"
|
|
333
|
+
},
|
|
334
|
+
select: {
|
|
335
|
+
runId: true,
|
|
336
|
+
seq: true
|
|
337
|
+
}
|
|
338
|
+
});
|
|
339
|
+
for (const row of rows) pending.add(checkpointKey(row.runId, row.seq));
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
const { wrote, writes } = decideIfPending(checkpoints, pending);
|
|
343
|
+
for (const [runId, seqs] of seqsByRun(writes)) {
|
|
344
|
+
for (let at = 0; at < seqs.length; at += 500) {
|
|
345
|
+
const chunk = seqs.slice(at, at + 500);
|
|
346
|
+
const { count } = await tx.durableStepCheckpoint.deleteMany({
|
|
347
|
+
where: {
|
|
348
|
+
runId,
|
|
349
|
+
seq: {
|
|
350
|
+
in: chunk
|
|
351
|
+
},
|
|
352
|
+
status: "pending"
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
if (count !== chunk.length) {
|
|
356
|
+
throw new Error("a checkpoint of the batch was settled while the batch was written");
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
for (let at = 0; at < writes.length; at += 500) {
|
|
361
|
+
await tx.durableStepCheckpoint.createMany({
|
|
362
|
+
data: writes.slice(at, at + 500).map(toCheckpointData)
|
|
295
363
|
});
|
|
296
|
-
wrote.push(count > 0);
|
|
297
364
|
}
|
|
298
365
|
return wrote;
|
|
299
366
|
});
|
|
300
367
|
}
|
|
368
|
+
async settleCheckpointsIfPending(settlements) {
|
|
369
|
+
const wrote = settlements.map(() => false);
|
|
370
|
+
if (settlements.length === 0) return wrote;
|
|
371
|
+
const seen = /* @__PURE__ */ new Set();
|
|
372
|
+
const firsts = [];
|
|
373
|
+
settlements.forEach((s, i) => {
|
|
374
|
+
const key = checkpointKey(s.runId, s.seq);
|
|
375
|
+
if (seen.has(key)) return;
|
|
376
|
+
seen.add(key);
|
|
377
|
+
firsts.push(i);
|
|
378
|
+
});
|
|
379
|
+
const dialect = rawSettleDialect(this.db);
|
|
380
|
+
if (!dialect) {
|
|
381
|
+
const settle = /* @__PURE__ */ __name(async (db) => {
|
|
382
|
+
for (const i of firsts) {
|
|
383
|
+
const s = settlements[i];
|
|
384
|
+
const { count } = await db.durableStepCheckpoint.updateMany({
|
|
385
|
+
where: {
|
|
386
|
+
runId: s.runId,
|
|
387
|
+
seq: s.seq,
|
|
388
|
+
status: "pending"
|
|
389
|
+
},
|
|
390
|
+
data: {
|
|
391
|
+
status: s.status,
|
|
392
|
+
output: jsonOrNull(s.output),
|
|
393
|
+
error: jsonOrNull(s.error),
|
|
394
|
+
events: jsonOrNull(s.events),
|
|
395
|
+
startedAt: s.startedAt,
|
|
396
|
+
finishedAt: s.finishedAt
|
|
397
|
+
}
|
|
398
|
+
});
|
|
399
|
+
wrote[i] = count > 0;
|
|
400
|
+
}
|
|
401
|
+
}, "settle");
|
|
402
|
+
if (firsts.length === 1) await settle(this.db);
|
|
403
|
+
else await this.db.$transaction((tx) => settle(tx));
|
|
404
|
+
return wrote;
|
|
405
|
+
}
|
|
406
|
+
await this.db.$transaction(async (tx) => {
|
|
407
|
+
const raw = tx;
|
|
408
|
+
for (let at = 0; at < firsts.length; at += 500) {
|
|
409
|
+
const chunk = firsts.slice(at, at + 500);
|
|
410
|
+
const won = await dialect(raw, chunk.map((i) => settlements[i]));
|
|
411
|
+
for (const i of chunk) {
|
|
412
|
+
const s = settlements[i];
|
|
413
|
+
wrote[i] = won.has(checkpointKey(s.runId, s.seq));
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
});
|
|
417
|
+
return wrote;
|
|
418
|
+
}
|
|
419
|
+
async saveCheckpoints(checkpoints) {
|
|
420
|
+
if (checkpoints.length === 0) return;
|
|
421
|
+
await this.db.$transaction((tx) => replaceCheckpoints(tx, checkpoints));
|
|
422
|
+
}
|
|
301
423
|
async transaction(work) {
|
|
302
424
|
return this.db.$transaction(async (tx) => work({
|
|
303
425
|
raw: tx,
|
|
@@ -661,6 +783,38 @@ var PrismaStateStore = class {
|
|
|
661
783
|
});
|
|
662
784
|
return rows.map(fromCheckpointRow);
|
|
663
785
|
}
|
|
786
|
+
async listCheckpointsForReplay(runId) {
|
|
787
|
+
const rows = await this.db.durableStepCheckpoint.findMany({
|
|
788
|
+
where: {
|
|
789
|
+
runId
|
|
790
|
+
},
|
|
791
|
+
orderBy: {
|
|
792
|
+
seq: "asc"
|
|
793
|
+
},
|
|
794
|
+
select: CHECKPOINT_SELECT_WITHOUT_INPUT
|
|
795
|
+
});
|
|
796
|
+
const open = rows.filter((row) => row.status !== "completed").map((row) => row.seq);
|
|
797
|
+
const inputs = /* @__PURE__ */ new Map();
|
|
798
|
+
for (let at = 0; at < open.length; at += 500) {
|
|
799
|
+
const found = await this.db.durableStepCheckpoint.findMany({
|
|
800
|
+
where: {
|
|
801
|
+
runId,
|
|
802
|
+
seq: {
|
|
803
|
+
in: open.slice(at, at + 500)
|
|
804
|
+
}
|
|
805
|
+
},
|
|
806
|
+
select: {
|
|
807
|
+
seq: true,
|
|
808
|
+
input: true
|
|
809
|
+
}
|
|
810
|
+
});
|
|
811
|
+
for (const row of found) inputs.set(row.seq, row.input);
|
|
812
|
+
}
|
|
813
|
+
return rows.map((row) => fromCheckpointRow({
|
|
814
|
+
...row,
|
|
815
|
+
input: inputs.get(row.seq) ?? null
|
|
816
|
+
}));
|
|
817
|
+
}
|
|
664
818
|
async getLatestCheckpointByName(runId, name) {
|
|
665
819
|
const row = await this.db.durableStepCheckpoint.findFirst({
|
|
666
820
|
where: {
|
|
@@ -891,36 +1045,25 @@ var PrismaStateStore = class {
|
|
|
891
1045
|
async registerChildWaits(entries) {
|
|
892
1046
|
if (entries.length === 0) return;
|
|
893
1047
|
await this.db.$transaction(async (tx) => {
|
|
894
|
-
for (
|
|
895
|
-
const
|
|
896
|
-
|
|
897
|
-
seq: waiter.seq,
|
|
898
|
-
parallelGroup: waiter.parallelGroup ?? null
|
|
899
|
-
};
|
|
900
|
-
await tx.durableSignalWaiter.upsert({
|
|
1048
|
+
for (let at = 0; at < entries.length; at += 500) {
|
|
1049
|
+
const chunk = entries.slice(at, at + 500);
|
|
1050
|
+
await tx.durableSignalWaiter.deleteMany({
|
|
901
1051
|
where: {
|
|
902
|
-
token:
|
|
903
|
-
|
|
904
|
-
create: {
|
|
905
|
-
token: waiter.token,
|
|
906
|
-
...data
|
|
907
|
-
},
|
|
908
|
-
update: data
|
|
909
|
-
});
|
|
910
|
-
}
|
|
911
|
-
for (const { checkpoint } of entries) {
|
|
912
|
-
const data = toCheckpointData(checkpoint);
|
|
913
|
-
await tx.durableStepCheckpoint.upsert({
|
|
914
|
-
where: {
|
|
915
|
-
runId_seq: {
|
|
916
|
-
runId: checkpoint.runId,
|
|
917
|
-
seq: checkpoint.seq
|
|
1052
|
+
token: {
|
|
1053
|
+
in: chunk.map(({ waiter }) => waiter.token)
|
|
918
1054
|
}
|
|
919
|
-
}
|
|
920
|
-
|
|
921
|
-
|
|
1055
|
+
}
|
|
1056
|
+
});
|
|
1057
|
+
await tx.durableSignalWaiter.createMany({
|
|
1058
|
+
data: chunk.map(({ waiter }) => ({
|
|
1059
|
+
token: waiter.token,
|
|
1060
|
+
runId: waiter.runId,
|
|
1061
|
+
seq: waiter.seq,
|
|
1062
|
+
parallelGroup: waiter.parallelGroup ?? null
|
|
1063
|
+
}))
|
|
922
1064
|
});
|
|
923
1065
|
}
|
|
1066
|
+
await replaceCheckpoints(tx, entries.map(({ checkpoint }) => checkpoint));
|
|
924
1067
|
});
|
|
925
1068
|
}
|
|
926
1069
|
async compactCheckpoints(runId, fromSeq, toSeq, into) {
|
|
@@ -1267,6 +1410,148 @@ function toCheckpointStat(nameKind, r, unenqueued) {
|
|
|
1267
1410
|
};
|
|
1268
1411
|
}
|
|
1269
1412
|
__name(toCheckpointStat, "toCheckpointStat");
|
|
1413
|
+
function seqsByRun(checkpoints) {
|
|
1414
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1415
|
+
for (const { runId, seq } of checkpoints) {
|
|
1416
|
+
const seqs = byRun.get(runId) ?? /* @__PURE__ */ new Set();
|
|
1417
|
+
seqs.add(seq);
|
|
1418
|
+
byRun.set(runId, seqs);
|
|
1419
|
+
}
|
|
1420
|
+
return new Map([
|
|
1421
|
+
...byRun
|
|
1422
|
+
].map(([runId, seqs]) => [
|
|
1423
|
+
runId,
|
|
1424
|
+
[
|
|
1425
|
+
...seqs
|
|
1426
|
+
]
|
|
1427
|
+
]));
|
|
1428
|
+
}
|
|
1429
|
+
__name(seqsByRun, "seqsByRun");
|
|
1430
|
+
var checkpointKey = /* @__PURE__ */ __name((runId, seq) => `${runId}\0${seq}`, "checkpointKey");
|
|
1431
|
+
function decideIfPending(checkpoints, pending) {
|
|
1432
|
+
const wrote = [];
|
|
1433
|
+
const writes = /* @__PURE__ */ new Map();
|
|
1434
|
+
for (const checkpoint of checkpoints) {
|
|
1435
|
+
const key = checkpointKey(checkpoint.runId, checkpoint.seq);
|
|
1436
|
+
if (!pending.has(key)) {
|
|
1437
|
+
wrote.push(false);
|
|
1438
|
+
continue;
|
|
1439
|
+
}
|
|
1440
|
+
wrote.push(true);
|
|
1441
|
+
writes.set(key, checkpoint);
|
|
1442
|
+
if (checkpoint.status !== "pending") pending.delete(key);
|
|
1443
|
+
}
|
|
1444
|
+
return {
|
|
1445
|
+
wrote,
|
|
1446
|
+
writes: [
|
|
1447
|
+
...writes.values()
|
|
1448
|
+
]
|
|
1449
|
+
};
|
|
1450
|
+
}
|
|
1451
|
+
__name(decideIfPending, "decideIfPending");
|
|
1452
|
+
async function replaceCheckpoints(tx, checkpoints) {
|
|
1453
|
+
for (const [runId, seqs] of seqsByRun(checkpoints)) {
|
|
1454
|
+
for (let at = 0; at < seqs.length; at += 500) {
|
|
1455
|
+
await tx.durableStepCheckpoint.deleteMany({
|
|
1456
|
+
where: {
|
|
1457
|
+
runId,
|
|
1458
|
+
seq: {
|
|
1459
|
+
in: seqs.slice(at, at + 500)
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
for (let at = 0; at < checkpoints.length; at += 500) {
|
|
1466
|
+
await tx.durableStepCheckpoint.createMany({
|
|
1467
|
+
data: checkpoints.slice(at, at + 500).map(toCheckpointData)
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
__name(replaceCheckpoints, "replaceCheckpoints");
|
|
1472
|
+
var CHECKPOINT_SELECT_WITHOUT_INPUT = {
|
|
1473
|
+
runId: true,
|
|
1474
|
+
seq: true,
|
|
1475
|
+
name: true,
|
|
1476
|
+
kind: true,
|
|
1477
|
+
stepId: true,
|
|
1478
|
+
status: true,
|
|
1479
|
+
output: true,
|
|
1480
|
+
error: true,
|
|
1481
|
+
events: true,
|
|
1482
|
+
attempts: true,
|
|
1483
|
+
workerGroup: true,
|
|
1484
|
+
parallelGroup: true,
|
|
1485
|
+
wakeAt: true,
|
|
1486
|
+
enqueuedAt: true,
|
|
1487
|
+
startedAt: true,
|
|
1488
|
+
finishedAt: true
|
|
1489
|
+
};
|
|
1490
|
+
var jsonText = /* @__PURE__ */ __name((v) => v == null ? null : JSON.stringify(v), "jsonText");
|
|
1491
|
+
function rawSettleDialect(db) {
|
|
1492
|
+
const client = db;
|
|
1493
|
+
if (typeof client.$queryRawUnsafe !== "function") return void 0;
|
|
1494
|
+
if (client._activeProvider === "postgresql") return settleRowsPostgres;
|
|
1495
|
+
if (client._activeProvider === "mysql") return settleRowsMysql;
|
|
1496
|
+
if (client._activeProvider === "sqlite") return settleRowsSqlite;
|
|
1497
|
+
return void 0;
|
|
1498
|
+
}
|
|
1499
|
+
__name(rawSettleDialect, "rawSettleDialect");
|
|
1500
|
+
var settleRowsPostgres = /* @__PURE__ */ __name(async (db, settlements) => {
|
|
1501
|
+
const values = [];
|
|
1502
|
+
const rows = settlements.map((s) => {
|
|
1503
|
+
const n = values.length;
|
|
1504
|
+
values.push(s.runId, s.seq, s.status, jsonText(s.output), jsonText(s.error), jsonText(s.events), s.startedAt.toISOString(), s.finishedAt.toISOString());
|
|
1505
|
+
const p = /* @__PURE__ */ __name((k) => `$${n + k}`, "p");
|
|
1506
|
+
return `(${p(1)}::text, ${p(2)}::integer, ${p(3)}::text, ${p(4)}::jsonb, ${p(5)}::jsonb, ${p(6)}::jsonb, (${p(7)}::timestamptz AT TIME ZONE 'UTC'), (${p(8)}::timestamptz AT TIME ZONE 'UTC'))`;
|
|
1507
|
+
});
|
|
1508
|
+
const written = await db.$queryRawUnsafe(`UPDATE "durable_step_checkpoints" AS c SET status = v.status, output = v.output,
|
|
1509
|
+
error = v.error, events = v.events, started_at = v.started_at, finished_at = v.finished_at
|
|
1510
|
+
FROM (VALUES ${rows.join(", ")}) AS v(run_id, seq, status, output, error, events, started_at,
|
|
1511
|
+
finished_at)
|
|
1512
|
+
WHERE c.run_id = v.run_id AND c.seq = v.seq AND c.status = 'pending'
|
|
1513
|
+
RETURNING c.run_id, c.seq`, ...values);
|
|
1514
|
+
return new Set(written.map((row) => checkpointKey(row.run_id, Number(row.seq))));
|
|
1515
|
+
}, "settleRowsPostgres");
|
|
1516
|
+
var settleRowsMysql = /* @__PURE__ */ __name(async (db, settlements) => {
|
|
1517
|
+
const pending = /* @__PURE__ */ new Set();
|
|
1518
|
+
for (const [runId, seqs] of seqsByRun(settlements)) {
|
|
1519
|
+
const rows2 = await db.$queryRawUnsafe(`SELECT run_id, seq FROM durable_step_checkpoints WHERE run_id = ? AND seq IN (${seqs.map(() => "?").join(", ")}) AND status = 'pending' FOR UPDATE`, runId, ...seqs);
|
|
1520
|
+
for (const row of rows2) pending.add(checkpointKey(row.run_id, Number(row.seq)));
|
|
1521
|
+
}
|
|
1522
|
+
const won = settlements.filter((s) => pending.has(checkpointKey(s.runId, s.seq)));
|
|
1523
|
+
if (won.length === 0) return pending;
|
|
1524
|
+
const at = /* @__PURE__ */ __name((d) => d.toISOString().replace("T", " ").replace("Z", ""), "at");
|
|
1525
|
+
const values = [];
|
|
1526
|
+
const rows = won.map((s, i) => {
|
|
1527
|
+
values.push(s.runId, s.seq, s.status, jsonText(s.output), jsonText(s.error), jsonText(s.events), at(s.startedAt), at(s.finishedAt));
|
|
1528
|
+
return i === 0 ? "SELECT ? AS run_id, ? AS seq, ? AS status, ? AS output, ? AS error, ? AS events, ? AS started_at, ? AS finished_at" : "SELECT ?, ?, ?, ?, ?, ?, ?, ?";
|
|
1529
|
+
});
|
|
1530
|
+
const updated = await db.$executeRawUnsafe(`UPDATE durable_step_checkpoints AS c JOIN (${rows.join(" UNION ALL ")}) AS v
|
|
1531
|
+
ON c.run_id = v.run_id AND c.seq = v.seq
|
|
1532
|
+
SET c.status = v.status, c.output = CAST(v.output AS JSON), c.error = CAST(v.error AS JSON),
|
|
1533
|
+
c.events = CAST(v.events AS JSON), c.started_at = v.started_at, c.finished_at = v.finished_at
|
|
1534
|
+
WHERE c.status = 'pending'`, ...values);
|
|
1535
|
+
if (updated !== won.length) {
|
|
1536
|
+
throw new Error("a checkpoint of the batch was settled while the batch was written");
|
|
1537
|
+
}
|
|
1538
|
+
return pending;
|
|
1539
|
+
}, "settleRowsMysql");
|
|
1540
|
+
var settleRowsSqlite = /* @__PURE__ */ __name(async (db, settlements) => {
|
|
1541
|
+
const values = [];
|
|
1542
|
+
const rows = settlements.map((s) => {
|
|
1543
|
+
values.push(s.runId, s.seq, s.status, jsonText(s.output), jsonText(s.error), jsonText(s.events), s.startedAt.getTime(), s.finishedAt.getTime());
|
|
1544
|
+
return "(?, ?, ?, ?, ?, ?, ?, ?)";
|
|
1545
|
+
});
|
|
1546
|
+
const written = await db.$queryRawUnsafe(`WITH v(run_id, seq, status, output, error, events, started_at, finished_at) AS (VALUES ${rows.join(", ")})
|
|
1547
|
+
UPDATE durable_step_checkpoints SET status = v.status, output = v.output, error = v.error,
|
|
1548
|
+
events = v.events, started_at = v.started_at, finished_at = v.finished_at
|
|
1549
|
+
FROM v
|
|
1550
|
+
WHERE durable_step_checkpoints.run_id = v.run_id AND durable_step_checkpoints.seq = v.seq
|
|
1551
|
+
AND durable_step_checkpoints.status = 'pending'
|
|
1552
|
+
RETURNING run_id, seq`, ...values);
|
|
1553
|
+
return new Set(written.map((row) => checkpointKey(row.run_id, Number(row.seq))));
|
|
1554
|
+
}, "settleRowsSqlite");
|
|
1270
1555
|
function toCheckpointData(cp) {
|
|
1271
1556
|
return {
|
|
1272
1557
|
runId: cp.runId,
|