@dudousxd/nestjs-durable-store-prisma 0.24.2 → 0.24.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -254,6 +254,44 @@ var PrismaStateStore = class {
254
254
  });
255
255
  return row ? fromCheckpointRow(row) : null;
256
256
  }
257
+ async getCheckpoints(runId, seqs) {
258
+ const unique = [
259
+ ...new Set(seqs)
260
+ ];
261
+ const rows = [];
262
+ for (let at = 0; at < unique.length; at += 500) {
263
+ rows.push(...await this.db.durableStepCheckpoint.findMany({
264
+ where: {
265
+ runId,
266
+ seq: {
267
+ in: unique.slice(at, at + 500)
268
+ }
269
+ }
270
+ }));
271
+ }
272
+ return rows.sort((a, b) => a.seq - b.seq).map(fromCheckpointRow);
273
+ }
274
+ async getCheckpointsForSettle(runId, seqs) {
275
+ const unique = [
276
+ ...new Set(seqs)
277
+ ];
278
+ const rows = [];
279
+ for (let at = 0; at < unique.length; at += 500) {
280
+ rows.push(...await this.db.durableStepCheckpoint.findMany({
281
+ where: {
282
+ runId,
283
+ seq: {
284
+ in: unique.slice(at, at + 500)
285
+ }
286
+ },
287
+ select: CHECKPOINT_SELECT_WITHOUT_INPUT
288
+ }));
289
+ }
290
+ return rows.sort((a, b) => a.seq - b.seq).map((row) => fromCheckpointRow({
291
+ ...row,
292
+ input: null
293
+ }));
294
+ }
257
295
  async saveCheckpoint(checkpoint) {
258
296
  const data = toCheckpointData(checkpoint);
259
297
  await this.db.durableStepCheckpoint.upsert({
@@ -282,22 +320,106 @@ var PrismaStateStore = class {
282
320
  async saveCheckpointsIfPending(checkpoints) {
283
321
  if (checkpoints.length === 0) return [];
284
322
  return this.db.$transaction(async (tx) => {
285
- const wrote = [];
286
- for (const checkpoint of checkpoints) {
287
- const { runId, seq, ...data } = toCheckpointData(checkpoint);
288
- const { count } = await tx.durableStepCheckpoint.updateMany({
289
- where: {
290
- runId,
291
- seq,
292
- status: "pending"
293
- },
294
- data
323
+ const pending = /* @__PURE__ */ new Set();
324
+ for (const [runId, seqs] of seqsByRun(checkpoints)) {
325
+ for (let at = 0; at < seqs.length; at += 500) {
326
+ const rows = await tx.durableStepCheckpoint.findMany({
327
+ where: {
328
+ runId,
329
+ seq: {
330
+ in: seqs.slice(at, at + 500)
331
+ },
332
+ status: "pending"
333
+ },
334
+ select: {
335
+ runId: true,
336
+ seq: true
337
+ }
338
+ });
339
+ for (const row of rows) pending.add(checkpointKey(row.runId, row.seq));
340
+ }
341
+ }
342
+ const { wrote, writes } = decideIfPending(checkpoints, pending);
343
+ for (const [runId, seqs] of seqsByRun(writes)) {
344
+ for (let at = 0; at < seqs.length; at += 500) {
345
+ const chunk = seqs.slice(at, at + 500);
346
+ const { count } = await tx.durableStepCheckpoint.deleteMany({
347
+ where: {
348
+ runId,
349
+ seq: {
350
+ in: chunk
351
+ },
352
+ status: "pending"
353
+ }
354
+ });
355
+ if (count !== chunk.length) {
356
+ throw new Error("a checkpoint of the batch was settled while the batch was written");
357
+ }
358
+ }
359
+ }
360
+ for (let at = 0; at < writes.length; at += 500) {
361
+ await tx.durableStepCheckpoint.createMany({
362
+ data: writes.slice(at, at + 500).map(toCheckpointData)
295
363
  });
296
- wrote.push(count > 0);
297
364
  }
298
365
  return wrote;
299
366
  });
300
367
  }
368
+ async settleCheckpointsIfPending(settlements) {
369
+ const wrote = settlements.map(() => false);
370
+ if (settlements.length === 0) return wrote;
371
+ const seen = /* @__PURE__ */ new Set();
372
+ const firsts = [];
373
+ settlements.forEach((s, i) => {
374
+ const key = checkpointKey(s.runId, s.seq);
375
+ if (seen.has(key)) return;
376
+ seen.add(key);
377
+ firsts.push(i);
378
+ });
379
+ const dialect = rawSettleDialect(this.db);
380
+ if (!dialect) {
381
+ const settle = /* @__PURE__ */ __name(async (db) => {
382
+ for (const i of firsts) {
383
+ const s = settlements[i];
384
+ const { count } = await db.durableStepCheckpoint.updateMany({
385
+ where: {
386
+ runId: s.runId,
387
+ seq: s.seq,
388
+ status: "pending"
389
+ },
390
+ data: {
391
+ status: s.status,
392
+ output: jsonOrNull(s.output),
393
+ error: jsonOrNull(s.error),
394
+ events: jsonOrNull(s.events),
395
+ startedAt: s.startedAt,
396
+ finishedAt: s.finishedAt
397
+ }
398
+ });
399
+ wrote[i] = count > 0;
400
+ }
401
+ }, "settle");
402
+ if (firsts.length === 1) await settle(this.db);
403
+ else await this.db.$transaction((tx) => settle(tx));
404
+ return wrote;
405
+ }
406
+ await this.db.$transaction(async (tx) => {
407
+ const raw = tx;
408
+ for (let at = 0; at < firsts.length; at += 500) {
409
+ const chunk = firsts.slice(at, at + 500);
410
+ const won = await dialect(raw, chunk.map((i) => settlements[i]));
411
+ for (const i of chunk) {
412
+ const s = settlements[i];
413
+ wrote[i] = won.has(checkpointKey(s.runId, s.seq));
414
+ }
415
+ }
416
+ });
417
+ return wrote;
418
+ }
419
+ async saveCheckpoints(checkpoints) {
420
+ if (checkpoints.length === 0) return;
421
+ await this.db.$transaction((tx) => replaceCheckpoints(tx, checkpoints));
422
+ }
301
423
  async transaction(work) {
302
424
  return this.db.$transaction(async (tx) => work({
303
425
  raw: tx,
@@ -661,6 +783,38 @@ var PrismaStateStore = class {
661
783
  });
662
784
  return rows.map(fromCheckpointRow);
663
785
  }
786
+ async listCheckpointsForReplay(runId) {
787
+ const rows = await this.db.durableStepCheckpoint.findMany({
788
+ where: {
789
+ runId
790
+ },
791
+ orderBy: {
792
+ seq: "asc"
793
+ },
794
+ select: CHECKPOINT_SELECT_WITHOUT_INPUT
795
+ });
796
+ const open = rows.filter((row) => row.status !== "completed").map((row) => row.seq);
797
+ const inputs = /* @__PURE__ */ new Map();
798
+ for (let at = 0; at < open.length; at += 500) {
799
+ const found = await this.db.durableStepCheckpoint.findMany({
800
+ where: {
801
+ runId,
802
+ seq: {
803
+ in: open.slice(at, at + 500)
804
+ }
805
+ },
806
+ select: {
807
+ seq: true,
808
+ input: true
809
+ }
810
+ });
811
+ for (const row of found) inputs.set(row.seq, row.input);
812
+ }
813
+ return rows.map((row) => fromCheckpointRow({
814
+ ...row,
815
+ input: inputs.get(row.seq) ?? null
816
+ }));
817
+ }
664
818
  async getLatestCheckpointByName(runId, name) {
665
819
  const row = await this.db.durableStepCheckpoint.findFirst({
666
820
  where: {
@@ -891,36 +1045,25 @@ var PrismaStateStore = class {
891
1045
  async registerChildWaits(entries) {
892
1046
  if (entries.length === 0) return;
893
1047
  await this.db.$transaction(async (tx) => {
894
- for (const { waiter } of entries) {
895
- const data = {
896
- runId: waiter.runId,
897
- seq: waiter.seq,
898
- parallelGroup: waiter.parallelGroup ?? null
899
- };
900
- await tx.durableSignalWaiter.upsert({
1048
+ for (let at = 0; at < entries.length; at += 500) {
1049
+ const chunk = entries.slice(at, at + 500);
1050
+ await tx.durableSignalWaiter.deleteMany({
901
1051
  where: {
902
- token: waiter.token
903
- },
904
- create: {
905
- token: waiter.token,
906
- ...data
907
- },
908
- update: data
909
- });
910
- }
911
- for (const { checkpoint } of entries) {
912
- const data = toCheckpointData(checkpoint);
913
- await tx.durableStepCheckpoint.upsert({
914
- where: {
915
- runId_seq: {
916
- runId: checkpoint.runId,
917
- seq: checkpoint.seq
1052
+ token: {
1053
+ in: chunk.map(({ waiter }) => waiter.token)
918
1054
  }
919
- },
920
- create: data,
921
- update: data
1055
+ }
1056
+ });
1057
+ await tx.durableSignalWaiter.createMany({
1058
+ data: chunk.map(({ waiter }) => ({
1059
+ token: waiter.token,
1060
+ runId: waiter.runId,
1061
+ seq: waiter.seq,
1062
+ parallelGroup: waiter.parallelGroup ?? null
1063
+ }))
922
1064
  });
923
1065
  }
1066
+ await replaceCheckpoints(tx, entries.map(({ checkpoint }) => checkpoint));
924
1067
  });
925
1068
  }
926
1069
  async compactCheckpoints(runId, fromSeq, toSeq, into) {
@@ -1267,6 +1410,148 @@ function toCheckpointStat(nameKind, r, unenqueued) {
1267
1410
  };
1268
1411
  }
1269
1412
  __name(toCheckpointStat, "toCheckpointStat");
1413
+ function seqsByRun(checkpoints) {
1414
+ const byRun = /* @__PURE__ */ new Map();
1415
+ for (const { runId, seq } of checkpoints) {
1416
+ const seqs = byRun.get(runId) ?? /* @__PURE__ */ new Set();
1417
+ seqs.add(seq);
1418
+ byRun.set(runId, seqs);
1419
+ }
1420
+ return new Map([
1421
+ ...byRun
1422
+ ].map(([runId, seqs]) => [
1423
+ runId,
1424
+ [
1425
+ ...seqs
1426
+ ]
1427
+ ]));
1428
+ }
1429
+ __name(seqsByRun, "seqsByRun");
1430
+ var checkpointKey = /* @__PURE__ */ __name((runId, seq) => `${runId}\0${seq}`, "checkpointKey");
1431
+ function decideIfPending(checkpoints, pending) {
1432
+ const wrote = [];
1433
+ const writes = /* @__PURE__ */ new Map();
1434
+ for (const checkpoint of checkpoints) {
1435
+ const key = checkpointKey(checkpoint.runId, checkpoint.seq);
1436
+ if (!pending.has(key)) {
1437
+ wrote.push(false);
1438
+ continue;
1439
+ }
1440
+ wrote.push(true);
1441
+ writes.set(key, checkpoint);
1442
+ if (checkpoint.status !== "pending") pending.delete(key);
1443
+ }
1444
+ return {
1445
+ wrote,
1446
+ writes: [
1447
+ ...writes.values()
1448
+ ]
1449
+ };
1450
+ }
1451
+ __name(decideIfPending, "decideIfPending");
1452
+ async function replaceCheckpoints(tx, checkpoints) {
1453
+ for (const [runId, seqs] of seqsByRun(checkpoints)) {
1454
+ for (let at = 0; at < seqs.length; at += 500) {
1455
+ await tx.durableStepCheckpoint.deleteMany({
1456
+ where: {
1457
+ runId,
1458
+ seq: {
1459
+ in: seqs.slice(at, at + 500)
1460
+ }
1461
+ }
1462
+ });
1463
+ }
1464
+ }
1465
+ for (let at = 0; at < checkpoints.length; at += 500) {
1466
+ await tx.durableStepCheckpoint.createMany({
1467
+ data: checkpoints.slice(at, at + 500).map(toCheckpointData)
1468
+ });
1469
+ }
1470
+ }
1471
+ __name(replaceCheckpoints, "replaceCheckpoints");
1472
+ var CHECKPOINT_SELECT_WITHOUT_INPUT = {
1473
+ runId: true,
1474
+ seq: true,
1475
+ name: true,
1476
+ kind: true,
1477
+ stepId: true,
1478
+ status: true,
1479
+ output: true,
1480
+ error: true,
1481
+ events: true,
1482
+ attempts: true,
1483
+ workerGroup: true,
1484
+ parallelGroup: true,
1485
+ wakeAt: true,
1486
+ enqueuedAt: true,
1487
+ startedAt: true,
1488
+ finishedAt: true
1489
+ };
1490
+ var jsonText = /* @__PURE__ */ __name((v) => v == null ? null : JSON.stringify(v), "jsonText");
1491
+ function rawSettleDialect(db) {
1492
+ const client = db;
1493
+ if (typeof client.$queryRawUnsafe !== "function") return void 0;
1494
+ if (client._activeProvider === "postgresql") return settleRowsPostgres;
1495
+ if (client._activeProvider === "mysql") return settleRowsMysql;
1496
+ if (client._activeProvider === "sqlite") return settleRowsSqlite;
1497
+ return void 0;
1498
+ }
1499
+ __name(rawSettleDialect, "rawSettleDialect");
1500
+ var settleRowsPostgres = /* @__PURE__ */ __name(async (db, settlements) => {
1501
+ const values = [];
1502
+ const rows = settlements.map((s) => {
1503
+ const n = values.length;
1504
+ values.push(s.runId, s.seq, s.status, jsonText(s.output), jsonText(s.error), jsonText(s.events), s.startedAt.toISOString(), s.finishedAt.toISOString());
1505
+ const p = /* @__PURE__ */ __name((k) => `$${n + k}`, "p");
1506
+ return `(${p(1)}::text, ${p(2)}::integer, ${p(3)}::text, ${p(4)}::jsonb, ${p(5)}::jsonb, ${p(6)}::jsonb, (${p(7)}::timestamptz AT TIME ZONE 'UTC'), (${p(8)}::timestamptz AT TIME ZONE 'UTC'))`;
1507
+ });
1508
+ const written = await db.$queryRawUnsafe(`UPDATE "durable_step_checkpoints" AS c SET status = v.status, output = v.output,
1509
+ error = v.error, events = v.events, started_at = v.started_at, finished_at = v.finished_at
1510
+ FROM (VALUES ${rows.join(", ")}) AS v(run_id, seq, status, output, error, events, started_at,
1511
+ finished_at)
1512
+ WHERE c.run_id = v.run_id AND c.seq = v.seq AND c.status = 'pending'
1513
+ RETURNING c.run_id, c.seq`, ...values);
1514
+ return new Set(written.map((row) => checkpointKey(row.run_id, Number(row.seq))));
1515
+ }, "settleRowsPostgres");
1516
+ var settleRowsMysql = /* @__PURE__ */ __name(async (db, settlements) => {
1517
+ const pending = /* @__PURE__ */ new Set();
1518
+ for (const [runId, seqs] of seqsByRun(settlements)) {
1519
+ const rows2 = await db.$queryRawUnsafe(`SELECT run_id, seq FROM durable_step_checkpoints WHERE run_id = ? AND seq IN (${seqs.map(() => "?").join(", ")}) AND status = 'pending' FOR UPDATE`, runId, ...seqs);
1520
+ for (const row of rows2) pending.add(checkpointKey(row.run_id, Number(row.seq)));
1521
+ }
1522
+ const won = settlements.filter((s) => pending.has(checkpointKey(s.runId, s.seq)));
1523
+ if (won.length === 0) return pending;
1524
+ const at = /* @__PURE__ */ __name((d) => d.toISOString().replace("T", " ").replace("Z", ""), "at");
1525
+ const values = [];
1526
+ const rows = won.map((s, i) => {
1527
+ values.push(s.runId, s.seq, s.status, jsonText(s.output), jsonText(s.error), jsonText(s.events), at(s.startedAt), at(s.finishedAt));
1528
+ return i === 0 ? "SELECT ? AS run_id, ? AS seq, ? AS status, ? AS output, ? AS error, ? AS events, ? AS started_at, ? AS finished_at" : "SELECT ?, ?, ?, ?, ?, ?, ?, ?";
1529
+ });
1530
+ const updated = await db.$executeRawUnsafe(`UPDATE durable_step_checkpoints AS c JOIN (${rows.join(" UNION ALL ")}) AS v
1531
+ ON c.run_id = v.run_id AND c.seq = v.seq
1532
+ SET c.status = v.status, c.output = CAST(v.output AS JSON), c.error = CAST(v.error AS JSON),
1533
+ c.events = CAST(v.events AS JSON), c.started_at = v.started_at, c.finished_at = v.finished_at
1534
+ WHERE c.status = 'pending'`, ...values);
1535
+ if (updated !== won.length) {
1536
+ throw new Error("a checkpoint of the batch was settled while the batch was written");
1537
+ }
1538
+ return pending;
1539
+ }, "settleRowsMysql");
1540
+ var settleRowsSqlite = /* @__PURE__ */ __name(async (db, settlements) => {
1541
+ const values = [];
1542
+ const rows = settlements.map((s) => {
1543
+ values.push(s.runId, s.seq, s.status, jsonText(s.output), jsonText(s.error), jsonText(s.events), s.startedAt.getTime(), s.finishedAt.getTime());
1544
+ return "(?, ?, ?, ?, ?, ?, ?, ?)";
1545
+ });
1546
+ const written = await db.$queryRawUnsafe(`WITH v(run_id, seq, status, output, error, events, started_at, finished_at) AS (VALUES ${rows.join(", ")})
1547
+ UPDATE durable_step_checkpoints SET status = v.status, output = v.output, error = v.error,
1548
+ events = v.events, started_at = v.started_at, finished_at = v.finished_at
1549
+ FROM v
1550
+ WHERE durable_step_checkpoints.run_id = v.run_id AND durable_step_checkpoints.seq = v.seq
1551
+ AND durable_step_checkpoints.status = 'pending'
1552
+ RETURNING run_id, seq`, ...values);
1553
+ return new Set(written.map((row) => checkpointKey(row.run_id, Number(row.seq))));
1554
+ }, "settleRowsSqlite");
1270
1555
  function toCheckpointData(cp) {
1271
1556
  return {
1272
1557
  runId: cp.runId,