@dudousxd/nestjs-durable-core 0.85.7 → 0.85.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -4989,13 +4989,17 @@ declare class WorkflowEngine {
4989
4989
  *
4990
4990
  * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
4991
4991
  * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
4992
- * few timer ticks without a long query or a big delete. A pass that reaches the end of the table
4993
- * rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
4994
- * poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
4995
- * namespace, and a scoped read would report another tenant's parent as gone), and on a store
4996
- * without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
4997
- */
4998
- sweepBufferedSignals(now?: number): Promise<number>;
4992
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
4993
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
4994
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
4995
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
4996
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
4997
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
4998
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
4999
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5000
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5001
+ */
5002
+ sweepBufferedSignals(now?: number, budgetMs?: number): Promise<number>;
4999
5003
  /**
5000
5004
  * Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
5001
5005
  * either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
package/dist/index.d.ts CHANGED
@@ -4989,13 +4989,17 @@ declare class WorkflowEngine {
4989
4989
  *
4990
4990
  * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
4991
4991
  * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
4992
- * few timer ticks without a long query or a big delete. A pass that reaches the end of the table
4993
- * rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
4994
- * poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
4995
- * namespace, and a scoped read would report another tenant's parent as gone), and on a store
4996
- * without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
4997
- */
4998
- sweepBufferedSignals(now?: number): Promise<number>;
4992
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
4993
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
4994
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
4995
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
4996
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
4997
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
4998
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
4999
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5000
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5001
+ */
5002
+ sweepBufferedSignals(now?: number, budgetMs?: number): Promise<number>;
4999
5003
  /**
5000
5004
  * Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
5001
5005
  * either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
package/dist/index.js CHANGED
@@ -937,7 +937,6 @@ var RETRYABLE = /* @__PURE__ */ new Set([
937
937
  "failed",
938
938
  "dead"
939
939
  ]);
940
- var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
941
940
  function canonicalJson(value) {
942
941
  const plain = JSON.stringify(value);
943
942
  return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
@@ -952,77 +951,235 @@ function sortedJson(value) {
952
951
  return JSON.stringify(value);
953
952
  }
954
953
  __name(sortedJson, "sortedJson");
955
- function settledNames(childId) {
954
+ var FAN_CHILD_PARTS = /^(.+\.all\.(\d+)\.)(?:w(\d+)\.)?(\d+)$/;
955
+ var SEQ_CHILD = /\.child\.(\d+)$/;
956
+ function slotOf(childId) {
956
957
  const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
957
- const fan = FAN_CHILD.exec(childId);
958
- if (!fan) return {
959
- signal
960
- };
961
- return {
962
- signal,
963
- fold: {
964
- name: `${FOLDED_PREFIX}${fan[1]}`,
965
- index: Number(fan[2])
958
+ const fan = FAN_CHILD_PARTS.exec(childId);
959
+ if (fan) {
960
+ const root = fan[1];
961
+ const wave = fan[3];
962
+ const firstSeq = Number(fan[2]);
963
+ const i = Number(fan[4]);
964
+ if (wave === void 0) {
965
+ return {
966
+ signal,
967
+ signalSeq: firstSeq + i,
968
+ fan: {
969
+ fold: `${FOLDED_PREFIX}${root}`,
970
+ firstSeq,
971
+ index: i
972
+ }
973
+ };
966
974
  }
975
+ return {
976
+ signal,
977
+ signalSeq: firstSeq + i,
978
+ fan: {
979
+ fold: `${FOLDED_PREFIX}${root}w${wave}.`,
980
+ firstSeq,
981
+ index: i,
982
+ wave: Number(wave),
983
+ waveZeroFold: `${FOLDED_PREFIX}${root}w0.`
984
+ }
985
+ };
986
+ }
987
+ const seq = SEQ_CHILD.exec(childId);
988
+ return seq ? {
989
+ signal,
990
+ signalSeq: Number(seq[1])
991
+ } : {
992
+ signal
967
993
  };
968
994
  }
969
- __name(settledNames, "settledNames");
970
- function parentHoldsChild(checkpoints, childId, completion) {
971
- const { signal, fold } = settledNames(childId);
972
- const expected = canonicalJson(completion);
973
- return checkpoints.some((cp) => {
974
- if (cp.status !== "completed") return false;
975
- if (cp.name === signal) return canonicalJson(cp.output) === expected;
976
- if (fold && cp.name === fold.name) {
977
- const { from, values } = cp.output ?? {};
978
- return typeof from === "number" && Array.isArray(values) && fold.index >= from && fold.index < from + values.length;
995
+ __name(slotOf, "slotOf");
996
+ function foldOf(cp, name) {
997
+ if (!cp || cp.status !== "completed" || cp.name !== name) return void 0;
998
+ const { from, values } = cp.output ?? {};
999
+ return typeof from === "number" && Array.isArray(values) ? {
1000
+ from,
1001
+ values
1002
+ } : void 0;
1003
+ }
1004
+ __name(foldOf, "foldOf");
1005
+ function covers(fold, index) {
1006
+ return !!fold && index >= fold.from && index < fold.from + fold.values.length;
1007
+ }
1008
+ __name(covers, "covers");
1009
+ function holdsSignal(cp, signal, expected) {
1010
+ return cp.status === "completed" && cp.name === signal && canonicalJson(cp.output) === expected;
1011
+ }
1012
+ __name(holdsSignal, "holdsSignal");
1013
+ async function checkpointsAt(store, runId, seqs) {
1014
+ const unique = [
1015
+ ...new Set(seqs)
1016
+ ];
1017
+ const bySeq = /* @__PURE__ */ new Map();
1018
+ if (unique.length === 0) return bySeq;
1019
+ const rows = store.getCheckpoints ? await store.getCheckpoints(runId, unique) : (await Promise.all(unique.map((seq) => store.getCheckpoint(runId, seq)))).filter((cp) => cp !== null);
1020
+ for (const cp of rows) bySeq.set(cp.seq, cp);
1021
+ return bySeq;
1022
+ }
1023
+ __name(checkpointsAt, "checkpointsAt");
1024
+ async function checkpointsNamed(store, runId, names) {
1025
+ const wanted = new Set(names);
1026
+ const rows = store.listCheckpointsByNamePrefix ? await store.listCheckpointsByNamePrefix(runId, [
1027
+ ...wanted
1028
+ ]) : await store.listCheckpoints(runId);
1029
+ return rows.filter((cp) => wanted.has(cp.name));
1030
+ }
1031
+ __name(checkpointsNamed, "checkpointsNamed");
1032
+ async function parentSettles(store, parentId, children) {
1033
+ const entries = children.map(({ childId, completion }) => ({
1034
+ slot: slotOf(childId),
1035
+ expected: canonicalJson(completion)
1036
+ }));
1037
+ const held = entries.map(() => void 0);
1038
+ const first = await checkpointsAt(store, parentId, entries.flatMap(({ slot }) => [
1039
+ ...slot.signalSeq === void 0 ? [] : [
1040
+ slot.signalSeq
1041
+ ],
1042
+ ...slot.fan ? [
1043
+ slot.fan.firstSeq
1044
+ ] : []
1045
+ ]));
1046
+ const waveFolds = [];
1047
+ const byName = [];
1048
+ entries.forEach(({ slot, expected }, at) => {
1049
+ const own = slot.signalSeq === void 0 ? void 0 : first.get(slot.signalSeq);
1050
+ if (own?.name === slot.signal) {
1051
+ held[at] = holdsSignal(own, slot.signal, expected);
1052
+ return;
979
1053
  }
980
- return false;
1054
+ const fan = slot.fan;
1055
+ if (fan) {
1056
+ if (covers(foldOf(own, fan.fold), fan.index)) {
1057
+ held[at] = true;
1058
+ return;
1059
+ }
1060
+ const atFirst = first.get(fan.firstSeq);
1061
+ if (fan.wave === void 0 || fan.wave === 0) {
1062
+ held[at] = covers(foldOf(atFirst, fan.fold), fan.index);
1063
+ return;
1064
+ }
1065
+ const waveZero = foldOf(atFirst, fan.waveZeroFold);
1066
+ if (waveZero && waveZero.values.length > 0) {
1067
+ waveFolds.push({
1068
+ at,
1069
+ seq: fan.firstSeq + fan.wave * waveZero.values.length
1070
+ });
1071
+ return;
1072
+ }
1073
+ byName.push(at);
1074
+ return;
1075
+ }
1076
+ if (slot.signalSeq !== void 0 && !own) {
1077
+ held[at] = false;
1078
+ return;
1079
+ }
1080
+ byName.push(at);
981
1081
  });
1082
+ if (waveFolds.length > 0) {
1083
+ const second = await checkpointsAt(store, parentId, waveFolds.map((w) => w.seq));
1084
+ for (const { at, seq } of waveFolds) {
1085
+ const fan = entries[at].slot.fan;
1086
+ held[at] = covers(foldOf(second.get(seq), fan.fold), fan.index);
1087
+ }
1088
+ }
1089
+ if (byName.length > 0) {
1090
+ const names = byName.flatMap((at) => {
1091
+ const { slot } = entries[at];
1092
+ return slot.fan ? [
1093
+ slot.signal,
1094
+ slot.fan.fold
1095
+ ] : [
1096
+ slot.signal
1097
+ ];
1098
+ });
1099
+ const named = await checkpointsNamed(store, parentId, names);
1100
+ for (const at of byName) {
1101
+ const { slot, expected } = entries[at];
1102
+ held[at] = named.some((cp) => holdsSignal(cp, slot.signal, expected) || !!slot.fan && covers(foldOf(cp, slot.fan.fold), slot.fan.index));
1103
+ }
1104
+ }
1105
+ return held.map((h) => h === true);
982
1106
  }
983
- __name(parentHoldsChild, "parentHoldsChild");
984
- async function parentCheckpoints(store, parentId, childId) {
985
- const { signal, fold } = settledNames(childId);
986
- const prefixes = fold ? [
987
- signal,
988
- fold.name
989
- ] : [
990
- signal
991
- ];
992
- return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
1107
+ __name(parentSettles, "parentSettles");
1108
+ function parentClosed(run) {
1109
+ return !run || TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status);
993
1110
  }
994
- __name(parentCheckpoints, "parentCheckpoints");
1111
+ __name(parentClosed, "parentClosed");
995
1112
  async function childCompletionNeedsBuffer(store, childId, completion, parent) {
996
1113
  const parentId = parentRunIdOf(childId);
997
1114
  if (!parentId) return false;
998
1115
  const run = parent === void 0 ? await store.getRun(parentId) : parent;
999
- if (!run) return false;
1000
- if (TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status)) return false;
1001
- return !parentHoldsChild(await parentCheckpoints(store, parentId, childId), childId, completion);
1116
+ if (parentClosed(run)) return false;
1117
+ const [held] = await parentSettles(store, parentId, [
1118
+ {
1119
+ childId,
1120
+ completion
1121
+ }
1122
+ ]);
1123
+ return !held;
1002
1124
  }
1003
1125
  __name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
1004
- async function unneededChildSignals(store, rows) {
1126
+ async function unneededChildSignals(store, rows, deadline = Number.POSITIVE_INFINITY) {
1127
+ const childOf = /* @__PURE__ */ __name((row) => row.token.startsWith(CHILD_TOKEN_PREFIX) ? row.token.slice(CHILD_TOKEN_PREFIX.length) : void 0, "childOf");
1005
1128
  const parentIds = rows.flatMap((row) => {
1006
- const parentId = parentRunIdOf(row.token.slice(CHILD_TOKEN_PREFIX.length));
1129
+ const childId = childOf(row);
1130
+ const parentId = childId === void 0 ? void 0 : parentRunIdOf(childId);
1007
1131
  return parentId ? [
1008
1132
  parentId
1009
1133
  ] : [];
1010
1134
  });
1011
1135
  const { runs, unread } = await readRunsByIds(store, parentIds);
1012
1136
  const unreadable = new Set(unread);
1013
- const doomed = [];
1014
- for (const row of rows) {
1015
- if (!row.token.startsWith(CHILD_TOKEN_PREFIX)) continue;
1016
- const childId = row.token.slice(CHILD_TOKEN_PREFIX.length);
1137
+ const doomed = /* @__PURE__ */ new Set();
1138
+ const open = /* @__PURE__ */ new Map();
1139
+ rows.forEach((row, at) => {
1140
+ const childId = childOf(row);
1141
+ if (childId === void 0) return;
1017
1142
  const parentId = parentRunIdOf(childId);
1018
- if (parentId && unreadable.has(parentId)) continue;
1019
- const parent = parentId ? runs.get(parentId) ?? null : null;
1143
+ if (!parentId) {
1144
+ doomed.add(row);
1145
+ return;
1146
+ }
1147
+ if (unreadable.has(parentId)) return;
1148
+ if (parentClosed(runs.get(parentId) ?? null)) {
1149
+ doomed.add(row);
1150
+ return;
1151
+ }
1152
+ const group = open.get(parentId) ?? [];
1153
+ group.push({
1154
+ at,
1155
+ row,
1156
+ childId
1157
+ });
1158
+ open.set(parentId, group);
1159
+ });
1160
+ let decided = rows.length;
1161
+ let checked = 0;
1162
+ for (const [parentId, group] of open) {
1163
+ if (checked > 0 && Date.now() >= deadline) {
1164
+ decided = group[0].at;
1165
+ break;
1166
+ }
1167
+ checked += 1;
1020
1168
  try {
1021
- if (!await childCompletionNeedsBuffer(store, childId, row.payload, parent)) doomed.push(row);
1169
+ const held = await parentSettles(store, parentId, group.map(({ row, childId }) => ({
1170
+ childId,
1171
+ completion: row.payload
1172
+ })));
1173
+ group.forEach(({ row }, k) => {
1174
+ if (held[k]) doomed.add(row);
1175
+ });
1022
1176
  } catch {
1023
1177
  }
1024
1178
  }
1025
- return doomed;
1179
+ return {
1180
+ doomed: rows.filter((row) => doomed.has(row)),
1181
+ decided
1182
+ };
1026
1183
  }
1027
1184
  __name(unneededChildSignals, "unneededChildSignals");
1028
1185
 
@@ -4679,6 +4836,7 @@ var WAKE_REQUEST_SCAN = 100;
4679
4836
  var CHILD_SIGNAL_PREFIX = "signal:child:";
4680
4837
  var BUFFERED_SWEEP_PAGE = 500;
4681
4838
  var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
4839
+ var BUFFERED_SWEEP_BUDGET_MS = 250;
4682
4840
  var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
4683
4841
  var DELIVER_CHUNK = 50;
4684
4842
  var RESULT_BATCH_MAX = 256;
@@ -5571,13 +5729,17 @@ var WorkflowEngine = class {
5571
5729
  *
5572
5730
  * Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
5573
5731
  * at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
5574
- * few timer ticks without a long query or a big delete. A pass that reaches the end of the table
5575
- * rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
5576
- * poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
5577
- * namespace, and a scoped read would report another tenant's parent as gone), and on a store
5578
- * without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5579
- */
5580
- async sweepBufferedSignals(now = this.clock()) {
5732
+ * few timer ticks without a long query or a big delete. Each page is decided set-based (see
5733
+ * `unneededChildSignals`): a handful of store calls per page and per open parent, none of them a
5734
+ * read by checkpoint name. A call stops once `budgetMs` of wall-clock time is spent (default
5735
+ * {@link BUFFERED_SWEEP_BUDGET_MS}), between pages or between two parents' reads, and the next
5736
+ * call resumes there. A pass that reaches the end of the table rests {@link BUFFERED_SWEEP_REST_MS}
5737
+ * before the next one. Call it every tick (the NestJS timer poller does); a call made while
5738
+ * another is running returns 0. Skipped while draining, on a namespaced engine (a buffered signal
5739
+ * carries no namespace, and a scoped read would report another tenant's parent as gone), and on a
5740
+ * store without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
5741
+ */
5742
+ async sweepBufferedSignals(now = this.clock(), budgetMs = BUFFERED_SWEEP_BUDGET_MS) {
5581
5743
  const { listBufferedSignals, deleteBufferedSignals } = this.store;
5582
5744
  if (!listBufferedSignals || !deleteBufferedSignals) return 0;
5583
5745
  if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
@@ -5585,15 +5747,21 @@ var WorkflowEngine = class {
5585
5747
  return 0;
5586
5748
  }
5587
5749
  this.bufferedSweepActive = true;
5750
+ const deadline = Date.now() + budgetMs;
5588
5751
  let deleted = 0;
5589
5752
  try {
5590
5753
  for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
5754
+ if (page > 0 && (Date.now() >= deadline || this.draining)) break;
5591
5755
  const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
5592
- const doomed = await unneededChildSignals(this.store, rows);
5756
+ const { doomed, decided } = await unneededChildSignals(this.store, rows, deadline);
5593
5757
  if (doomed.length > 0) {
5594
5758
  await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
5595
5759
  deleted += doomed.length;
5596
5760
  }
5761
+ if (decided < rows.length) {
5762
+ if (decided > 0) this.bufferedSweepCursor = rows[decided - 1].id;
5763
+ break;
5764
+ }
5597
5765
  const last = rows.at(-1);
5598
5766
  if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
5599
5767
  this.bufferedSweepCursor = void 0;