@dudousxd/nestjs-durable-core 0.85.12 → 0.86.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3268,6 +3268,11 @@ function legacySingletonHolders(active, runId, limit) {
3268
3268
  return scanned.filter((run) => run.id !== runId && !run.tags?.includes(SINGLETON_ADMITTED_TAG) && (!run.tags?.includes(SINGLETON_TRACKED_TAG) || window.has(run.id)));
3269
3269
  }
3270
3270
  __name(legacySingletonHolders, "legacySingletonHolders");
3271
+ var SINGLETON_LOCK_CHUNK = 500;
3272
+ function singletonClaimantIds(rows, tag) {
3273
+ return rows.filter((row) => Array.isArray(row.tags) && row.tags.includes(tag)).map((row) => row.id);
3274
+ }
3275
+ __name(singletonClaimantIds, "singletonClaimantIds");
3271
3276
  function singletonAdmissionTags(runs, runId, tag, workflow, limit) {
3272
3277
  if (!Number.isInteger(limit) || limit < 1) throw new Error("singleton limit must be a positive integer");
3273
3278
  const active = runs.filter((run) => run.workflow === workflow && SINGLETON_ACTIVE_STATUSES.includes(run.status) && run.tags?.includes(tag));
@@ -3312,6 +3317,41 @@ __name(singletonAdmissionDenialPatch, "singletonAdmissionDenialPatch");
3312
3317
  // src/singleton-gate.ts
3313
3318
  var SINGLETON_RETRY_MS = 1e3;
3314
3319
  var SINGLETON_RETRY_JITTER_MS = 250;
3320
+ var ADMISSION_ATTEMPTS = 3;
3321
+ var ADMISSION_BACKOFF_MS = 40;
3322
+ var LOCK_CONFLICT_CODES = /* @__PURE__ */ new Set([
3323
+ "40P01",
3324
+ "40001",
3325
+ "ER_LOCK_DEADLOCK",
3326
+ "1213",
3327
+ "P2034",
3328
+ "TransactionWriteConflict"
3329
+ ]);
3330
+ function isLockConflict(error2) {
3331
+ const seen = /* @__PURE__ */ new Set();
3332
+ const pending = [
3333
+ error2
3334
+ ];
3335
+ while (pending.length > 0) {
3336
+ const current = pending.pop();
3337
+ if (typeof current !== "object" || current === null || seen.has(current)) continue;
3338
+ seen.add(current);
3339
+ const e = current;
3340
+ for (const code of [
3341
+ e.code,
3342
+ e.errno,
3343
+ e.sqlState,
3344
+ e.sqlstate,
3345
+ e.originalCode,
3346
+ e.kind
3347
+ ]) {
3348
+ if (LOCK_CONFLICT_CODES.has(String(code))) return true;
3349
+ }
3350
+ pending.push(e.cause, e.driverError, e.originalError, e.meta, e.driverAdapterError);
3351
+ }
3352
+ return false;
3353
+ }
3354
+ __name(isLockConflict, "isLockConflict");
3315
3355
  var SingletonGate = class {
3316
3356
  static {
3317
3357
  __name(this, "SingletonGate");
@@ -3353,7 +3393,7 @@ var SingletonGate = class {
3353
3393
  if (!this.deps.store.tryAdmitSingleton) {
3354
3394
  throw new Error("Singleton workflows require StateStore.tryAdmitSingleton for atomic durable admission");
3355
3395
  }
3356
- const admitted = await this.deps.store.tryAdmitSingleton(run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
3396
+ const admitted = await this.claim(run, cfg);
3357
3397
  if (admitted && !run.tags?.includes(SINGLETON_ADMITTED_TAG)) {
3358
3398
  run.tags = [
3359
3399
  ...(run.tags ?? []).filter((tag) => tag !== SINGLETON_TRACKED_TAG),
@@ -3363,6 +3403,26 @@ var SingletonGate = class {
3363
3403
  return admitted;
3364
3404
  }
3365
3405
  /**
3406
+ * One claim, retried when the database aborts it as a lock conflict. A conflict that outlasts the
3407
+ * retries answers "not admitted", so the engine releases this turn's lease, instead of stranding
3408
+ * the run until the lease lapses. It writes nothing to the run: this turn's lease may have lapsed
3409
+ * during the claim, and another instance may now hold the run, its slot and newer tags. A later
3410
+ * sweep re-drives the run: recovery once it is unleased, or its timer if it is parked.
3411
+ */
3412
+ async claim(run, cfg) {
3413
+ const admit = this.deps.store.tryAdmitSingleton;
3414
+ for (let attempt = 1; ; attempt += 1) {
3415
+ try {
3416
+ return await admit.call(this.deps.store, run.id, this.tag(cfg, run.input), run.workflow, cfg.limit ?? 1, this.retryWakeAt());
3417
+ } catch (error2) {
3418
+ if (!isLockConflict(error2)) throw error2;
3419
+ if (attempt >= ADMISSION_ATTEMPTS) return false;
3420
+ const backoff = ADMISSION_BACKOFF_MS * attempt * (0.5 + Math.random());
3421
+ await new Promise((resolve) => setTimeout(resolve, backoff));
3422
+ }
3423
+ }
3424
+ }
3425
+ /**
3366
3426
  * Notify-on-release: a singleton run settled (freeing a slot), so dispatch the oldest gated
3367
3427
  * (`suspended` + same tag) waiters now instead of waiting for their ~1s retry timer. Each re-checks
3368
3428
  * admission in the executor and runs only if it actually wins a slot, so FIFO/race-free guarantees
@@ -4949,6 +5009,22 @@ var NamespaceMismatch = class NamespaceMismatch2 extends Error {
4949
5009
  super("namespace-mismatch");
4950
5010
  }
4951
5011
  };
5012
+ var WorkflowNotRegistered = class extends Error {
5013
+ static {
5014
+ __name(this, "WorkflowNotRegistered");
5015
+ }
5016
+ constructor(workflow, version) {
5017
+ super(`workflow ${workflow}@${version} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
5018
+ this.name = "WorkflowNotRegistered";
5019
+ }
5020
+ };
5021
+ var UNREGISTERED_DEAD_LETTER_FROM = [
5022
+ "pending",
5023
+ "running",
5024
+ "suspended",
5025
+ "cancelling",
5026
+ "blocked"
5027
+ ];
4952
5028
  var WorkflowEngine = class {
4953
5029
  static {
4954
5030
  __name(this, "WorkflowEngine");
@@ -4968,6 +5044,7 @@ var WorkflowEngine = class {
4968
5044
  */
4969
5045
  ownLeaseCutoffMs;
4970
5046
  maxRecoveryAttempts;
5047
+ unregisteredWorkflowGraceMs;
4971
5048
  remoteAdvanceSilenceMs;
4972
5049
  reconcileMs;
4973
5050
  timerBatchSize;
@@ -5154,6 +5231,8 @@ var WorkflowEngine = class {
5154
5231
  this.leaseMs = deps.leaseMs ?? 3e4;
5155
5232
  this.ownLeaseCutoffMs = this.clock() + this.leaseMs;
5156
5233
  this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
5234
+ const grace = deps.unregisteredWorkflowGraceMs;
5235
+ this.unregisteredWorkflowGraceMs = grace !== void 0 && Number.isFinite(grace) && grace >= 0 ? grace : 24 * 60 * 60 * 1e3;
5157
5236
  this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
5158
5237
  this.reconcileMs = deps.reconcileMs === void 0 ? 3e5 : deps.reconcileMs || void 0;
5159
5238
  const stalledAfter = deps.stalledAfterMs;
@@ -5470,13 +5549,19 @@ var WorkflowEngine = class {
5470
5549
  * that is lost brings it back due after that period. False when the transport refused the wakes,
5471
5550
  * so the caller resumes them itself.
5472
5551
  */
5473
- async handOffDue(runs, nowMs) {
5552
+ async handOffDue(runs, nowMs, settled) {
5474
5553
  const claimed = [];
5475
5554
  for (const run of runs) {
5476
5555
  if (this.draining) break;
5477
5556
  if (!await this.store.tryLockRun(run.id, this.instanceId, nowMs + this.leaseMs, nowMs)) {
5478
5557
  continue;
5479
5558
  }
5559
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
5560
+ const result = await this.settleUnregistered(run, nowMs);
5561
+ if (result) settled?.set(run.id, result);
5562
+ else if (result === void 0) this.logUnregisteredSkip(run);
5563
+ continue;
5564
+ }
5480
5565
  try {
5481
5566
  const patch = {
5482
5567
  wakeAt: nowMs + this.leaseMs
@@ -6250,9 +6335,7 @@ var WorkflowEngine = class {
6250
6335
  });
6251
6336
  }
6252
6337
  const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
6253
- if (!registered) {
6254
- throw new Error(`workflow ${run.workflow}@${run.workflowVersion} is not registered \u2014 keep the prior version deployed so in-flight runs can drain (skew protection)`);
6255
- }
6338
+ if (!registered) throw new WorkflowNotRegistered(run.workflow, run.workflowVersion);
6256
6339
  return this.track(this.execute(run, registered.fn));
6257
6340
  }
6258
6341
  /**
@@ -6667,6 +6750,15 @@ var WorkflowEngine = class {
6667
6750
  };
6668
6751
  }
6669
6752
  }
6753
+ if (!this.isRegisteredHere(run) && !await this.canResolveWorkflow(run)) {
6754
+ const settled2 = await this.settleUnregistered(run, nowMs);
6755
+ if (settled2) return {
6756
+ result: settled2,
6757
+ dispatch: false
6758
+ };
6759
+ if (settled2 === void 0) this.logUnregisteredSkip(run);
6760
+ return void 0;
6761
+ }
6670
6762
  const settled = await this.countRecovery(run);
6671
6763
  if (settled) return {
6672
6764
  result: settled,
@@ -6738,25 +6830,10 @@ var WorkflowEngine = class {
6738
6830
  updatedAt: /* @__PURE__ */ new Date()
6739
6831
  });
6740
6832
  await this.store.releaseRunLock(run.id);
6741
- this.emit({
6742
- type: "run.failed",
6743
- runId: run.id,
6744
- workflow: run.workflow,
6745
- namespace: run.namespace,
6746
- error: error2
6747
- });
6748
- this.notifyDead({
6833
+ return this.announceDead({
6749
6834
  ...run,
6750
- status: "dead",
6751
- error: error2,
6752
6835
  recoveryAttempts: attempts
6753
- });
6754
- this.notifyParent(run.id, failedCompletion(error2));
6755
- return {
6756
- runId: run.id,
6757
- status: "dead",
6758
- error: error2
6759
- };
6836
+ }, error2);
6760
6837
  }
6761
6838
  await this.store.updateRun(run.id, {
6762
6839
  recoveryAttempts: attempts,
@@ -6764,6 +6841,85 @@ var WorkflowEngine = class {
6764
6841
  });
6765
6842
  return void 0;
6766
6843
  }
6844
+ /** Tell everyone a run was just written `dead`: events, dead-letter routing, a waiting parent. */
6845
+ announceDead(run, error2) {
6846
+ this.emit({
6847
+ type: "run.failed",
6848
+ runId: run.id,
6849
+ workflow: run.workflow,
6850
+ namespace: run.namespace,
6851
+ error: error2
6852
+ });
6853
+ this.notifyDead({
6854
+ ...run,
6855
+ status: "dead",
6856
+ error: error2
6857
+ });
6858
+ this.notifyParent(run.id, failedCompletion(error2));
6859
+ return {
6860
+ runId: run.id,
6861
+ status: "dead",
6862
+ error: error2
6863
+ };
6864
+ }
6865
+ /** The synchronous half of {@link canResolveWorkflow}: callers check it first, so a registered run
6866
+ * pays no await (same rule as {@link resume}). */
6867
+ isRegisteredHere(run) {
6868
+ return this.workflows.has(versionKey(run.workflow, run.workflowVersion));
6869
+ }
6870
+ /**
6871
+ * Whether this instance can run `run`'s pinned `workflow@version`: registered here, inherited from a
6872
+ * remote ancestor, or routed by convention to a live worker group (the lookup {@link resume} does).
6873
+ */
6874
+ async canResolveWorkflow(run) {
6875
+ if (this.isRegisteredHere(run)) return true;
6876
+ return await this.findInheritedRegistration(run) !== void 0 || await this.resolveRemoteByConvention(run) !== void 0;
6877
+ }
6878
+ /**
6879
+ * A run whose lease this instance holds but whose `workflow@version` nothing here can run (see
6880
+ * {@link WorkflowNotRegistered}). Within {@link unregisteredWorkflowGraceMs} of becoming due it is a
6881
+ * rolling deploy's skew: the lease is released at once so the pod that still has the version takes
6882
+ * the run now, not after the lease expires. Past it, the workflow is gone from the whole fleet (a
6883
+ * capable pod would have taken the run long ago): dead-letter it.
6884
+ *
6885
+ * The age is read off `updatedAt` / `wakeAt`, so every path that would touch either (a recovery
6886
+ * count, a re-pend, a hand-off's lease-long `wakeAt`) must call this BEFORE writing — otherwise an
6887
+ * orphan never ages.
6888
+ *
6889
+ * Returns the `dead` result, `null` when the run left its status meanwhile (a lease-free
6890
+ * `cancel()`: settled, not ours to touch), or `undefined` when it was left for later.
6891
+ */
6892
+ async settleUnregistered(listed, nowMs) {
6893
+ const run = await this.store.getRun(listed.id);
6894
+ if (!run || !UNREGISTERED_DEAD_LETTER_FROM.includes(run.status)) {
6895
+ await this.store.releaseRunLock(listed.id);
6896
+ return null;
6897
+ }
6898
+ const dueSince = Math.max(run.updatedAt.getTime(), run.wakeAt ?? 0);
6899
+ if (this.unregisteredWorkflowGraceMs === 0 || nowMs - dueSince < this.unregisteredWorkflowGraceMs) {
6900
+ await this.store.releaseRunLock(run.id);
6901
+ return void 0;
6902
+ }
6903
+ const error2 = {
6904
+ message: `workflow ${run.workflow}@${run.workflowVersion} is not registered on any live instance (unresumable for over ${this.unregisteredWorkflowGraceMs}ms) \u2014 moved to dead-letter`,
6905
+ code: "workflow_not_registered"
6906
+ };
6907
+ const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, [
6908
+ run.status
6909
+ ], {
6910
+ status: "dead",
6911
+ error: error2,
6912
+ updatedAt: /* @__PURE__ */ new Date()
6913
+ });
6914
+ await this.store.releaseRunLock(run.id);
6915
+ if (!applied) return null;
6916
+ console.warn(`[nestjs-durable] run ${run.id}: ${error2.message}`);
6917
+ return this.announceDead(run, error2);
6918
+ }
6919
+ /** Log a run skipped for an unregistered workflow — still there, still not progressing. */
6920
+ logUnregisteredSkip(run) {
6921
+ console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, new WorkflowNotRegistered(run.workflow, run.workflowVersion));
6922
+ }
6767
6923
  /**
6768
6924
  * Resume every suspended run whose durable timer is due. Call periodically (a poller) and on
6769
6925
  * boot. A run still not due re-suspends cheaply without running new work.
@@ -6788,11 +6944,12 @@ var WorkflowEngine = class {
6788
6944
  ...due,
6789
6945
  ...blocked
6790
6946
  ];
6791
- if (await this.handOffDue(runs, nowMs)) {
6792
- return runs.map((run) => ({
6947
+ const settled = /* @__PURE__ */ new Map();
6948
+ if (await this.handOffDue(runs, nowMs, settled)) {
6949
+ return runs.map((run) => settled.get(run.id) ?? {
6793
6950
  runId: run.id,
6794
6951
  status: run.status
6795
- }));
6952
+ });
6796
6953
  }
6797
6954
  }
6798
6955
  const waiterByRun = indexWaitersByRun(await listSignalWaitersOfRuns(this.store, [
@@ -6873,8 +7030,13 @@ var WorkflowEngine = class {
6873
7030
  const nowMs = this.clock();
6874
7031
  const acquired = await this.store.tryLockRun(runId, this.instanceId, nowMs + this.leaseMs, nowMs);
6875
7032
  if (!acquired) return null;
6876
- return this.resumeImplicit(runId).catch((err) => {
7033
+ return this.resumeImplicit(runId).catch(async (err) => {
6877
7034
  if (err instanceof NamespaceMismatch) return null;
7035
+ if (err instanceof WorkflowNotRegistered) {
7036
+ const run = await this.store.getRun(runId);
7037
+ const settled = run ? await this.settleUnregistered(run, nowMs) : null;
7038
+ if (settled !== void 0) return settled;
7039
+ }
6878
7040
  throw err;
6879
7041
  });
6880
7042
  }
@@ -7129,7 +7291,8 @@ var WorkflowEngine = class {
7129
7291
  * Resume each run only if this instance can acquire its recovery lease — so when several
7130
7292
  * replicas recover or poll at once, each run is picked up by exactly one of them.
7131
7293
  *
7132
- * A run this instance cannot resume is SKIPPED, not allowed to abort the batch. The canonical
7294
+ * A run this instance cannot resume is SKIPPED, not allowed to abort the batch (a run whose workflow
7295
+ * stays registered nowhere past `unregisteredWorkflowGraceMs` is dead-lettered instead). The canonical
7133
7296
  * case is skew protection: mid-rolling-deploy a pod leases a run whose workflow it does not have
7134
7297
  * and `resume` throws by design — but the rest of the batch is runs it can drive perfectly well,
7135
7298
  * and every caller here is a recovery sweep. Letting the first such run end the loop means one
@@ -7153,6 +7316,11 @@ var WorkflowEngine = class {
7153
7316
  const settled = onLocked ? await onLocked(run) : void 0;
7154
7317
  results.push(settled ?? await this.resumeImplicit(run.id));
7155
7318
  } catch (error2) {
7319
+ if (error2 instanceof WorkflowNotRegistered) {
7320
+ const settled = await this.settleUnregistered(run, nowMs).catch(() => void 0);
7321
+ if (settled) results.push(settled);
7322
+ if (settled !== void 0) continue;
7323
+ }
7156
7324
  console.error(`[nestjs-durable] could not resume run ${run.id} (${run.workflow}@${run.workflowVersion}); leaving it for another instance:`, error2);
7157
7325
  }
7158
7326
  }
@@ -8169,7 +8337,7 @@ var WorkflowEngine = class {
8169
8337
  const current = await this.store.getRun(run.id);
8170
8338
  if (!this.isFenced(run.id)) {
8171
8339
  this.handOffLease(run.id);
8172
- await this.store.releaseRunLock(run.id);
8340
+ await this.store.releaseRunLock(run.id, this.instanceId);
8173
8341
  }
8174
8342
  if (!current) throw new Error(`run ${run.id} was deleted during singleton admission`);
8175
8343
  if (current.status === "suspended") {
@@ -12895,6 +13063,7 @@ export {
12895
13063
  RunGateway,
12896
13064
  SINGLETON_ACTIVE_STATUSES,
12897
13065
  SINGLETON_ADMITTED_TAG,
13066
+ SINGLETON_LOCK_CHUNK,
12898
13067
  SINGLETON_RESERVED_TAGS,
12899
13068
  SINGLETON_TRACKED_TAG,
12900
13069
  STATE_STORE,
@@ -12913,6 +13082,7 @@ export {
12913
13082
  WORKFLOW_NAME_KEY,
12914
13083
  WorkflowBlocked,
12915
13084
  WorkflowEngine,
13085
+ WorkflowNotRegistered,
12916
13086
  WorkflowSuspended,
12917
13087
  aggregateAnnouncements,
12918
13088
  aggregatesCheckpoints,
@@ -13027,6 +13197,7 @@ export {
13027
13197
  signTenantToken,
13028
13198
  singletonAdmissionDenialPatch,
13029
13199
  singletonAdmissionTags,
13200
+ singletonClaimantIds,
13030
13201
  sqlComparator,
13031
13202
  stepConfigOf,
13032
13203
  stepId,