@dudousxd/nestjs-durable-core 0.79.0 → 0.80.1

package/dist/index.cjs

@@ -4052,6 +4052,16 @@ function isNewerVersion(a, b) {
   }) > 0;
 }
 __name(isNewerVersion, "isNewerVersion");
+var SETTLE_ELIGIBLE_STATUSES = [
+  "pending",
+  "running",
+  "suspended",
+  "blocked",
+  "failed",
+  "cancelling"
+];
+var SUSPEND_ELIGIBLE_STATUSES = SETTLE_ELIGIBLE_STATUSES.filter((status) => status !== "cancelling");
+var RECOVERY_BATCH = 500;
 var NamespaceMismatch = class NamespaceMismatch2 extends Error {
   static {
     __name(this, "NamespaceMismatch");
@@ -4151,6 +4161,18 @@ var WorkflowEngine = class {
   awaitedStepFans = /* @__PURE__ */ new Map();
   /** Step fans whose join barrier this instance opened, by `runId:firstSeq` — see {@link settledHere}. */
   stepFansFiredHere = /* @__PURE__ */ new Map();
+  /**
+  * The in-process execution gate, by run id: the execution of the run currently in flight in THIS
+  * process (`tail`), and at most one resume queued behind it (`queued`). See {@link resumeRun}.
+  *
+  * The run lease is keyed by engine instance, so it fences OTHER processes only: to the store, a
+  * second execution in this same process holds the very same lease (and {@link execute} skips
+  * `tryLockRun` when its snapshot says this instance holds it). Without this gate, a resume landing
+  * while this process was still executing the run (a signal, a late step result, a child wake, an
+  * explicit `resume()`) went straight past the lease and ran the body concurrently: a
+  * not-yet-checkpointed step executed twice.
+  */
+  runGates = /* @__PURE__ */ new Map();
   /** The tail of each run's queue of results being handled on this instance — see {@link serializedPerRun}. */
   resultChains = /* @__PURE__ */ new Map();
   /** Each run's results waiting to be settled on this instance — see {@link completeRemoteResult}. */
@@ -4619,11 +4641,16 @@ var WorkflowEngine = class {
     if (registered.singleton) {
       await this.singletons.assertCapacity(name, registered.singleton, input);
     }
+    const startAtMs = opts?.startAt instanceof Date ? opts.startAt.getTime() : opts?.startAt;
+    const delayed = startAtMs !== void 0 && Number.isFinite(startAtMs) && startAtMs > this.clock();
     const run = {
       id: runId,
       workflow: name,
       workflowVersion: registered.version,
-      status: "pending",
+      status: delayed ? "suspended" : "pending",
+      ...delayed ? {
+        wakeAt: startAtMs
+      } : {},
       namespace: opts?.namespace ?? this.namespace,
       // DERIVED from the registration, exactly like `namespace` is derived from this engine — there is
       // deliberately no `opts.origin`, so a caller cannot label its run as another package's work.
@@ -4649,6 +4676,18 @@ var WorkflowEngine = class {
         error: winner.error
       };
     }
+    if (delayed) {
+      this.emit({
+        type: "run.suspended",
+        runId,
+        workflow: name,
+        namespace: run.namespace
+      });
+      return {
+        runId,
+        status: "suspended"
+      };
+    }
     await this.runDispatcher.dispatch(runId);
     if (this.controlPlane) {
       void this.controlPlane.publishControl({
@@ -4686,7 +4725,104 @@ var WorkflowEngine = class {
   resumeImplicit(runId) {
     return this.resumeRun(runId, false);
   }
-  async resumeRun(runId, allowFailed) {
+  /**
+  * One execution of a run at a time in this process. A resume that lands while the run executes
+  * here neither joins that execution (the double-run) nor no-ops against its held lease (a lost
+  * wake: its checkpoint may postdate the running turn's snapshot): it is QUEUED, and re-drives the
+  * run once the running execution has settled and released its lease, re-reading the run then
+  * (terminal: echoed, never re-executed). Resumes landing while one is already queued coalesce
+  * into it: it has not started yet, so it will observe every write they made. Their callers get
+  * the queued execution's result, i.e. the run's state after their own wake was applied.
+  *
+  * Across processes, exclusion stays the lease's job: {@link execute} still `tryLockRun`s when it
+  * does not hold the lease.
+  */
+  resumeRun(runId, allowFailed) {
+    const gate = this.runGates.get(runId);
+    if (!gate) {
+      const tail = this.resumeRunNow(runId, allowFailed);
+      const fresh = {
+        tail
+      };
+      this.runGates.set(runId, fresh);
+      this.closeGateAfter(runId, fresh, tail);
+      return tail;
+    }
+    let queued = gate.queued;
+    if (queued) {
+      queued.allowFailed ||= allowFailed;
+    } else {
+      const entry = {
+        allowFailed
+      };
+      const result = gate.tail.catch(() => void 0).then(() => {
+        if (gate.queued === entry) gate.queued = void 0;
+        return this.resumeRunNow(runId, entry.allowFailed);
+      });
+      entry.result = result;
+      queued = entry;
+      gate.queued = queued;
+      gate.tail = result;
+      this.closeGateAfter(runId, gate, result);
+      void this.track(result).catch(() => void 0);
+    }
+    if (currentWorkflowCtx()?.runId === runId) return this.echoCurrentStatus(runId);
+    return queued.result;
+  }
+  /** Drop `gate` once `tail` settles, unless another execution queued behind it meanwhile. */
+  closeGateAfter(runId, gate, tail) {
+    const close = /* @__PURE__ */ __name(() => {
+      if (this.runGates.get(runId) === gate && gate.tail === tail) this.runGates.delete(runId);
+    }, "close");
+    void tail.then(close, close);
+  }
+  /**
+  * Release a run's lease AFTER its outcome is decided (persisted, or about to be returned). Best
+  * effort, and that matters most on the failure path: the thing that failed the run is frequently
+  * the very store this release talks to (a step that violated a DB constraint leaves Postgres
+  * refusing every further statement on that connection), so the release throws too. A throwing
+  * release in a `finally` REPLACES the run's real outcome with a lease write nobody asked about,
+  * skips the post-turn re-drives after it, and, for a turn driven as a background resume, surfaces
+  * as an unhandled rejection instead of a failed run.
+  *
+  * Letting it go is safe: the lease expires on its own and `recoverIncomplete` re-drives whatever
+  * is still held. A lost release costs one lease window of latency; a masked error costs the
+  * diagnosis.
+  */
+  async releaseLeaseBestEffort(runId) {
+    try {
+      await this.store.releaseRunLock(runId);
+    } catch (err) {
+      console.warn(`[nestjs-durable] releasing the lease of run ${runId} failed (it expires on its own and recovery re-drives): ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  /**
+  * The run's persisted state as a {@link RunResult}, without executing anything: the answer to a
+  * resume issued from inside the run's own execution, and to a settle whose compare-and-set lost
+  * (the path that won already emitted and notified). A run that is gone reads as `cancelled`, the
+  * safest terminal reading, never a fabricated success.
+  */
+  async echoCurrentStatus(runId) {
+    const run = await this.store.getRun(runId);
+    return {
+      runId,
+      status: run?.status ?? "cancelled",
+      output: run?.output,
+      error: run?.error
+    };
+  }
+  /**
+  * The engine's stand-in for {@link StateStore.updateRunIf} on a store without it, called with the
+  * store as `this`: read-then-write, which narrows the race to the gap between the read and the
+  * write (the pre-CAS behavior, now applied to every settle) but cannot close it.
+  */
+  async updateRunIfFallback(runId, expected, patch) {
+    const current = await this.getRun(runId);
+    if (!current || !expected.includes(current.status)) return false;
+    await this.updateRun(runId, patch);
+    return true;
+  }
+  async resumeRunNow(runId, allowFailed) {
     const run = await this.store.getRun(runId);
     if (!run) throw new Error(`run ${runId} not found`);
     if (run.status === "cancelled" || run.status === "completed" || run.status === "dead" || run.status === "failed" && !allowFailed) {
@@ -4992,7 +5128,7 @@ var WorkflowEngine = class {
   async recoverIncomplete(nowMs = this.clock()) {
     if (this.draining) return [];
     const results = [];
-    for (const listed of await this.store.listIncompleteRuns(this.namespace)) {
+    for (const listed of await this.orphanedRuns(nowMs)) {
       if (this.draining) break;
       const acquired = await this.store.tryLockRun(listed.id, this.instanceId, nowMs + this.leaseMs, nowMs);
       if (!acquired) continue;
@@ -5038,6 +5174,21 @@ var WorkflowEngine = class {
     return results;
   }
   /**
+  * The runs {@link recoverIncomplete} should try to reclaim: incomplete runs whose lease is free or
+  * expired, never the healthy ones a live turn is renewing. Every worker runs recovery on every
+  * timer tick, so listing every running run and lock-probing each (one doomed UPDATE per healthy
+  * run per worker per tick) was pure database load. The store answers it directly when it can
+  * ({@link StateStore.listOrphanedRuns}, bounded by {@link RECOVERY_BATCH}); else the full list is
+  * filtered here, which still spares the probes. Shuffled, like the due-timer pass, so instances
+  * recovering at once spread over the batch instead of contending for the same leases.
+  */
+  async orphanedRuns(nowMs) {
+    if (this.store.listOrphanedRuns) {
+      return shuffle(await this.store.listOrphanedRuns(nowMs, RECOVERY_BATCH, this.namespace));
+    }
+    return shuffle((await this.store.listIncompleteRuns(this.namespace)).filter((run) => run.lockedUntil == null || run.lockedUntil <= nowMs).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, RECOVERY_BATCH));
+  }
+  /**
   * The remote step this run dispatched and is still awaiting a result for, if any — the signature
   * that tells {@link recoverIncomplete} "this run's work is ON THE TRANSPORT, not in a crashed
   * process". A `pending` remote checkpoint is written BEFORE the dispatch and settled by the
@@ -6554,12 +6705,13 @@ ${run.workflowVersion}`;
   async settleRun(run, outcome) {
     const updatedAt = /* @__PURE__ */ new Date();
     if (outcome.kind === "completed") {
-      await this.store.updateRun(run.id, {
+      const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
         status: "completed",
         output: outcome.output,
         error: void 0,
         updatedAt
       });
+      if (!applied2) return this.echoCurrentStatus(run.id);
       this.emit({
         type: "run.completed",
         runId: run.id,
@@ -6581,11 +6733,12 @@ ${run.workflowVersion}`;
       const handed = await this.routeFailure(run, outcome.error);
       if (handed && handed.kind !== "failed") return this.settleRun(run, handed);
       const error2 = handed?.error ?? outcome.error;
-      await this.store.updateRun(run.id, {
+      const applied2 = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SETTLE_ELIGIBLE_STATUSES, {
         status: "failed",
         error: error2,
         updatedAt
       });
+      if (!applied2) return this.echoCurrentStatus(run.id);
       this.emit({
         type: "run.failed",
         runId: run.id,
@@ -6601,19 +6754,12 @@ ${run.workflowVersion}`;
       };
     }
     if (outcome.kind === "blocked") return this.parkBlocked(run, outcome.blocked);
-    const latest = await this.store.getRun(run.id);
-    if (latest?.status === "cancelled") {
-      return {
-        runId: run.id,
-        status: "cancelled",
-        error: latest.error
-      };
-    }
-    await this.store.updateRun(run.id, {
+    const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, {
       status: "suspended",
       wakeAt: this.reconcileWakeAt(outcome.wakeAt),
       updatedAt
     });
+    if (!applied) return this.echoCurrentStatus(run.id);
     this.emit({
       type: "run.suspended",
       runId: run.id,
@@ -6660,7 +6806,7 @@ ${run.workflowVersion}`;
           from: this.instanceId
         }).catch(() => void 0);
       }
-      await this.store.releaseRunLock(run.id);
+      await this.releaseLeaseBestEffort(run.id);
       return {
         runId: run.id,
         status: "cancelled",
@@ -6690,7 +6836,7 @@ ${run.workflowVersion}`;
     const handoff = liveHandoff(run.id, checkpoints);
     if (handoff) {
       const result = await this.settleRun(run, await this.followHandoff(run, handoff));
-      await this.store.releaseRunLock(run.id);
+      await this.releaseLeaseBestEffort(run.id);
       return result;
     }
     const history = await this.remoteHistory(run.id, checkpoints);
@@ -6728,7 +6874,7 @@ ${run.workflowVersion}`;
       decision = silenceMs != null ? await this.awaitWithLivenessDeadline(run.id, advance, silenceMs, () => new RemoteWorkflowTimeout(run.id, silenceMs)) : await advance;
     } catch (err) {
       if (err instanceof RemoteWorkflowTimeout) {
-        await this.store.releaseRunLock(run.id);
+        await this.releaseLeaseBestEffort(run.id);
         return {
           runId: run.id,
           status: "running"
@@ -7399,7 +7545,7 @@ ${run.workflowVersion}`;
         this.awaitedRemoteSteps.delete(run.id);
       }
       if (this.awaitedStepFans.get(run.id) === awaitedFans) this.awaitedStepFans.delete(run.id);
-      await this.store.releaseRunLock(run.id);
+      await this.releaseLeaseBestEffort(run.id);
       if (suspendedOnFan) {
         void this.redriveSettledChildren(run.id, suspendedOnFan).catch(() => void 0);
       }
@@ -7770,27 +7916,21 @@ ${run.workflowVersion}`;
   * own `finally` also releases (the run-execution path) is safe.
   */
   async parkBlocked(run, blocked) {
-    const current = await this.store.getRun(run.id);
-    if (current && (current.status === "cancelled" || current.status === "completed")) {
-      await this.store.releaseRunLock(run.id);
-      return {
-        runId: run.id,
-        status: current.status,
-        output: current.output,
-        error: current.error
-      };
-    }
     const error2 = {
       message: blocked.reason,
       code: blocked.code,
       retryable: true
     };
-    await this.store.updateRun(run.id, {
+    const applied = await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, run.id, SUSPEND_ELIGIBLE_STATUSES, {
       status: "blocked",
       error: error2,
       wakeAt: this.clock() + this.blockedPollMs,
       updatedAt: /* @__PURE__ */ new Date()
     });
+    if (!applied) {
+      await this.releaseLeaseBestEffort(run.id);
+      return this.echoCurrentStatus(run.id);
+    }
     this.emit({
       type: "run.blocked",
       runId: run.id,
@@ -7800,7 +7940,7 @@ ${run.workflowVersion}`;
       requires: blocked.requires,
       diagnostics: blocked.diagnostics
     });
-    await this.store.releaseRunLock(run.id);
+    await this.releaseLeaseBestEffort(run.id);
     return {
       runId: run.id,
       status: "blocked",
@@ -9148,6 +9288,12 @@ var CodecStateStore = class {
   /** Forwarded verbatim (deletes touch no payload). Bound only when the inner store has it, so the
   *  engine's per-run fallback still kicks in through the wrapper. */
   deleteRuns;
+  /** Decodes the runs like {@link listIncompleteRuns}. Bound only when the inner store has it, so
+  *  the engine's list-and-filter fallback still kicks in through the wrapper. */
+  listOrphanedRuns;
+  /** Encodes the patch like {@link updateRun}. Bound only when the inner store has it, so the
+  *  engine's read-then-write fallback still kicks in through the wrapper. */
+  updateRunIf;
   /** Encodes like {@link saveCheckpoint}. Bound only when the inner store has it, so the engine's
   *  plain-save fallback still kicks in through the wrapper. */
   saveCheckpointIfPending;
@@ -9188,6 +9334,10 @@ var CodecStateStore = class {
     if (inner.tryAdmitSingleton) this.tryAdmitSingleton = inner.tryAdmitSingleton.bind(inner);
     const deleteRuns = inner.deleteRuns;
     if (deleteRuns) this.deleteRuns = (ids) => deleteRuns.call(inner, ids);
+    const orphans = inner.listOrphanedRuns;
+    if (orphans) this.listOrphanedRuns = async (nowMs, limit, namespace) => (await orphans.call(inner, nowMs, limit, namespace)).map((r) => this.decRun(r));
+    const updateIf = inner.updateRunIf;
+    if (updateIf) this.updateRunIf = (runId, expected, patch) => updateIf.call(inner, runId, expected, this.encRunPatch(patch));
     const saveIfPending = inner.saveCheckpointIfPending;
     if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
     const saveManyIfPending = inner.saveCheckpointsIfPending;
@@ -9294,12 +9444,15 @@ var CodecStateStore = class {
     return this.inner.createRun(this.encRun(run));
   }
   updateRun(runId, patch) {
+    return this.inner.updateRun(runId, this.encRunPatch(patch));
+  }
+  encRunPatch(patch) {
     const next = {
       ...patch
     };
     if ("input" in patch) next.input = this.enc(patch.input);
     if ("output" in patch) next.output = this.enc(patch.output);
-    return this.inner.updateRun(runId, next);
+    return next;
   }
   async getRun(runId) {
     return this.decRun(await this.inner.getRun(runId));
@@ -9708,6 +9861,14 @@ var InMemoryStateStore = class {
     this.reindexAttributes(run.id, run.searchAttributes);
   }
   async updateRun(runId, inputPatch) {
+    this.applyRunPatch(runId, inputPatch);
+  }
+  updateRunIf(runId, expectedStatuses, patch) {
+    const existing = this.runs.get(runId);
+    if (!existing || !expectedStatuses.includes(existing.status)) return Promise.resolve(false);
+    return this.updateRun(runId, patch).then(() => true);
+  }
+  applyRunPatch(runId, inputPatch) {
     const patch = inputPatch.status && TERMINAL_RUN_STATUSES.includes(inputPatch.status) ? {
       ...inputPatch,
       tags: ("tags" in inputPatch ? inputPatch.tags : this.runs.get(runId)?.tags)?.filter((tag) => tag !== SINGLETON_ADMITTED_TAG)
@@ -9798,6 +9959,13 @@ var InMemoryStateStore = class {
       return true;
     });
   }
+  async listOrphanedRuns(nowMs, limit, namespace) {
+    return [
+      ...this.runs.values()
+    ].filter((r) => (r.status === "running" || r.status === "cancelling") && (r.lockedUntil == null || r.lockedUntil <= nowMs) && (namespace === void 0 || r.namespace === namespace)).sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime()).slice(0, Math.max(0, limit)).map((r) => ({
+      ...r
+    }));
+  }
   async listIncompleteRuns(namespace) {
     return [
       ...this.runs.values()