@dudousxd/nestjs-durable-core 0.32.1 → 0.34.0

package/dist/index.cjs

@@ -29,6 +29,7 @@ __export(index_exports, {
   Entities: () => Entities,
   EventAccumulators: () => EventAccumulators,
   FatalError: () => FatalError,
+  GatherError: () => GatherError,
   InMemoryAdmissionBackend: () => InMemoryAdmissionBackend,
   InMemoryStateStore: () => InMemoryStateStore,
   InMemoryTransport: () => InMemoryTransport,
@@ -37,6 +38,7 @@ __export(index_exports, {
   RemoteStepError: () => RemoteStepError,
   RemoteStepTimeout: () => RemoteStepTimeout,
   RemoteWorkflowExecutor: () => RemoteWorkflowExecutor,
+  RemoteWorkflowTimeout: () => RemoteWorkflowTimeout,
   STATE_STORE: () => STATE_STORE,
   STATE_STORE_CANONICAL: () => STATE_STORE_CANONICAL,
   SignalTimeoutError: () => SignalTimeoutError,
@@ -51,6 +53,7 @@ __export(index_exports, {
   attributeOperand: () => attributeOperand,
   breakpointToken: () => breakpointToken,
   collectMetrics: () => collectMetrics,
+  createStepLogger: () => createStepLogger,
   eventMatchOf: () => eventMatchOf,
   eventMatches: () => eventMatches,
   eventPrefix: () => eventPrefix,
@@ -453,6 +456,7 @@ function stepCheckpoint(p) {
     attempts: p.attempts,
     workerGroup: p.workerGroup,
     wakeAt: p.wakeAt,
+    parallelGroup: p.parallelGroup,
     enqueuedAt: p.enqueuedAt,
     startedAt: p.startedAt,
     finishedAt: p.finishedAt
@@ -470,6 +474,7 @@ function instantCheckpoint(p) {
     status: p.status ?? "completed",
     output: p.output,
     wakeAt: p.wakeAt,
+    parallelGroup: p.parallelGroup,
     attempts: 1,
     enqueuedAt: at,
     startedAt: at,
@@ -531,6 +536,19 @@ var RemoteStepTimeout = class extends Error {
     this.timeoutMs = timeoutMs;
   }
 };
+var RemoteWorkflowTimeout = class extends Error {
+  static {
+    __name(this, "RemoteWorkflowTimeout");
+  }
+  taskId;
+  timeoutMs;
+  constructor(taskId, timeoutMs) {
+    super(`remote workflow task ${taskId} produced no decision within ${timeoutMs}ms \u2014 presumed dropped; re-driving via recovery`);
+    this.name = "RemoteWorkflowTimeout";
+    this.taskId = taskId;
+    this.timeoutMs = timeoutMs;
+  }
+};
 var NonDeterminismError = class extends Error {
   static {
     __name(this, "NonDeterminismError");
@@ -544,6 +562,19 @@ var NonDeterminismError = class extends Error {
     this.seq = seq;
   }
 };
+var GatherError = class extends Error {
+  static {
+    __name(this, "GatherError");
+  }
+  failures;
+  constructor(failures, total) {
+    const ids = failures.map((f) => f.id).join(", ");
+    const denom = total ?? failures.length;
+    super(`ctx.all: ${failures.length} of ${denom} child(ren) failed: ${ids}`);
+    this.name = "GatherError";
+    this.failures = failures;
+  }
+};
 var WorkflowSuspended = class extends Error {
   static {
     __name(this, "WorkflowSuspended");
@@ -1213,6 +1244,72 @@ function createWorkflowCtx(host, runId, compensations, workflow = "") {
     }
     throw new WorkflowSuspended();
   }, "child");
+  const all = /* @__PURE__ */ __name(async (workflow2, inputs, opts) => {
+    if (inputs.length === 0) return [];
+    const mode = opts?.mode ?? "waitAll";
+    const name = workflowName(workflow2);
+    const positions = inputs.map(() => pos.next());
+    const group = `all:${positions[0]}`;
+    const id = /* @__PURE__ */ __name((i) => `${runId}.all.${positions[0]}.${i}`, "id");
+    const existing = await Promise.all(positions.map((seq) => readCheckpoint(seq)));
+    if (mode === "failFast") {
+      for (let i = 0; i < inputs.length; i += 1) {
+        const cp = existing[i];
+        const c = cp?.status === "completed" ? cp.output : null;
+        if (c && c.ok === false) {
+          throw new GatherError([
+            {
+              index: i,
+              id: id(i),
+              error: c.error ?? "unknown"
+            }
+          ], inputs.length);
+        }
+      }
+    }
+    let pending = false;
+    for (let i = 0; i < inputs.length; i += 1) {
+      const cp = existing[i];
+      if (cp?.status === "completed") continue;
+      pending = true;
+      const seq = positions[i];
+      const childId = id(i);
+      await store.putSignalWaiter({
+        token: `child:${childId}`,
+        runId,
+        seq
+      });
+      if (!await store.getRun(childId)) host.startChild(name, inputs[i], childId);
+      if (!cp) {
+        await writeCheckpoint(instantCheckpoint({
+          runId,
+          seq,
+          name: `signal:child:${childId}`,
+          kind: "signal",
+          status: "running",
+          parallelGroup: group
+        }));
+      }
+    }
+    if (pending) throw new WorkflowSuspended();
+    const outputs = [];
+    const failures = [];
+    for (let i = 0; i < inputs.length; i += 1) {
+      const c = existing[i]?.output;
+      if (c && typeof c === "object" && "ok" in c && c.ok === false) {
+        failures.push({
+          index: i,
+          id: id(i),
+          error: c.error ?? "unknown"
+        });
+        outputs.push(void 0);
+      } else {
+        outputs.push(c?.value);
+      }
+    }
+    if (failures.length > 0) throw new GatherError(failures, inputs.length);
+    return outputs;
+  }, "all");
   const startChild = /* @__PURE__ */ __name(async (workflow2, input, options) => {
     const { childId, priority } = normalizeChildOptions(options);
     const current = pos.next();
@@ -1347,6 +1444,7 @@ function createWorkflowCtx(host, runId, compensations, workflow = "") {
     waitForEvent,
     task,
     child,
+    all,
     startChild,
     breakpoint,
     webhook,
@@ -1404,6 +1502,7 @@ var WorkflowEngine = class {
   instanceId;
   leaseMs;
   maxRecoveryAttempts;
+  remoteAdvanceSilenceMs;
   webhookUrl;
   traceparent;
   context;
@@ -1428,7 +1527,9 @@ var WorkflowEngine = class {
   accumulators;
   /** In-flight remote steps awaiting a worker result, keyed by stepId. */
   pending = /* @__PURE__ */ new Map();
-  /** Per-step "reset the liveness timer" callbacks, called when a heartbeat arrives. */
+  /** "Reset the liveness timer" callbacks keyed by the beat's target — a stepId (in-flight remote step)
+  *  or a runId (in-flight remote workflow turn) — invoked when a matching heartbeat arrives. The two id
+  *  shapes don't collide in practice: a stepId is `${runId}:${seq}`, a run key is the bare runId. */
   heartbeatResets = /* @__PURE__ */ new Map();
   listeners = /* @__PURE__ */ new Set();
   /** Step interceptors (onion middleware around real local-step execution), first = outermost. */
@@ -1467,6 +1568,7 @@ var WorkflowEngine = class {
     this.instanceId = deps.instanceId ?? globalThis.crypto.randomUUID();
     this.leaseMs = deps.leaseMs ?? 3e4;
     this.maxRecoveryAttempts = deps.maxRecoveryAttempts;
+    this.remoteAdvanceSilenceMs = deps.remoteAdvanceSilenceMs;
     this.webhookUrl = deps.webhookUrl;
     this.traceparent = deps.traceparent;
     this.context = deps.context;
@@ -1503,9 +1605,11 @@ var WorkflowEngine = class {
         }
         await this.completeRemoteResult(result);
       },
-      // A heartbeat for an in-flight long step resets its liveness window (see callRemote).
+      // A heartbeat resets the liveness window for whatever it targets: an in-flight long STEP (keyed
+      // by stepId, see callRemote) or — when stepId is absent — the in-flight workflow TURN (keyed by
+      // runId, see runRemoteExecution's heartbeat-rearmed advance).
       async (beat) => {
-        this.heartbeatResets.get(beat.stepId)?.();
+        this.heartbeatResets.get(beat.stepId ?? beat.runId)?.();
       },
       // A remote workflow worker streams each local step's lifecycle (running → completed/failed) so
       // it's checkpointed live, not all-at-once when the long turn ends.
@@ -2336,7 +2440,8 @@ var WorkflowEngine = class {
         attempts: 1,
         enqueuedAt: startedAt,
         startedAt,
-        finishedAt: startedAt
+        finishedAt: startedAt,
+        parallelGroup: event.parallelGroup
       }));
       this.emit({
         type: "step.started",
@@ -2360,7 +2465,8 @@ var WorkflowEngine = class {
       attempts: 1,
       enqueuedAt: startedAt,
       startedAt,
-      finishedAt: event.finishedAt != null ? new Date(event.finishedAt) : /* @__PURE__ */ new Date()
+      finishedAt: event.finishedAt != null ? new Date(event.finishedAt) : /* @__PURE__ */ new Date(),
+      parallelGroup: event.parallelGroup
     }));
     this.emit({
       type: failed ? "step.failed" : "step.completed",
@@ -2595,8 +2701,17 @@ var WorkflowEngine = class {
     const history = await this.remoteHistory(run.id);
     let decision;
     try {
-      decision = await remote.executor.advance(run, history);
+      const silenceMs = this.remoteAdvanceSilenceMs;
+      const advance = remote.executor.advance(run, history);
+      decision = silenceMs != null ? await this.awaitWithLivenessDeadline(run.id, advance, silenceMs, () => new RemoteWorkflowTimeout(run.id, silenceMs)) : await advance;
     } catch (err) {
+      if (err instanceof RemoteWorkflowTimeout) {
+        await this.store.releaseRunLock(run.id);
+        return {
+          runId: run.id,
+          status: "running"
+        };
+      }
       const error = {
         message: err instanceof Error ? err.message : String(err),
         stack: err instanceof Error ? err.stack : void 0
@@ -2716,7 +2831,11 @@ var WorkflowEngine = class {
           attempts: 1,
           enqueuedAt: startedAt,
           startedAt,
-          finishedAt
+          finishedAt,
+          // Carry the worker's `parallelGroup` (a Python `ctx.gather` tags every step in the fan with
+          // the same group) onto the checkpoint, so the dashboard can render the steps as one parallel
+          // group instead of a sequential list. Additive: undefined for ordinary sequential steps.
+          parallelGroup: cmd.parallelGroup
         }));
         this.emit({
           type: cmd.error ? "step.failed" : "step.completed",
@@ -3245,11 +3364,13 @@ var WorkflowEngine = class {
     }
   }
   /**
-  * Await a remote result, but reject with `RemoteStepTimeout` if neither the result nor a heartbeat
-  * arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat`) rearms the
-  * window, so a worker that keeps beating stays alive past `timeoutMs`.
+  * Await `resultPromise`, but reject with `onTimeout()` if neither it settles nor a heartbeat for `id`
+  * arrives within `timeoutMs`. Each heartbeat (delivered via `transport.onHeartbeat`, looked up in
+  * `heartbeatResets` by `id`) REARMS the window, so a worker that keeps beating stays alive past
+  * `timeoutMs`. Shared by the remote-STEP liveness (keyed by stepId) and the remote-WORKFLOW advance
+  * liveness (keyed by runId); the caller supplies the timeout error and any extra cleanup via `onTimeout`.
   */
-  awaitWithHeartbeat(id, resultPromise, timeoutMs) {
+  awaitWithLivenessDeadline(id, resultPromise, timeoutMs, onTimeout) {
     return new Promise((resolve, reject) => {
       let timer;
       const cleanup = /* @__PURE__ */ __name(() => {
@@ -3259,8 +3380,7 @@ var WorkflowEngine = class {
       const arm = /* @__PURE__ */ __name(() => {
         timer = setTimeout(() => {
           cleanup();
-          this.pending.delete(id);
-          reject(new RemoteStepTimeout(id, timeoutMs));
+          reject(onTimeout());
         }, timeoutMs);
         timer.unref?.();
       }, "arm");
@@ -3278,6 +3398,14 @@ var WorkflowEngine = class {
       });
     });
   }
+  /** Remote-STEP liveness: await a worker result, timing out (with the step's pending-map cleanup) to
+  *  `RemoteStepTimeout` if it neither lands nor beats within `timeoutMs`. See {@link awaitWithLivenessDeadline}. */
+  awaitWithHeartbeat(id, resultPromise, timeoutMs) {
+    return this.awaitWithLivenessDeadline(id, resultPromise, timeoutMs, () => {
+      this.pending.delete(id);
+      return new RemoteStepTimeout(id, timeoutMs);
+    });
+  }
 };
 var RemoteStepError = class extends Error {
   static {
@@ -3351,11 +3479,12 @@ var RemoteWorkflowExecutor = class {
     };
     const decision = new Promise((resolve, reject) => {
       this.pending.set(taskId, resolve);
-      if (this.opts.timeoutMs) {
+      const { timeoutMs } = this.opts;
+      if (timeoutMs) {
         const timer = setTimeout(() => {
           this.pending.delete(taskId);
-          reject(new Error(`workflow task ${taskId} timed out after ${this.opts.timeoutMs}ms`));
-        }, this.opts.timeoutMs);
+          reject(new RemoteWorkflowTimeout(taskId, timeoutMs));
+        }, timeoutMs);
         timer.unref?.();
       }
     });
@@ -4042,6 +4171,7 @@ var InMemoryTransport = class {
   }
   handlers = /* @__PURE__ */ new Map();
   resultHandler;
+  heartbeatHandler;
   stepEventHandler;
   controlHandlers = /* @__PURE__ */ new Set();
   /** Register a fake worker handler for a step name. */
@@ -4056,7 +4186,12 @@ var InMemoryTransport = class {
   onResult(handler) {
     this.resultHandler = handler;
   }
-  onHeartbeat(_handler) {
+  onHeartbeat(handler) {
+    this.heartbeatHandler = handler;
+  }
+  /** Test hook: deliver a liveness heartbeat to the engine, as a real broker would on its channel. */
+  async emitHeartbeat(beat) {
+    await this.heartbeatHandler?.(beat);
   }
   /** Workflow step lifecycle: deliver to the engine's handler (async, mirroring a real broker). */
   async dispatchStepEvent(event) {
@@ -4083,6 +4218,7 @@ var InMemoryTransport = class {
   Entities,
   EventAccumulators,
   FatalError,
+  GatherError,
   InMemoryAdmissionBackend,
   InMemoryStateStore,
   InMemoryTransport,
@@ -4091,6 +4227,7 @@ var InMemoryTransport = class {
   RemoteStepError,
   RemoteStepTimeout,
   RemoteWorkflowExecutor,
+  RemoteWorkflowTimeout,
   STATE_STORE,
   STATE_STORE_CANONICAL,
   SignalTimeoutError,
@@ -4105,6 +4242,7 @@ var InMemoryTransport = class {
   attributeOperand,
   breakpointToken,
   collectMetrics,
+  createStepLogger,
   eventMatchOf,
   eventMatches,
   eventPrefix,