@blokjs/runner 0.6.21 → 0.7.0

package/dist/TriggerBase.js

@@ -14,9 +14,12 @@ import { resolveIdempotencyKey } from "./idempotency/resolveIdempotencyKey";
 import { CircuitBreaker } from "./monitoring/CircuitBreaker";
 import { ConcurrencyMetrics } from "./monitoring/ConcurrencyMetrics";
 import { HealthCheck } from "./monitoring/HealthCheck";
+import { ProcessErrorMetrics } from "./monitoring/ProcessErrorMetrics";
 import { PrometheusMetricsBridge } from "./monitoring/PrometheusMetricsBridge";
 import { RateLimiter } from "./monitoring/RateLimiter";
 import { TriggerMetricsCollector } from "./monitoring/TriggerMetricsCollector";
+import { captureError, setErrorSink } from "./observability/ErrorSink";
+import { createSentryErrorSink } from "./observability/SentryIntegration";
 import { DebounceCoordinator } from "./scheduling/DebounceCoordinator";
 import { DeferredDispatchSignal } from "./scheduling/DeferredDispatchSignal";
 import { DeferredRunScheduler } from "./scheduling/DeferredRunScheduler";
@@ -257,6 +260,12 @@ export default class TriggerBase extends Trigger {
             return;
         TriggerBase.crashHandlersInstalled = true;
         const onUncaught = (err) => {
+            // OBS-06 T8 — count the fatal-error event before the flip/rethrow.
+            ProcessErrorMetrics.getInstance().recordUnhandledRejection({
+                trigger_type: "process",
+                reason_class: err?.constructor?.name ?? "Error",
+            });
+            captureError(err, { source: "uncaughtException" });
             try {
                 const flipped = RunTracker.getInstance().markAllRunningRunsAsCrashed(err);
                 logger?.error?.(`[blok][crash-autoflip] uncaughtException — flipped ${flipped} running run(s) to crashed: ${err.stack || err.message}`);
@@ -273,6 +282,12 @@ export default class TriggerBase extends Trigger {
         };
         const onRejection = (reason) => {
             const err = reason instanceof Error ? reason : new Error(String(reason));
+            // OBS-06 T8 — count the rejection event.
+            ProcessErrorMetrics.getInstance().recordUnhandledRejection({
+                trigger_type: "process",
+                reason_class: err.constructor?.name ?? "Error",
+            });
+            captureError(err, { source: "unhandledRejection" });
             try {
                 const flipped = RunTracker.getInstance().markAllRunningRunsAsCrashed(err);
                 logger?.error?.(`[blok][crash-autoflip] unhandledRejection — flipped ${flipped} running run(s) to crashed: ${err.stack || err.message}`);
@@ -350,35 +365,6 @@ export default class TriggerBase extends Trigger {
                 catch {
                     // Best-effort.
                 }
-                // 4. Disconnect cross-process concurrency backend.
-                //
-                // PR 3 D5 — wrap disconnect() in a Promise.race timeout so a
-                // slow NATS drain doesn't hang past the SIGTERM-to-SIGKILL
-                // window. Default 10s; configurable via
-                // BLOK_BACKEND_DISCONNECT_TIMEOUT_MS. Timer is .unref()'d so
-                // it doesn't keep the event loop alive after a successful
-                // disconnect.
-                const backend = RunTracker.getInstance().getConcurrencyBackend();
-                if (backend) {
-                    const disconnectTimeoutMs = (() => {
-                        const raw = process.env.BLOK_BACKEND_DISCONNECT_TIMEOUT_MS;
-                        if (!raw || !/^\d+$/.test(raw))
-                            return 10_000;
-                        return Number(raw);
-                    })();
-                    try {
-                        await Promise.race([
-                            backend.disconnect(),
-                            new Promise((_, reject) => {
-                                const t = setTimeout(() => reject(new Error(`backend.disconnect() timed out after ${disconnectTimeoutMs}ms`)), disconnectTimeoutMs);
-                                t.unref?.();
-                            }),
-                        ]);
-                    }
-                    catch (err) {
-                        logger?.error?.(`[blok][shutdown] backend disconnect failed (or timed out): ${err instanceof Error ? err.message : String(err)}`);
-                    }
-                }
                 logger?.log?.("[blok][shutdown] graceful shutdown complete");
             }
             catch (err) {
@@ -466,6 +452,19 @@ export default class TriggerBase extends Trigger {
         catch (err) {
             logger?.error?.(`[blok][shutdown] setup failed: ${err instanceof Error ? err.message : String(err)}`);
         }
+        // Error sink (MO-ALERTS) — opt-in via SENTRY_DSN. Forwards uncaught
+        // exceptions / unhandled rejections to Sentry. Unset DSN → no sink
+        // installed → zero behaviour change. Setup failure is logged, never fatal.
+        try {
+            const dsn = process.env.SENTRY_DSN;
+            if (dsn) {
+                setErrorSink(createSentryErrorSink(dsn));
+                logger?.log?.("[blok][error-sink] Sentry error sink initialized (SENTRY_DSN set).");
+            }
+        }
+        catch (err) {
+            logger?.error?.(`[blok][error-sink] setup failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
     }
     /**
      * F6 — populate the `WorkflowRegistry` from `this.nodeMap.workflows`
@@ -652,6 +651,176 @@ export default class TriggerBase extends Trigger {
             this.hmr = null;
         }
     }
+    /**
+     * Re-entry trace setup (deferred timer fire). Rehydrates ctx.state +
+     * iteration cursors from the persisted snapshot and re-registers the
+     * AbortController. Extracted verbatim from run(); see the inline notes.
+     */
+    rehydrateDeferredRun(ctx, tracker) {
+        const ctxRecord = ctx;
+        const traceRunId = ctxRecord._traceRunId;
+        // Logger wrapping was already applied on the first pass — no
+        // need to re-wrap (and re-wrapping would double-route logs).
+        // PR 1 follow-up · A2 fix. The first-pass `finally` block
+        // unregisters the AbortController via `tracker.unregisterAbortController`.
+        // Without re-registering on re-entry, `tracker.abortRunningRun(runId)`
+        // can't fire the controller — the controller stays on
+        // `ctx._PRIVATE_.abortController` but the tracker's lookup
+        // returns undefined. Operator cancel of a `running` run that
+        // came from delayed/queued/debounced flips status to "cancelled"
+        // but the in-flight step never sees `ctx.signal.aborted`.
+        // Re-register here mirroring the first-pass branch below.
+        if (traceRunId) {
+            const privateSlot = ctx._PRIVATE_;
+            if (privateSlot?.abortController) {
+                tracker.registerAbortController(traceRunId, privateSlot.abortController);
+            }
+            // v0.6 prerequisite for wait-inside-primitives Phase 2 —
+            // rehydrate `ctx.state` from the persisted snapshot the
+            // runner took at the wait throw site. Two re-entry paths
+            // converge here:
+            //   1. In-process timer fire — same `ctx`, state already
+            //      populated. Rehydrate is a no-op (the parsed
+            //      snapshot equals current state); we still apply it
+            //      for uniformity and to forgive any micro-drift
+            //      between snapshot and current state if a malicious
+            //      caller re-enters with a tampered ctx.
+            //   2. Cross-process recovery (`recoverDispatches` →
+            //      `restoreDispatch` → `dispatchDeferred` with a
+            //      fresh ctx). Without rehydrate, state is empty and
+            //      forEach iteration index / loop accumulator / saga
+            //      progress are all lost.
+            //
+            // Mutates `ctx.state` IN PLACE rather than reassigning so
+            // the `vars: state` alias set up in `createContext` keeps
+            // pointing at the same object. Authors writing
+            // `ctx.vars[k] = v` continue to mutate the canonical
+            // store; otherwise we'd silently fork the two views.
+            const persistedRun = tracker.getStore().getRun(traceRunId);
+            if (persistedRun?.stateSnapshot) {
+                try {
+                    const parsed = JSON.parse(persistedRun.stateSnapshot);
+                    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                        const stateObj = ctx.state;
+                        for (const k of Object.keys(stateObj)) {
+                            delete stateObj[k];
+                        }
+                        Object.assign(stateObj, parsed);
+                    }
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    ctx.logger.logLevel("warn", `[blok][wait] failed to rehydrate ctx.state from snapshot: ${msg}. Proceeding with the ctx the runner was given.`);
+                }
+            }
+            // v0.6 wait-inside-primitives — rehydrate every NodeRun's
+            // `iteration_context` into a Map keyed by step NAME
+            // (not NodeRun id) so each primitive's run() can look
+            // itself up across defer/resume cycles. NodeRun ids
+            // CHANGE on every dispatchDeferred re-entry (tracker
+            // creates a fresh NodeRun per pass), but step names are
+            // stable across the run. Phase 4 — multiple primitives'
+            // cursors coexist (e.g. forEach > forEach > wait), one
+            // entry per primitive name. When two NodeRuns share a
+            // name (the rare case of two siblings at the same depth
+            // with the same step id), the LATEST wins via the
+            // startedAt-with-insertion-order tiebreak.
+            try {
+                const nodeRuns = tracker.getStore().getNodeRuns(traceRunId);
+                const sortedDesc = nodeRuns
+                    .map((n, idx) => ({ n, idx }))
+                    .filter(({ n }) => n.iterationContext !== undefined)
+                    .sort((a, b) => {
+                    const dt = b.n.startedAt - a.n.startedAt;
+                    return dt !== 0 ? dt : b.idx - a.idx;
+                });
+                const cursorMap = new Map();
+                for (const { n } of sortedDesc) {
+                    if (n.iterationContext === undefined)
+                        continue;
+                    // First write per name wins because sortedDesc is
+                    // latest-first; this gives each primitive its most
+                    // recent cursor.
+                    if (!cursorMap.has(n.nodeName)) {
+                        cursorMap.set(n.nodeName, n.iterationContext);
+                    }
+                }
+                if (cursorMap.size > 0) {
+                    ctx._blokIterationCursors = cursorMap;
+                }
+                // Back-compat `_blokIterationResume` (single-slot)
+                // keeps legacy callers working. Populated from the
+                // most recent cursor across the run.
+                const top = sortedDesc[0]?.n.iterationContext;
+                if (top) {
+                    ctx._blokIterationResume = top;
+                }
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                ctx.logger.logLevel("warn", `[blok][wait] failed to rehydrate iteration_context: ${msg}. primitives will resume from iteration 0.`);
+            }
+        }
+        return traceRunId;
+    }
+    /**
+     * First-pass trace setup. Starts the run record (replay + sub-workflow
+     * lineage from headers), registers the AbortController, and wraps the
+     * logger. Extracted verbatim from run().
+     */
+    startRunTrace(ctx, cfg, tracker) {
+        const ctxRecord = ctx;
+        const runner = this.getRunner(cfg);
+        const stepCount = runner.getStepCount?.() ?? cfg.steps?.length ?? 0;
+        // Tier 1 · replay lineage. The replay endpoint
+        // (TraceRouter.POST /__blok/runs/:id/replay) sets
+        // `X-Blok-Replay-Of: <originalRunId>` on the dispatched HTTP
+        // request. Read it here so the new run carries `replayOf` and
+        // Studio can render a "Replay of #..." breadcrumb.
+        const reqHeaders = (ctx.request?.headers ?? {});
+        const replayOfHeader = reqHeaders["x-blok-replay-of"] ?? reqHeaders["X-Blok-Replay-Of"];
+        const replayOf = Array.isArray(replayOfHeader)
+            ? replayOfHeader[0]
+            : typeof replayOfHeader === "string"
+                ? replayOfHeader
+                : undefined;
+        // G2 (v0.6) · sub-workflow lineage across the HTTP boundary.
+        // `SubworkflowNode.dispatchHttpSelf` sets these headers on the
+        // outbound self-call so the receiver's run record carries the
+        // parent ids. Without this, an http-self child would appear
+        // as a fresh top-level run with no Studio breadcrumb.
+        const parentRunId = pickHeader(reqHeaders, "x-blok-parent-run-id");
+        const parentNodeRunId = pickHeader(reqHeaders, "x-blok-parent-node-run-id");
+        const run = tracker.startRun({
+            workflowName: cfg.name || ctx.workflow_name || "unknown",
+            workflowPath: ctx.workflow_path || "",
+            triggerType: this.constructor.name.replace("Trigger", "").toLowerCase() || "unknown",
+            triggerSummary: this.buildTraceTriggerSummary(ctx),
+            nodeCount: stepCount,
+            replayOf,
+            parentRunId,
+            parentNodeRunId,
+        });
+        const traceRunId = run.id;
+        ctxRecord._traceRunId = run.id;
+        // Carry the sub-workflow depth across the HTTP hop so the
+        // recursion guard in nested children still fires.
+        const depthHeader = pickHeader(reqHeaders, "x-blok-subworkflow-depth");
+        const parsedDepth = depthHeader ? Number.parseInt(depthHeader, 10) : Number.NaN;
+        if (Number.isFinite(parsedDepth) && parsedDepth > 0) {
+            ctxRecord._subworkflowDepth = parsedDepth;
+        }
+        // Tier 2 follow-up · register the ctx's AbortController so the
+        // cancel API can fire it for `running` runs. Stashed on
+        // _PRIVATE_ by createContext; lookup via the optional shape.
+        const privateSlot = ctx._PRIVATE_;
+        if (privateSlot?.abortController) {
+            tracker.registerAbortController(run.id, privateSlot.abortController);
+        }
+        // Wrap logger to forward log entries to RunTracker
+        ctx.logger = new TracingLogger(ctx.logger, run.id, tracker);
+        return traceRunId;
+    }
     async run(ctx, configuration = this.configuration) {
         this.inFlightRequests++;
         const runStart = performance.now();
@@ -678,160 +847,10 @@ export default class TriggerBase extends Trigger {
         const ctxRecord = ctx;
         const isReentryAtTrace = ctxRecord._blokDispatchReentry === true;
         if (tracker.active && isReentryAtTrace) {
-            traceRunId = ctxRecord._traceRunId;
-            // Logger wrapping was already applied on the first pass — no
-            // need to re-wrap (and re-wrapping would double-route logs).
-            // PR 1 follow-up · A2 fix. The first-pass `finally` block
-            // unregisters the AbortController via `tracker.unregisterAbortController`.
-            // Without re-registering on re-entry, `tracker.abortRunningRun(runId)`
-            // can't fire the controller — the controller stays on
-            // `ctx._PRIVATE_.abortController` but the tracker's lookup
-            // returns undefined. Operator cancel of a `running` run that
-            // came from delayed/queued/debounced flips status to "cancelled"
-            // but the in-flight step never sees `ctx.signal.aborted`.
-            // Re-register here mirroring the first-pass branch below.
-            if (traceRunId) {
-                const privateSlot = ctx._PRIVATE_;
-                if (privateSlot?.abortController) {
-                    tracker.registerAbortController(traceRunId, privateSlot.abortController);
-                }
-                // v0.6 prerequisite for wait-inside-primitives Phase 2 —
-                // rehydrate `ctx.state` from the persisted snapshot the
-                // runner took at the wait throw site. Two re-entry paths
-                // converge here:
-                //   1. In-process timer fire — same `ctx`, state already
-                //      populated. Rehydrate is a no-op (the parsed
-                //      snapshot equals current state); we still apply it
-                //      for uniformity and to forgive any micro-drift
-                //      between snapshot and current state if a malicious
-                //      caller re-enters with a tampered ctx.
-                //   2. Cross-process recovery (`recoverDispatches` →
-                //      `restoreDispatch` → `dispatchDeferred` with a
-                //      fresh ctx). Without rehydrate, state is empty and
-                //      forEach iteration index / loop accumulator / saga
-                //      progress are all lost.
-                //
-                // Mutates `ctx.state` IN PLACE rather than reassigning so
-                // the `vars: state` alias set up in `createContext` keeps
-                // pointing at the same object. Authors writing
-                // `ctx.vars[k] = v` continue to mutate the canonical
-                // store; otherwise we'd silently fork the two views.
-                const persistedRun = tracker.getStore().getRun(traceRunId);
-                if (persistedRun?.stateSnapshot) {
-                    try {
-                        const parsed = JSON.parse(persistedRun.stateSnapshot);
-                        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
-                            const stateObj = ctx.state;
-                            for (const k of Object.keys(stateObj)) {
-                                delete stateObj[k];
-                            }
-                            Object.assign(stateObj, parsed);
-                        }
-                    }
-                    catch (err) {
-                        const msg = err instanceof Error ? err.message : String(err);
-                        ctx.logger.logLevel("warn", `[blok][wait] failed to rehydrate ctx.state from snapshot: ${msg}. Proceeding with the ctx the runner was given.`);
-                    }
-                }
-                // v0.6 wait-inside-primitives — rehydrate every NodeRun's
-                // `iteration_context` into a Map keyed by step NAME
-                // (not NodeRun id) so each primitive's run() can look
-                // itself up across defer/resume cycles. NodeRun ids
-                // CHANGE on every dispatchDeferred re-entry (tracker
-                // creates a fresh NodeRun per pass), but step names are
-                // stable across the run. Phase 4 — multiple primitives'
-                // cursors coexist (e.g. forEach > forEach > wait), one
-                // entry per primitive name. When two NodeRuns share a
-                // name (the rare case of two siblings at the same depth
-                // with the same step id), the LATEST wins via the
-                // startedAt-with-insertion-order tiebreak.
-                try {
-                    const nodeRuns = tracker.getStore().getNodeRuns(traceRunId);
-                    const sortedDesc = nodeRuns
-                        .map((n, idx) => ({ n, idx }))
-                        .filter(({ n }) => n.iterationContext !== undefined)
-                        .sort((a, b) => {
-                        const dt = b.n.startedAt - a.n.startedAt;
-                        return dt !== 0 ? dt : b.idx - a.idx;
-                    });
-                    const cursorMap = new Map();
-                    for (const { n } of sortedDesc) {
-                        if (n.iterationContext === undefined)
-                            continue;
-                        // First write per name wins because sortedDesc is
-                        // latest-first; this gives each primitive its most
-                        // recent cursor.
-                        if (!cursorMap.has(n.nodeName)) {
-                            cursorMap.set(n.nodeName, n.iterationContext);
-                        }
-                    }
-                    if (cursorMap.size > 0) {
-                        ctx._blokIterationCursors = cursorMap;
-                    }
-                    // Back-compat `_blokIterationResume` (single-slot)
-                    // keeps legacy callers working. Populated from the
-                    // most recent cursor across the run.
-                    const top = sortedDesc[0]?.n.iterationContext;
-                    if (top) {
-                        ctx._blokIterationResume = top;
-                    }
-                }
-                catch (err) {
-                    const msg = err instanceof Error ? err.message : String(err);
-                    ctx.logger.logLevel("warn", `[blok][wait] failed to rehydrate iteration_context: ${msg}. primitives will resume from iteration 0.`);
-                }
-            }
+            traceRunId = this.rehydrateDeferredRun(ctx, tracker);
         }
         else if (tracker.active) {
-            const runner = this.getRunner(cfg);
-            const stepCount = runner.getStepCount?.() ?? cfg.steps?.length ?? 0;
-            // Tier 1 · replay lineage. The replay endpoint
-            // (TraceRouter.POST /__blok/runs/:id/replay) sets
-            // `X-Blok-Replay-Of: <originalRunId>` on the dispatched HTTP
-            // request. Read it here so the new run carries `replayOf` and
-            // Studio can render a "Replay of #..." breadcrumb.
-            const reqHeaders = (ctx.request?.headers ?? {});
-            const replayOfHeader = reqHeaders["x-blok-replay-of"] ?? reqHeaders["X-Blok-Replay-Of"];
-            const replayOf = Array.isArray(replayOfHeader)
-                ? replayOfHeader[0]
-                : typeof replayOfHeader === "string"
-                    ? replayOfHeader
-                    : undefined;
-            // G2 (v0.6) · sub-workflow lineage across the HTTP boundary.
-            // `SubworkflowNode.dispatchHttpSelf` sets these headers on the
-            // outbound self-call so the receiver's run record carries the
-            // parent ids. Without this, an http-self child would appear
-            // as a fresh top-level run with no Studio breadcrumb.
-            const parentRunId = pickHeader(reqHeaders, "x-blok-parent-run-id");
-            const parentNodeRunId = pickHeader(reqHeaders, "x-blok-parent-node-run-id");
-            const run = tracker.startRun({
-                workflowName: cfg.name || ctx.workflow_name || "unknown",
-                workflowPath: ctx.workflow_path || "",
-                triggerType: this.constructor.name.replace("Trigger", "").toLowerCase() || "unknown",
-                triggerSummary: this.buildTraceTriggerSummary(ctx),
-                nodeCount: stepCount,
-                replayOf,
-                parentRunId,
-                parentNodeRunId,
-            });
-            traceRunId = run.id;
-            ctxRecord._traceRunId = run.id;
-            // Carry the sub-workflow depth across the HTTP hop so the
-            // recursion guard in nested children still fires.
-            const depthHeader = pickHeader(reqHeaders, "x-blok-subworkflow-depth");
-            const parsedDepth = depthHeader ? Number.parseInt(depthHeader, 10) : Number.NaN;
-            if (Number.isFinite(parsedDepth) && parsedDepth > 0) {
-                ctxRecord._subworkflowDepth = parsedDepth;
-            }
-            // Tier 2 follow-up · register the ctx's AbortController so the
-            // cancel API can fire it for `running` runs. Stashed on
-            // _PRIVATE_ by createContext; lookup via the optional shape.
-            const privateSlot = ctx._PRIVATE_;
-            if (privateSlot?.abortController) {
-                tracker.registerAbortController(run.id, privateSlot.abortController);
-            }
-            // Wrap logger to forward log entries to RunTracker
-            ctx.logger = new TracingLogger(ctx.logger, run.id, tracker);
+            traceRunId = this.startRunTrace(ctx, cfg, tracker);
         }
         try {
             // --- Scheduling gates (Tier 2 #5 + #7) ---
@@ -1206,6 +1225,22 @@ export default class TriggerBase extends Trigger {
                 !(err instanceof WaitDispatchRequest)) {
                 tracker.failRun(traceRunId, err instanceof Error ? err : new Error(String(err)));
             }
+            // OBS-05 T2 — emit blok_workflow_errors_total with the resolved
+            // terminal status so dashboards separate failed / crashed /
+            // timedOut / throttled / cancelled. DeferredDispatchSignal and
+            // WaitDispatchRequest are deferral control-flow, not errors —
+            // skip them. The status is read back from the run record AFTER
+            // the upstream markers (markRunTimedOut / markRunThrottled /
+            // abortRunningRun / failRun) have flipped it.
+            if (traceRunId && !(err instanceof DeferredDispatchSignal) && !(err instanceof WaitDispatchRequest)) {
+                const resolvedStatus = tracker.getRun(traceRunId)?.status;
+                this.metricsBridge.recordError(err instanceof Error ? err.constructor.name : "Error", {
+                    workflow_name: cfg.name || "",
+                    workflow_version: `${cfg.version}`,
+                    env: process.env.NODE_ENV || "development",
+                    status: resolvedStatus,
+                });
+            }
             throw err;
         }
         finally {