@blokjs/runner 0.2.1 → 0.4.0

package/dist/tracing/RunTracker.js

@@ -5,6 +5,86 @@ import https from "node:https";
 import { v4 as uuid } from "uuid";
 import { InMemoryRunStore } from "./InMemoryRunStore";
 import { createStore } from "./createStore";
+import { sanitize } from "./sanitize";
+/**
+ * Cap on the number of `NODE_ATTEMPT_FAILED` entries kept on a single
+ * `NodeRun.attempts` array. Bounds store growth on extreme retry counts —
+ * a runaway loop generating 1000 attempts can't bloat the run store. The
+ * latest attempts are always preserved (older ones are dropped).
+ */
+const MAX_STORED_ATTEMPTS = 10;
+/**
+ * PR 1 follow-up · terminal status guard.
+ *
+ * Once a run reaches a terminal status, late-arriving completeRun/failRun
+ * calls (e.g., from a runner that didn't see a parallel cancel) must NOT
+ * overwrite it. Cancellation, expiry, throttling, crashes, and timeouts
+ * all win over a stale "the steps finished" signal.
+ */
+const TERMINAL_STATUSES = new Set([
+    "completed",
+    "failed",
+    "cancelled",
+    "throttled",
+    "expired",
+    "crashed",
+    "timedOut",
+]);
+/**
+ * Build a {@link RunErrorDetail} from any thrown error. When the source is
+ * a typed `BlokError` (master plan §17), all 17+ structured fields are
+ * preserved; otherwise the legacy `{message, stack}` shape falls through.
+ *
+ * Detection is duck-typed against the `category` field (BlokError carries
+ * a `category` enum value like `"DEPENDENCY"`; vanilla `Error` never
+ * does). This avoids a hard import dependency from the tracing layer
+ * onto `@blokjs/shared`.
+ */
+function toRunErrorDetail(error) {
+    if (error === null || error === undefined) {
+        return { message: "unknown error" };
+    }
+    if (typeof error !== "object") {
+        return { message: String(error) };
+    }
+    const e = error;
+    const detail = {
+        message: typeof e.message === "string" ? e.message : "unknown error",
+    };
+    if (typeof e.stack === "string")
+        detail.stack = e.stack;
+    // Structured BlokError fields. We accept either runner-side
+    // (`errorCode` getter on BlokError) or raw NodeErrorPayload (`code`)
+    // shapes — failNode is called with the BlokError instance, but the
+    // payload variant covers RunStore re-hydration paths.
+    const code = e.errorCode ?? e.code;
+    if (typeof code === "string" && code.length > 0)
+        detail.code = code;
+    if (typeof e.category === "string")
+        detail.category = e.category;
+    if (typeof e.severity === "string")
+        detail.severity = e.severity;
+    if (typeof e.httpStatus === "number")
+        detail.httpStatus = e.httpStatus;
+    if (typeof e.retryable === "boolean")
+        detail.retryable = e.retryable;
+    if (typeof e.retryAfterMs === "number")
+        detail.retryAfterMs = e.retryAfterMs;
+    if (typeof e.description === "string" && e.description.length > 0)
+        detail.description = e.description;
+    if (typeof e.remediation === "string" && e.remediation.length > 0)
+        detail.remediation = e.remediation;
+    if (typeof e.docUrl === "string" && e.docUrl.length > 0)
+        detail.docUrl = e.docUrl;
+    if (e.details !== undefined && e.details !== null)
+        detail.details = e.details;
+    if (e.contextSnapshot !== undefined && e.contextSnapshot !== null)
+        detail.contextSnapshot = e.contextSnapshot;
+    if (Array.isArray(e.causes) && e.causes.length > 0) {
+        detail.causes = e.causes.filter((c) => typeof c === "object" && c !== null);
+    }
+    return detail;
+}
 export class RunTracker extends EventEmitter {
     store;
     maxRuns;
@@ -41,6 +121,12 @@ export class RunTracker extends EventEmitter {
     }
     // === Workflow Lifecycle ===
     startRun(opts) {
+        // Phase 2.1 · environment scoping. Read `BLOK_ENV` (default
+        // `production`) so every run carries the env it was triggered
+        // against. Studio's EnvChip (`useEnvScope.current`) filters
+        // list views by this field. Old runs without the field still
+        // match `production` via the post-filter default.
+        const environment = (process.env.BLOK_ENV || "production").trim() || "production";
         const run = {
             id: `run_${uuid().replace(/-/g, "").slice(0, 12)}`,
             workflowName: opts.workflowName,
@@ -53,6 +139,15 @@ export class RunTracker extends EventEmitter {
             completedNodes: 0,
             tags: opts.tags,
             metadata: opts.metadata,
+            environment,
+            replayOf: opts.replayOf,
+            parentRunId: opts.parentRunId,
+            parentNodeRunId: opts.parentNodeRunId,
+            scheduledAt: opts.scheduledAt,
+            expiresAt: opts.expiresAt,
+            debounceKey: opts.debounceKey,
+            debounceMode: opts.debounceMode,
+            pingCount: opts.pingCount,
         };
         this.store.saveRun(run);
         this.emitEvent(run.id, run.workflowName, "RUN_STARTED", undefined, undefined, {
@@ -68,6 +163,13 @@ export class RunTracker extends EventEmitter {
         const run = this.store.getRun(runId);
         if (!run)
             return;
+        // PR 1 follow-up · terminal-status guard. Don't overwrite a run that
+        // has already reached a terminal status (cancelled / expired / etc.)
+        // — a late completeRun from a runner that didn't see a parallel
+        // cancel must not flip the status back. Defense in depth against the
+        // REVIEW.md A2 class of bug.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
         const finishedAt = Date.now();
         const durationMs = finishedAt - run.startedAt;
         this.store.updateRun(runId, {
@@ -85,22 +187,468 @@ export class RunTracker extends EventEmitter {
         const run = this.store.getRun(runId);
         if (!run)
             return;
+        // PR 1 follow-up · terminal-status guard. Same rationale as completeRun.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
         const finishedAt = Date.now();
         const durationMs = finishedAt - run.startedAt;
         this.store.updateRun(runId, {
             status: "failed",
             finishedAt,
             durationMs,
-            error: {
-                message: error.message,
-                stack: error.stack,
-            },
+            error: toRunErrorDetail(error),
         });
         this.emitEvent(runId, run.workflowName, "RUN_FAILED", undefined, undefined, {
             durationMs,
-            error: { message: error.message, stack: error.stack },
+            error: toRunErrorDetail(error),
+        });
+    }
+    /**
+     * Tier 2 #6 — mark a run as throttled because the concurrency gate
+     * denied it before any step executed. Distinct from `failRun` because
+     * no step ran; nothing produced an error. Studio surfaces a Throttled
+     * badge and SSE subscribers see a granular `RUN_THROTTLED` event.
+     */
+    markRunThrottled(runId, info) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        // Review fix-up · BUG-1. Don't overwrite a terminal status. A
+        // concurrent operator-cancel or crash auto-flip might have flipped
+        // the run between read and write; preserve the earlier terminal
+        // outcome rather than re-marking as throttled.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
+        const finishedAt = Date.now();
+        const durationMs = finishedAt - run.startedAt;
+        this.store.updateRun(runId, {
+            status: "throttled",
+            finishedAt,
+            durationMs,
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_THROTTLED", undefined, undefined, {
+            durationMs,
+            concurrencyKey: info.concurrencyKey,
+            concurrencyLimit: info.concurrencyLimit,
+            currentInFlight: info.currentInFlight,
+        });
+    }
+    /**
+     * Tier 2 #6 follow-up — mark a run as queued because the concurrency
+     * gate denied it AND the trigger is configured with `onLimit: "queue"`.
+     * The run will be re-attempted after `scheduledAt`; `scheduledAt` is
+     * persisted on the run record so Studio can render a "queued · retries
+     * at <time>" badge.
+     *
+     * Distinct from `markRunThrottled` because queued runs WILL eventually
+     * execute (or stay queued indefinitely until a slot frees), while
+     * throttled runs are terminal and `failRun` semantics are skipped.
+     *
+     * Caller is responsible for actually scheduling the retry via
+     * `DeferredRunScheduler`. This method only flips status + emits the
+     * `RUN_QUEUED` event. Re-marking with a later `scheduledAt` updates
+     * the field (used when re-defer happens after a timer-fired re-acquire
+     * also fails).
+     */
+    markRunQueued(runId, info) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        // Review fix-up · BUG-1. Don't overwrite a terminal status (e.g.,
+        // `cancelled` from a concurrent operator-cancel during the
+        // onLimit:queue re-defer race). The TTL-expired path is handled
+        // separately in TriggerBase via QueueExpiredError.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
+        this.store.updateRun(runId, {
+            status: "queued",
+            scheduledAt: info.scheduledAt,
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_QUEUED", undefined, undefined, {
+            concurrencyKey: info.concurrencyKey,
+            concurrencyLimit: info.concurrencyLimit,
+            currentInFlight: info.currentInFlight,
+            scheduledAt: info.scheduledAt,
+        });
+    }
+    // === Scheduling lifecycle (Tier 2 #5 + #7) ===
+    /**
+     * Tier 2 #5 — mark a run as `delayed`. Called immediately after
+     * `startRun` for runs that should be deferred. The run record carries
+     * `scheduledAt` (and optionally `expiresAt`) so Studio can render a
+     * "Delayed → fires at <time>" badge.
+     *
+     * Caller is responsible for actually scheduling the dispatch via
+     * `DeferredRunScheduler`. This method only flips status + emits the
+     * `RUN_DELAYED` event.
+     */
+    markRunDelayed(runId, info) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        // Review fix-up · BUG-1. Don't overwrite a terminal status — e.g.,
+        // a wait.for() re-entry race where the operator cancelled the run
+        // while WaitDispatchRequest was being thrown.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
+        this.store.updateRun(runId, {
+            status: "delayed",
+            scheduledAt: info.scheduledAt,
+            expiresAt: info.expiresAt,
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_DELAYED", undefined, undefined, {
+            scheduledAt: info.scheduledAt,
+            delayMs: info.delayMs,
+            expiresAt: info.expiresAt,
         });
     }
+    /**
+     * Tier 2 #5 — mark a run as `expired` because its TTL was exceeded
+     * before dispatch. Distinct from `failed` (no step ran) and
+     * `cancelled` (operator action — TTL is automatic).
+     */
+    markRunExpired(runId, info) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        // Review fix-up · BUG-1. Don't overwrite a terminal status. A
+        // run that was cancelled by an operator before the dispatch timer
+        // fired should stay `cancelled`, not flip to `expired`.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
+        const finishedAt = info.expiredAt;
+        const durationMs = finishedAt - run.startedAt;
+        const lateBy = info.expiredAt - info.expiresAt;
+        this.store.updateRun(runId, {
+            status: "expired",
+            finishedAt,
+            durationMs,
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_EXPIRED", undefined, undefined, {
+            expiresAt: info.expiresAt,
+            expiredAt: info.expiredAt,
+            lateBy,
+        });
+    }
+    /**
+     * Tier 2 #7 — mark a run as `debounced`. In **leading** mode this is
+     * terminal: the ping was suppressed because a sibling fired
+     * immediately (`intoRunId` carries the sibling's id). In **trailing**
+     * mode this is transient: the same run is marked `debounced` while
+     * the timer is active and flips to `running` when the window closes
+     * (no separate transition method needed — `tracker` updates status
+     * directly via store before invoking the runner).
+     */
+    markRunDebounced(runId, info) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        // Review fix-up · BUG-1. Don't overwrite a terminal status. A
+        // trailing debounce timer firing into a cancelled active run
+        // should NOT flip the run back to debounced.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
+        const isTerminal = info.mode === "leading" && info.intoRunId !== undefined;
+        const finishedAt = isTerminal ? Date.now() : undefined;
+        const durationMs = isTerminal && finishedAt ? finishedAt - run.startedAt : undefined;
+        this.store.updateRun(runId, {
+            status: "debounced",
+            debounceKey: info.debounceKey,
+            debounceMode: info.mode,
+            pingCount: info.pingCount,
+            scheduledAt: info.scheduledAt,
+            ...(isTerminal ? { finishedAt, durationMs } : {}),
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_DEBOUNCED", undefined, undefined, {
+            debounceKey: info.debounceKey,
+            mode: info.mode,
+            intoRunId: info.intoRunId,
+            pingCount: info.pingCount,
+            scheduledAt: info.scheduledAt,
+        });
+    }
+    /**
+     * Tier 2 quick-wins — mark a run as `crashed` (uncaught exception,
+     * OOM, signal). Distinct from `failRun` because the failure was
+     * NOT a step's `process()` throwing — it was the runner itself
+     * giving up. Currently manual; call from custom triggers / ops
+     * harnesses when uncaught failures are detected.
+     */
+    markRunCrashed(runId, info) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        // Review fix-up · BUG-1. Don't overwrite a terminal status. A
+        // run that was already cancelled / failed / timedOut shouldn't
+        // be flipped to crashed by the boot orphan-recovery pass.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
+        const finishedAt = Date.now();
+        const durationMs = finishedAt - run.startedAt;
+        this.store.updateRun(runId, {
+            status: "crashed",
+            finishedAt,
+            durationMs,
+            error: toRunErrorDetail(info.error),
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_CRASHED", undefined, undefined, {
+            durationMs,
+            error: toRunErrorDetail(info.error),
+        });
+    }
+    /**
+     * Tier 2 quick-wins follow-up — bulk-flip every run currently in
+     * `running` status to `crashed`. Returns the count flipped.
+     *
+     * Used by:
+     * - Process-level uncaught-exception handlers
+     *   (`TriggerBase.installCrashHandlers`) — flip in-flight runs
+     *   before the process dies.
+     * - Boot recovery (`TriggerBase.recoverOrphanedRuns`) — flip runs
+     *   that were `running` from the previous (dead) process.
+     *
+     * Synchronous + safe to call from a `process.on("uncaughtException")`
+     * handler (which can't await). Backed by sync sqlite/in-memory
+     * writes that complete before the handler returns.
+     *
+     * Optional `opts.maxStartedAt` filter — only flip runs whose
+     * `startedAt` is at or before this timestamp. Used by boot recovery
+     * to avoid flipping runs from the current (live) process.
+     */
+    markAllRunningRunsAsCrashed(error, opts) {
+        // PR 1 follow-up · A1 fix. `getRuns` defaults `opts?.limit ?? 50` in
+        // SqliteRunStore — left unbounded, this method silently flips at
+        // most 50 orphans per call. Loop until the store returns fewer rows
+        // than the page size (= no more matches under the LIMIT).
+        //
+        // Bounded outer loop: cap at 1000 iterations defensively. With the
+        // 50-row page size that's 50K orphans handled per single call —
+        // well above any realistic boot-recovery scenario.
+        let totalFlipped = 0;
+        const PAGE_SIZE = 50; // mirrors SqliteRunStore.getRuns default LIMIT
+        const MAX_PAGES = 1000;
+        for (let page = 0; page < MAX_PAGES; page++) {
+            // Snapshot the runs first — markRunCrashed mutates the store and
+            // could perturb iteration if we read+update inline.
+            const { runs } = this.store.getRuns({ status: "running" });
+            const candidates = opts?.maxStartedAt !== undefined ? runs.filter((r) => r.startedAt <= opts.maxStartedAt) : runs;
+            if (candidates.length === 0)
+                break;
+            for (const run of candidates) {
+                this.markRunCrashed(run.id, { error });
+            }
+            totalFlipped += candidates.length;
+            // If we got fewer rows than the page size, the store has no more
+            // matches under the LIMIT — exit early.
+            if (runs.length < PAGE_SIZE)
+                break;
+        }
+        return totalFlipped;
+    }
+    /**
+     * Tier 2 quick-wins — mark a run as `timedOut` because a step's
+     * final retry attempt exceeded its `maxDuration` cap. Distinct
+     * from `failed` so SLA dashboards can separate timeout-driven
+     * failures (network / capacity) from logic failures (bugs).
+     * Auto-called by `RunnerSteps` on final-attempt `StepTimeoutError`.
+     */
+    markRunTimedOut(runId, info) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        // Review fix-up · BUG-1. Don't overwrite a terminal status — a
+        // run that was cancelled mid-step shouldn't flip to timedOut
+        // when the maxDuration timer fires after the cancel.
+        if (TERMINAL_STATUSES.has(run.status))
+            return;
+        const finishedAt = Date.now();
+        const durationMs = finishedAt - run.startedAt;
+        this.store.updateRun(runId, {
+            status: "timedOut",
+            finishedAt,
+            durationMs,
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_TIMED_OUT", undefined, undefined, {
+            durationMs,
+            stepId: info.stepId,
+            maxDurationMs: info.maxDurationMs,
+            attemptsExhausted: info.attemptsExhausted,
+        });
+    }
+    /**
+     * Tier 2 #7 — record an additional ping into an existing trailing-mode
+     * debounce window. Increments `pingCount` and updates `scheduledAt`.
+     * Does NOT emit a new event (avoid event-stream bloat under burst).
+     */
+    recordDebouncePing(runId, opts) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        this.store.updateRun(runId, {
+            pingCount: opts.pingCount,
+            scheduledAt: opts.scheduledAt,
+        });
+    }
+    /**
+     * Tier 2 #7 — transition a `delayed`/`debounced` run into `running`
+     * when its timer fires. Studio sees the status change via the
+     * existing run-update SSE stream.
+     */
+    transitionRunToRunning(runId) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return;
+        this.store.updateRun(runId, {
+            status: "running",
+            startedAt: run.startedAt, // preserve the original submission time
+        });
+    }
+    /**
+     * Tier 2 polish — cancel a pending (delayed/debounced/queued) run.
+     * Idempotent. Returns true when the run existed AND was in a cancellable
+     * state; false when the run doesn't exist OR is already running/completed/
+     * failed/throttled/expired/crashed/timedOut/cancelled.
+     *
+     * **Caller responsibility**: this method only updates the run record
+     * (status → `"cancelled"`) and emits `RUN_CANCELLED`. The caller must
+     * separately clear any pending scheduler timers via
+     * `DeferredRunScheduler.getInstance().cancel(runId)` and (when applicable)
+     * `DebounceCoordinator.getInstance().cancel(workflowName, debounceKey)`.
+     * Done this way to avoid an import cycle from tracing → scheduling.
+     */
+    cancelRun(runId, options) {
+        const run = this.store.getRun(runId);
+        if (!run)
+            return false;
+        // Tier 2 follow-up · "running" added so cooperative AbortSignal
+        // cancellation can flip status to "cancelled" before the in-flight
+        // step throws `RunCancelledError`. The tracker's `abortRunningRun`
+        // calls this method right after firing the AbortController.
+        const cancellable = ["delayed", "debounced", "queued", "running"];
+        if (!cancellable.includes(run.status))
+            return false;
+        const previousStatus = run.status;
+        const finishedAt = Date.now();
+        const durationMs = finishedAt - run.startedAt;
+        this.store.updateRun(runId, {
+            status: "cancelled",
+            finishedAt,
+            durationMs,
+        });
+        this.emitEvent(runId, run.workflowName, "RUN_CANCELLED", undefined, undefined, {
+            durationMs,
+            previousStatus,
+        });
+        // PR 5 G1 — cascade to fire-and-forget children. Sub-workflow
+        // children with `wait: true` (sync) cancel automatically via the
+        // AbortSignal chain in createChildContext; children with
+        // `wait: false` (async / fire-and-forget) need explicit cascade
+        // because the parent step has already returned before the cancel.
+        // Walk getRunsByParent recursively (bounded by
+        // BLOK_MAX_SUBWORKFLOW_DEPTH).
+        if (options?.cascade !== false) {
+            const children = this.store.getRunsByParent(runId);
+            for (const child of children) {
+                if (cancellable.includes(child.status)) {
+                    // Recursive — bounded by max-depth; each level reduces
+                    // the candidate pool until none remain.
+                    this.cancelRun(child.id, { cascade: true });
+                }
+            }
+        }
+        return true;
+    }
+    // === Cooperative cancellation (Tier 2 follow-up) ===
+    /**
+     * Per-process map from runId to the AbortController owned by the
+     * trigger's createContext call. Populated by TriggerBase right after
+     * `startRun()`; cleared in TriggerBase's finally block. Used by
+     * `abortRunningRun` to fire the signal when an operator cancels a
+     * `running` run via the cancel API.
+     */
+    abortControllers = new Map();
+    registerAbortController(runId, controller) {
+        this.abortControllers.set(runId, controller);
+    }
+    unregisterAbortController(runId) {
+        this.abortControllers.delete(runId);
+    }
+    /**
+     * Tier 2 follow-up · cooperative cancellation for `running` runs.
+     *
+     * Fires the run's AbortController (so `ctx.signal.aborted` becomes
+     * true and any node consulting it can abort early) AND flips the run
+     * status to `"cancelled"` immediately via `cancelRun`. RunnerSteps'
+     * between-step abort check throws `RunCancelledError` shortly after,
+     * which TriggerBase catches without re-flipping the status.
+     *
+     * Returns true when an AbortController was registered for this run
+     * AND the status was successfully flipped; false otherwise (run not
+     * found, run not in `running` status, or no controller registered —
+     * e.g. controller already cleaned up).
+     */
+    abortRunningRun(runId) {
+        const run = this.store.getRun(runId);
+        if (!run || run.status !== "running")
+            return false;
+        const controller = this.abortControllers.get(runId);
+        if (controller) {
+            try {
+                controller.abort();
+            }
+            catch {
+                // AbortController.abort never throws on first call; double-abort is safe.
+            }
+        }
+        // Flip status now so polls return cancelled immediately. The
+        // in-flight step's throw will land in TriggerBase.run's catch
+        // shortly; the catch sees status is already terminal and skips
+        // failRun (RunCancelledError instanceof check).
+        return this.cancelRun(runId);
+    }
+    // === Concurrency gate pass-throughs (Tier 2 #6) ===
+    /**
+     * Tier 2 #6 follow-up · cross-process concurrency backend.
+     *
+     * When set (via {@link setConcurrencyBackend}), the tracker's
+     * `acquireConcurrencySlot` and `releaseConcurrencySlot` methods
+     * delegate to the backend instead of the local sync `RunStore` impl.
+     * Used to coordinate across processes via NATS KV / Redis.
+     *
+     * Default `null` — preserves zero-overhead in-process behavior.
+     * Trigger packages install a backend in `listen()` when the operator
+     * sets `BLOK_CONCURRENCY_BACKEND=nats-kv`.
+     */
+    concurrencyBackend = null;
+    setConcurrencyBackend(backend) {
+        this.concurrencyBackend = backend;
+    }
+    getConcurrencyBackend() {
+        return this.concurrencyBackend;
+    }
+    /**
+     * Acquire a concurrency slot for `(workflowName, concurrencyKey)`.
+     * Delegates to the configured cross-process backend when set; falls
+     * back to the local sync `RunStore` impl otherwise.
+     *
+     * Async — the cross-process backend (NATS KV) is async-only. The
+     * sync fallback is wrapped in `Promise.resolve()` so the call site
+     * is uniform.
+     */
+    async acquireConcurrencySlot(workflowName, concurrencyKey, concurrencyLimit, runId, leaseExpiresAt) {
+        if (this.concurrencyBackend) {
+            return this.concurrencyBackend.acquireSlot(workflowName, concurrencyKey, concurrencyLimit, runId, leaseExpiresAt);
+        }
+        return this.store.acquireConcurrencySlot(workflowName, concurrencyKey, concurrencyLimit, runId, leaseExpiresAt);
+    }
+    /** Release a slot acquired via `acquireConcurrencySlot`. Idempotent. */
+    async releaseConcurrencySlot(workflowName, concurrencyKey, runId) {
+        if (this.concurrencyBackend) {
+            await this.concurrencyBackend.releaseSlot(workflowName, concurrencyKey, runId);
+            return;
+        }
+        this.store.releaseConcurrencySlot(workflowName, concurrencyKey, runId);
+    }
     // === Node Lifecycle ===
     startNode(runId, opts) {
         const nodeRun = {
@@ -115,6 +663,8 @@ export class RunTracker extends EventEmitter {
             parentNodeId: opts.parentNodeId,
             depth: opts.depth,
             stepIndex: opts.stepIndex,
+            wait: opts.wait,
+            subworkflowDepth: opts.subworkflowDepth,
         };
         this.store.saveNodeRun(nodeRun);
         const run = this.store.getRun(runId);
@@ -150,25 +700,87 @@ export class RunTracker extends EventEmitter {
             metrics: nodeMetrics,
         });
     }
+    /**
+     * Tier 1 idempotency cache hit. Marks the node as completed without
+     * having actually run, attaches the source-run/source-node lineage so
+     * Studio can render a CACHED badge with click-through, and emits a
+     * `NODE_CACHED` event so SSE subscribers see the short-circuit live.
+     *
+     * Caller is responsible for replaying the cached result through
+     * `PersistenceHelper.applyStepOutput` — this method only records the
+     * tracing side. Caching layers ABOVE persistence, never within it.
+     */
+    markNodeCached(nodeRunId, source, outputs) {
+        const nodeRun = this.store.getNodeRun(nodeRunId);
+        if (!nodeRun)
+            return;
+        const finishedAt = Date.now();
+        const durationMs = finishedAt - nodeRun.startedAt;
+        // Security review FW-10 — the idempotency cache holds raw step
+        // output (correct: downstream steps need actual values to run),
+        // but trace storage of a cache hit must mirror the live-run path
+        // where `completeNode` calls `sanitize(ctx.response.data)`.
+        // Without this, a cached step's outputs row could contain raw
+        // `password`/`token` fields that the live run would have redacted.
+        const sanitizedOutputs = outputs === undefined ? undefined : sanitize(outputs);
+        this.store.updateNodeRun(nodeRunId, {
+            status: "completed",
+            finishedAt,
+            durationMs,
+            outputs: sanitizedOutputs,
+            cached: { ...source },
+        });
+        const run = this.store.getRun(nodeRun.runId);
+        if (run) {
+            this.store.updateRun(nodeRun.runId, {
+                completedNodes: run.completedNodes + 1,
+            });
+        }
+        this.emitEvent(nodeRun.runId, run?.workflowName || "", "NODE_CACHED", nodeRun.nodeName, nodeRunId, {
+            durationMs,
+            source: { ...source },
+        });
+    }
+    /**
+     * Tier 1 retry: record a single failed attempt before the next retry. The
+     * node stays in `running` status — `failNode` is the terminal call that
+     * fires only after `retry.maxAttempts` is exhausted.
+     *
+     * Per-node attempt history is capped at {@link MAX_STORED_ATTEMPTS} (10)
+     * to bound store growth on extreme retry counts. The cap matches the
+     * risk-register decision in `tier1-idempotency-replay-retry.md`.
+     */
+    recordNodeAttemptFailed(nodeRunId, info) {
+        const nodeRun = this.store.getNodeRun(nodeRunId);
+        if (!nodeRun)
+            return;
+        const errorDetail = toRunErrorDetail(info.error);
+        const next = [...(nodeRun.attempts ?? []), { attempt: info.attempt, error: errorDetail, timestamp: Date.now() }];
+        const capped = next.length > MAX_STORED_ATTEMPTS ? next.slice(-MAX_STORED_ATTEMPTS) : next;
+        this.store.updateNodeRun(nodeRunId, { attempts: capped });
+        const run = this.store.getRun(nodeRun.runId);
+        this.emitEvent(nodeRun.runId, run?.workflowName || "", "NODE_ATTEMPT_FAILED", nodeRun.nodeName, nodeRunId, {
+            attempt: info.attempt,
+            error: errorDetail,
+        });
+    }
     failNode(nodeRunId, error) {
         const nodeRun = this.store.getNodeRun(nodeRunId);
         if (!nodeRun)
             return;
         const finishedAt = Date.now();
         const durationMs = finishedAt - nodeRun.startedAt;
+        const errorDetail = toRunErrorDetail(error);
         this.store.updateNodeRun(nodeRunId, {
             status: "failed",
             finishedAt,
             durationMs,
-            error: {
-                message: error.message,
-                stack: error.stack,
-            },
+            error: errorDetail,
         });
         const run = this.store.getRun(nodeRun.runId);
         this.emitEvent(nodeRun.runId, run?.workflowName || "", "NODE_FAILED", nodeRun.nodeName, nodeRunId, {
             durationMs,
-            error: { message: error.message, stack: error.stack },
+            error: errorDetail,
         });
     }
     skipNode(runId, nodeName, stepIndex, reason) {
@@ -178,11 +790,71 @@ export class RunTracker extends EventEmitter {
             stepIndex,
         });
     }
+    /**
+     * Record a streaming `Progress` frame for an in-flight node. Overwrites
+     * any previous progress (only the latest milestone is preserved on
+     * the {@link NodeRun} record). Emits a `NODE_PROGRESS` event so SSE
+     * subscribers (Studio) get the live update too.
+     *
+     * Master plan §17 Phase 5 follow-up — wires the proto `Progress`
+     * frame from `ExecuteStream` into the trace store + Studio.
+     *
+     * @param percent 0–100; values outside the range are clamped.
+     * @param phase optional free-form phase label (may be empty).
+     */
+    recordProgress(nodeRunId, percent, phase) {
+        const nodeRun = this.store.getNodeRun(nodeRunId);
+        if (!nodeRun)
+            return;
+        const clamped = Math.max(0, Math.min(100, Math.round(percent)));
+        const updatedAt = Date.now();
+        this.store.updateNodeRun(nodeRunId, {
+            progress: {
+                percent: clamped,
+                phase: phase ?? "",
+                updatedAt,
+            },
+        });
+        const run = this.store.getRun(nodeRun.runId);
+        this.emitEvent(nodeRun.runId, run?.workflowName || "", "NODE_PROGRESS", nodeRun.nodeName, nodeRunId, {
+            percent: clamped,
+            phase: phase ?? "",
+            updatedAt,
+        });
+    }
+    /**
+     * Record a streaming `PartialResult` snapshot for an in-flight node.
+     * Overwrites any previous snapshot. Emits a `NODE_PARTIAL_RESULT`
+     * event for SSE subscribers.
+     *
+     * Master plan §17 Phase 5 follow-up.
+     */
+    recordPartialResult(nodeRunId, snapshot) {
+        const nodeRun = this.store.getNodeRun(nodeRunId);
+        if (!nodeRun)
+            return;
+        const updatedAt = Date.now();
+        this.store.updateNodeRun(nodeRunId, {
+            partialResult: { snapshot, updatedAt },
+        });
+        const run = this.store.getRun(nodeRun.runId);
+        this.emitEvent(nodeRun.runId, run?.workflowName || "", "NODE_PARTIAL_RESULT", nodeRun.nodeName, nodeRunId, {
+            snapshot,
+            updatedAt,
+        });
+    }
     // === Logging ===
     addLog(entry) {
+        // Security review FW-6 — pipe arbitrary log payload through
+        // the sensitive-field redactor before persisting or emitting.
+        // `ctx.logger.logLevel("warn", "x", { password: "..." })` lands
+        // here; without sanitize the secret would persist + stream via
+        // SSE to anyone with /__blok/runs/:id/events access.
+        const sanitizedData = entry.data === undefined ? undefined : sanitize(entry.data);
         const log = {
             id: `log_${uuid().replace(/-/g, "").slice(0, 12)}`,
             ...entry,
+            data: sanitizedData,
             timestamp: Date.now(),
         };
         this.store.saveLog(log);
@@ -190,7 +862,7 @@ export class RunTracker extends EventEmitter {
         this.emitEvent(entry.runId, run?.workflowName || "", "LOG_ENTRY", entry.nodeName, entry.nodeId, {
             level: entry.level,
             message: entry.message,
-            data: entry.data,
+            data: sanitizedData,
         });
     }
     // === Vars Updated ===
@@ -214,6 +886,14 @@ export class RunTracker extends EventEmitter {
     getEvents(runId, since) {
         return this.store.getEvents(runId, since);
     }
+    /**
+     * Tier 2 sub-workflow lineage. Returns every run that was started by
+     * a `subworkflow:` step inside the given parent run. Powers Studio's
+     * "Sub-runs" list on a parent's run-detail page.
+     */
+    getRunsByParent(parentRunId) {
+        return this.store.getRunsByParent(parentRunId);
+    }
     getLogs(runId, nodeId) {
         return this.store.getLogs(runId, nodeId);
     }