@voltro/workflow 0.69.0 → 0.69.1

package/dist/index.d.ts

@@ -687,6 +687,48 @@ export declare interface CloseWorkflowChildrenForParentOptions {
     readonly interruptChild: (workflowName: string, executionId: string, policy: Exclude<WorkflowParentClosePolicy, 'abandon'>) => Promise<void>;
 }
 
+/** The engine's own re-read rule: an unprocessed message whose `last_read` is
+ *  older than this is re-delivered by the shard's owner on its next poll. A
+ *  message older than this that is STILL unread has no owner reading it. */
+export declare const CLUSTER_REREAD_WINDOW_MS: number;
+
+export declare class ClusterDiagnostics extends ClusterDiagnostics_base {
+}
+
+declare const ClusterDiagnostics_base: Context.TagClass<ClusterDiagnostics, "@voltro/workflow/ClusterDiagnostics", ClusterDiagnosticsOps>;
+
+export declare interface ClusterDiagnosticsOps {
+    /** This runner's advertised address — what keys its locks and its runner row. */
+    readonly ownAddress: string;
+    readonly shardLockMode: 'advisory' | 'row' | 'none';
+    /** Every message addressed to an entity — a workflow's `run` and its activities. */
+    readonly messagesFor: (entityId: string) => Effect.Effect<ReadonlyArray<ClusterMessageState>>;
+    /** Unprocessed, deliverable messages no runner has read for longer than
+     *  `olderThanMs` (default: the engine's re-read window). */
+    readonly unreadMessages: (olderThanMs?: number) => Effect.Effect<ReadonlyArray<UnreadMessage>>;
+    /** Who holds the locks on these shards. Empty where the dialect cannot say
+     *  (an advisory lock on MySQL has no holder view the framework reads). */
+    readonly shardLockHolders: (shardIds: ReadonlyArray<string>) => Effect.Effect<ReadonlyArray<ShardLockHolder>>;
+    /** The runner table — every address that registered, with its heartbeat. */
+    readonly runners: () => Effect.Effect<ReadonlyArray<ClusterRunner>>;
+}
+
+export declare interface ClusterMessageState {
+    /** The rpc tag — `run` for the workflow itself, `activity/<name>/<attempt>` for a step. */
+    readonly tag: string | null;
+    readonly entityType: string;
+    readonly entityId: string;
+    readonly shardId: string;
+    readonly processed: boolean;
+    readonly lastRead: Date | null;
+}
+
+export declare interface ClusterRunner {
+    readonly address: string;
+    readonly lastHeartbeat: Date | null;
+    readonly healthy: boolean;
+}
+
 /**
  * Order pending rows for admission.
  *
@@ -918,6 +960,15 @@ export declare const deferringControlsOf: (control: ResolvedFlowControl | undefi
  *  own. Use {@link discardIntent} for a start that is being dropped. */
 export declare const deleteIntent: (store: AdmissionDataStore, id: string) => Promise<boolean>;
 
+/**
+ * The cluster half of a stalled run's report, built from the diagnostics: the
+ * run's messages and, for the unprocessed ones, who holds their shards. Pure
+ * over the two reads, so the sweep's caller and a test agree on its shape.
+ * `undefined` when the engine has no message for the run at all — an
+ * in-memory engine, or a run whose journal was cleared.
+ */
+export declare const describeStalledRun: (ops: ClusterDiagnosticsOps, executionId: string, now?: number) => Effect.Effect<StalledRunClusterState | undefined>;
+
 export declare interface DiscardDecision {
     readonly intentId: string;
     readonly reason: string;
@@ -977,14 +1028,23 @@ export declare interface DrainOutcome {
      */
     readonly examined: boolean;
     /**
-     * When the earliest not-yet-due pending row comes due, as an epoch ms — the
+     * When the earliest not-yet-due pending row comes due — OR the earliest
+     * `timeouts.finish` deadline of a run still running — as an epoch ms: the
      * number that lets the drainer arm for that instant instead of polling until
      * it arrives.
      *
-     * `undefined` means "nothing in the read window is waiting". It is a LOWER
-     * bound on the true next deadline (the window is capped by `batchSize`), which
-     * is the safe direction: an understated deadline costs one early tick, an
-     * overstated one misses a start.
+     * The finish deadline is part of it because the drain tick is what ENFORCES
+     * `timeouts.finish`, and a running run writes nothing to the queue: a drainer
+     * that disarmed on an empty queue while a run approached its bound would
+     * not cancel it at the bound but at the next arrival that happened to wake
+     * it — measured as a cancellation landing tens of minutes after the declared
+     * limit, by an amount that depended on unrelated traffic.
+     *
+     * `undefined` means "nothing in the read window is waiting and no controlled
+     * run is under a finish bound". It is a LOWER bound on the true next deadline
+     * (the window is capped by `batchSize`), which is the safe direction: an
+     * understated deadline costs one early tick, an overstated one misses a
+     * start or a cancellation.
      */
     readonly nextDueAt?: number;
 }
@@ -1836,6 +1896,18 @@ export declare const serialiseWorkflowRowForWire: (row: Record<string, unknown>)
  */
 export declare const setSuspendSignalHintMs: (ms: number | undefined) => void;
 
+export declare interface ShardLockHolder {
+    readonly shardId: string;
+    readonly mode: 'advisory' | 'row';
+    /** Who holds it: `pid <n> from <client>` for a postgres advisory lock, the
+     *  runner address for a row lease. */
+    readonly holder: string;
+    /** `true` when the holder is THIS runner's own lock session/address. */
+    readonly own: boolean;
+    readonly since: Date | null;
+    readonly detail?: Record<string, unknown>;
+}
+
 /**
  * Durable sleep — wake time journaled in the cluster. Wrapper around
  * `DurableClock.sleep` that emits `timer-set` / `timer-fired` events
@@ -1896,6 +1968,15 @@ export declare interface StalenessSweepDeps {
     /** App-supplied reaction — page someone, open a ticket. Must not throw;
      *  a rejection is collected as a failure and the sweep continues. */
     readonly onStalled?: (run: StalledRun) => Promise<void>;
+    /**
+     * What else the report should carry about THIS run — the cluster engine's
+     * view of it (its messages, who holds their shards), attached by the CLI
+     * from `ClusterDiagnostics`. Best-effort: a rejection is collected as a
+     * failure and the report goes out without it. The fields land on the
+     * `run-stalled` event payload and on the warning line, so the report says
+     * where the run is stuck and not only that it is.
+     */
+    readonly describeRun?: (run: StalledRun) => Promise<Record<string, unknown> | undefined>;
     readonly now?: () => number;
     readonly log?: {
         readonly info: (message: string, fields?: Record<string, unknown>) => void;
@@ -1940,6 +2021,20 @@ export declare interface StalledRun {
     readonly reason: StallReason;
 }
 
+/** A `run-stalled` report's cluster half, as the sweep attaches it. */
+export declare interface StalledRunClusterState {
+    readonly runner: string;
+    readonly shardLockMode: ClusterDiagnosticsOps['shardLockMode'];
+    readonly messages: ReadonlyArray<{
+        readonly tag: string | null;
+        readonly shardId: string;
+        readonly processed: boolean;
+        readonly lastRead: string | null;
+        readonly unreadForMs: number | null;
+    }>;
+    readonly shardLocks: ReadonlyArray<ShardLockHolder>;
+}
+
 /** Why the sweep believes this run is not moving. Diagnostic, not a taxonomy
  *  the caller should branch on for correctness. */
 export declare type StallReason = 
@@ -2150,6 +2245,11 @@ export declare const sweepStalledRuns: (deps: StalenessSweepDeps, options?: Stal
 
 export declare const truncateWorkflowValue: (value: unknown, bytes?: number) => unknown;
 
+export declare interface UnreadMessage extends ClusterMessageState {
+    /** How long ago the last runner read it. */
+    readonly unreadForMs: number;
+}
+
 /**
  * Validate + resolve a declaration. Throws at DEFINITION time (module load,
  * i.e. boot) with a message naming the workflow and the field.
@@ -2499,12 +2599,22 @@ export declare interface WorkflowExecuteRecordingOptions extends WorkflowRunReco
      *
      * Called on success AND on failure, and DELIBERATELY not on suspend: a
      * suspended run is still holding the resource it was admitted for.
+     *
+     * On a failure it also carries the payload and the error, so the caller
+     * can raise the workflow's `onFailure` — the one route into that signal
+     * that produces a `failed` row, and the one that was not wired: the
+     * flow-control paths (an expired start, a finish timeout, a dead-lettered
+     * intent) notified, and a body that failed did not.
      */
     readonly onTerminal?: (input: {
         readonly workflowName: string;
         readonly executionId: string;
         readonly runId?: string;
         readonly status: 'succeeded' | 'failed';
+        /** The run's payload — present on `failed`, for the failure report. */
+        readonly payload?: unknown;
+        readonly errorTag?: string | null;
+        readonly errorMessage?: string | null;
     }) => Promise<void>;
     /**
      * CRASH-LOOP BREAKER — how many CONSECUTIVE runner deaths this run may cost

package/dist/index.js

@@ -1,3 +1,4 @@
 import { A as e, B as t, C as n, D as r, E as i, G as a, H as o, I as s, L as c, M as l, N as u, O as d, P as f, R as p, S as m, T as h, U as g, V as _, W as v, _ as y, a as b, b as x, c as S, d as C, f as w, g as T, h as E, i as D, j as O, k, l as A, m as j, n as M, o as N, p as P, r as F, s as I, t as L, u as R, v as z, w as B, x as V, y as H, z as U } from "./primitives-CqHxfASq.js";
-import { $ as W, A as G, At as K, B as q, Bt as J, C as Y, Ct as X, D as Z, Dt as Q, E as $, Et as ee, F as te, Ft as ne, G as re, H as ie, I as ae, It as oe, J as se, K as ce, L as le, Lt as ue, M as de, Mt as fe, N as pe, Nt as me, O as he, Ot as ge, P as _e, Pt as ve, Q as ye, R as be, Rt as xe, S as Se, St as Ce, T as we, Tt as Te, U as Ee, V as De, Vt as Oe, W as ke, X as Ae, Y as je, Z as Me, _ as Ne, _t as Pe, a as Fe, at as Ie, b as Le, bt as Re, c as ze, ct as Be, d as Ve, dt as He, et as Ue, f as We, ft as Ge, g as Ke, gt as qe, h as Je, ht as Ye, i as Xe, it as Ze, j as Qe, jt as $e, k as et, kt as tt, l as nt, lt as rt, m as it, mt as at, n as ot, nt as st, o as ct, ot as lt, p as ut, pt as dt, q as ft, r as pt, rt as mt, s as ht, st as gt, t as _t, tt as vt, u as yt, ut as bt, v as xt, vt as St, w as Ct, wt, x as Tt, xt as Et, y as Dt, yt as Ot, z as kt, zt as At } from "./src-Dj8r1Kcn.js";
-export { mt as ADMISSIONS_TABLE, ut as BUDGET_HOLDS_TABLE, it as BudgetHoldExpired, ke as CANCEL_ON_WATERMARK, l as CurrentWorkflowExecutionId, u as CurrentWorkflowPatches, B as CurrentWorkflowReplayShape, f as CurrentWorkflowRunId, Je as DEFAULT_BUDGET_HOLD_TIMEOUT_MS, re as DEFAULT_COLD_LOOKBACK_MS, be as DEFAULT_DRAIN_BATCH, ce as DEFAULT_EVENT_BATCH, Ze as DEFAULT_LEASE_MS, Xe as DEFAULT_MAX_RUN_RECLAIMS, Fe as DEFAULT_REPLAY_SHAPE_LIMIT, q as DEFAULT_STALL_AFTER_MS, De as DEFAULT_STALL_EVENT_LOOKBACK, ie as DEFAULT_STALL_RUN_PAGE, pe as DEFAULT_SUSPEND_HINT_MS, d as DEFAULT_WORKFLOW_SCHEDULE_MAX_RUNTIME_MS, t as DEFERRING_CONTROLS, ft as EVENTS_TABLE, ee as FlowControlKeyError, Ie as PAUSES_TABLE, lt as PENDING_TABLE, se as WATERMARKS_TABLE, L as WorkflowFlowControlProperty, M as WorkflowMessagesProperty, s as WorkflowRunRecorder, k as WorkflowScheduleProperty, c as WorkflowStepInterceptorTag, F as WorkflowVersionTypeId, D as WorkflowWorkerLayerTypeId, Ke as _voltroBudgetHoldsTable, ve as _voltroWorkflowAdmissionsTable, ne as _voltroWorkflowPausesTable, oe as _voltroWorkflowPendingTable, xe as _voltroWorkflowRunEventsTable, At as _voltroWorkflowRunStepsTable, J as _voltroWorkflowRunsTable, ue as _voltroWorkflowStartContextsTable, Ue as _voltroWorkflowWatermarksTable, vt as admitStart, gt as allIntents, _ as assertConsistentPools, Qe as awaitEvent, de as awaitSignal, Ct as awaitSignalSuspending, We as awaitUpdate, Ne as budgetHoldGeneration, xt as budgetHoldKey, Dt as budgetHoldSignalName, pt as closeWorkflowChildrenForParent, Q as comparePendingOrder, we as completeSuspendingSignal, Be as deadLetterIntent, ge as decideAdmission, o as deferringControlsOf, rt as deleteIntent, bt as discardIntent, kt as drainTick, He as dueIntents, b as durableClock, N as durableQueue, I as durableQueueModule, S as durableRateLimiterModule, Ge as evaluateAdmission, p as getCurrentWorkflowExecutionId, U as getCurrentWorkflowRunId, A as getWorkflowFlowControl, e as getWorkflowSchedule, R as getWorkflowVersionMetadata, g as hasCancelOn, v as hasFlowControl, Le as heldBudgets, Oe as inMemoryWorkflowEngineLayer, ot as inspectWorkflow, tt as isFinishExpired, K as isStartExpired, C as isWorkflowWorkerLayer, st as linkExecution, _t as makeInMemoryRecorder, h as makeReplayShapeTracker, ct as makeWorkflowRunRecorder, Z as makeWorkflowUpdateId, _e as maybeHintSuspendingSignal, ye as newestEventAt, $e as nextSlotAt, i as nondeterminismEventPayload, ht as normaliseWorkflowPatches, dt as noteIntentAttempt, at as parkDelayedStart, w as patch, Ye as pauseWorkflow, Tt as pendingBudgetHolds, qe as pendingSlotId, Pe as pendingWindow, St as persistDefer, W as planCancellations, fe as pooledConcurrencyKey, P as processQueue, Ot as pruneAdmissions, j as queueWorker, E as rateLimit, Re as readAdmissionState, Et as readPausedWorkflows, je as readWatermark, Ce as recordAdmission, Se as releaseBudgetHolds, X as releaseLease, r as reportNondeterminism, te as resetSuspendHintsForTest, me as resolveAdmissionKeys, ze as resolveRunGuardTuning, ae as resolveSuspendHintMs, he as resolveWorkflowMessageRun, wt as resumeWorkflow, Te as retryDeadLetterIntent, et as sendWorkflowSignal, G as sendWorkflowUpdate, nt as serialiseWorkflowRowForWire, le as setSuspendSignalHintMs, T as sleep, y as sleepUntil, z as step, H as stepIdempotencyKey, x as stepModule, Y as suspendForBudget, $ as suspendingSignalDeferredName, Ae as sweepCancelOn, Ee as sweepStalledRuns, yt as truncateWorkflowValue, a as validateFlowControl, O as validateWorkflowSchedule, V as withCompensation, m as workflow, n as workflowModule, Ve as wrapWorkflowExecuteWithRunRecording, Me as writeWatermark };
+import { $ as W, A as G, At as K, B as q, Bt as J, C as Y, Ct as X, D as Z, Dt as Q, E as $, Et as ee, F as te, Ft as ne, G as re, H as ie, I as ae, It as oe, J as se, K as ce, L as le, Lt as ue, M as de, Mt as fe, N as pe, Nt as me, O as he, Ot as ge, P as _e, Pt as ve, Q as ye, R as be, Rt as xe, S as Se, St as Ce, T as we, Tt as Te, U as Ee, V as De, Vt as Oe, W as ke, X as Ae, Y as je, Z as Me, _ as Ne, _t as Pe, a as Fe, at as Ie, b as Le, bt as Re, c as ze, ct as Be, d as Ve, dt as He, et as Ue, f as We, ft as Ge, g as Ke, gt as qe, h as Je, ht as Ye, i as Xe, it as Ze, j as Qe, jt as $e, k as et, kt as tt, l as nt, lt as rt, m as it, mt as at, n as ot, nt as st, o as ct, ot as lt, p as ut, pt as dt, q as ft, r as pt, rt as mt, s as ht, st as gt, t as _t, tt as vt, u as yt, ut as bt, v as xt, vt as St, w as Ct, wt, x as Tt, xt as Et, y as Dt, yt as Ot, z as kt, zt as At } from "./src-TRpoU54e.js";
+import { n as jt, r as Mt, t as Nt } from "./clusterDiagnostics-CNtl5LX9.js";
+export { mt as ADMISSIONS_TABLE, ut as BUDGET_HOLDS_TABLE, it as BudgetHoldExpired, ke as CANCEL_ON_WATERMARK, Nt as CLUSTER_REREAD_WINDOW_MS, jt as ClusterDiagnostics, l as CurrentWorkflowExecutionId, u as CurrentWorkflowPatches, B as CurrentWorkflowReplayShape, f as CurrentWorkflowRunId, Je as DEFAULT_BUDGET_HOLD_TIMEOUT_MS, re as DEFAULT_COLD_LOOKBACK_MS, be as DEFAULT_DRAIN_BATCH, ce as DEFAULT_EVENT_BATCH, Ze as DEFAULT_LEASE_MS, Xe as DEFAULT_MAX_RUN_RECLAIMS, Fe as DEFAULT_REPLAY_SHAPE_LIMIT, q as DEFAULT_STALL_AFTER_MS, De as DEFAULT_STALL_EVENT_LOOKBACK, ie as DEFAULT_STALL_RUN_PAGE, pe as DEFAULT_SUSPEND_HINT_MS, d as DEFAULT_WORKFLOW_SCHEDULE_MAX_RUNTIME_MS, t as DEFERRING_CONTROLS, ft as EVENTS_TABLE, ee as FlowControlKeyError, Ie as PAUSES_TABLE, lt as PENDING_TABLE, se as WATERMARKS_TABLE, L as WorkflowFlowControlProperty, M as WorkflowMessagesProperty, s as WorkflowRunRecorder, k as WorkflowScheduleProperty, c as WorkflowStepInterceptorTag, F as WorkflowVersionTypeId, D as WorkflowWorkerLayerTypeId, Ke as _voltroBudgetHoldsTable, ve as _voltroWorkflowAdmissionsTable, ne as _voltroWorkflowPausesTable, oe as _voltroWorkflowPendingTable, xe as _voltroWorkflowRunEventsTable, At as _voltroWorkflowRunStepsTable, J as _voltroWorkflowRunsTable, ue as _voltroWorkflowStartContextsTable, Ue as _voltroWorkflowWatermarksTable, vt as admitStart, gt as allIntents, _ as assertConsistentPools, Qe as awaitEvent, de as awaitSignal, Ct as awaitSignalSuspending, We as awaitUpdate, Ne as budgetHoldGeneration, xt as budgetHoldKey, Dt as budgetHoldSignalName, pt as closeWorkflowChildrenForParent, Q as comparePendingOrder, we as completeSuspendingSignal, Be as deadLetterIntent, ge as decideAdmission, o as deferringControlsOf, rt as deleteIntent, Mt as describeStalledRun, bt as discardIntent, kt as drainTick, He as dueIntents, b as durableClock, N as durableQueue, I as durableQueueModule, S as durableRateLimiterModule, Ge as evaluateAdmission, p as getCurrentWorkflowExecutionId, U as getCurrentWorkflowRunId, A as getWorkflowFlowControl, e as getWorkflowSchedule, R as getWorkflowVersionMetadata, g as hasCancelOn, v as hasFlowControl, Le as heldBudgets, Oe as inMemoryWorkflowEngineLayer, ot as inspectWorkflow, tt as isFinishExpired, K as isStartExpired, C as isWorkflowWorkerLayer, st as linkExecution, _t as makeInMemoryRecorder, h as makeReplayShapeTracker, ct as makeWorkflowRunRecorder, Z as makeWorkflowUpdateId, _e as maybeHintSuspendingSignal, ye as newestEventAt, $e as nextSlotAt, i as nondeterminismEventPayload, ht as normaliseWorkflowPatches, dt as noteIntentAttempt, at as parkDelayedStart, w as patch, Ye as pauseWorkflow, Tt as pendingBudgetHolds, qe as pendingSlotId, Pe as pendingWindow, St as persistDefer, W as planCancellations, fe as pooledConcurrencyKey, P as processQueue, Ot as pruneAdmissions, j as queueWorker, E as rateLimit, Re as readAdmissionState, Et as readPausedWorkflows, je as readWatermark, Ce as recordAdmission, Se as releaseBudgetHolds, X as releaseLease, r as reportNondeterminism, te as resetSuspendHintsForTest, me as resolveAdmissionKeys, ze as resolveRunGuardTuning, ae as resolveSuspendHintMs, he as resolveWorkflowMessageRun, wt as resumeWorkflow, Te as retryDeadLetterIntent, et as sendWorkflowSignal, G as sendWorkflowUpdate, nt as serialiseWorkflowRowForWire, le as setSuspendSignalHintMs, T as sleep, y as sleepUntil, z as step, H as stepIdempotencyKey, x as stepModule, Y as suspendForBudget, $ as suspendingSignalDeferredName, Ae as sweepCancelOn, Ee as sweepStalledRuns, yt as truncateWorkflowValue, a as validateFlowControl, O as validateWorkflowSchedule, V as withCompensation, m as workflow, n as workflowModule, Ve as wrapWorkflowExecuteWithRunRecording, Me as writeWatermark };