@voltro/workflow 0.33.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1801 -0
- package/dist/clusterTestSuite.js +28 -28
- package/dist/index.d.ts +509 -6
- package/dist/index.js +3 -3
- package/dist/{primitives-Bp98_F5L.js → primitives-oLKOTbvJ.js} +131 -76
- package/dist/primitives.d.ts +145 -5
- package/dist/primitives.js +2 -2
- package/dist/{src-CP61KZ9V.js → src-BXObaqb-.js} +912 -493
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Cause } from 'effect';
|
|
2
2
|
import { Context } from 'effect';
|
|
3
|
+
import { DataStore } from '@voltro/database';
|
|
3
4
|
import { DurableClock as durableClock } from '@effect/workflow';
|
|
4
5
|
import { DurableQueue as durableQueueModule } from '@effect/workflow';
|
|
5
6
|
import { DurableRateLimiter as durableRateLimiterModule } from '@effect/workflow';
|
|
@@ -446,6 +447,54 @@ export declare interface AwaitUpdateOptions<A, B = A> {
|
|
|
446
447
|
readonly timeoutMs?: number;
|
|
447
448
|
}
|
|
448
449
|
|
|
450
|
+
export declare const BUDGET_HOLDS_TABLE = "_voltro_budget_holds";
|
|
451
|
+
|
|
452
|
+
/** Raised into the workflow when a hold outlived its ceiling. */
|
|
453
|
+
export declare class BudgetHoldExpired extends Error {
|
|
454
|
+
readonly budget: string;
|
|
455
|
+
readonly holdKey: string;
|
|
456
|
+
constructor(budget: string, holdKey: string, timeoutMs: number);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** How many holds this (execution, step, budget) has already taken. */
|
|
460
|
+
export declare const budgetHoldGeneration: (store: DataStore, input: {
|
|
461
|
+
readonly executionId: string;
|
|
462
|
+
readonly stepName: string;
|
|
463
|
+
readonly budget: string;
|
|
464
|
+
}) => Promise<number>;
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* The replay-stable identity of one hold within one run.
|
|
468
|
+
*
|
|
469
|
+
* The `generation` is the part that is easy to leave out and fatal to leave out.
|
|
470
|
+
* A run can be held MORE THAN ONCE at the same step: it is released, the body
|
|
471
|
+
* replays, it re-reads the counter, and the budget is still over — because the
|
|
472
|
+
* window rolled for a tenant that immediately spent again, or because an
|
|
473
|
+
* operator lifted the wrong hold. With a generation-free key the second park
|
|
474
|
+
* would await the deferred the FIRST release had already completed, get an
|
|
475
|
+
* instant resolution, and spend straight through a budget that was still
|
|
476
|
+
* exceeded — the AI-Flows constant-signal-name collision, one level up and with
|
|
477
|
+
* money on the other side of it.
|
|
478
|
+
*
|
|
479
|
+
* `generation` is the number of holds already recorded for this
|
|
480
|
+
* (execution, step, budget), which is durable, monotone, and unchanged by a
|
|
481
|
+
* crash-replay that took no new hold — so it is stable exactly when replay
|
|
482
|
+
* safety needs it to be and different exactly when correctness needs it to be.
|
|
483
|
+
*/
|
|
484
|
+
export declare const budgetHoldKey: (input: {
|
|
485
|
+
readonly executionId: string;
|
|
486
|
+
readonly stepName: string;
|
|
487
|
+
readonly budget: string;
|
|
488
|
+
readonly generation: number;
|
|
489
|
+
}) => string;
|
|
490
|
+
|
|
491
|
+
/**
|
|
492
|
+
* The signal name ONE hold parks on. Derived from the hold key, so the releaser
|
|
493
|
+
* reconstructs it from the row with no shared state — the same discipline
|
|
494
|
+
* `suspendingSignalDeferredName` and `humanResponseSignalName` follow.
|
|
495
|
+
*/
|
|
496
|
+
export declare const budgetHoldSignalName: (holdKey: string) => string;
|
|
497
|
+
|
|
449
498
|
export declare const CANCEL_ON_WATERMARK = "cancelOn";
|
|
450
499
|
|
|
451
500
|
export declare interface CancelDecision {
|
|
@@ -659,6 +708,29 @@ export declare interface CompleteSuspendingSignalInput {
|
|
|
659
708
|
* engine-assigned executionId, not our row id. */
|
|
660
709
|
export declare const CurrentWorkflowExecutionId: FiberRef.FiberRef<string | undefined>;
|
|
661
710
|
|
|
711
|
+
/**
|
|
712
|
+
* FiberRef carrying the patch markers that were declared on the workflow at the
|
|
713
|
+
* moment THIS RUN STARTED — read back from `_voltro_workflow_runs.
|
|
714
|
+
* workflowPatches` on a resume, taken from the current declaration on a first
|
|
715
|
+
* start. This is what makes `patch('id')` (see `primitives.ts`) answer the
|
|
716
|
+
* Temporal question "was this marker in effect when this execution began?"
|
|
717
|
+
* rather than "is it in the code I am running now".
|
|
718
|
+
*
|
|
719
|
+
* Empty outside a recorded workflow body, so `patch()` is `false` there — the
|
|
720
|
+
* safe direction: an un-recorded context gets the PRE-patch branch.
|
|
721
|
+
*/
|
|
722
|
+
export declare const CurrentWorkflowPatches: FiberRef.FiberRef<readonly string[]>;
|
|
723
|
+
|
|
724
|
+
/**
|
|
725
|
+
* The tracker for the CURRENT body entry, or `undefined` when there is nothing
|
|
726
|
+
* to compare against.
|
|
727
|
+
*
|
|
728
|
+
* Installed by `wrapWorkflowExecuteWithRunRecording` ONLY on a re-entry (a run
|
|
729
|
+
* whose row already exists). A first entry has no prior shape, so the whole
|
|
730
|
+
* mechanism costs one `FiberRef` read per step and nothing else.
|
|
731
|
+
*/
|
|
732
|
+
export declare const CurrentWorkflowReplayShape: FiberRef.FiberRef<ReplayShapeTracker | undefined>;
|
|
733
|
+
|
|
662
734
|
/** FiberRef carrying the active workflow run's id. The outer
|
|
663
735
|
* `wrapWithRunRecording` in dev.ts sets it via
|
|
664
736
|
* `Effect.locally(CurrentWorkflowRunId, runId)`. */
|
|
@@ -697,6 +769,11 @@ declare type DecodedPayload<P> = P extends Schema.Struct.Fields ? Schema.Struct.
|
|
|
697
769
|
readonly Type: infer T;
|
|
698
770
|
} ? T : never;
|
|
699
771
|
|
|
772
|
+
/** Default ceiling on a hold — 7 days, matching the HITL park bound. The wait
|
|
773
|
+
* itself is free (no worker is held), and a run that gives up on a monthly
|
|
774
|
+
* budget after an hour is a run somebody has to re-drive by hand. */
|
|
775
|
+
export declare const DEFAULT_BUDGET_HOLD_TIMEOUT_MS: number;
|
|
776
|
+
|
|
700
777
|
/** How far back a COLD start looks when no watermark exists yet — one hour.
|
|
701
778
|
* Not the whole journal: a first boot against an old deployment would
|
|
702
779
|
* otherwise read every event ever published to decide about runs that mostly
|
|
@@ -727,6 +804,33 @@ export declare const DEFAULT_EVENT_BATCH = 500;
|
|
|
727
804
|
*/
|
|
728
805
|
export declare const DEFAULT_LEASE_MS: number;
|
|
729
806
|
|
|
807
|
+
/** Consecutive runner deaths a single run may cost before it is parked. Three,
|
|
808
|
+
* not one: a single crash is frequently the node and not the payload, and
|
|
809
|
+
* parking on it would turn every spot-instance eviction into an operator
|
|
810
|
+
* ticket. Three in a row is a property of the work. */
|
|
811
|
+
export declare const DEFAULT_MAX_RUN_RECLAIMS = 3;
|
|
812
|
+
|
|
813
|
+
/** Recorded step rows the replay-shape snapshot will load for one run. */
|
|
814
|
+
export declare const DEFAULT_REPLAY_SHAPE_LIMIT = 2000;
|
|
815
|
+
|
|
816
|
+
/** How long a live run may make no progress before it is reported.
|
|
817
|
+
*
|
|
818
|
+
* Thirty minutes, not five: the population this must not drown in is
|
|
819
|
+
* long-running steps (a big import, a slow provider), and a threshold under
|
|
820
|
+
* the longest legitimate step turns the feature into noise. Runs that wait
|
|
821
|
+
* legitimately for HOURS wait on a durable timer, which is excluded outright,
|
|
822
|
+
* so the threshold only has to clear the slowest single step. */
|
|
823
|
+
export declare const DEFAULT_STALL_AFTER_MS: number;
|
|
824
|
+
|
|
825
|
+
/** Lifecycle events read per candidate. Enough to see the timer handshake and
|
|
826
|
+
* a prior `run-stalled` on a run that is, by construction, not doing much. */
|
|
827
|
+
export declare const DEFAULT_STALL_EVENT_LOOKBACK = 20;
|
|
828
|
+
|
|
829
|
+
/** Live runs examined per tick. Bounded for the reason every sweep here is:
|
|
830
|
+
* oldest-first, so a deployment past the bound makes progress on the next
|
|
831
|
+
* tick instead of re-reading the same page. */
|
|
832
|
+
export declare const DEFAULT_STALL_RUN_PAGE = 200;
|
|
833
|
+
|
|
730
834
|
export declare type DeferMode = DeferringControl | 'paused';
|
|
731
835
|
|
|
732
836
|
/**
|
|
@@ -917,6 +1021,13 @@ export declare const hasCancelOn: (control: ResolvedFlowControl | undefined) =>
|
|
|
917
1021
|
* two writes per start for a decision that is always "admit". */
|
|
918
1022
|
export declare const hasFlowControl: (control: ResolvedFlowControl | undefined) => boolean;
|
|
919
1023
|
|
|
1024
|
+
/** The distinct budgets currently holding at least one run — what a poller
|
|
1025
|
+
* iterates so it asks about nothing else. */
|
|
1026
|
+
export declare const heldBudgets: (store: DataStore) => Promise<ReadonlyArray<{
|
|
1027
|
+
readonly budget: string;
|
|
1028
|
+
readonly tenantId: string | null;
|
|
1029
|
+
}>>;
|
|
1030
|
+
|
|
920
1031
|
export declare interface InMemoryRecorder {
|
|
921
1032
|
readonly layer: Layer.Layer<WorkflowRunRecorder>;
|
|
922
1033
|
readonly readSteps: () => ReadonlyArray<RecordedStep>;
|
|
@@ -1019,6 +1130,17 @@ export declare const linkExecution: (store: AdmissionDataStore, ledgerId: string
|
|
|
1019
1130
|
|
|
1020
1131
|
export declare const makeInMemoryRecorder: () => InMemoryRecorder;
|
|
1021
1132
|
|
|
1133
|
+
/**
|
|
1134
|
+
* Build a tracker from the step rows this run recorded on EARLIER body entries.
|
|
1135
|
+
*
|
|
1136
|
+
* Pass the rows as they are: one row per step ATTEMPT, so a step that retried
|
|
1137
|
+
* three times contributes three. Counting rows rather than distinct names is
|
|
1138
|
+
* what makes the per-name sequence rule work, and the asymmetry is deliberate —
|
|
1139
|
+
* only reaching a name MORE often than recorded is a finding. Fewer is normal
|
|
1140
|
+
* (a crash mid-step records two rows for one logical reach).
|
|
1141
|
+
*/
|
|
1142
|
+
export declare const makeReplayShapeTracker: (recordedRows: ReadonlyArray<RecordedStepShape>) => ReplayShapeTracker;
|
|
1143
|
+
|
|
1022
1144
|
export declare const makeWorkflowRunRecorder: (options: WorkflowRunRecorderOptions) => WorkflowRunRecorderService;
|
|
1023
1145
|
|
|
1024
1146
|
export declare const makeWorkflowUpdateId: () => string;
|
|
@@ -1051,6 +1173,23 @@ export declare const newestEventAt: (events: ReadonlyArray<DomainEventRow>) => D
|
|
|
1051
1173
|
*/
|
|
1052
1174
|
export declare const nextSlotAt: (recentAdmissions: ReadonlyArray<number>, limit: number, periodMs: number, now: number) => number | undefined;
|
|
1053
1175
|
|
|
1176
|
+
/** The `nondeterminism-suspected` event body. Kept here so the two emit sites
|
|
1177
|
+
* (the step wrapper, and the run wrapper's `settle`) cannot drift. */
|
|
1178
|
+
export declare const nondeterminismEventPayload: (finding: NondeterminismFinding) => Record<string, unknown>;
|
|
1179
|
+
|
|
1180
|
+
export declare interface NondeterminismFinding {
|
|
1181
|
+
readonly kind: NondeterminismKind;
|
|
1182
|
+
readonly stepName: string;
|
|
1183
|
+
/** How many rows this run recorded for `stepName` on EARLIER body entries. */
|
|
1184
|
+
readonly recorded: number;
|
|
1185
|
+
/** How many times the CURRENT body entry reached it. */
|
|
1186
|
+
readonly reached: number;
|
|
1187
|
+
/** One sentence an operator can act on, without opening the code. */
|
|
1188
|
+
readonly detail: string;
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
export declare type NondeterminismKind = 'unreached-step' | 'extra-step-occurrence';
|
|
1192
|
+
|
|
1054
1193
|
declare type NormaliseWorkflowMessages<M extends WorkflowMessageSchemas | undefined> = {
|
|
1055
1194
|
readonly signals: M extends {
|
|
1056
1195
|
readonly signals: infer Signals;
|
|
@@ -1058,13 +1197,68 @@ declare type NormaliseWorkflowMessages<M extends WorkflowMessageSchemas | undefi
|
|
|
1058
1197
|
readonly updates: M extends {
|
|
1059
1198
|
readonly updates: infer Updates;
|
|
1060
1199
|
} ? NonNullable<Updates> & Readonly<Record<string, WorkflowMessagePairSchemas>> : {};
|
|
1061
|
-
readonly queries: M extends {
|
|
1062
|
-
readonly queries: infer Queries;
|
|
1063
|
-
} ? NonNullable<Queries> & Readonly<Record<string, WorkflowMessagePairSchemas>> : {};
|
|
1064
1200
|
};
|
|
1065
1201
|
|
|
1202
|
+
/** `workflowPatches` comes back off a `json()` column, so it is whatever the
|
|
1203
|
+
* dialect's driver decoded — an array, a JSON string, or null. Anything that
|
|
1204
|
+
* is not a list of strings means "no markers", which is the safe answer:
|
|
1205
|
+
* `patch()` then reports the PRE-patch branch. */
|
|
1206
|
+
export declare const normaliseWorkflowPatches: (value: unknown) => ReadonlyArray<string> | undefined;
|
|
1207
|
+
|
|
1066
1208
|
export declare const noteIntentAttempt: (store: AdmissionDataStore, intent: PendingIntent, error: string, now: number) => Promise<void>;
|
|
1067
1209
|
|
|
1210
|
+
/**
|
|
1211
|
+
* Is a named change marker in effect for the run currently executing?
|
|
1212
|
+
*
|
|
1213
|
+
* This is the escape valve for the versioning trap. `compatibleWith` is
|
|
1214
|
+
* all-or-nothing: leave an old version out and every in-flight run on it is
|
|
1215
|
+
* terminally failed; leave it in and the old runs replay against the new body
|
|
1216
|
+
* with no detection. `patch` is the third option — the body itself branches, so
|
|
1217
|
+
* runs that started BEFORE the change keep taking the old path to completion
|
|
1218
|
+
* while new runs take the new one.
|
|
1219
|
+
*
|
|
1220
|
+
* ```ts
|
|
1221
|
+
* export const Charge = workflow({
|
|
1222
|
+
* name: 'billing.charge',
|
|
1223
|
+
* payload: { orderId: Schema.String },
|
|
1224
|
+
* idempotencyKey: ({ orderId }) => `billing.charge:${orderId}`,
|
|
1225
|
+
* patches: ['split-tax-calculation'], // ← declare it
|
|
1226
|
+
* })
|
|
1227
|
+
*
|
|
1228
|
+
* // inside the body:
|
|
1229
|
+
* if (yield* patch('split-tax-calculation')) {
|
|
1230
|
+
* const net = yield* step({ name: 'net-total', execute })
|
|
1231
|
+
* const tax = yield* step({ name: 'tax', execute })
|
|
1232
|
+
* return net + tax
|
|
1233
|
+
* }
|
|
1234
|
+
* return yield* step({ name: 'total', execute }) // the pre-patch path
|
|
1235
|
+
* ```
|
|
1236
|
+
*
|
|
1237
|
+
* ── The answer is pinned to the RUN, not to the code ────────────────────────
|
|
1238
|
+
*
|
|
1239
|
+
* `patches` is stamped onto `_voltro_workflow_runs.workflowPatches` when the
|
|
1240
|
+
* run starts and read back from that row on every resume. So a run started
|
|
1241
|
+
* before you added the marker answers `false` for the rest of its life,
|
|
1242
|
+
* however many times it replays and whatever the deployed code says — which is
|
|
1243
|
+
* the property that makes the branch deterministic across a redeploy.
|
|
1244
|
+
*
|
|
1245
|
+
* Under `@effect/workflow` the naive implementation does the opposite: an
|
|
1246
|
+
* activity that is ABSENT from an old run's journal EXECUTES on replay (the
|
|
1247
|
+
* journal is keyed name/attempt), so a marker journaled by an activity would
|
|
1248
|
+
* answer `true` for exactly the old runs it must answer `false` for. Reading
|
|
1249
|
+
* the run row sidesteps that entirely.
|
|
1250
|
+
*
|
|
1251
|
+
* ── Retiring a patch ───────────────────────────────────────────────────────
|
|
1252
|
+
*
|
|
1253
|
+
* Once no run predating the marker can still be in flight, delete the old
|
|
1254
|
+
* branch and the entry from `patches`. Runs that stamped it keep the marker on
|
|
1255
|
+
* their row for the audit trail; `patch()` simply stops being called.
|
|
1256
|
+
*
|
|
1257
|
+
* Outside a recorded workflow body (a unit test, a bare `step()` call) this is
|
|
1258
|
+
* `false` — the pre-patch branch, which is the safe direction.
|
|
1259
|
+
*/
|
|
1260
|
+
export declare const patch: (id: string) => Effect.Effect<boolean>;
|
|
1261
|
+
|
|
1068
1262
|
export declare const PAUSES_TABLE = "_voltro_workflow_pauses";
|
|
1069
1263
|
|
|
1070
1264
|
export declare const pauseWorkflow: (store: AdmissionDataStore, workflowName: string, by: string | null, reason: string | null) => Promise<void>;
|
|
@@ -1074,6 +1268,20 @@ declare type PayloadSchemaOf<P> = P extends Schema.Struct.Fields ? Schema.Struct
|
|
|
1074
1268
|
|
|
1075
1269
|
export declare const PENDING_TABLE = "_voltro_workflow_pending";
|
|
1076
1270
|
|
|
1271
|
+
/** Every hold currently parked — the operator's view, and the poller's input. */
|
|
1272
|
+
export declare const pendingBudgetHolds: (store: DataStore, options?: {
|
|
1273
|
+
readonly budget?: string;
|
|
1274
|
+
}) => Promise<ReadonlyArray<{
|
|
1275
|
+
readonly id: string;
|
|
1276
|
+
readonly holdKey: string;
|
|
1277
|
+
readonly budget: string;
|
|
1278
|
+
readonly tenantId: string | null;
|
|
1279
|
+
readonly executionId: string;
|
|
1280
|
+
readonly workflowName: string;
|
|
1281
|
+
readonly stepName: string;
|
|
1282
|
+
readonly heldAt: string;
|
|
1283
|
+
}>>;
|
|
1284
|
+
|
|
1077
1285
|
/** A pending row in domain shape. */
|
|
1078
1286
|
export declare interface PendingIntent {
|
|
1079
1287
|
readonly id: string;
|
|
@@ -1295,6 +1503,44 @@ export declare interface RecordedStep {
|
|
|
1295
1503
|
durationMs: number | null;
|
|
1296
1504
|
}
|
|
1297
1505
|
|
|
1506
|
+
/** The slice of a `_voltro_workflow_run_steps` row this needs. */
|
|
1507
|
+
export declare interface RecordedStepShape {
|
|
1508
|
+
readonly stepName: string;
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
/**
|
|
1512
|
+
* Resume every run parked on `budget`.
|
|
1513
|
+
*
|
|
1514
|
+
* Requires the `WorkflowEngine` — run it through the same runner that executes
|
|
1515
|
+
* workflows (`workflowRuntime.runPromise`), exactly like `completeSuspendingSignal`.
|
|
1516
|
+
*
|
|
1517
|
+
* A release WAKES a run; it does not authorise a spend. The resumed body
|
|
1518
|
+
* re-reads the budget and parks again on a fresh generation if it is still over
|
|
1519
|
+
* — see `suspendForBudget`. So this is safe to call optimistically (on a
|
|
1520
|
+
* `recovered` signal, or from an operator's console) without knowing whether
|
|
1521
|
+
* every held run will actually proceed.
|
|
1522
|
+
*
|
|
1523
|
+
* The row is closed BEFORE the deferred is completed, deliberately: the resumed
|
|
1524
|
+
* body counts existing holds to derive its next generation, and a row still
|
|
1525
|
+
* reading `releasedAt: null` would make the re-park collide with the hold it was
|
|
1526
|
+
* just released from.
|
|
1527
|
+
*/
|
|
1528
|
+
export declare const releaseBudgetHolds: (input: ReleaseBudgetHoldsInput) => Effect.Effect<{
|
|
1529
|
+
readonly released: number;
|
|
1530
|
+
}, never, WorkflowEngine.WorkflowEngine>;
|
|
1531
|
+
|
|
1532
|
+
export declare interface ReleaseBudgetHoldsInput {
|
|
1533
|
+
readonly store: DataStore;
|
|
1534
|
+
/** Which budget regained headroom. */
|
|
1535
|
+
readonly budget: string;
|
|
1536
|
+
/** Restrict to one tenant. Omitted → every tenant held on this budget (a
|
|
1537
|
+
* window rollover is per-tenant in the accountant but the counter key rotates
|
|
1538
|
+
* for everyone, so both shapes are legitimate). */
|
|
1539
|
+
readonly tenantId?: string | null | undefined;
|
|
1540
|
+
/** `released` = the budget itself recovered; `lifted` = an operator decided. */
|
|
1541
|
+
readonly outcome?: 'released' | 'lifted';
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1298
1544
|
/**
|
|
1299
1545
|
* Free the slot an execution holds. Called when a run reaches a terminal state.
|
|
1300
1546
|
*
|
|
@@ -1304,6 +1550,34 @@ export declare interface RecordedStep {
|
|
|
1304
1550
|
*/
|
|
1305
1551
|
export declare const releaseLease: (store: AdmissionDataStore, executionId: string, now: number) => Promise<boolean>;
|
|
1306
1552
|
|
|
1553
|
+
export declare interface ReplayShapeTracker {
|
|
1554
|
+
/**
|
|
1555
|
+
* The body reached `stepName`. Returns a finding the FIRST time that reach
|
|
1556
|
+
* is anomalous, `undefined` otherwise — so a step in a hot loop reports once
|
|
1557
|
+
* rather than once per iteration.
|
|
1558
|
+
*/
|
|
1559
|
+
readonly reach: (stepName: string) => NondeterminismFinding | undefined;
|
|
1560
|
+
/**
|
|
1561
|
+
* The body ran to a COMPLETE outcome. Returns the set-membership findings —
|
|
1562
|
+
* recorded names the body never reached.
|
|
1563
|
+
*
|
|
1564
|
+
* Call this ONLY on success/failure, never on a suspend: a suspended body
|
|
1565
|
+
* stopped partway through on purpose, so everything after the suspension
|
|
1566
|
+
* point is legitimately unreached.
|
|
1567
|
+
*/
|
|
1568
|
+
readonly settle: () => ReadonlyArray<NondeterminismFinding>;
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
/**
|
|
1572
|
+
* Emit findings from INSIDE the workflow body, where the run id and the
|
|
1573
|
+
* recorder are only reachable through the fiber context.
|
|
1574
|
+
*
|
|
1575
|
+
* Best-effort in the strongest sense: no recorder, no run id, or a rejecting
|
|
1576
|
+
* write all resolve to "nothing happened". A tripwire that can break the run it
|
|
1577
|
+
* is watching is not a tripwire.
|
|
1578
|
+
*/
|
|
1579
|
+
export declare const reportNondeterminism: (findings: ReadonlyArray<NondeterminismFinding>) => Effect.Effect<void>;
|
|
1580
|
+
|
|
1307
1581
|
/** Resolve every declared key against one payload. Pure; throws only
|
|
1308
1582
|
* {@link FlowControlKeyError}. */
|
|
1309
1583
|
export declare const resolveAdmissionKeys: (control: ResolvedFlowControl, payload: unknown) => AdmissionKeys;
|
|
@@ -1376,6 +1650,20 @@ export declare interface ResolvedFlowControl {
|
|
|
1376
1650
|
readonly cancelOn?: ReadonlyArray<ResolvedCancelOn>;
|
|
1377
1651
|
}
|
|
1378
1652
|
|
|
1653
|
+
/**
|
|
1654
|
+
* Read the run-guard knobs from the environment, for the boot paths to spread
|
|
1655
|
+
* into {@link wrapWorkflowExecuteWithRunRecording}. Mirrors
|
|
1656
|
+
* `resolveFailoverTuning` in `clusterLayer.ts` — same shape, same
|
|
1657
|
+
* malformed-value-is-ignored rule, so an operator learns one convention:
|
|
1658
|
+
*
|
|
1659
|
+
* - `VOLTRO_WORKFLOW_MAX_RECLAIMS` (→ `maxRunReclaims`, default 3)
|
|
1660
|
+
* - `VOLTRO_WORKFLOW_REPLAY_SHAPE_LIMIT` (→ `replayShapeLimit`, default 2000)
|
|
1661
|
+
*/
|
|
1662
|
+
export declare const resolveRunGuardTuning: () => {
|
|
1663
|
+
maxRunReclaims?: number;
|
|
1664
|
+
replayShapeLimit?: number;
|
|
1665
|
+
};
|
|
1666
|
+
|
|
1379
1667
|
export declare const resolveWorkflowMessageRun: (store: WorkflowMessageStore, target: WorkflowMessageTarget) => Promise<WorkflowResolvedRun>;
|
|
1380
1668
|
|
|
1381
1669
|
export declare const resumeWorkflow: (store: AdmissionDataStore, workflowName: string) => Promise<boolean>;
|
|
@@ -1441,6 +1729,82 @@ export declare const sleepUntil: (options: {
|
|
|
1441
1729
|
readonly until: Date | number;
|
|
1442
1730
|
}) => Effect.Effect<void, never, never>;
|
|
1443
1731
|
|
|
1732
|
+
/** The narrow store slice this needs. Structural, like `CancelOnDataStore`. */
|
|
1733
|
+
export declare interface StalenessDataStore {
|
|
1734
|
+
query(descriptor: QueryDescriptor): Promise<ReadonlyArray<Row>>;
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
export declare interface StalenessSweepDeps {
|
|
1738
|
+
readonly store: StalenessDataStore;
|
|
1739
|
+
/**
|
|
1740
|
+
* Record the `run-stalled` event. Wired to the run recorder's `recordEvent`.
|
|
1741
|
+
* Absent → detection without a durable trace (the handler still fires), which
|
|
1742
|
+
* is the right shape for a `voltro doctor` style one-shot check.
|
|
1743
|
+
*/
|
|
1744
|
+
readonly recordEvent?: (input: {
|
|
1745
|
+
readonly runId: string;
|
|
1746
|
+
readonly eventType: 'run-stalled';
|
|
1747
|
+
readonly payload: Record<string, unknown>;
|
|
1748
|
+
}) => Promise<unknown>;
|
|
1749
|
+
/** App-supplied reaction — page someone, open a ticket. Must not throw;
|
|
1750
|
+
* a rejection is collected as a failure and the sweep continues. */
|
|
1751
|
+
readonly onStalled?: (run: StalledRun) => Promise<void>;
|
|
1752
|
+
readonly now?: () => number;
|
|
1753
|
+
readonly log?: {
|
|
1754
|
+
readonly info: (message: string, fields?: Record<string, unknown>) => void;
|
|
1755
|
+
readonly warn: (message: string, fields?: Record<string, unknown>) => void;
|
|
1756
|
+
};
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
export declare interface StalenessSweepOptions {
|
|
1760
|
+
readonly stallAfterMs?: number;
|
|
1761
|
+
readonly runPage?: number;
|
|
1762
|
+
readonly eventLookback?: number;
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
export declare interface StalenessSweepResult {
|
|
1766
|
+
/** Live runs old enough to be worth examining. */
|
|
1767
|
+
readonly examined: number;
|
|
1768
|
+
/** Newly reported this tick. */
|
|
1769
|
+
readonly stalled: ReadonlyArray<StalledRun>;
|
|
1770
|
+
/** Stalled, but already reported since their last progress — reported
|
|
1771
|
+
* separately rather than folded into `stalled`, so a caller can tell "the
|
|
1772
|
+
* wedge persists" from "nothing is stuck". */
|
|
1773
|
+
readonly alreadyReported: number;
|
|
1774
|
+
/** Excluded because they are inside a durable timer that has not come due. */
|
|
1775
|
+
readonly waitingOnTimer: number;
|
|
1776
|
+
/** The candidate page filled up: more live old runs exist and the next tick
|
|
1777
|
+
* will reach them. Reported rather than inferred — the counts alone cannot
|
|
1778
|
+
* distinguish it from "nothing else was old". */
|
|
1779
|
+
readonly sawFullRunPage: boolean;
|
|
1780
|
+
readonly failures: ReadonlyArray<{
|
|
1781
|
+
readonly subject: string;
|
|
1782
|
+
readonly detail: string;
|
|
1783
|
+
}>;
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
export declare interface StalledRun {
|
|
1787
|
+
readonly runId: string;
|
|
1788
|
+
readonly tag: string;
|
|
1789
|
+
readonly executionId: string;
|
|
1790
|
+
readonly status: string;
|
|
1791
|
+
readonly lastProgressAt: Date;
|
|
1792
|
+
readonly idleMs: number;
|
|
1793
|
+
readonly reason: StallReason;
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
/** Why the sweep believes this run is not moving. Diagnostic, not a taxonomy
|
|
1797
|
+
* the caller should branch on for correctness. */
|
|
1798
|
+
export declare type StallReason =
|
|
1799
|
+
/** Suspended with a `signal-awaited` outstanding — the classic wedge: the
|
|
1800
|
+
* sender never came. */
|
|
1801
|
+
'awaiting-signal'
|
|
1802
|
+
/** Suspended, but not on a signal and not on a timer. */
|
|
1803
|
+
| 'suspended'
|
|
1804
|
+
/** Status `running` with no step or event movement — a step that never
|
|
1805
|
+
* returns, or a runner that vanished without the reclaim path noticing. */
|
|
1806
|
+
| 'no-progress';
|
|
1807
|
+
|
|
1444
1808
|
export declare const step: <R, Success extends Schema.Schema.Any = typeof Schema.Void, Error extends Schema.Schema.All = typeof Schema.Never>(options: StepOptions<R, Success, Error>) => ReturnType<typeof stepModule.make<R, Success, Error>>;
|
|
1445
1809
|
|
|
1446
1810
|
/**
|
|
@@ -1543,6 +1907,62 @@ export declare interface StepRetryPolicy {
|
|
|
1543
1907
|
readonly note?: string;
|
|
1544
1908
|
}
|
|
1545
1909
|
|
|
1910
|
+
/**
|
|
1911
|
+
* Park the run until the budget has headroom.
|
|
1912
|
+
*
|
|
1913
|
+
* Returns (successfully) only when `hasHeadroom` answers true — so the caller
|
|
1914
|
+
* can spend immediately afterwards without a second check of its own. Fails with
|
|
1915
|
+
* `BudgetHoldExpired` once the total `timeoutMs` is used up, having spent
|
|
1916
|
+
* nothing.
|
|
1917
|
+
*
|
|
1918
|
+
* ── Why it is a LOOP, and why the loop is not a spin ─────────────────────────
|
|
1919
|
+
*
|
|
1920
|
+
* Two independent things can end one park: an external release, and the
|
|
1921
|
+
* recheck clock. Neither proves there is headroom (a window can roll over for a
|
|
1922
|
+
* tenant that immediately spends again; an operator can lift the wrong hold), so
|
|
1923
|
+
* each wake re-reads and — if still over — parks AGAIN, on a FRESH generation.
|
|
1924
|
+
*
|
|
1925
|
+
* It is not a spin because every iteration is a real `Workflow.suspend`: the
|
|
1926
|
+
* fiber is released, the wake is a `DurableClock`, and the minimum time between
|
|
1927
|
+
* two iterations is `recheckEveryMs`. The generation is what makes that true —
|
|
1928
|
+
* re-parking on the SAME key would await a deferred the previous release had
|
|
1929
|
+
* already completed, return instantly, and turn this loop into exactly the
|
|
1930
|
+
* busy-wait it looks like.
|
|
1931
|
+
*
|
|
1932
|
+
* The deadline is evaluated against the WALL CLOCK at each wake, from the FIRST
|
|
1933
|
+
* hold's durable `heldAt` rather than a captured `Date.now()`. A replay reading a
|
|
1934
|
+
* later clock therefore gives up sooner-or-equal, never later — which is the
|
|
1935
|
+
* safe direction for a control whose purpose is to stop spending.
|
|
1936
|
+
*/
|
|
1937
|
+
export declare const suspendForBudget: (options: SuspendForBudgetOptions) => Effect.Effect<void, BudgetHoldExpired>;
|
|
1938
|
+
|
|
1939
|
+
export declare interface SuspendForBudgetOptions {
|
|
1940
|
+
readonly store: DataStore;
|
|
1941
|
+
readonly budget: string;
|
|
1942
|
+
readonly tenantId: string | null;
|
|
1943
|
+
readonly executionId: string;
|
|
1944
|
+
readonly workflowName: string;
|
|
1945
|
+
readonly stepName: string;
|
|
1946
|
+
readonly spentMicroUsd?: number | undefined;
|
|
1947
|
+
readonly limitMicroUsd?: number | undefined;
|
|
1948
|
+
/** Total time the run may stay held across ALL parks. Default 7 days;
|
|
1949
|
+
* `Number.POSITIVE_INFINITY` waits forever (a legitimate choice for a
|
|
1950
|
+
* monthly cap). */
|
|
1951
|
+
readonly timeoutMs?: number | undefined;
|
|
1952
|
+
/** Self-recheck cadence. Default 15 minutes. */
|
|
1953
|
+
readonly recheckEveryMs?: number | undefined;
|
|
1954
|
+
/**
|
|
1955
|
+
* Does the budget have headroom NOW?
|
|
1956
|
+
*
|
|
1957
|
+
* Re-read on every wake, and it is the only thing that ends the hold. A
|
|
1958
|
+
* release — from an operator, or from the finops accountant's `recovered`
|
|
1959
|
+
* signal — WAKES the run; it does not authorise the spend. Those are different
|
|
1960
|
+
* facts, and conflating them is how a run resumes into a budget that is still
|
|
1961
|
+
* exceeded and spends anyway.
|
|
1962
|
+
*/
|
|
1963
|
+
readonly hasHeadroom: Effect.Effect<boolean>;
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1546
1966
|
/**
|
|
1547
1967
|
* The stable `DurableDeferred` name for a suspending signal wait. It is a pure
|
|
1548
1968
|
* function of the workflow name + the signal name, so the awaiting body and an
|
|
@@ -1562,6 +1982,15 @@ export declare const suspendingSignalDeferredName: (workflowName: string, signal
|
|
|
1562
1982
|
*/
|
|
1563
1983
|
export declare const sweepCancelOn: (deps: CancelOnSweepDeps, options: CancelOnSweepOptions) => Promise<CancelOnSweepResult>;
|
|
1564
1984
|
|
|
1985
|
+
/**
|
|
1986
|
+
* One staleness tick. Never throws.
|
|
1987
|
+
*
|
|
1988
|
+
* Same contract as `sweepCancelOn`: one malformed row must not take the whole
|
|
1989
|
+
* scheduled tick down and stop detection for every workflow until somebody
|
|
1990
|
+
* notices. Failures are collected and reported.
|
|
1991
|
+
*/
|
|
1992
|
+
export declare const sweepStalledRuns: (deps: StalenessSweepDeps, options?: StalenessSweepOptions) => Promise<StalenessSweepResult>;
|
|
1993
|
+
|
|
1565
1994
|
export declare const truncateWorkflowValue: (value: unknown, bytes?: number) => unknown;
|
|
1566
1995
|
|
|
1567
1996
|
/**
|
|
@@ -1581,6 +2010,14 @@ export declare interface ValidateFlowControlInput {
|
|
|
1581
2010
|
readonly control: (WorkflowFlowControl<never> & WorkflowCancelOnDeclared) | undefined;
|
|
1582
2011
|
}
|
|
1583
2012
|
|
|
2013
|
+
/**
|
|
2014
|
+
* One parked run, waiting on budget headroom.
|
|
2015
|
+
*
|
|
2016
|
+
* Reactive so a dashboard can show "3 runs held on `ai-usd`" without polling —
|
|
2017
|
+
* which is the operator's entry point to lifting one.
|
|
2018
|
+
*/
|
|
2019
|
+
export declare const _voltroBudgetHoldsTable: TableLike;
|
|
2020
|
+
|
|
1584
2021
|
/**
|
|
1585
2022
|
* `_voltro_workflow_admissions` — append-only ledger, and the lease register.
|
|
1586
2023
|
*
|
|
@@ -1904,6 +2341,37 @@ export declare interface WorkflowExecuteRecordingOptions extends WorkflowRunReco
|
|
|
1904
2341
|
readonly runId?: string;
|
|
1905
2342
|
readonly status: 'succeeded' | 'failed';
|
|
1906
2343
|
}) => Promise<void>;
|
|
2344
|
+
/**
|
|
2345
|
+
* CRASH-LOOP BREAKER — how many CONSECUTIVE runner deaths this run may cost
|
|
2346
|
+
* the fleet before it is parked instead of reclaimed again.
|
|
2347
|
+
*
|
|
2348
|
+
* The failure it bounds has no error to catch: a step that kills the process
|
|
2349
|
+
* (OOM, a native crash, a `process.exit` in a dependency) leaves the shard
|
|
2350
|
+
* lease to age out, a survivor claims it, and executes the same payload —
|
|
2351
|
+
* forever, across every replica in turn. Poison handling exists at ADMISSION
|
|
2352
|
+
* (`admissionTables.ts`), which is the wrong side of the boundary: the run
|
|
2353
|
+
* was admitted long ago and dies on the way out.
|
|
2354
|
+
*
|
|
2355
|
+
* At the cap the run becomes `suspended` with a `run-crashlooped` event, the
|
|
2356
|
+
* body is NOT entered, and the fleet stops rotating. An operator resume
|
|
2357
|
+
* re-arms it — and any clean re-entry (a durable sleep waking, a redrive)
|
|
2358
|
+
* resets the counter to 0, so this measures a loop, not a lifetime.
|
|
2359
|
+
*
|
|
2360
|
+
* Default {@link DEFAULT_MAX_RUN_RECLAIMS}; `VOLTRO_WORKFLOW_MAX_RECLAIMS`
|
|
2361
|
+
* overrides it via {@link resolveRunGuardTuning}.
|
|
2362
|
+
*/
|
|
2363
|
+
readonly maxRunReclaims?: number;
|
|
2364
|
+
/**
|
|
2365
|
+
* Step rows read to build the replay-shape snapshot for the nondeterminism
|
|
2366
|
+
* tripwire. A run with more recorded steps than this SKIPS the tripwire
|
|
2367
|
+
* entirely rather than comparing against a truncated history — a partial
|
|
2368
|
+
* prior shape would report the steps it failed to load as `unreached-step`,
|
|
2369
|
+
* which is a false positive manufactured by the bound itself.
|
|
2370
|
+
*
|
|
2371
|
+
* Default {@link DEFAULT_REPLAY_SHAPE_LIMIT};
|
|
2372
|
+
* `VOLTRO_WORKFLOW_REPLAY_SHAPE_LIMIT` overrides it.
|
|
2373
|
+
*/
|
|
2374
|
+
readonly replayShapeLimit?: number;
|
|
1907
2375
|
}
|
|
1908
2376
|
|
|
1909
2377
|
/**
|
|
@@ -2066,16 +2534,24 @@ export declare interface WorkflowMessagesCarrier<M extends WorkflowMessagesMetad
|
|
|
2066
2534
|
readonly [WorkflowMessagesProperty]: M;
|
|
2067
2535
|
}
|
|
2068
2536
|
|
|
2537
|
+
/**
|
|
2538
|
+
* The message channels a workflow declares.
|
|
2539
|
+
*
|
|
2540
|
+
* TWO, not three. `queries` used to sit here as a third channel and there was
|
|
2541
|
+
* never a send path for it — no `sendWorkflowQuery`, no `awaitQuery`, nothing
|
|
2542
|
+
* to receive one. It was normalised into metadata, projected into the generated
|
|
2543
|
+
* rpcGroup and carried on the client's `WorkflowState`, so a user got a fully
|
|
2544
|
+
* typed record they could not invoke from anywhere. Signals (fire-and-forget)
|
|
2545
|
+
* and updates (synchronous, with a result) are the channels that exist.
|
|
2546
|
+
*/
|
|
2069
2547
|
export declare interface WorkflowMessageSchemas {
|
|
2070
2548
|
readonly signals?: Readonly<Record<string, Schema.Schema.Any>>;
|
|
2071
2549
|
readonly updates?: Readonly<Record<string, WorkflowMessagePairSchemas>>;
|
|
2072
|
-
readonly queries?: Readonly<Record<string, WorkflowMessagePairSchemas>>;
|
|
2073
2550
|
}
|
|
2074
2551
|
|
|
2075
2552
|
export declare interface WorkflowMessagesMetadata {
|
|
2076
2553
|
readonly signals: Readonly<Record<string, Schema.Schema.Any>>;
|
|
2077
2554
|
readonly updates: Readonly<Record<string, WorkflowMessagePairSchemas>>;
|
|
2078
|
-
readonly queries: Readonly<Record<string, WorkflowMessagePairSchemas>>;
|
|
2079
2555
|
}
|
|
2080
2556
|
|
|
2081
2557
|
export declare const WorkflowMessagesProperty = "__voltroWorkflowMessages";
|
|
@@ -2224,7 +2700,23 @@ export declare type WorkflowRunEventType = 'run-started' | 'run-suspended' | 'ru
|
|
|
2224
2700
|
/** Emitted by `awaitUpdate(...)` when schema validation or the
|
|
2225
2701
|
* update handler fails. `payload` carries `{ updateId, updateName,
|
|
2226
2702
|
* errorTag, errorMessage }`; the caller's update promise rejects. */
|
|
2227
|
-
| 'update-failed'
|
|
2703
|
+
| 'update-failed'
|
|
2704
|
+
/** The replay tripwire fired: this run is re-executing against a body whose
|
|
2705
|
+
* step shape no longer matches what the run journaled. `payload` carries
|
|
2706
|
+
* `{ kind, stepName, recorded, reached, detail }` — see
|
|
2707
|
+
* `nondeterminism.ts`. NEVER accompanied by a state change: the run keeps
|
|
2708
|
+
* going, because a false positive that killed a run would be worse than the
|
|
2709
|
+
* divergence it suspects. */
|
|
2710
|
+
| 'nondeterminism-suspected'
|
|
2711
|
+
/** The crash-loop breaker parked this run: its runner died mid-body
|
|
2712
|
+
* `maxRunReclaims` times in a row and a survivor reclaimed it each time.
|
|
2713
|
+
* `payload` carries `{ reclaimCount, maxRunReclaims }`. The run row is
|
|
2714
|
+
* `suspended` — an operator resume re-arms it with a fresh budget. */
|
|
2715
|
+
| 'run-crashlooped'
|
|
2716
|
+
/** The staleness sweep saw no progress on a live run for longer than the
|
|
2717
|
+
* configured threshold. `payload` carries `{ idleMs, stallAfterMs,
|
|
2718
|
+
* lastProgressAt, reason }`. Diagnostic only — the run is untouched. */
|
|
2719
|
+
| 'run-stalled';
|
|
2228
2720
|
|
|
2229
2721
|
/** A workflow run's TERMINAL outcome, handed to the metrics hook. Kept local to
|
|
2230
2722
|
* this package (structurally matching `@voltro/runtime`'s `WorkflowRunRecord`) so
|
|
@@ -2461,6 +2953,17 @@ export declare interface WorkflowVersionMetadata {
|
|
|
2461
2953
|
export declare interface WorkflowVersionOptions {
|
|
2462
2954
|
readonly version?: string | number;
|
|
2463
2955
|
readonly compatibleWith?: ReadonlyArray<string | number>;
|
|
2956
|
+
/**
|
|
2957
|
+
* Named change markers this workflow's body may branch on, via
|
|
2958
|
+
* {@link patch}. Declaring one here is what makes `yield* patch('id')`
|
|
2959
|
+
* answer `true` for runs started from now on and `false` for runs that were
|
|
2960
|
+
* already in flight — see {@link patch} for the semantics and the worked
|
|
2961
|
+
* example.
|
|
2962
|
+
*
|
|
2963
|
+
* The declared set is stamped onto `_voltro_workflow_runs.workflowPatches`
|
|
2964
|
+
* at run start and read back from THERE on every resume, so the answer is
|
|
2965
|
+
* pinned to the run and cannot change under a redeploy.
|
|
2966
|
+
*/
|
|
2464
2967
|
readonly patches?: ReadonlyArray<string>;
|
|
2465
2968
|
/**
|
|
2466
2969
|
* When `true`, a workflow whose top-level body FAILS does not become a
|