@rivetkit/workflow-engine 0.0.0-0-0-0-preview-guard-stops.9d82529

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/context.ts ADDED
@@ -0,0 +1,2677 @@
1
+ import type { Logger } from "pino";
2
+ import type { EngineDriver } from "./driver.js";
3
+ import {
4
+ extractErrorInfo,
5
+ getErrorEventTag,
6
+ markErrorReported,
7
+ } from "./error-utils.js";
8
+ import {
9
+ CancelledError,
10
+ CriticalError,
11
+ EntryInProgressError,
12
+ EvictedError,
13
+ HistoryDivergedError,
14
+ JoinError,
15
+ MessageWaitError,
16
+ RaceError,
17
+ RollbackCheckpointError,
18
+ RollbackError,
19
+ RollbackStopError,
20
+ SleepError,
21
+ StepExhaustedError,
22
+ StepFailedError,
23
+ } from "./errors.js";
24
+ import { buildEntryMetadataKey, buildLoopIterationRange } from "./keys.js";
25
+ import {
26
+ appendLoopIteration,
27
+ appendName,
28
+ emptyLocation,
29
+ isLocationPrefix,
30
+ locationToKey,
31
+ registerName,
32
+ } from "./location.js";
33
+ import {
34
+ createEntry,
35
+ deleteEntriesWithPrefix,
36
+ flush,
37
+ getOrCreateMetadata,
38
+ loadMetadata,
39
+ type PendingDeletions,
40
+ setEntry,
41
+ } from "./storage.js";
42
+ import type {
43
+ BranchConfig,
44
+ BranchOutput,
45
+ Entry,
46
+ EntryKindType,
47
+ EntryMetadata,
48
+ Location,
49
+ LoopConfig,
50
+ LoopIterationResult,
51
+ LoopResult,
52
+ Message,
53
+ RollbackContextInterface,
54
+ StepConfig,
55
+ Storage,
56
+ TryBlockCatchKind,
57
+ TryBlockConfig,
58
+ TryBlockFailure,
59
+ TryBlockResult,
60
+ TryStepCatchKind,
61
+ TryStepConfig,
62
+ TryStepFailure,
63
+ TryStepResult,
64
+ WorkflowContextInterface,
65
+ WorkflowError,
66
+ WorkflowErrorEvent,
67
+ WorkflowErrorHandler,
68
+ WorkflowMessageDriver,
69
+ WorkflowQueue,
70
+ WorkflowQueueMessage,
71
+ WorkflowQueueNextBatchOptions,
72
+ WorkflowQueueNextOptions,
73
+ } from "./types.js";
74
+
75
+ /**
76
+ * Default values for step configuration.
77
+ * These are exported so users can reference them when overriding.
78
+ */
79
+ export const DEFAULT_MAX_RETRIES = 3;
80
+ export const DEFAULT_RETRY_BACKOFF_BASE = 100;
81
+ export const DEFAULT_RETRY_BACKOFF_MAX = 30000;
82
+ export const DEFAULT_LOOP_HISTORY_PRUNE_INTERVAL = 20;
83
+ export const DEFAULT_STEP_TIMEOUT = 30000; // 30 seconds
84
+ const DEFAULT_TRY_STEP_CATCH: readonly TryStepCatchKind[] = [
85
+ "critical",
86
+ "timeout",
87
+ "exhausted",
88
+ ];
89
+ const DEFAULT_TRY_BLOCK_CATCH: readonly TryBlockCatchKind[] = [
90
+ "step",
91
+ "join",
92
+ "race",
93
+ ];
94
+
95
+ const QUEUE_HISTORY_MESSAGE_MARKER = "__rivetWorkflowQueueMessage";
96
+ const TRY_STEP_FAILURE_SYMBOL = Symbol("workflow.try-step.failure");
97
+ const TRY_BLOCK_FAILURE_SYMBOL = Symbol("workflow.try-block.failure");
98
+
99
+ /**
100
+ * Calculate backoff delay with exponential backoff.
101
+ * Uses deterministic calculation (no jitter) for replay consistency.
102
+ */
103
+ function calculateBackoff(attempts: number, base: number, max: number): number {
104
+ // Exponential backoff without jitter for determinism
105
+ return Math.min(max, base * 2 ** attempts);
106
+ }
107
+
108
+ /**
109
+ * Error thrown when a step times out.
110
+ */
111
+ export class StepTimeoutError extends Error {
112
+ constructor(
113
+ public readonly stepName: string,
114
+ public readonly timeoutMs: number,
115
+ ) {
116
+ super(`Step "${stepName}" timed out after ${timeoutMs}ms`);
117
+ this.name = "StepTimeoutError";
118
+ }
119
+ }
120
+
121
+ type SchedulerYieldState = {
122
+ deadline?: number;
123
+ messageNames: Set<string>;
124
+ };
125
+
126
+ type TryBlockFailureInfo = Pick<TryBlockFailure, "source" | "name">;
127
+
128
+ function attachTryStepFailure<T extends Error>(
129
+ error: T,
130
+ failure: TryStepFailure,
131
+ ): T {
132
+ (
133
+ error as T & {
134
+ [TRY_STEP_FAILURE_SYMBOL]?: TryStepFailure;
135
+ }
136
+ )[TRY_STEP_FAILURE_SYMBOL] = failure;
137
+ return error;
138
+ }
139
+
140
+ function readTryStepFailure(error: unknown): TryStepFailure | undefined {
141
+ if (!(error instanceof Error)) {
142
+ return undefined;
143
+ }
144
+
145
+ return (
146
+ error as Error & {
147
+ [TRY_STEP_FAILURE_SYMBOL]?: TryStepFailure;
148
+ }
149
+ )[TRY_STEP_FAILURE_SYMBOL];
150
+ }
151
+
152
+ function attachTryBlockFailure<T extends Error>(
153
+ error: T,
154
+ failure: TryBlockFailureInfo,
155
+ ): T {
156
+ (
157
+ error as T & {
158
+ [TRY_BLOCK_FAILURE_SYMBOL]?: TryBlockFailureInfo;
159
+ }
160
+ )[TRY_BLOCK_FAILURE_SYMBOL] = failure;
161
+ return error;
162
+ }
163
+
164
+ function readTryBlockFailure(error: unknown): TryBlockFailureInfo | undefined {
165
+ if (!(error instanceof Error)) {
166
+ return undefined;
167
+ }
168
+
169
+ return (
170
+ error as Error & {
171
+ [TRY_BLOCK_FAILURE_SYMBOL]?: TryBlockFailureInfo;
172
+ }
173
+ )[TRY_BLOCK_FAILURE_SYMBOL];
174
+ }
175
+
176
+ function shouldRethrowTryError(error: unknown): boolean {
177
+ return (
178
+ error instanceof StepFailedError ||
179
+ error instanceof SleepError ||
180
+ error instanceof MessageWaitError ||
181
+ error instanceof EvictedError ||
182
+ error instanceof HistoryDivergedError ||
183
+ error instanceof EntryInProgressError ||
184
+ error instanceof RollbackCheckpointError ||
185
+ error instanceof RollbackStopError
186
+ );
187
+ }
188
+
189
+ function shouldCatchTryStepFailure(
190
+ failure: TryStepFailure,
191
+ catchKinds?: readonly TryStepCatchKind[],
192
+ ): boolean {
193
+ const effectiveCatch = catchKinds ?? DEFAULT_TRY_STEP_CATCH;
194
+ return effectiveCatch.includes(failure.kind);
195
+ }
196
+
197
+ function shouldCatchTryBlockFailure(
198
+ failure: TryBlockFailure,
199
+ catchKinds?: readonly TryBlockCatchKind[],
200
+ ): boolean {
201
+ const effectiveCatch = catchKinds ?? DEFAULT_TRY_BLOCK_CATCH;
202
+
203
+ if (failure.source === "step") {
204
+ return failure.step?.kind === "rollback"
205
+ ? effectiveCatch.includes("rollback")
206
+ : effectiveCatch.includes("step");
207
+ }
208
+ if (failure.source === "join") {
209
+ return effectiveCatch.includes("join");
210
+ }
211
+ if (failure.source === "race") {
212
+ return effectiveCatch.includes("race");
213
+ }
214
+ return effectiveCatch.includes("rollback");
215
+ }
216
+
217
+ function parseStoredWorkflowError(message: string | undefined): WorkflowError {
218
+ if (!message) {
219
+ return {
220
+ name: "Error",
221
+ message: "unknown error",
222
+ };
223
+ }
224
+
225
+ const match = /^([^:]+):\s*(.*)$/s.exec(message);
226
+ if (!match) {
227
+ return {
228
+ name: "Error",
229
+ message,
230
+ };
231
+ }
232
+
233
+ return {
234
+ name: match[1],
235
+ message: match[2],
236
+ };
237
+ }
238
+
239
+ function getTryStepFailureFromExhaustedError(
240
+ stepName: string,
241
+ attempts: number,
242
+ error: StepExhaustedError,
243
+ ): TryStepFailure {
244
+ return {
245
+ kind: "exhausted",
246
+ stepName,
247
+ attempts,
248
+ error: parseStoredWorkflowError(error.lastError),
249
+ };
250
+ }
251
+
252
+ function mergeSchedulerYield(
253
+ state: SchedulerYieldState | undefined,
254
+ error: SleepError | MessageWaitError | StepFailedError,
255
+ ): SchedulerYieldState {
256
+ const nextState: SchedulerYieldState = state ?? {
257
+ messageNames: new Set<string>(),
258
+ };
259
+
260
+ if (error instanceof SleepError) {
261
+ nextState.deadline =
262
+ nextState.deadline === undefined
263
+ ? error.deadline
264
+ : Math.min(nextState.deadline, error.deadline);
265
+ for (const messageName of error.messageNames ?? []) {
266
+ nextState.messageNames.add(messageName);
267
+ }
268
+ return nextState;
269
+ }
270
+
271
+ if (error instanceof MessageWaitError) {
272
+ for (const messageName of error.messageNames) {
273
+ nextState.messageNames.add(messageName);
274
+ }
275
+ return nextState;
276
+ }
277
+
278
+ nextState.deadline =
279
+ nextState.deadline === undefined
280
+ ? error.retryAt
281
+ : Math.min(nextState.deadline, error.retryAt);
282
+ return nextState;
283
+ }
284
+
285
+ function buildSchedulerYieldError(
286
+ state: SchedulerYieldState,
287
+ ): SleepError | MessageWaitError {
288
+ const messageNames = [...state.messageNames];
289
+ if (state.deadline !== undefined) {
290
+ return new SleepError(
291
+ state.deadline,
292
+ messageNames.length > 0 ? messageNames : undefined,
293
+ );
294
+ }
295
+ return new MessageWaitError(messageNames);
296
+ }
297
+
298
+ function controlFlowErrorPriority(error: Error): number {
299
+ if (error instanceof EvictedError) {
300
+ return 0;
301
+ }
302
+ if (error instanceof HistoryDivergedError) {
303
+ return 1;
304
+ }
305
+ if (error instanceof EntryInProgressError) {
306
+ return 2;
307
+ }
308
+ if (error instanceof RollbackCheckpointError) {
309
+ return 3;
310
+ }
311
+ if (error instanceof RollbackStopError) {
312
+ return 4;
313
+ }
314
+ return 5;
315
+ }
316
+
317
+ function selectControlFlowError(
318
+ current: Error | undefined,
319
+ candidate: Error,
320
+ ): Error {
321
+ if (!current) {
322
+ return candidate;
323
+ }
324
+ return controlFlowErrorPriority(candidate) <
325
+ controlFlowErrorPriority(current)
326
+ ? candidate
327
+ : current;
328
+ }
329
+
330
+ /**
331
+ * Internal representation of a rollback handler.
332
+ */
333
+ export interface RollbackAction<T = unknown> {
334
+ entryId: string;
335
+ name: string;
336
+ output: T;
337
+ rollback: (ctx: RollbackContextInterface, output: T) => Promise<void>;
338
+ }
339
+
340
+ /**
341
+ * Internal implementation of WorkflowContext.
342
+ */
343
+ export class WorkflowContextImpl implements WorkflowContextInterface {
344
+ private entryInProgress = false;
345
+ private abortController: AbortController;
346
+ private currentLocation: Location;
347
+ private visitedKeys: Set<string>;
348
+ private mode: "forward" | "rollback";
349
+ private rollbackActions?: RollbackAction[];
350
+ private rollbackCheckpointSet: boolean;
351
+ /** Track names used in current execution to detect duplicates */
352
+ private usedNamesInExecution = new Set<string>();
353
+ private pendingCompletableMessageIds = new Set<string>();
354
+ private historyNotifier?: () => void;
355
+ private onError?: WorkflowErrorHandler;
356
+ private logger?: Logger;
357
+
358
+ constructor(
359
+ public readonly workflowId: string,
360
+ private storage: Storage,
361
+ private driver: EngineDriver,
362
+ private messageDriver: WorkflowMessageDriver,
363
+ location: Location = emptyLocation(),
364
+ abortController?: AbortController,
365
+ mode: "forward" | "rollback" = "forward",
366
+ rollbackActions?: RollbackAction[],
367
+ rollbackCheckpointSet = false,
368
+ historyNotifier?: () => void,
369
+ onError?: WorkflowErrorHandler,
370
+ logger?: Logger,
371
+ visitedKeys?: Set<string>,
372
+ ) {
373
+ this.currentLocation = location;
374
+ this.abortController = abortController ?? new AbortController();
375
+ this.mode = mode;
376
+ this.rollbackActions = rollbackActions;
377
+ this.rollbackCheckpointSet = rollbackCheckpointSet;
378
+ this.historyNotifier = historyNotifier;
379
+ this.onError = onError;
380
+ this.logger = logger;
381
+ this.visitedKeys = visitedKeys ?? new Set();
382
+ }
383
+
384
+ get abortSignal(): AbortSignal {
385
+ return this.abortController.signal;
386
+ }
387
+
388
+ get queue(): WorkflowQueue {
389
+ return {
390
+ next: async (name, opts) => await this.queueNext(name, opts),
391
+ nextBatch: async (name, opts) =>
392
+ await this.queueNextBatch(name, opts),
393
+ send: async (name, body) => await this.queueSend(name, body),
394
+ };
395
+ }
396
+
397
+ isEvicted(): boolean {
398
+ return this.abortSignal.aborted;
399
+ }
400
+
401
+ private assertNotInProgress(): void {
402
+ if (this.entryInProgress) {
403
+ throw new EntryInProgressError();
404
+ }
405
+ }
406
+
407
+ private checkEvicted(): void {
408
+ if (this.abortSignal.aborted) {
409
+ throw new EvictedError();
410
+ }
411
+ }
412
+
413
+ private async flushStorage(): Promise<void> {
414
+ await flush(this.storage, this.driver, this.historyNotifier);
415
+ }
416
+
417
+ /**
418
+ * Create a new branch context for parallel/nested execution.
419
+ */
420
+ createBranch(
421
+ location: Location,
422
+ abortController?: AbortController,
423
+ ): WorkflowContextImpl {
424
+ return new WorkflowContextImpl(
425
+ this.workflowId,
426
+ this.storage,
427
+ this.driver,
428
+ this.messageDriver,
429
+ location,
430
+ abortController ?? this.abortController,
431
+ this.mode,
432
+ this.rollbackActions,
433
+ this.rollbackCheckpointSet,
434
+ this.historyNotifier,
435
+ this.onError,
436
+ this.logger,
437
+ this.visitedKeys,
438
+ );
439
+ }
440
+
441
+ /**
442
+ * Log a debug message using the configured logger.
443
+ */
444
+ private log(
445
+ level: "debug" | "info" | "warn" | "error",
446
+ data: Record<string, unknown>,
447
+ ): void {
448
+ if (!this.logger) return;
449
+ this.logger[level](data);
450
+ }
451
+
452
+ private async notifyError(event: WorkflowErrorEvent): Promise<void> {
453
+ if (!this.onError) {
454
+ return;
455
+ }
456
+
457
+ try {
458
+ await this.onError(event);
459
+ } catch (error) {
460
+ this.log("warn", {
461
+ msg: "workflow error hook failed",
462
+ hookEventType: getErrorEventTag(event),
463
+ error: extractErrorInfo(error),
464
+ });
465
+ }
466
+ }
467
+
468
+ private async notifyStepError<T>(
469
+ config: StepConfig<T>,
470
+ attempt: number,
471
+ error: unknown,
472
+ opts: {
473
+ willRetry: boolean;
474
+ retryDelay?: number;
475
+ retryAt?: number;
476
+ },
477
+ ): Promise<void> {
478
+ const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
479
+ await this.notifyError({
480
+ step: {
481
+ workflowId: this.workflowId,
482
+ stepName: config.name,
483
+ attempt,
484
+ maxRetries,
485
+ remainingRetries: Math.max(0, maxRetries - (attempt - 1)),
486
+ willRetry: opts.willRetry,
487
+ retryDelay: opts.retryDelay,
488
+ retryAt: opts.retryAt,
489
+ error: extractErrorInfo(error),
490
+ },
491
+ });
492
+ }
493
+
494
+ /**
495
+ * Mark a key as visited.
496
+ */
497
+ private markVisited(key: string): void {
498
+ this.visitedKeys.add(key);
499
+ }
500
+
501
+ /**
502
+ * Check if a name has already been used at the current location in this execution.
503
+ * Throws HistoryDivergedError if duplicate detected.
504
+ */
505
+ private checkDuplicateName(name: string): void {
506
+ const fullKey = `${locationToKey(this.storage, this.currentLocation)}/${name}`;
507
+ if (this.usedNamesInExecution.has(fullKey)) {
508
+ throw new HistoryDivergedError(
509
+ `Duplicate entry name "${name}" at location "${locationToKey(this.storage, this.currentLocation)}". ` +
510
+ `Each step/loop/sleep/queue.next/join/race must have a unique name within its scope.`,
511
+ );
512
+ }
513
+ this.usedNamesInExecution.add(fullKey);
514
+ }
515
+
516
+ private stopRollback(): never {
517
+ throw new RollbackStopError();
518
+ }
519
+
520
+ private stopRollbackIfMissing(entry: Entry | undefined): void {
521
+ if (this.mode === "rollback" && !entry) {
522
+ this.stopRollback();
523
+ }
524
+ }
525
+
526
+ private stopRollbackIfIncomplete(condition: boolean): void {
527
+ if (this.mode === "rollback" && condition) {
528
+ this.stopRollback();
529
+ }
530
+ }
531
+
532
+ private registerRollbackAction<T>(
533
+ config: StepConfig<T>,
534
+ entryId: string,
535
+ output: T,
536
+ metadata: EntryMetadata,
537
+ ): void {
538
+ if (!config.rollback) {
539
+ return;
540
+ }
541
+ if (metadata.rollbackCompletedAt !== undefined) {
542
+ return;
543
+ }
544
+ this.rollbackActions?.push({
545
+ entryId,
546
+ name: config.name,
547
+ output: output as unknown,
548
+ rollback: config.rollback as (
549
+ ctx: RollbackContextInterface,
550
+ output: unknown,
551
+ ) => Promise<void>,
552
+ });
553
+ }
554
+
555
+ /**
556
+ * Ensure a rollback checkpoint exists before registering rollback handlers.
557
+ */
558
+ private ensureRollbackCheckpoint<T>(config: StepConfig<T>): void {
559
+ if (!config.rollback) {
560
+ return;
561
+ }
562
+ if (!this.rollbackCheckpointSet) {
563
+ throw new RollbackCheckpointError();
564
+ }
565
+ }
566
+
567
+ /**
568
+ * Validate that all expected entries in the branch were visited.
569
+ * Throws HistoryDivergedError if there are unvisited entries.
570
+ */
571
+ validateComplete(): void {
572
+ const prefix = locationToKey(this.storage, this.currentLocation);
573
+
574
+ for (const key of this.storage.history.entries.keys()) {
575
+ // Check if this key is under our current location prefix
576
+ // Handle root prefix (empty string) specially - all keys are under root
577
+ const isUnderPrefix =
578
+ prefix === ""
579
+ ? true // Root: all keys are children
580
+ : key.startsWith(`${prefix}/`) || key === prefix;
581
+
582
+ if (isUnderPrefix) {
583
+ if (!this.visitedKeys.has(key)) {
584
+ // Entry exists in history but wasn't visited
585
+ // This means workflow code may have changed
586
+ throw new HistoryDivergedError(
587
+ `Entry "${key}" exists in history but was not visited. ` +
588
+ `Workflow code may have changed. Use ctx.removed() to handle migrations.`,
589
+ );
590
+ }
591
+ }
592
+ }
593
+ }
594
+
595
+ /**
596
+ * Evict the workflow.
597
+ */
598
+ evict(): void {
599
+ this.abortController.abort(new EvictedError());
600
+ }
601
+
602
+ /**
603
+ * Wait for `ms`, rejecting early with EvictedError if the workflow is
604
+ * evicted. Both the timer and the abort listener are torn down on either
605
+ * outcome, so a completed sleep never leaves a dangling listener on the
606
+ * long-lived run abort signal.
607
+ */
608
+ private async sleepOrEvict(ms: number): Promise<void> {
609
+ if (this.abortSignal.aborted) {
610
+ throw new EvictedError();
611
+ }
612
+ let timer: ReturnType<typeof setTimeout> | undefined;
613
+ let onAbort: (() => void) | undefined;
614
+ try {
615
+ await new Promise<void>((resolve, reject) => {
616
+ timer = setTimeout(resolve, ms);
617
+ onAbort = () => reject(new EvictedError());
618
+ this.abortSignal.addEventListener("abort", onAbort, {
619
+ once: true,
620
+ });
621
+ });
622
+ } finally {
623
+ if (timer !== undefined) {
624
+ clearTimeout(timer);
625
+ }
626
+ if (onAbort) {
627
+ this.abortSignal.removeEventListener("abort", onAbort);
628
+ }
629
+ }
630
+ }
631
+
632
+ // === Step ===
633
+
634
+ async step<T>(
635
+ nameOrConfig: string | StepConfig<T>,
636
+ run?: () => Promise<T>,
637
+ ): Promise<T> {
638
+ this.assertNotInProgress();
639
+ this.checkEvicted();
640
+
641
+ const config: StepConfig<T> =
642
+ typeof nameOrConfig === "string"
643
+ ? { name: nameOrConfig, run: run! }
644
+ : nameOrConfig;
645
+
646
+ this.entryInProgress = true;
647
+ try {
648
+ return await this.executeStep(config);
649
+ } finally {
650
+ this.entryInProgress = false;
651
+ }
652
+ }
653
+
654
+ async tryStep<T>(
655
+ nameOrConfig: string | TryStepConfig<T>,
656
+ run?: () => Promise<T>,
657
+ ): Promise<TryStepResult<T>> {
658
+ const config =
659
+ typeof nameOrConfig === "string"
660
+ ? ({
661
+ name: nameOrConfig,
662
+ run: run!,
663
+ } satisfies TryStepConfig<T>)
664
+ : nameOrConfig;
665
+
666
+ try {
667
+ return {
668
+ ok: true,
669
+ value: await this.step(config),
670
+ };
671
+ } catch (error) {
672
+ if (shouldRethrowTryError(error)) {
673
+ throw error;
674
+ }
675
+
676
+ const failure = readTryStepFailure(error);
677
+ if (!failure || !shouldCatchTryStepFailure(failure, config.catch)) {
678
+ throw error;
679
+ }
680
+
681
+ return {
682
+ ok: false,
683
+ failure,
684
+ };
685
+ }
686
+ }
687
+
688
+ async try<T>(
689
+ nameOrConfig: string | TryBlockConfig<T>,
690
+ run?: (ctx: WorkflowContextInterface) => Promise<T>,
691
+ ): Promise<TryBlockResult<T>> {
692
+ this.assertNotInProgress();
693
+ this.checkEvicted();
694
+
695
+ const config =
696
+ typeof nameOrConfig === "string"
697
+ ? ({
698
+ name: nameOrConfig,
699
+ run: run!,
700
+ } satisfies TryBlockConfig<T>)
701
+ : nameOrConfig;
702
+
703
+ this.entryInProgress = true;
704
+ try {
705
+ return await this.executeTry(config);
706
+ } finally {
707
+ this.entryInProgress = false;
708
+ }
709
+ }
710
+
711
+ private async executeTry<T>(
712
+ config: TryBlockConfig<T>,
713
+ ): Promise<TryBlockResult<T>> {
714
+ this.checkDuplicateName(config.name);
715
+
716
+ const location = appendName(
717
+ this.storage,
718
+ this.currentLocation,
719
+ config.name,
720
+ );
721
+ const blockCtx = this.createBranch(location);
722
+
723
+ try {
724
+ const value = await config.run(blockCtx);
725
+ blockCtx.validateComplete();
726
+ return {
727
+ ok: true,
728
+ value,
729
+ };
730
+ } catch (error) {
731
+ if (shouldRethrowTryError(error)) {
732
+ throw error;
733
+ }
734
+
735
+ const stepFailure = readTryStepFailure(error);
736
+ if (stepFailure) {
737
+ const failure: TryBlockFailure = {
738
+ source: "step",
739
+ name: stepFailure.stepName,
740
+ error: stepFailure.error,
741
+ step: stepFailure,
742
+ };
743
+ if (!shouldCatchTryBlockFailure(failure, config.catch)) {
744
+ throw error;
745
+ }
746
+ return {
747
+ ok: false,
748
+ failure,
749
+ };
750
+ }
751
+
752
+ const operationFailure = readTryBlockFailure(error);
753
+ if (operationFailure) {
754
+ const failure: TryBlockFailure = {
755
+ ...operationFailure,
756
+ error: extractErrorInfo(error),
757
+ };
758
+ if (!shouldCatchTryBlockFailure(failure, config.catch)) {
759
+ throw error;
760
+ }
761
+ return {
762
+ ok: false,
763
+ failure,
764
+ };
765
+ }
766
+
767
+ if (error instanceof RollbackError) {
768
+ const failure: TryBlockFailure = {
769
+ source: "block",
770
+ name: config.name,
771
+ error: extractErrorInfo(error),
772
+ };
773
+ if (!shouldCatchTryBlockFailure(failure, config.catch)) {
774
+ throw error;
775
+ }
776
+ return {
777
+ ok: false,
778
+ failure,
779
+ };
780
+ }
781
+
782
+ throw error;
783
+ }
784
+ }
785
+
786
+ private async executeStep<T>(config: StepConfig<T>): Promise<T> {
787
+ this.ensureRollbackCheckpoint(config);
788
+ if (this.mode === "rollback") {
789
+ return await this.executeStepRollback(config);
790
+ }
791
+
792
+ // Check for duplicate name in current execution
793
+ this.checkDuplicateName(config.name);
794
+
795
+ const location = appendName(
796
+ this.storage,
797
+ this.currentLocation,
798
+ config.name,
799
+ );
800
+ const key = locationToKey(this.storage, location);
801
+ const existing = this.storage.history.entries.get(key);
802
+
803
+ // Mark this entry as visited for validateComplete
804
+ this.markVisited(key);
805
+
806
+ if (existing) {
807
+ if (existing.kind.type !== "step") {
808
+ throw new HistoryDivergedError(
809
+ `Expected step "${config.name}" at ${key}, found ${existing.kind.type}`,
810
+ );
811
+ }
812
+
813
+ const stepData = existing.kind.data;
814
+
815
+ const metadata = await loadMetadata(
816
+ this.storage,
817
+ this.driver,
818
+ existing.id,
819
+ );
820
+
821
+ // Replay successful result (including void steps).
822
+ if (
823
+ metadata.status === "completed" ||
824
+ stepData.output !== undefined
825
+ ) {
826
+ return stepData.output as T;
827
+ }
828
+
829
+ // Check if we should retry
830
+ const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
831
+
832
+ if (metadata.attempts > maxRetries) {
833
+ const lastError = metadata.error;
834
+ const exhaustedError = new StepExhaustedError(
835
+ config.name,
836
+ lastError,
837
+ );
838
+ attachTryStepFailure(
839
+ exhaustedError,
840
+ getTryStepFailureFromExhaustedError(
841
+ config.name,
842
+ metadata.attempts,
843
+ exhaustedError,
844
+ ),
845
+ );
846
+ markErrorReported(exhaustedError);
847
+ if (metadata.status !== "exhausted") {
848
+ metadata.status = "exhausted";
849
+ metadata.dirty = true;
850
+ await this.flushStorage();
851
+ await this.notifyStepError(
852
+ config,
853
+ metadata.attempts,
854
+ exhaustedError,
855
+ { willRetry: false },
856
+ );
857
+ }
858
+ throw exhaustedError;
859
+ }
860
+
861
+ // Calculate backoff and yield to scheduler
862
+ // This allows the workflow to be evicted during backoff
863
+ const backoffDelay = calculateBackoff(
864
+ metadata.attempts,
865
+ config.retryBackoffBase ?? DEFAULT_RETRY_BACKOFF_BASE,
866
+ config.retryBackoffMax ?? DEFAULT_RETRY_BACKOFF_MAX,
867
+ );
868
+ const retryAt = metadata.lastAttemptAt + backoffDelay;
869
+ const now = Date.now();
870
+
871
+ if (now < retryAt) {
872
+ // Yield to scheduler - will be woken up at retryAt
873
+ throw new SleepError(retryAt);
874
+ }
875
+ }
876
+
877
+ // Execute the step
878
+ const entry =
879
+ existing ?? createEntry(location, { type: "step", data: {} });
880
+ if (!existing) {
881
+ // New entry - register name
882
+ this.log("debug", {
883
+ msg: "executing new step",
884
+ step: config.name,
885
+ key,
886
+ });
887
+ const nameIndex = registerName(this.storage, config.name);
888
+ entry.location = [...location];
889
+ entry.location[entry.location.length - 1] = nameIndex;
890
+ setEntry(this.storage, location, entry);
891
+ } else {
892
+ this.log("debug", { msg: "retrying step", step: config.name, key });
893
+ }
894
+
895
+ const metadata = getOrCreateMetadata(this.storage, entry.id);
896
+ const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
897
+ const retryBackoffBase =
898
+ config.retryBackoffBase ?? DEFAULT_RETRY_BACKOFF_BASE;
899
+ const retryBackoffMax =
900
+ config.retryBackoffMax ?? DEFAULT_RETRY_BACKOFF_MAX;
901
+ metadata.status = "running";
902
+ metadata.attempts++;
903
+ metadata.lastAttemptAt = Date.now();
904
+ metadata.dirty = true;
905
+
906
+ // Get timeout configuration
907
+ const timeout = config.timeout ?? DEFAULT_STEP_TIMEOUT;
908
+
909
+ try {
910
+ // Execute with timeout
911
+ const output = await this.executeWithTimeout(
912
+ config.run(),
913
+ timeout,
914
+ config.name,
915
+ );
916
+
917
+ if (entry.kind.type === "step") {
918
+ entry.kind.data.output = output;
919
+ }
920
+ entry.dirty = true;
921
+ metadata.status = "completed";
922
+ metadata.error = undefined;
923
+ metadata.completedAt = Date.now();
924
+
925
+ // Ephemeral steps don't trigger an immediate flush. This avoids the
926
+ // synchronous write overhead for transient operations. Note that the
927
+ // step's entry is still marked dirty and WILL be persisted on the
928
+ // next flush from a non-ephemeral operation. The purpose of ephemeral
929
+ // is to batch writes, not to avoid persistence entirely.
930
+ if (!config.ephemeral) {
931
+ this.log("debug", {
932
+ msg: "flushing step",
933
+ step: config.name,
934
+ key,
935
+ });
936
+ await this.flushStorage();
937
+ }
938
+
939
+ this.log("debug", {
940
+ msg: "step completed",
941
+ step: config.name,
942
+ key,
943
+ });
944
+ return output;
945
+ } catch (error) {
946
+ if (entry.kind.type === "step") {
947
+ entry.kind.data.error = String(error);
948
+ }
949
+ entry.dirty = true;
950
+
951
+ // Timeout errors are treated as critical by default. Steps opt
952
+ // into retrying on timeout with retryOnTimeout: true.
953
+ if (error instanceof StepTimeoutError && !config.retryOnTimeout) {
954
+ metadata.status = "exhausted";
955
+ metadata.error = String(error);
956
+ await this.notifyStepError(config, metadata.attempts, error, {
957
+ willRetry: false,
958
+ });
959
+ throw markErrorReported(
960
+ attachTryStepFailure(new CriticalError(error.message), {
961
+ kind: "timeout",
962
+ stepName: config.name,
963
+ attempts: metadata.attempts,
964
+ error: extractErrorInfo(error),
965
+ }),
966
+ );
967
+ }
968
+
969
+ if (
970
+ error instanceof CriticalError ||
971
+ error instanceof RollbackError
972
+ ) {
973
+ metadata.status = "exhausted";
974
+ metadata.error = String(error);
975
+ await this.notifyStepError(config, metadata.attempts, error, {
976
+ willRetry: false,
977
+ });
978
+ throw markErrorReported(
979
+ attachTryStepFailure(error, {
980
+ kind:
981
+ error instanceof RollbackError
982
+ ? "rollback"
983
+ : "critical",
984
+ stepName: config.name,
985
+ attempts: metadata.attempts,
986
+ error: extractErrorInfo(error),
987
+ }),
988
+ );
989
+ }
990
+
991
+ const willRetry = metadata.attempts <= maxRetries;
992
+ metadata.status = willRetry ? "failed" : "exhausted";
993
+ metadata.error = String(error);
994
+
995
+ if (willRetry) {
996
+ const retryDelay = calculateBackoff(
997
+ metadata.attempts,
998
+ retryBackoffBase,
999
+ retryBackoffMax,
1000
+ );
1001
+ const retryAt = metadata.lastAttemptAt + retryDelay;
1002
+ await this.notifyStepError(config, metadata.attempts, error, {
1003
+ willRetry: true,
1004
+ retryDelay,
1005
+ retryAt,
1006
+ });
1007
+ throw new StepFailedError(
1008
+ config.name,
1009
+ error,
1010
+ metadata.attempts,
1011
+ retryAt,
1012
+ );
1013
+ }
1014
+
1015
+ const exhaustedError = markErrorReported(
1016
+ attachTryStepFailure(
1017
+ new StepExhaustedError(config.name, String(error)),
1018
+ {
1019
+ kind:
1020
+ error instanceof StepTimeoutError
1021
+ ? "timeout"
1022
+ : "exhausted",
1023
+ stepName: config.name,
1024
+ attempts: metadata.attempts,
1025
+ error: extractErrorInfo(error),
1026
+ },
1027
+ ),
1028
+ );
1029
+ await this.notifyStepError(config, metadata.attempts, error, {
1030
+ willRetry: false,
1031
+ });
1032
+ throw exhaustedError;
1033
+ }
1034
+ }
1035
+
1036
+ /**
1037
+ * Execute a promise with timeout.
1038
+ *
1039
+ * Note: This does NOT cancel the underlying operation. JavaScript Promises
1040
+ * cannot be cancelled once started. When a timeout occurs:
1041
+ * - The step is rejected with StepTimeoutError. By default this is treated
1042
+ * as a critical failure with no retry. Set retryOnTimeout: true on the
1043
+ * step config to retry timeouts like any other error.
1044
+ * - The underlying async operation continues running in the background
1045
+ * - Any side effects from the operation may still occur
1046
+ *
1047
+ * For cancellable operations, pass ctx.abortSignal to APIs that support AbortSignal:
1048
+ *
1049
+ * return fetch(url, { signal: ctx.abortSignal });
1050
+
1051
+ * });
1052
+ *
1053
+ * Or check ctx.isEvicted() periodically in long-running loops.
1054
+ */
1055
+ private async executeStepRollback<T>(config: StepConfig<T>): Promise<T> {
1056
+ this.checkDuplicateName(config.name);
1057
+ this.ensureRollbackCheckpoint(config);
1058
+
1059
+ const location = appendName(
1060
+ this.storage,
1061
+ this.currentLocation,
1062
+ config.name,
1063
+ );
1064
+ const key = locationToKey(this.storage, location);
1065
+ const existing = this.storage.history.entries.get(key);
1066
+
1067
+ this.markVisited(key);
1068
+
1069
+ if (!existing || existing.kind.type !== "step") {
1070
+ this.stopRollback();
1071
+ }
1072
+
1073
+ const metadata = await loadMetadata(
1074
+ this.storage,
1075
+ this.driver,
1076
+ existing.id,
1077
+ );
1078
+ if (metadata.status !== "completed") {
1079
+ this.stopRollback();
1080
+ }
1081
+
1082
+ const output = existing.kind.data.output as T;
1083
+ this.registerRollbackAction(config, existing.id, output, metadata);
1084
+
1085
+ return output;
1086
+ }
1087
+
1088
+ private async executeWithTimeout<T>(
1089
+ promise: Promise<T>,
1090
+ timeoutMs: number,
1091
+ stepName: string,
1092
+ ): Promise<T> {
1093
+ if (timeoutMs <= 0) {
1094
+ return promise;
1095
+ }
1096
+
1097
+ let timeoutId: ReturnType<typeof setTimeout> | undefined;
1098
+ const timeoutPromise = new Promise<never>((_, reject) => {
1099
+ timeoutId = setTimeout(() => {
1100
+ reject(new StepTimeoutError(stepName, timeoutMs));
1101
+ }, timeoutMs);
1102
+ });
1103
+
1104
+ try {
1105
+ return await Promise.race([promise, timeoutPromise]);
1106
+ } finally {
1107
+ if (timeoutId !== undefined) {
1108
+ clearTimeout(timeoutId);
1109
+ }
1110
+ }
1111
+ }
1112
+
1113
+ // === Loop ===
1114
+
1115
+ async loop<S, T>(
1116
+ nameOrConfig: string | LoopConfig<S, T>,
1117
+ run?: (
1118
+ ctx: WorkflowContextInterface,
1119
+ ) => LoopIterationResult<undefined, T>,
1120
+ ): Promise<T> {
1121
+ this.assertNotInProgress();
1122
+ this.checkEvicted();
1123
+
1124
+ const config: LoopConfig<S, T> =
1125
+ typeof nameOrConfig === "string"
1126
+ ? { name: nameOrConfig, run: run as LoopConfig<S, T>["run"] }
1127
+ : nameOrConfig;
1128
+
1129
+ this.entryInProgress = true;
1130
+ try {
1131
+ return await this.executeLoop(config);
1132
+ } finally {
1133
+ this.entryInProgress = false;
1134
+ }
1135
+ }
1136
+
1137
+ private async executeLoop<S, T>(config: LoopConfig<S, T>): Promise<T> {
1138
+ // Check for duplicate name in current execution
1139
+ this.checkDuplicateName(config.name);
1140
+
1141
+ const location = appendName(
1142
+ this.storage,
1143
+ this.currentLocation,
1144
+ config.name,
1145
+ );
1146
+ const key = locationToKey(this.storage, location);
1147
+ const existing = this.storage.history.entries.get(key);
1148
+
1149
+ // Mark this entry as visited for validateComplete
1150
+ this.markVisited(key);
1151
+
1152
+ let entry: Entry;
1153
+ let metadata: EntryMetadata | undefined;
1154
+ let state: S;
1155
+ let iteration: number;
1156
+ let rollbackSingleIteration = false;
1157
+ let rollbackIterationRan = false;
1158
+ let rollbackOutput: T | undefined;
1159
+ const rollbackMode = this.mode === "rollback";
1160
+
1161
+ if (existing) {
1162
+ if (existing.kind.type !== "loop") {
1163
+ throw new HistoryDivergedError(
1164
+ `Expected loop "${config.name}" at ${key}, found ${existing.kind.type}`,
1165
+ );
1166
+ }
1167
+
1168
+ const loopData = existing.kind.data;
1169
+ metadata = await loadMetadata(
1170
+ this.storage,
1171
+ this.driver,
1172
+ existing.id,
1173
+ );
1174
+
1175
+ if (rollbackMode) {
1176
+ if (loopData.output !== undefined) {
1177
+ return loopData.output as T;
1178
+ }
1179
+ rollbackSingleIteration = true;
1180
+ rollbackIterationRan = false;
1181
+ rollbackOutput = undefined;
1182
+ }
1183
+
1184
+ if (metadata.status === "completed") {
1185
+ return loopData.output as T;
1186
+ }
1187
+
1188
+ // Loop already completed
1189
+ if (loopData.output !== undefined) {
1190
+ return loopData.output as T;
1191
+ }
1192
+
1193
+ // Resume from saved state
1194
+ entry = existing;
1195
+ state = loopData.state as S;
1196
+ iteration = loopData.iteration;
1197
+ if (rollbackMode) {
1198
+ rollbackOutput = loopData.output as T | undefined;
1199
+ rollbackIterationRan = rollbackOutput !== undefined;
1200
+ }
1201
+ } else {
1202
+ this.stopRollbackIfIncomplete(true);
1203
+
1204
+ // New loop
1205
+ state = config.state as S;
1206
+ iteration = 0;
1207
+ entry = createEntry(location, {
1208
+ type: "loop",
1209
+ data: { state, iteration },
1210
+ });
1211
+ setEntry(this.storage, location, entry);
1212
+ metadata = getOrCreateMetadata(this.storage, entry.id);
1213
+ }
1214
+
1215
+ if (metadata) {
1216
+ metadata.status = "running";
1217
+ metadata.error = undefined;
1218
+ metadata.dirty = true;
1219
+ }
1220
+
1221
+ const historyPruneInterval =
1222
+ config.historyPruneInterval ??
1223
+ config.commitInterval ??
1224
+ config.historyEvery ??
1225
+ DEFAULT_LOOP_HISTORY_PRUNE_INTERVAL;
1226
+ const historySize =
1227
+ config.historySize ?? config.historyKeep ?? historyPruneInterval;
1228
+
1229
+ // Track the last iteration we pruned up to so we only delete
1230
+ // newly-expired iterations instead of re-scanning from 0.
1231
+ let lastPrunedUpTo = 0;
1232
+
1233
+ // Deferred flush promise from the previous prune cycle. Awaited at the
1234
+ // start of the next iteration so the flush runs in parallel with user code.
1235
+ let deferredFlush: Promise<void> | null = null;
1236
+
1237
+ // Execute loop iterations
1238
+ while (true) {
1239
+ // Await any deferred flush from the previous prune cycle
1240
+ if (deferredFlush) {
1241
+ await deferredFlush;
1242
+ deferredFlush = null;
1243
+ }
1244
+
1245
+ if (rollbackMode && rollbackSingleIteration) {
1246
+ if (rollbackIterationRan) {
1247
+ return rollbackOutput as T;
1248
+ }
1249
+ this.stopRollbackIfIncomplete(true);
1250
+ }
1251
+ this.checkEvicted();
1252
+
1253
+ // Create branch for this iteration
1254
+ const iterationLocation = appendLoopIteration(
1255
+ this.storage,
1256
+ location,
1257
+ config.name,
1258
+ iteration,
1259
+ );
1260
+ const branchCtx = this.createBranch(iterationLocation);
1261
+
1262
+ // Execute iteration
1263
+ const iterationResult = await config.run(branchCtx, state);
1264
+ if (iterationResult === undefined && state !== undefined) {
1265
+ throw new Error(
1266
+ `Loop "${config.name}" returned undefined for a stateful iteration. Return Loop.continue(state) or Loop.break(value).`,
1267
+ );
1268
+ }
1269
+ const result =
1270
+ iterationResult === undefined
1271
+ ? ({ continue: true, state } as LoopResult<S, T>)
1272
+ : iterationResult;
1273
+
1274
+ // Validate branch completed cleanly
1275
+ branchCtx.validateComplete();
1276
+
1277
+ if ("break" in result && result.break) {
1278
+ // Loop complete
1279
+ if (entry.kind.type === "loop") {
1280
+ entry.kind.data.output = result.value;
1281
+ entry.kind.data.state = state;
1282
+ entry.kind.data.iteration = iteration;
1283
+ }
1284
+ entry.dirty = true;
1285
+ if (metadata) {
1286
+ metadata.status = "completed";
1287
+ metadata.completedAt = Date.now();
1288
+ metadata.dirty = true;
1289
+ }
1290
+
1291
+ // Collect pruning deletions and flush
1292
+ const deletions = this.collectLoopPruning(
1293
+ location,
1294
+ iteration + 1,
1295
+ historySize,
1296
+ lastPrunedUpTo,
1297
+ );
1298
+ await this.flushStorageWithDeletions(deletions);
1299
+
1300
+ if (rollbackMode && rollbackSingleIteration) {
1301
+ rollbackOutput = result.value;
1302
+ rollbackIterationRan = true;
1303
+ continue;
1304
+ }
1305
+
1306
+ return result.value;
1307
+ }
1308
+
1309
+ // Continue with new state
1310
+ if ("continue" in result && result.continue) {
1311
+ state = result.state;
1312
+ }
1313
+ iteration++;
1314
+
1315
+ if (!rollbackMode) {
1316
+ if (entry.kind.type === "loop") {
1317
+ entry.kind.data.state = state;
1318
+ entry.kind.data.iteration = iteration;
1319
+ }
1320
+ entry.dirty = true;
1321
+ }
1322
+
1323
+ // Periodically defer the flush so the next iteration can overlap
1324
+ // with loop pruning and any pending dirty state writes.
1325
+ if (iteration % historyPruneInterval === 0) {
1326
+ const deletions = this.collectLoopPruning(
1327
+ location,
1328
+ iteration,
1329
+ historySize,
1330
+ lastPrunedUpTo,
1331
+ );
1332
+ lastPrunedUpTo = Math.max(0, iteration - historySize);
1333
+
1334
+ // Defer the flush to run in parallel with the next iteration
1335
+ deferredFlush = this.flushStorageWithDeletions(deletions);
1336
+ }
1337
+ }
1338
+ }
1339
+
1340
+ /**
1341
+ * Collect pending deletions for loop history pruning.
1342
+ *
1343
+ * Only deletes iterations in the range [fromIteration, keepFrom) where
1344
+ * keepFrom = currentIteration - historySize. This avoids re-scanning
1345
+ * already-deleted iterations.
1346
+ */
1347
+ private collectLoopPruning(
1348
+ loopLocation: Location,
1349
+ currentIteration: number,
1350
+ historySize: number,
1351
+ fromIteration: number,
1352
+ ): PendingDeletions | undefined {
1353
+ if (currentIteration <= historySize) {
1354
+ return undefined;
1355
+ }
1356
+
1357
+ const keepFrom = Math.max(0, currentIteration - historySize);
1358
+ if (fromIteration >= keepFrom) {
1359
+ return undefined;
1360
+ }
1361
+
1362
+ const loopSegment = loopLocation[loopLocation.length - 1];
1363
+ if (typeof loopSegment !== "number") {
1364
+ throw new Error("Expected loop location to end with a name index");
1365
+ }
1366
+
1367
+ const range = buildLoopIterationRange(
1368
+ loopLocation,
1369
+ loopSegment,
1370
+ fromIteration,
1371
+ keepFrom,
1372
+ );
1373
+ const metadataKeys: Uint8Array[] = [];
1374
+
1375
+ for (const [key, entry] of this.storage.history.entries) {
1376
+ if (!isLocationPrefix(loopLocation, entry.location)) {
1377
+ continue;
1378
+ }
1379
+
1380
+ const iterationSegment = entry.location[loopLocation.length];
1381
+ if (
1382
+ !iterationSegment ||
1383
+ typeof iterationSegment === "number" ||
1384
+ iterationSegment.loop !== loopSegment ||
1385
+ iterationSegment.iteration < fromIteration ||
1386
+ iterationSegment.iteration >= keepFrom
1387
+ ) {
1388
+ continue;
1389
+ }
1390
+
1391
+ metadataKeys.push(buildEntryMetadataKey(entry.id));
1392
+ this.storage.entryMetadata.delete(entry.id);
1393
+ this.storage.history.entries.delete(key);
1394
+ }
1395
+
1396
+ return {
1397
+ prefixes: [],
1398
+ keys: metadataKeys,
1399
+ ranges: [range],
1400
+ };
1401
+ }
1402
+
1403
+ /**
1404
+ * Flush storage with optional pending deletions so pruning
1405
+ * happens alongside the state write.
1406
+ */
1407
+ private async flushStorageWithDeletions(
1408
+ deletions?: PendingDeletions,
1409
+ ): Promise<void> {
1410
+ await flush(this.storage, this.driver, this.historyNotifier, deletions);
1411
+ }
1412
+
1413
+ // === Sleep ===
1414
+
1415
+ async sleep(name: string, durationMs: number): Promise<void> {
1416
+ const deadline = Date.now() + durationMs;
1417
+ return this.sleepUntil(name, deadline);
1418
+ }
1419
+
1420
+ async sleepUntil(name: string, timestampMs: number): Promise<void> {
1421
+ this.assertNotInProgress();
1422
+ this.checkEvicted();
1423
+
1424
+ this.entryInProgress = true;
1425
+ try {
1426
+ await this.executeSleep(name, timestampMs);
1427
+ } finally {
1428
+ this.entryInProgress = false;
1429
+ }
1430
+ }
1431
+
1432
+ private async executeSleep(name: string, deadline: number): Promise<void> {
1433
+ // Check for duplicate name in current execution
1434
+ this.checkDuplicateName(name);
1435
+
1436
+ const location = appendName(this.storage, this.currentLocation, name);
1437
+ const key = locationToKey(this.storage, location);
1438
+ const existing = this.storage.history.entries.get(key);
1439
+
1440
+ // Mark this entry as visited for validateComplete
1441
+ this.markVisited(key);
1442
+
1443
+ let entry: Entry;
1444
+
1445
+ if (existing) {
1446
+ if (existing.kind.type !== "sleep") {
1447
+ throw new HistoryDivergedError(
1448
+ `Expected sleep "${name}" at ${key}, found ${existing.kind.type}`,
1449
+ );
1450
+ }
1451
+
1452
+ const sleepData = existing.kind.data;
1453
+
1454
+ if (this.mode === "rollback") {
1455
+ this.stopRollbackIfIncomplete(sleepData.state === "pending");
1456
+ return;
1457
+ }
1458
+
1459
+ // Already completed or interrupted
1460
+ if (sleepData.state !== "pending") {
1461
+ return;
1462
+ }
1463
+
1464
+ // Use stored deadline
1465
+ deadline = sleepData.deadline;
1466
+ entry = existing;
1467
+ } else {
1468
+ this.stopRollbackIfIncomplete(true);
1469
+
1470
+ entry = createEntry(location, {
1471
+ type: "sleep",
1472
+ data: { deadline, state: "pending" },
1473
+ });
1474
+ setEntry(this.storage, location, entry);
1475
+ entry.dirty = true;
1476
+ await this.flushStorage();
1477
+ }
1478
+
1479
+ const now = Date.now();
1480
+ const remaining = deadline - now;
1481
+
1482
+ if (remaining <= 0) {
1483
+ // Deadline passed
1484
+ if (entry.kind.type === "sleep") {
1485
+ entry.kind.data.state = "completed";
1486
+ }
1487
+ entry.dirty = true;
1488
+ await this.flushStorage();
1489
+ return;
1490
+ }
1491
+
1492
+ // Short sleep: wait in memory
1493
+ if (remaining < this.driver.workerPollInterval) {
1494
+ await this.sleepOrEvict(remaining);
1495
+
1496
+ this.checkEvicted();
1497
+
1498
+ if (entry.kind.type === "sleep") {
1499
+ entry.kind.data.state = "completed";
1500
+ }
1501
+ entry.dirty = true;
1502
+ await this.flushStorage();
1503
+ return;
1504
+ }
1505
+
1506
+ // Long sleep: yield to scheduler
1507
+ throw new SleepError(deadline);
1508
+ }
1509
+
1510
+ // === Rollback Checkpoint ===
1511
+
1512
+ async rollbackCheckpoint(name: string): Promise<void> {
1513
+ this.assertNotInProgress();
1514
+ this.checkEvicted();
1515
+
1516
+ this.entryInProgress = true;
1517
+ try {
1518
+ await this.executeRollbackCheckpoint(name);
1519
+ } finally {
1520
+ this.entryInProgress = false;
1521
+ }
1522
+ }
1523
+
1524
+ private async executeRollbackCheckpoint(name: string): Promise<void> {
1525
+ this.checkDuplicateName(name);
1526
+
1527
+ const location = appendName(this.storage, this.currentLocation, name);
1528
+ const key = locationToKey(this.storage, location);
1529
+ const existing = this.storage.history.entries.get(key);
1530
+
1531
+ this.markVisited(key);
1532
+
1533
+ if (existing) {
1534
+ if (existing.kind.type !== "rollback_checkpoint") {
1535
+ throw new HistoryDivergedError(
1536
+ `Expected rollback checkpoint "${name}" at ${key}, found ${existing.kind.type}`,
1537
+ );
1538
+ }
1539
+ this.rollbackCheckpointSet = true;
1540
+ return;
1541
+ }
1542
+
1543
+ if (this.mode === "rollback") {
1544
+ throw new HistoryDivergedError(
1545
+ `Missing rollback checkpoint "${name}" at ${key}`,
1546
+ );
1547
+ }
1548
+
1549
+ const entry = createEntry(location, {
1550
+ type: "rollback_checkpoint",
1551
+ data: { name },
1552
+ });
1553
+ setEntry(this.storage, location, entry);
1554
+ entry.dirty = true;
1555
+ await this.flushStorage();
1556
+
1557
+ this.rollbackCheckpointSet = true;
1558
+ }
1559
+
1560
+ // === Queue ===
1561
+
1562
+ private async queueSend(name: string, body: unknown): Promise<void> {
1563
+ const message: Message = {
1564
+ id: crypto.randomUUID(),
1565
+ name,
1566
+ data: body,
1567
+ sentAt: Date.now(),
1568
+ };
1569
+ await this.messageDriver.addMessage(message);
1570
+ }
1571
+
1572
+ private async queueNext<T>(
1573
+ name: string,
1574
+ opts?: WorkflowQueueNextOptions,
1575
+ ): Promise<WorkflowQueueMessage<T>> {
1576
+ const messages = await this.queueNextBatch<T>(name, {
1577
+ ...(opts ?? {}),
1578
+ count: 1,
1579
+ });
1580
+ const message = messages[0];
1581
+ if (!message) {
1582
+ throw new Error(
1583
+ `queue.next("${name}") timed out before receiving a message. Use queue.nextBatch(...) for optional/time-limited reads.`,
1584
+ );
1585
+ }
1586
+ return message;
1587
+ }
1588
+
1589
+ private async queueNextBatch<T>(
1590
+ name: string,
1591
+ opts?: WorkflowQueueNextBatchOptions,
1592
+ ): Promise<Array<WorkflowQueueMessage<T>>> {
1593
+ this.assertNotInProgress();
1594
+ this.checkEvicted();
1595
+
1596
+ this.entryInProgress = true;
1597
+ try {
1598
+ return await this.executeQueueNextBatch<T>(name, opts);
1599
+ } finally {
1600
+ this.entryInProgress = false;
1601
+ }
1602
+ }
1603
+
1604
+ private async executeQueueNextBatch<T>(
1605
+ name: string,
1606
+ opts?: WorkflowQueueNextBatchOptions,
1607
+ ): Promise<Array<WorkflowQueueMessage<T>>> {
1608
+ if (this.pendingCompletableMessageIds.size > 0) {
1609
+ throw new Error(
1610
+ "Previous completable queue message is not completed. Call `message.complete(...)` before receiving the next message.",
1611
+ );
1612
+ }
1613
+
1614
+ const resolvedOpts = opts ?? {};
1615
+ const messageNames = this.normalizeQueueNames(resolvedOpts.names);
1616
+ const messageNameLabel = this.messageNamesLabel(messageNames);
1617
+ const count = Math.max(1, resolvedOpts.count ?? 1);
1618
+ const completable = resolvedOpts.completable === true;
1619
+
1620
+ this.checkDuplicateName(name);
1621
+
1622
+ const countLocation = appendName(
1623
+ this.storage,
1624
+ this.currentLocation,
1625
+ `${name}:count`,
1626
+ );
1627
+ const countKey = locationToKey(this.storage, countLocation);
1628
+ const existingCount = this.storage.history.entries.get(countKey);
1629
+ this.markVisited(countKey);
1630
+ this.stopRollbackIfMissing(existingCount);
1631
+
1632
+ let deadline: number | undefined;
1633
+ let deadlineEntry: Entry | undefined;
1634
+ if (resolvedOpts.timeout !== undefined) {
1635
+ const deadlineLocation = appendName(
1636
+ this.storage,
1637
+ this.currentLocation,
1638
+ `${name}:deadline`,
1639
+ );
1640
+ const deadlineKey = locationToKey(this.storage, deadlineLocation);
1641
+ deadlineEntry = this.storage.history.entries.get(deadlineKey);
1642
+ this.markVisited(deadlineKey);
1643
+ this.stopRollbackIfMissing(deadlineEntry);
1644
+
1645
+ if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
1646
+ deadline = deadlineEntry.kind.data.deadline;
1647
+ } else {
1648
+ deadline = Date.now() + resolvedOpts.timeout;
1649
+ const created = createEntry(deadlineLocation, {
1650
+ type: "sleep",
1651
+ data: { deadline, state: "pending" },
1652
+ });
1653
+ setEntry(this.storage, deadlineLocation, created);
1654
+ created.dirty = true;
1655
+ await this.flushStorage();
1656
+ deadlineEntry = created;
1657
+ }
1658
+ }
1659
+
1660
+ if (existingCount && existingCount.kind.type === "message") {
1661
+ const replayCount = existingCount.kind.data.data as number;
1662
+ return await this.readReplayQueueMessages<T>(
1663
+ name,
1664
+ replayCount,
1665
+ completable,
1666
+ );
1667
+ }
1668
+
1669
+ const now = Date.now();
1670
+ if (deadline !== undefined && now >= deadline) {
1671
+ if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
1672
+ deadlineEntry.kind.data.state = "completed";
1673
+ deadlineEntry.dirty = true;
1674
+ }
1675
+ await this.recordQueueCountEntry(
1676
+ countLocation,
1677
+ `${messageNameLabel}:count`,
1678
+ 0,
1679
+ );
1680
+ return [];
1681
+ }
1682
+
1683
+ const received = await this.receiveMessagesNow(
1684
+ messageNames,
1685
+ count,
1686
+ completable,
1687
+ );
1688
+ if (received.length > 0) {
1689
+ const historyMessages = received.map((message) =>
1690
+ this.toWorkflowQueueMessage<T>(message),
1691
+ );
1692
+ if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
1693
+ deadlineEntry.kind.data.state = "interrupted";
1694
+ deadlineEntry.dirty = true;
1695
+ }
1696
+ await this.recordQueueMessages(
1697
+ name,
1698
+ countLocation,
1699
+ messageNames,
1700
+ historyMessages,
1701
+ );
1702
+ const queueMessages = received.map((message, index) =>
1703
+ this.createQueueMessage<T>(message, completable, {
1704
+ historyLocation: appendName(
1705
+ this.storage,
1706
+ this.currentLocation,
1707
+ `${name}:${index}`,
1708
+ ),
1709
+ }),
1710
+ );
1711
+ return queueMessages;
1712
+ }
1713
+
1714
+ if (deadline === undefined) {
1715
+ throw new MessageWaitError(messageNames);
1716
+ }
1717
+ throw new SleepError(deadline, messageNames);
1718
+ }
1719
+
1720
+ private normalizeQueueNames(names?: readonly string[]): string[] {
1721
+ if (!names || names.length === 0) {
1722
+ return [];
1723
+ }
1724
+ const deduped: string[] = [];
1725
+ const seen = new Set<string>();
1726
+ for (const name of names) {
1727
+ if (seen.has(name)) {
1728
+ continue;
1729
+ }
1730
+ seen.add(name);
1731
+ deduped.push(name);
1732
+ }
1733
+ return deduped;
1734
+ }
1735
+
1736
+ private messageNamesLabel(messageNames: string[]): string {
1737
+ if (messageNames.length === 0) {
1738
+ return "*";
1739
+ }
1740
+ return messageNames.length === 1
1741
+ ? messageNames[0]
1742
+ : messageNames.join("|");
1743
+ }
1744
+
1745
+ private async receiveMessagesNow(
1746
+ messageNames: string[],
1747
+ count: number,
1748
+ completable: boolean,
1749
+ ): Promise<Message[]> {
1750
+ return await this.messageDriver.receiveMessages({
1751
+ names: messageNames.length > 0 ? messageNames : undefined,
1752
+ count,
1753
+ completable,
1754
+ });
1755
+ }
1756
+
1757
+ private async recordQueueMessages<T>(
1758
+ name: string,
1759
+ countLocation: Location,
1760
+ messageNames: string[],
1761
+ messages: Array<WorkflowQueueMessage<T>>,
1762
+ ): Promise<void> {
1763
+ for (let i = 0; i < messages.length; i++) {
1764
+ const messageLocation = appendName(
1765
+ this.storage,
1766
+ this.currentLocation,
1767
+ `${name}:${i}`,
1768
+ );
1769
+ const messageEntry = createEntry(messageLocation, {
1770
+ type: "message",
1771
+ data: {
1772
+ name: messages[i].name,
1773
+ data: this.toHistoryQueueMessage(messages[i]),
1774
+ },
1775
+ });
1776
+ setEntry(this.storage, messageLocation, messageEntry);
1777
+ this.markVisited(locationToKey(this.storage, messageLocation));
1778
+ }
1779
+ await this.recordQueueCountEntry(
1780
+ countLocation,
1781
+ `${this.messageNamesLabel(messageNames)}:count`,
1782
+ messages.length,
1783
+ );
1784
+ }
1785
+
1786
+ private async recordQueueCountEntry(
1787
+ countLocation: Location,
1788
+ countLabel: string,
1789
+ count: number,
1790
+ ): Promise<void> {
1791
+ const countEntry = createEntry(countLocation, {
1792
+ type: "message",
1793
+ data: {
1794
+ name: countLabel,
1795
+ data: count,
1796
+ },
1797
+ });
1798
+ setEntry(this.storage, countLocation, countEntry);
1799
+ await this.flushStorage();
1800
+ }
1801
+
1802
+ private async readReplayQueueMessages<T>(
1803
+ name: string,
1804
+ count: number,
1805
+ completable: boolean,
1806
+ ): Promise<Array<WorkflowQueueMessage<T>>> {
1807
+ const results: Array<WorkflowQueueMessage<T>> = [];
1808
+ for (let i = 0; i < count; i++) {
1809
+ const messageLocation = appendName(
1810
+ this.storage,
1811
+ this.currentLocation,
1812
+ `${name}:${i}`,
1813
+ );
1814
+ const messageKey = locationToKey(this.storage, messageLocation);
1815
+ this.markVisited(messageKey);
1816
+ const existingMessage =
1817
+ this.storage.history.entries.get(messageKey);
1818
+ if (!existingMessage || existingMessage.kind.type !== "message") {
1819
+ throw new HistoryDivergedError(
1820
+ `Expected queue message "${name}:${i}" in history`,
1821
+ );
1822
+ }
1823
+ const parsed = this.fromHistoryQueueMessage(
1824
+ existingMessage.kind.data.name,
1825
+ existingMessage.kind.data.data,
1826
+ );
1827
+ results.push(
1828
+ this.createQueueMessage<T>(parsed.message, completable, {
1829
+ historyLocation: messageLocation,
1830
+ completed: parsed.completed,
1831
+ replay: true,
1832
+ }),
1833
+ );
1834
+ }
1835
+ return results;
1836
+ }
1837
+
1838
+ private toWorkflowQueueMessage<T>(
1839
+ message: Message,
1840
+ ): WorkflowQueueMessage<T> {
1841
+ return {
1842
+ id: message.id,
1843
+ name: message.name,
1844
+ body: message.data as T,
1845
+ createdAt: message.sentAt,
1846
+ };
1847
+ }
1848
+
1849
+ private createQueueMessage<T>(
1850
+ message: Message,
1851
+ completable: boolean,
1852
+ opts?: {
1853
+ historyLocation?: Location;
1854
+ completed?: boolean;
1855
+ replay?: boolean;
1856
+ },
1857
+ ): WorkflowQueueMessage<T> {
1858
+ const queueMessage = this.toWorkflowQueueMessage<T>(message);
1859
+ if (!completable) {
1860
+ return queueMessage;
1861
+ }
1862
+
1863
+ if (opts?.replay && opts.completed) {
1864
+ return {
1865
+ ...queueMessage,
1866
+ complete: async () => {
1867
+ // No-op: this message was already completed in a prior run.
1868
+ },
1869
+ };
1870
+ }
1871
+
1872
+ const messageId = message.id;
1873
+ this.pendingCompletableMessageIds.add(messageId);
1874
+ let completed = false;
1875
+
1876
+ return {
1877
+ ...queueMessage,
1878
+ complete: async (response?: unknown) => {
1879
+ if (completed) {
1880
+ throw new Error("Queue message already completed");
1881
+ }
1882
+ completed = true;
1883
+ try {
1884
+ await this.completeMessage(message, response);
1885
+ await this.markQueueMessageCompleted(opts?.historyLocation);
1886
+ this.pendingCompletableMessageIds.delete(messageId);
1887
+ } catch (error) {
1888
+ completed = false;
1889
+ throw error;
1890
+ }
1891
+ },
1892
+ };
1893
+ }
1894
+
1895
+ private async markQueueMessageCompleted(
1896
+ historyLocation: Location | undefined,
1897
+ ): Promise<void> {
1898
+ if (!historyLocation) {
1899
+ return;
1900
+ }
1901
+ const key = locationToKey(this.storage, historyLocation);
1902
+ const entry = this.storage.history.entries.get(key);
1903
+ if (!entry || entry.kind.type !== "message") {
1904
+ return;
1905
+ }
1906
+ const parsed = this.fromHistoryQueueMessage(
1907
+ entry.kind.data.name,
1908
+ entry.kind.data.data,
1909
+ );
1910
+ entry.kind.data.data = this.toHistoryQueueMessage(
1911
+ this.toWorkflowQueueMessage(parsed.message),
1912
+ true,
1913
+ );
1914
+ entry.dirty = true;
1915
+ await this.flushStorage();
1916
+ }
1917
+
1918
+ private async completeMessage(
1919
+ message: Message,
1920
+ response?: unknown,
1921
+ ): Promise<void> {
1922
+ if (message.complete) {
1923
+ await message.complete(response);
1924
+ return;
1925
+ }
1926
+ await this.messageDriver.completeMessage(message.id, response);
1927
+ }
1928
+
1929
+ private toHistoryQueueMessage(
1930
+ message: WorkflowQueueMessage<unknown>,
1931
+ completed = false,
1932
+ ): unknown {
1933
+ return {
1934
+ [QUEUE_HISTORY_MESSAGE_MARKER]: 1,
1935
+ id: message.id,
1936
+ name: message.name,
1937
+ body: message.body,
1938
+ createdAt: message.createdAt,
1939
+ completed,
1940
+ };
1941
+ }
1942
+
1943
+ private fromHistoryQueueMessage(
1944
+ name: string,
1945
+ value: unknown,
1946
+ ): { message: Message; completed: boolean } {
1947
+ if (
1948
+ typeof value === "object" &&
1949
+ value !== null &&
1950
+ (value as Record<string, unknown>)[QUEUE_HISTORY_MESSAGE_MARKER] ===
1951
+ 1
1952
+ ) {
1953
+ const serialized = value as Record<string, unknown>;
1954
+ const id = typeof serialized.id === "string" ? serialized.id : "";
1955
+ const serializedName =
1956
+ typeof serialized.name === "string" ? serialized.name : name;
1957
+ const createdAt =
1958
+ typeof serialized.createdAt === "number"
1959
+ ? serialized.createdAt
1960
+ : 0;
1961
+ const completed =
1962
+ typeof serialized.completed === "boolean"
1963
+ ? serialized.completed
1964
+ : false;
1965
+ return {
1966
+ message: {
1967
+ id,
1968
+ name: serializedName,
1969
+ data: serialized.body,
1970
+ sentAt: createdAt,
1971
+ },
1972
+ completed,
1973
+ };
1974
+ }
1975
+ return {
1976
+ message: {
1977
+ id: "",
1978
+ name,
1979
+ data: value,
1980
+ sentAt: 0,
1981
+ },
1982
+ completed: false,
1983
+ };
1984
+ }
1985
+
1986
+ // === Join ===
1987
+
1988
+ async join<T extends Record<string, BranchConfig<unknown>>>(
1989
+ name: string,
1990
+ branches: T,
1991
+ ): Promise<{ [K in keyof T]: BranchOutput<T[K]> }> {
1992
+ this.assertNotInProgress();
1993
+ this.checkEvicted();
1994
+
1995
+ this.entryInProgress = true;
1996
+ try {
1997
+ return await this.executeJoin(name, branches);
1998
+ } finally {
1999
+ this.entryInProgress = false;
2000
+ }
2001
+ }
2002
+
2003
+ private async executeJoin<T extends Record<string, BranchConfig<unknown>>>(
2004
+ name: string,
2005
+ branches: T,
2006
+ ): Promise<{ [K in keyof T]: BranchOutput<T[K]> }> {
2007
+ // Check for duplicate name in current execution
2008
+ this.checkDuplicateName(name);
2009
+
2010
+ const location = appendName(this.storage, this.currentLocation, name);
2011
+ const key = locationToKey(this.storage, location);
2012
+ const existing = this.storage.history.entries.get(key);
2013
+
2014
+ // Mark this entry as visited for validateComplete
2015
+ this.markVisited(key);
2016
+
2017
+ this.stopRollbackIfMissing(existing);
2018
+
2019
+ let entry: Entry;
2020
+
2021
+ if (existing) {
2022
+ if (existing.kind.type !== "join") {
2023
+ throw new HistoryDivergedError(
2024
+ `Expected join "${name}" at ${key}, found ${existing.kind.type}`,
2025
+ );
2026
+ }
2027
+ entry = existing;
2028
+ } else {
2029
+ entry = createEntry(location, {
2030
+ type: "join",
2031
+ data: {
2032
+ branches: Object.fromEntries(
2033
+ Object.keys(branches).map((k) => [
2034
+ k,
2035
+ { status: "pending" as const },
2036
+ ]),
2037
+ ),
2038
+ },
2039
+ });
2040
+ setEntry(this.storage, location, entry);
2041
+ entry.dirty = true;
2042
+ // Flush immediately to persist entry before branches execute
2043
+ await this.flushStorage();
2044
+ }
2045
+
2046
+ if (entry.kind.type !== "join") {
2047
+ throw new HistoryDivergedError("Entry type mismatch");
2048
+ }
2049
+
2050
+ this.stopRollbackIfIncomplete(
2051
+ Object.values(entry.kind.data.branches).some(
2052
+ (branch) => branch.status !== "completed",
2053
+ ),
2054
+ );
2055
+
2056
+ const joinData = entry.kind.data;
2057
+ const results: Record<string, unknown> = {};
2058
+ const errors: Record<string, Error> = {};
2059
+ let schedulerYieldState: SchedulerYieldState | undefined;
2060
+ let propagatedError: Error | undefined;
2061
+
2062
+ for (const [branchName, branchStatus] of Object.entries(
2063
+ joinData.branches,
2064
+ )) {
2065
+ if (branchStatus.status === "completed") {
2066
+ results[branchName] = branchStatus.output;
2067
+ continue;
2068
+ }
2069
+
2070
+ if (branchStatus.status === "failed") {
2071
+ errors[branchName] = new Error(
2072
+ branchStatus.error ?? "branch failed",
2073
+ );
2074
+ }
2075
+ }
2076
+
2077
+ // Execute all branches in parallel
2078
+ const branchPromises = Object.entries(branches).map(
2079
+ async ([branchName, config]) => {
2080
+ const branchStatus = joinData.branches[branchName];
2081
+ if (!branchStatus) {
2082
+ throw new HistoryDivergedError(
2083
+ `Expected join branch "${branchName}" in "${name}"`,
2084
+ );
2085
+ }
2086
+
2087
+ // Already completed
2088
+ if (branchStatus.status === "completed") {
2089
+ results[branchName] = branchStatus.output;
2090
+ return;
2091
+ }
2092
+
2093
+ // Already failed
2094
+ if (branchStatus.status === "failed") {
2095
+ errors[branchName] = new Error(branchStatus.error);
2096
+ return;
2097
+ }
2098
+
2099
+ // Execute branch
2100
+ const branchLocation = appendName(
2101
+ this.storage,
2102
+ location,
2103
+ branchName,
2104
+ );
2105
+ const branchCtx = this.createBranch(branchLocation);
2106
+
2107
+ branchStatus.status = "running";
2108
+ branchStatus.error = undefined;
2109
+ entry.dirty = true;
2110
+
2111
+ try {
2112
+ const output = await config.run(branchCtx);
2113
+ branchCtx.validateComplete();
2114
+
2115
+ branchStatus.status = "completed";
2116
+ branchStatus.output = output;
2117
+ branchStatus.error = undefined;
2118
+ results[branchName] = output;
2119
+ } catch (error) {
2120
+ if (
2121
+ error instanceof SleepError ||
2122
+ error instanceof MessageWaitError ||
2123
+ error instanceof StepFailedError
2124
+ ) {
2125
+ schedulerYieldState = mergeSchedulerYield(
2126
+ schedulerYieldState,
2127
+ error,
2128
+ );
2129
+ branchStatus.status = "running";
2130
+ branchStatus.error = undefined;
2131
+ entry.dirty = true;
2132
+ return;
2133
+ }
2134
+
2135
+ if (
2136
+ error instanceof EvictedError ||
2137
+ error instanceof HistoryDivergedError ||
2138
+ error instanceof EntryInProgressError ||
2139
+ error instanceof RollbackCheckpointError ||
2140
+ error instanceof RollbackStopError
2141
+ ) {
2142
+ propagatedError = selectControlFlowError(
2143
+ propagatedError,
2144
+ error,
2145
+ );
2146
+ branchStatus.status = "running";
2147
+ branchStatus.error = undefined;
2148
+ entry.dirty = true;
2149
+ return;
2150
+ }
2151
+
2152
+ branchStatus.status = "failed";
2153
+ branchStatus.output = undefined;
2154
+ branchStatus.error = String(error);
2155
+ errors[branchName] = error as Error;
2156
+ }
2157
+
2158
+ entry.dirty = true;
2159
+ },
2160
+ );
2161
+
2162
+ // Wait for ALL branches (no short-circuit on error)
2163
+ await Promise.allSettled(branchPromises);
2164
+ await this.flushStorage();
2165
+
2166
+ if (propagatedError) {
2167
+ throw propagatedError;
2168
+ }
2169
+
2170
+ if (
2171
+ Object.values(joinData.branches).some(
2172
+ (branch) =>
2173
+ branch.status === "pending" || branch.status === "running",
2174
+ )
2175
+ ) {
2176
+ if (!schedulerYieldState) {
2177
+ throw new Error(
2178
+ `Join "${name}" has pending branches without a scheduler yield`,
2179
+ );
2180
+ }
2181
+ throw buildSchedulerYieldError(schedulerYieldState);
2182
+ }
2183
+
2184
+ // Throw if any branches failed
2185
+ if (Object.keys(errors).length > 0) {
2186
+ throw attachTryBlockFailure(new JoinError(errors), {
2187
+ source: "join",
2188
+ name,
2189
+ });
2190
+ }
2191
+
2192
+ return results as { [K in keyof T]: BranchOutput<T[K]> };
2193
+ }
2194
+
2195
+ // === Race ===
2196
+
2197
+ async race<T>(
2198
+ name: string,
2199
+ branches: Array<{
2200
+ name: string;
2201
+ run: (ctx: WorkflowContextInterface) => Promise<T>;
2202
+ }>,
2203
+ ): Promise<{ winner: string; value: T }> {
2204
+ this.assertNotInProgress();
2205
+ this.checkEvicted();
2206
+
2207
+ this.entryInProgress = true;
2208
+ try {
2209
+ return await this.executeRace(name, branches);
2210
+ } finally {
2211
+ this.entryInProgress = false;
2212
+ }
2213
+ }
2214
+
2215
+ private async executeRace<T>(
2216
+ name: string,
2217
+ branches: Array<{
2218
+ name: string;
2219
+ run: (ctx: WorkflowContextInterface) => Promise<T>;
2220
+ }>,
2221
+ ): Promise<{ winner: string; value: T }> {
2222
+ // Check for duplicate name in current execution
2223
+ this.checkDuplicateName(name);
2224
+
2225
+ const location = appendName(this.storage, this.currentLocation, name);
2226
+ const key = locationToKey(this.storage, location);
2227
+ const existing = this.storage.history.entries.get(key);
2228
+
2229
+ // Mark this entry as visited for validateComplete
2230
+ this.markVisited(key);
2231
+
2232
+ this.stopRollbackIfMissing(existing);
2233
+
2234
+ let entry: Entry;
2235
+
2236
+ if (existing) {
2237
+ if (existing.kind.type !== "race") {
2238
+ throw new HistoryDivergedError(
2239
+ `Expected race "${name}" at ${key}, found ${existing.kind.type}`,
2240
+ );
2241
+ }
2242
+ entry = existing;
2243
+
2244
+ // Check if we already have a winner
2245
+ const raceKind = existing.kind;
2246
+ if (raceKind.data.winner !== null) {
2247
+ const winnerStatus =
2248
+ raceKind.data.branches[raceKind.data.winner];
2249
+ return {
2250
+ winner: raceKind.data.winner,
2251
+ value: winnerStatus.output as T,
2252
+ };
2253
+ }
2254
+
2255
+ this.stopRollbackIfIncomplete(true);
2256
+ } else {
2257
+ entry = createEntry(location, {
2258
+ type: "race",
2259
+ data: {
2260
+ winner: null,
2261
+ branches: Object.fromEntries(
2262
+ branches.map((b) => [
2263
+ b.name,
2264
+ { status: "pending" as const },
2265
+ ]),
2266
+ ),
2267
+ },
2268
+ });
2269
+ setEntry(this.storage, location, entry);
2270
+ entry.dirty = true;
2271
+ // Flush immediately to persist entry before branches execute
2272
+ await this.flushStorage();
2273
+ }
2274
+
2275
+ if (entry.kind.type !== "race") {
2276
+ throw new HistoryDivergedError("Entry type mismatch");
2277
+ }
2278
+
2279
+ const raceData = entry.kind.data;
2280
+
2281
+ // Create abort controller for cancellation
2282
+ const raceAbortController = new AbortController();
2283
+
2284
+ // Track all branch promises to wait for cleanup
2285
+ const branchPromises: Promise<void>[] = [];
2286
+
2287
+ // Track winner info
2288
+ let winnerName: string | null = null;
2289
+ let winnerValue: T | undefined;
2290
+ let hasWinner = false;
2291
+ const errors: Record<string, Error> = {};
2292
+ const lateErrors: Array<{ name: string; error: string }> = [];
2293
+ let schedulerYieldState: SchedulerYieldState | undefined;
2294
+ let propagatedError: Error | undefined;
2295
+
2296
+ // Check for replay winners first
2297
+ for (const branch of branches) {
2298
+ const branchStatus = raceData.branches[branch.name];
2299
+ if (!branchStatus) {
2300
+ throw new HistoryDivergedError(
2301
+ `Expected race branch "${branch.name}" in "${name}"`,
2302
+ );
2303
+ }
2304
+ if (
2305
+ branchStatus.status !== "pending" &&
2306
+ branchStatus.status !== "running"
2307
+ ) {
2308
+ if (branchStatus.status === "failed") {
2309
+ errors[branch.name] = new Error(
2310
+ branchStatus.error ?? "branch failed",
2311
+ );
2312
+ }
2313
+ if (branchStatus.status === "completed" && !hasWinner) {
2314
+ hasWinner = true;
2315
+ winnerName = branch.name;
2316
+ winnerValue = branchStatus.output as T;
2317
+ }
2318
+ }
2319
+ }
2320
+
2321
+ // If we found a replay winner, return immediately
2322
+ if (hasWinner && winnerName !== null) {
2323
+ return { winner: winnerName, value: winnerValue as T };
2324
+ }
2325
+
2326
+ // Execute branches that need to run
2327
+ for (const branch of branches) {
2328
+ const branchStatus = raceData.branches[branch.name];
2329
+ if (!branchStatus) {
2330
+ throw new HistoryDivergedError(
2331
+ `Expected race branch "${branch.name}" in "${name}"`,
2332
+ );
2333
+ }
2334
+
2335
+ // Skip already completed/cancelled
2336
+ if (
2337
+ branchStatus.status !== "pending" &&
2338
+ branchStatus.status !== "running"
2339
+ ) {
2340
+ continue;
2341
+ }
2342
+
2343
+ const branchLocation = appendName(
2344
+ this.storage,
2345
+ location,
2346
+ branch.name,
2347
+ );
2348
+ const branchCtx = this.createBranch(
2349
+ branchLocation,
2350
+ raceAbortController,
2351
+ );
2352
+
2353
+ branchStatus.status = "running";
2354
+ branchStatus.error = undefined;
2355
+ entry.dirty = true;
2356
+
2357
+ const branchPromise = branch.run(branchCtx).then(
2358
+ async (output) => {
2359
+ if (hasWinner) {
2360
+ // This branch completed after a winner was determined
2361
+ // Still record the completion for observability
2362
+ branchStatus.status = "completed";
2363
+ branchStatus.output = output;
2364
+ branchStatus.error = undefined;
2365
+ entry.dirty = true;
2366
+ return;
2367
+ }
2368
+
2369
+ if (propagatedError) {
2370
+ branchStatus.status = "completed";
2371
+ branchStatus.output = output;
2372
+ branchStatus.error = undefined;
2373
+ entry.dirty = true;
2374
+ return;
2375
+ }
2376
+
2377
+ hasWinner = true;
2378
+ winnerName = branch.name;
2379
+ winnerValue = output;
2380
+
2381
+ branchCtx.validateComplete();
2382
+
2383
+ branchStatus.status = "completed";
2384
+ branchStatus.output = output;
2385
+ branchStatus.error = undefined;
2386
+ raceData.winner = branch.name;
2387
+ entry.dirty = true;
2388
+
2389
+ // Cancel other branches
2390
+ raceAbortController.abort();
2391
+ },
2392
+ (error) => {
2393
+ if (hasWinner) {
2394
+ if (
2395
+ error instanceof CancelledError ||
2396
+ error instanceof EvictedError
2397
+ ) {
2398
+ branchStatus.status = "cancelled";
2399
+ } else {
2400
+ lateErrors.push({
2401
+ name: branch.name,
2402
+ error: String(error),
2403
+ });
2404
+ }
2405
+ entry.dirty = true;
2406
+ return;
2407
+ }
2408
+
2409
+ if (
2410
+ error instanceof SleepError ||
2411
+ error instanceof MessageWaitError ||
2412
+ error instanceof StepFailedError
2413
+ ) {
2414
+ schedulerYieldState = mergeSchedulerYield(
2415
+ schedulerYieldState,
2416
+ error,
2417
+ );
2418
+ branchStatus.status = "running";
2419
+ branchStatus.error = undefined;
2420
+ entry.dirty = true;
2421
+ return;
2422
+ }
2423
+
2424
+ if (
2425
+ error instanceof EvictedError ||
2426
+ error instanceof HistoryDivergedError ||
2427
+ error instanceof EntryInProgressError ||
2428
+ error instanceof RollbackCheckpointError ||
2429
+ error instanceof RollbackStopError
2430
+ ) {
2431
+ propagatedError = selectControlFlowError(
2432
+ propagatedError,
2433
+ error,
2434
+ );
2435
+ branchStatus.status = "running";
2436
+ branchStatus.error = undefined;
2437
+ entry.dirty = true;
2438
+ return;
2439
+ }
2440
+
2441
+ if (error instanceof CancelledError) {
2442
+ branchStatus.status = "cancelled";
2443
+ } else {
2444
+ branchStatus.status = "failed";
2445
+ branchStatus.output = undefined;
2446
+ branchStatus.error = String(error);
2447
+ errors[branch.name] = error as Error;
2448
+ }
2449
+ entry.dirty = true;
2450
+ },
2451
+ );
2452
+
2453
+ branchPromises.push(branchPromise);
2454
+ }
2455
+
2456
+ // Wait for all branches to complete or be cancelled
2457
+ await Promise.allSettled(branchPromises);
2458
+
2459
+ if (propagatedError) {
2460
+ await this.flushStorage();
2461
+ throw propagatedError;
2462
+ }
2463
+
2464
+ if (
2465
+ !hasWinner &&
2466
+ Object.values(raceData.branches).some(
2467
+ (branch) =>
2468
+ branch.status === "pending" || branch.status === "running",
2469
+ )
2470
+ ) {
2471
+ await this.flushStorage();
2472
+ if (!schedulerYieldState) {
2473
+ throw new Error(
2474
+ `Race "${name}" has pending branches without a scheduler yield`,
2475
+ );
2476
+ }
2477
+ throw buildSchedulerYieldError(schedulerYieldState);
2478
+ }
2479
+
2480
+ // Clean up entries from non-winning branches
2481
+ if (hasWinner && winnerName !== null) {
2482
+ for (const branch of branches) {
2483
+ if (branch.name !== winnerName) {
2484
+ const branchLocation = appendName(
2485
+ this.storage,
2486
+ location,
2487
+ branch.name,
2488
+ );
2489
+ await deleteEntriesWithPrefix(
2490
+ this.storage,
2491
+ this.driver,
2492
+ branchLocation,
2493
+ this.historyNotifier,
2494
+ );
2495
+ }
2496
+ }
2497
+ }
2498
+
2499
+ // Flush final state
2500
+ await this.flushStorage();
2501
+
2502
+ // Log late errors if any (these occurred after a winner was determined)
2503
+ if (lateErrors.length > 0) {
2504
+ console.warn(
2505
+ `Race "${name}" had ${lateErrors.length} branch(es) fail after winner was determined:`,
2506
+ lateErrors,
2507
+ );
2508
+ }
2509
+
2510
+ // Return result or throw error
2511
+ if (hasWinner && winnerName !== null) {
2512
+ return { winner: winnerName, value: winnerValue as T };
2513
+ }
2514
+
2515
+ // All branches failed
2516
+ throw attachTryBlockFailure(
2517
+ new RaceError(
2518
+ "All branches failed",
2519
+ Object.entries(errors).map(([branchName, error]) => ({
2520
+ name: branchName,
2521
+ error: String(error),
2522
+ })),
2523
+ ),
2524
+ {
2525
+ source: "race",
2526
+ name,
2527
+ },
2528
+ );
2529
+ }
2530
+
2531
+ // === Removed ===
2532
+
2533
+ async removed(name: string, originalType: EntryKindType): Promise<void> {
2534
+ this.assertNotInProgress();
2535
+ this.checkEvicted();
2536
+
2537
+ this.entryInProgress = true;
2538
+ try {
2539
+ await this.executeRemoved(name, originalType);
2540
+ } finally {
2541
+ this.entryInProgress = false;
2542
+ }
2543
+ }
2544
+
2545
+ private async executeRemoved(
2546
+ name: string,
2547
+ originalType: EntryKindType,
2548
+ ): Promise<void> {
2549
+ // Check for duplicate name in current execution
2550
+ this.checkDuplicateName(name);
2551
+
2552
+ const location = appendName(this.storage, this.currentLocation, name);
2553
+ const key = locationToKey(this.storage, location);
2554
+ const existing = this.storage.history.entries.get(key);
2555
+
2556
+ // Mark this entry as visited for validateComplete
2557
+ this.markVisited(key);
2558
+
2559
+ this.stopRollbackIfMissing(existing);
2560
+
2561
+ if (existing) {
2562
+ // Validate the existing entry matches what we expect
2563
+ if (
2564
+ existing.kind.type !== "removed" &&
2565
+ existing.kind.type !== originalType
2566
+ ) {
2567
+ throw new HistoryDivergedError(
2568
+ `Expected ${originalType} or removed at ${key}, found ${existing.kind.type}`,
2569
+ );
2570
+ }
2571
+
2572
+ // If it's not already marked as removed, we just skip it
2573
+ return;
2574
+ }
2575
+
2576
+ // Create a removed entry placeholder
2577
+ const entry = createEntry(location, {
2578
+ type: "removed",
2579
+ data: { originalType, originalName: name },
2580
+ });
2581
+ setEntry(this.storage, location, entry);
2582
+ await this.flushStorage();
2583
+ }
2584
+
2585
+ // === Version ===
2586
+
2587
+ async getVersion(name: string, latest: number): Promise<number> {
2588
+ this.assertNotInProgress();
2589
+ this.checkEvicted();
2590
+
2591
+ this.entryInProgress = true;
2592
+ try {
2593
+ return await this.executeGetVersion(name, latest);
2594
+ } finally {
2595
+ this.entryInProgress = false;
2596
+ }
2597
+ }
2598
+
2599
+ private async executeGetVersion(
2600
+ name: string,
2601
+ latest: number,
2602
+ ): Promise<number> {
2603
+ if (!Number.isInteger(latest) || latest < 1) {
2604
+ throw new Error(
2605
+ `getVersion("${name}", ${latest}): latest must be an integer >= 1`,
2606
+ );
2607
+ }
2608
+
2609
+ // Check for duplicate name in current execution
2610
+ this.checkDuplicateName(name);
2611
+
2612
+ const location = appendName(this.storage, this.currentLocation, name);
2613
+ const key = locationToKey(this.storage, location);
2614
+ const existing = this.storage.history.entries.get(key);
2615
+
2616
+ // Mark this entry as visited for validateComplete
2617
+ this.markVisited(key);
2618
+
2619
+ this.stopRollbackIfMissing(existing);
2620
+
2621
+ if (existing) {
2622
+ if (existing.kind.type !== "version_check") {
2623
+ throw new HistoryDivergedError(
2624
+ `Expected version_check at ${key}, found ${existing.kind.type}`,
2625
+ );
2626
+ }
2627
+ // Pure replay: this instance is already pinned at this location.
2628
+ return existing.kind.data.resolved;
2629
+ }
2630
+
2631
+ // No recorded version at this location. Decide whether this instance
2632
+ // already executed past this point under older code (old in-flight) or
2633
+ // is reaching it fresh at the live frontier.
2634
+ //
2635
+ // The discriminator is "is there any unvisited history entry under the
2636
+ // current scope?". Entries created earlier in this same run are already
2637
+ // marked visited, so the only unvisited entries under the scope are
2638
+ // leftovers from a prior run, which proves this scope already executed
2639
+ // under code that predates this gate. Such instances resolve to the
2640
+ // implicit floor version 1 (old branch); fresh instances resolve to
2641
+ // `latest`.
2642
+ const resolved = this.hasUnvisitedUnderCurrentScope(key) ? 1 : latest;
2643
+
2644
+ const entry = createEntry(location, {
2645
+ type: "version_check",
2646
+ data: { resolved, latest },
2647
+ });
2648
+ setEntry(this.storage, location, entry);
2649
+ await this.flushStorage();
2650
+
2651
+ return resolved;
2652
+ }
2653
+
2654
+ /**
2655
+ * Returns true if any history entry under the current location scope
2656
+ * (other than `excludeKey`) has not yet been visited this run. Used by
2657
+ * getVersion to detect an old in-flight instance whose scope was already
2658
+ * executed by code that predates a version gate.
2659
+ */
2660
+ private hasUnvisitedUnderCurrentScope(excludeKey: string): boolean {
2661
+ const prefix = locationToKey(this.storage, this.currentLocation);
2662
+
2663
+ for (const key of this.storage.history.entries.keys()) {
2664
+ if (key === excludeKey) {
2665
+ continue;
2666
+ }
2667
+ const isUnderPrefix =
2668
+ prefix === ""
2669
+ ? true
2670
+ : key.startsWith(`${prefix}/`) || key === prefix;
2671
+ if (isUnderPrefix && !this.visitedKeys.has(key)) {
2672
+ return true;
2673
+ }
2674
+ }
2675
+ return false;
2676
+ }
2677
+ }