@rivetkit/workflow-engine 0.0.0-pr.4600.32b0fc8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/context.ts ADDED
@@ -0,0 +1,2585 @@
1
+ import type { Logger } from "pino";
2
+ import type { EngineDriver } from "./driver.js";
3
+ import {
4
+ extractErrorInfo,
5
+ getErrorEventTag,
6
+ markErrorReported,
7
+ } from "./error-utils.js";
8
+ import {
9
+ CancelledError,
10
+ CriticalError,
11
+ EntryInProgressError,
12
+ EvictedError,
13
+ HistoryDivergedError,
14
+ JoinError,
15
+ MessageWaitError,
16
+ RaceError,
17
+ RollbackCheckpointError,
18
+ RollbackError,
19
+ RollbackStopError,
20
+ SleepError,
21
+ StepExhaustedError,
22
+ StepFailedError,
23
+ } from "./errors.js";
24
+ import { buildLoopIterationRange, buildEntryMetadataKey } from "./keys.js";
25
+ import {
26
+ appendLoopIteration,
27
+ appendName,
28
+ emptyLocation,
29
+ isLocationPrefix,
30
+ locationToKey,
31
+ registerName,
32
+ } from "./location.js";
33
+ import {
34
+ createEntry,
35
+ deleteEntriesWithPrefix,
36
+ flush,
37
+ getOrCreateMetadata,
38
+ loadMetadata,
39
+ setEntry,
40
+ type PendingDeletions,
41
+ } from "./storage.js";
42
+ import type {
43
+ BranchConfig,
44
+ BranchOutput,
45
+ BranchStatus,
46
+ Entry,
47
+ EntryKindType,
48
+ EntryMetadata,
49
+ Location,
50
+ LoopConfig,
51
+ LoopIterationResult,
52
+ LoopResult,
53
+ Message,
54
+ RollbackContextInterface,
55
+ StepConfig,
56
+ Storage,
57
+ TryBlockCatchKind,
58
+ TryBlockConfig,
59
+ TryBlockFailure,
60
+ TryBlockResult,
61
+ TryStepCatchKind,
62
+ TryStepConfig,
63
+ TryStepFailure,
64
+ TryStepResult,
65
+ WorkflowContextInterface,
66
+ WorkflowError,
67
+ WorkflowErrorEvent,
68
+ WorkflowErrorHandler,
69
+ WorkflowQueue,
70
+ WorkflowQueueMessage,
71
+ WorkflowQueueNextBatchOptions,
72
+ WorkflowQueueNextOptions,
73
+ WorkflowMessageDriver,
74
+ } from "./types.js";
75
+ import { sleep } from "./utils.js";
76
+
77
+ /**
78
+ * Default values for step configuration.
79
+ * These are exported so users can reference them when overriding.
80
+ */
81
+ export const DEFAULT_MAX_RETRIES = 3;
82
+ export const DEFAULT_RETRY_BACKOFF_BASE = 100;
83
+ export const DEFAULT_RETRY_BACKOFF_MAX = 30000;
84
+ export const DEFAULT_LOOP_HISTORY_PRUNE_INTERVAL = 20;
85
+ export const DEFAULT_STEP_TIMEOUT = 30000; // 30 seconds
86
+ const DEFAULT_TRY_STEP_CATCH: readonly TryStepCatchKind[] = [
87
+ "critical",
88
+ "timeout",
89
+ "exhausted",
90
+ ];
91
+ const DEFAULT_TRY_BLOCK_CATCH: readonly TryBlockCatchKind[] = [
92
+ "step",
93
+ "join",
94
+ "race",
95
+ ];
96
+
97
+ const QUEUE_HISTORY_MESSAGE_MARKER = "__rivetWorkflowQueueMessage";
98
+ const TRY_STEP_FAILURE_SYMBOL = Symbol("workflow.try-step.failure");
99
+ const TRY_BLOCK_FAILURE_SYMBOL = Symbol("workflow.try-block.failure");
100
+
101
+ /**
102
+ * Calculate backoff delay with exponential backoff.
103
+ * Uses deterministic calculation (no jitter) for replay consistency.
104
+ */
105
+ function calculateBackoff(attempts: number, base: number, max: number): number {
106
+ // Exponential backoff without jitter for determinism
107
+ return Math.min(max, base * 2 ** attempts);
108
+ }
109
+
110
+ /**
111
+ * Error thrown when a step times out.
112
+ */
113
+ export class StepTimeoutError extends Error {
114
+ constructor(
115
+ public readonly stepName: string,
116
+ public readonly timeoutMs: number,
117
+ ) {
118
+ super(`Step "${stepName}" timed out after ${timeoutMs}ms`);
119
+ this.name = "StepTimeoutError";
120
+ }
121
+ }
122
+
123
+ type SchedulerYieldState = {
124
+ deadline?: number;
125
+ messageNames: Set<string>;
126
+ };
127
+
128
+ type TryBlockFailureInfo = Pick<TryBlockFailure, "source" | "name">;
129
+
130
+ function attachTryStepFailure<T extends Error>(
131
+ error: T,
132
+ failure: TryStepFailure,
133
+ ): T {
134
+ (
135
+ error as T & {
136
+ [TRY_STEP_FAILURE_SYMBOL]?: TryStepFailure;
137
+ }
138
+ )[TRY_STEP_FAILURE_SYMBOL] = failure;
139
+ return error;
140
+ }
141
+
142
+ function readTryStepFailure(error: unknown): TryStepFailure | undefined {
143
+ if (!(error instanceof Error)) {
144
+ return undefined;
145
+ }
146
+
147
+ return (
148
+ error as Error & {
149
+ [TRY_STEP_FAILURE_SYMBOL]?: TryStepFailure;
150
+ }
151
+ )[TRY_STEP_FAILURE_SYMBOL];
152
+ }
153
+
154
+ function attachTryBlockFailure<T extends Error>(
155
+ error: T,
156
+ failure: TryBlockFailureInfo,
157
+ ): T {
158
+ (
159
+ error as T & {
160
+ [TRY_BLOCK_FAILURE_SYMBOL]?: TryBlockFailureInfo;
161
+ }
162
+ )[TRY_BLOCK_FAILURE_SYMBOL] = failure;
163
+ return error;
164
+ }
165
+
166
+ function readTryBlockFailure(error: unknown): TryBlockFailureInfo | undefined {
167
+ if (!(error instanceof Error)) {
168
+ return undefined;
169
+ }
170
+
171
+ return (
172
+ error as Error & {
173
+ [TRY_BLOCK_FAILURE_SYMBOL]?: TryBlockFailureInfo;
174
+ }
175
+ )[TRY_BLOCK_FAILURE_SYMBOL];
176
+ }
177
+
178
+ function shouldRethrowTryError(error: unknown): boolean {
179
+ return (
180
+ error instanceof StepFailedError ||
181
+ error instanceof SleepError ||
182
+ error instanceof MessageWaitError ||
183
+ error instanceof EvictedError ||
184
+ error instanceof HistoryDivergedError ||
185
+ error instanceof EntryInProgressError ||
186
+ error instanceof RollbackCheckpointError ||
187
+ error instanceof RollbackStopError
188
+ );
189
+ }
190
+
191
+ function shouldCatchTryStepFailure(
192
+ failure: TryStepFailure,
193
+ catchKinds?: readonly TryStepCatchKind[],
194
+ ): boolean {
195
+ const effectiveCatch = catchKinds ?? DEFAULT_TRY_STEP_CATCH;
196
+ return effectiveCatch.includes(failure.kind);
197
+ }
198
+
199
+ function shouldCatchTryBlockFailure(
200
+ failure: TryBlockFailure,
201
+ catchKinds?: readonly TryBlockCatchKind[],
202
+ ): boolean {
203
+ const effectiveCatch = catchKinds ?? DEFAULT_TRY_BLOCK_CATCH;
204
+
205
+ if (failure.source === "step") {
206
+ return failure.step?.kind === "rollback"
207
+ ? effectiveCatch.includes("rollback")
208
+ : effectiveCatch.includes("step");
209
+ }
210
+ if (failure.source === "join") {
211
+ return effectiveCatch.includes("join");
212
+ }
213
+ if (failure.source === "race") {
214
+ return effectiveCatch.includes("race");
215
+ }
216
+ return effectiveCatch.includes("rollback");
217
+ }
218
+
219
+ function parseStoredWorkflowError(message: string | undefined): WorkflowError {
220
+ if (!message) {
221
+ return {
222
+ name: "Error",
223
+ message: "unknown error",
224
+ };
225
+ }
226
+
227
+ const match = /^([^:]+):\s*(.*)$/s.exec(message);
228
+ if (!match) {
229
+ return {
230
+ name: "Error",
231
+ message,
232
+ };
233
+ }
234
+
235
+ return {
236
+ name: match[1],
237
+ message: match[2],
238
+ };
239
+ }
240
+
241
+ function getTryStepFailureFromExhaustedError(
242
+ stepName: string,
243
+ attempts: number,
244
+ error: StepExhaustedError,
245
+ ): TryStepFailure {
246
+ return {
247
+ kind: "exhausted",
248
+ stepName,
249
+ attempts,
250
+ error: parseStoredWorkflowError(error.lastError),
251
+ };
252
+ }
253
+
254
+ function mergeSchedulerYield(
255
+ state: SchedulerYieldState | undefined,
256
+ error: SleepError | MessageWaitError | StepFailedError,
257
+ ): SchedulerYieldState {
258
+ const nextState: SchedulerYieldState = state ?? {
259
+ messageNames: new Set<string>(),
260
+ };
261
+
262
+ if (error instanceof SleepError) {
263
+ nextState.deadline =
264
+ nextState.deadline === undefined
265
+ ? error.deadline
266
+ : Math.min(nextState.deadline, error.deadline);
267
+ for (const messageName of error.messageNames ?? []) {
268
+ nextState.messageNames.add(messageName);
269
+ }
270
+ return nextState;
271
+ }
272
+
273
+ if (error instanceof MessageWaitError) {
274
+ for (const messageName of error.messageNames) {
275
+ nextState.messageNames.add(messageName);
276
+ }
277
+ return nextState;
278
+ }
279
+
280
+ nextState.deadline =
281
+ nextState.deadline === undefined
282
+ ? error.retryAt
283
+ : Math.min(nextState.deadline, error.retryAt);
284
+ return nextState;
285
+ }
286
+
287
+ function buildSchedulerYieldError(
288
+ state: SchedulerYieldState,
289
+ ): SleepError | MessageWaitError {
290
+ const messageNames = [...state.messageNames];
291
+ if (state.deadline !== undefined) {
292
+ return new SleepError(
293
+ state.deadline,
294
+ messageNames.length > 0 ? messageNames : undefined,
295
+ );
296
+ }
297
+ return new MessageWaitError(messageNames);
298
+ }
299
+
300
+ function controlFlowErrorPriority(error: Error): number {
301
+ if (error instanceof EvictedError) {
302
+ return 0;
303
+ }
304
+ if (error instanceof HistoryDivergedError) {
305
+ return 1;
306
+ }
307
+ if (error instanceof EntryInProgressError) {
308
+ return 2;
309
+ }
310
+ if (error instanceof RollbackCheckpointError) {
311
+ return 3;
312
+ }
313
+ if (error instanceof RollbackStopError) {
314
+ return 4;
315
+ }
316
+ return 5;
317
+ }
318
+
319
+ function selectControlFlowError(
320
+ current: Error | undefined,
321
+ candidate: Error,
322
+ ): Error {
323
+ if (!current) {
324
+ return candidate;
325
+ }
326
+ return controlFlowErrorPriority(candidate) <
327
+ controlFlowErrorPriority(current)
328
+ ? candidate
329
+ : current;
330
+ }
331
+
332
+ /**
333
+ * Internal representation of a rollback handler.
334
+ */
335
+ export interface RollbackAction<T = unknown> {
336
+ entryId: string;
337
+ name: string;
338
+ output: T;
339
+ rollback: (ctx: RollbackContextInterface, output: T) => Promise<void>;
340
+ }
341
+
342
+ /**
343
+ * Internal implementation of WorkflowContext.
344
+ */
345
+ export class WorkflowContextImpl implements WorkflowContextInterface {
346
+ private entryInProgress = false;
347
+ private abortController: AbortController;
348
+ private currentLocation: Location;
349
+ private visitedKeys: Set<string>;
350
+ private mode: "forward" | "rollback";
351
+ private rollbackActions?: RollbackAction[];
352
+ private rollbackCheckpointSet: boolean;
353
+ /** Track names used in current execution to detect duplicates */
354
+ private usedNamesInExecution = new Set<string>();
355
+ private pendingCompletableMessageIds = new Set<string>();
356
+ private historyNotifier?: () => void;
357
+ private onError?: WorkflowErrorHandler;
358
+ private logger?: Logger;
359
+
360
+ constructor(
361
+ public readonly workflowId: string,
362
+ private storage: Storage,
363
+ private driver: EngineDriver,
364
+ private messageDriver: WorkflowMessageDriver,
365
+ location: Location = emptyLocation(),
366
+ abortController?: AbortController,
367
+ mode: "forward" | "rollback" = "forward",
368
+ rollbackActions?: RollbackAction[],
369
+ rollbackCheckpointSet = false,
370
+ historyNotifier?: () => void,
371
+ onError?: WorkflowErrorHandler,
372
+ logger?: Logger,
373
+ visitedKeys?: Set<string>,
374
+ ) {
375
+ this.currentLocation = location;
376
+ this.abortController = abortController ?? new AbortController();
377
+ this.mode = mode;
378
+ this.rollbackActions = rollbackActions;
379
+ this.rollbackCheckpointSet = rollbackCheckpointSet;
380
+ this.historyNotifier = historyNotifier;
381
+ this.onError = onError;
382
+ this.logger = logger;
383
+ this.visitedKeys = visitedKeys ?? new Set();
384
+ }
385
+
386
+ get abortSignal(): AbortSignal {
387
+ return this.abortController.signal;
388
+ }
389
+
390
+ get queue(): WorkflowQueue {
391
+ return {
392
+ next: async (name, opts) => await this.queueNext(name, opts),
393
+ nextBatch: async (name, opts) =>
394
+ await this.queueNextBatch(name, opts),
395
+ send: async (name, body) => await this.queueSend(name, body),
396
+ };
397
+ }
398
+
399
+ isEvicted(): boolean {
400
+ return this.abortSignal.aborted;
401
+ }
402
+
403
+ private assertNotInProgress(): void {
404
+ if (this.entryInProgress) {
405
+ throw new EntryInProgressError();
406
+ }
407
+ }
408
+
409
+ private checkEvicted(): void {
410
+ if (this.abortSignal.aborted) {
411
+ throw new EvictedError();
412
+ }
413
+ }
414
+
415
+ private async flushStorage(): Promise<void> {
416
+ await flush(this.storage, this.driver, this.historyNotifier);
417
+ }
418
+
419
+ /**
420
+ * Create a new branch context for parallel/nested execution.
421
+ */
422
+ createBranch(
423
+ location: Location,
424
+ abortController?: AbortController,
425
+ ): WorkflowContextImpl {
426
+ return new WorkflowContextImpl(
427
+ this.workflowId,
428
+ this.storage,
429
+ this.driver,
430
+ this.messageDriver,
431
+ location,
432
+ abortController ?? this.abortController,
433
+ this.mode,
434
+ this.rollbackActions,
435
+ this.rollbackCheckpointSet,
436
+ this.historyNotifier,
437
+ this.onError,
438
+ this.logger,
439
+ this.visitedKeys,
440
+ );
441
+ }
442
+
443
+ /**
444
+ * Log a debug message using the configured logger.
445
+ */
446
+ private log(
447
+ level: "debug" | "info" | "warn" | "error",
448
+ data: Record<string, unknown>,
449
+ ): void {
450
+ if (!this.logger) return;
451
+ this.logger[level](data);
452
+ }
453
+
454
+ private async notifyError(event: WorkflowErrorEvent): Promise<void> {
455
+ if (!this.onError) {
456
+ return;
457
+ }
458
+
459
+ try {
460
+ await this.onError(event);
461
+ } catch (error) {
462
+ this.log("warn", {
463
+ msg: "workflow error hook failed",
464
+ hookEventType: getErrorEventTag(event),
465
+ error: extractErrorInfo(error),
466
+ });
467
+ }
468
+ }
469
+
470
+ private async notifyStepError<T>(
471
+ config: StepConfig<T>,
472
+ attempt: number,
473
+ error: unknown,
474
+ opts: {
475
+ willRetry: boolean;
476
+ retryDelay?: number;
477
+ retryAt?: number;
478
+ },
479
+ ): Promise<void> {
480
+ const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
481
+ await this.notifyError({
482
+ step: {
483
+ workflowId: this.workflowId,
484
+ stepName: config.name,
485
+ attempt,
486
+ maxRetries,
487
+ remainingRetries: Math.max(0, maxRetries - (attempt - 1)),
488
+ willRetry: opts.willRetry,
489
+ retryDelay: opts.retryDelay,
490
+ retryAt: opts.retryAt,
491
+ error: extractErrorInfo(error),
492
+ },
493
+ });
494
+ }
495
+
496
+ /**
497
+ * Mark a key as visited.
498
+ */
499
+ private markVisited(key: string): void {
500
+ this.visitedKeys.add(key);
501
+ }
502
+
503
+ /**
504
+ * Check if a name has already been used at the current location in this execution.
505
+ * Throws HistoryDivergedError if duplicate detected.
506
+ */
507
+ private checkDuplicateName(name: string): void {
508
+ const fullKey =
509
+ locationToKey(this.storage, this.currentLocation) + "/" + name;
510
+ if (this.usedNamesInExecution.has(fullKey)) {
511
+ throw new HistoryDivergedError(
512
+ `Duplicate entry name "${name}" at location "${locationToKey(this.storage, this.currentLocation)}". ` +
513
+ `Each step/loop/sleep/queue.next/join/race must have a unique name within its scope.`,
514
+ );
515
+ }
516
+ this.usedNamesInExecution.add(fullKey);
517
+ }
518
+
519
+ private stopRollback(): never {
520
+ throw new RollbackStopError();
521
+ }
522
+
523
+ private stopRollbackIfMissing(entry: Entry | undefined): void {
524
+ if (this.mode === "rollback" && !entry) {
525
+ this.stopRollback();
526
+ }
527
+ }
528
+
529
+ private stopRollbackIfIncomplete(condition: boolean): void {
530
+ if (this.mode === "rollback" && condition) {
531
+ this.stopRollback();
532
+ }
533
+ }
534
+
535
+ private registerRollbackAction<T>(
536
+ config: StepConfig<T>,
537
+ entryId: string,
538
+ output: T,
539
+ metadata: EntryMetadata,
540
+ ): void {
541
+ if (!config.rollback) {
542
+ return;
543
+ }
544
+ if (metadata.rollbackCompletedAt !== undefined) {
545
+ return;
546
+ }
547
+ this.rollbackActions?.push({
548
+ entryId,
549
+ name: config.name,
550
+ output: output as unknown,
551
+ rollback: config.rollback as (
552
+ ctx: RollbackContextInterface,
553
+ output: unknown,
554
+ ) => Promise<void>,
555
+ });
556
+ }
557
+
558
+ /**
559
+ * Ensure a rollback checkpoint exists before registering rollback handlers.
560
+ */
561
+ private ensureRollbackCheckpoint<T>(config: StepConfig<T>): void {
562
+ if (!config.rollback) {
563
+ return;
564
+ }
565
+ if (!this.rollbackCheckpointSet) {
566
+ throw new RollbackCheckpointError();
567
+ }
568
+ }
569
+
570
+ /**
571
+ * Validate that all expected entries in the branch were visited.
572
+ * Throws HistoryDivergedError if there are unvisited entries.
573
+ */
574
+ validateComplete(): void {
575
+ const prefix = locationToKey(this.storage, this.currentLocation);
576
+
577
+ for (const key of this.storage.history.entries.keys()) {
578
+ // Check if this key is under our current location prefix
579
+ // Handle root prefix (empty string) specially - all keys are under root
580
+ const isUnderPrefix =
581
+ prefix === ""
582
+ ? true // Root: all keys are children
583
+ : key.startsWith(prefix + "/") || key === prefix;
584
+
585
+ if (isUnderPrefix) {
586
+ if (!this.visitedKeys.has(key)) {
587
+ // Entry exists in history but wasn't visited
588
+ // This means workflow code may have changed
589
+ throw new HistoryDivergedError(
590
+ `Entry "${key}" exists in history but was not visited. ` +
591
+ `Workflow code may have changed. Use ctx.removed() to handle migrations.`,
592
+ );
593
+ }
594
+ }
595
+ }
596
+ }
597
+
598
+ /**
599
+ * Evict the workflow.
600
+ */
601
+ evict(): void {
602
+ this.abortController.abort(new EvictedError());
603
+ }
604
+
605
+ /**
606
+ * Wait for eviction message.
607
+ *
608
+ * The event listener uses { once: true } to auto-remove after firing,
609
+ * preventing memory leaks if this method is called multiple times.
610
+ */
611
+ waitForEviction(): Promise<never> {
612
+ return new Promise((_, reject) => {
613
+ if (this.abortSignal.aborted) {
614
+ reject(new EvictedError());
615
+ return;
616
+ }
617
+ this.abortSignal.addEventListener(
618
+ "abort",
619
+ () => {
620
+ reject(new EvictedError());
621
+ },
622
+ { once: true },
623
+ );
624
+ });
625
+ }
626
+
627
+ // === Step ===
628
+
629
+ async step<T>(
630
+ nameOrConfig: string | StepConfig<T>,
631
+ run?: () => Promise<T>,
632
+ ): Promise<T> {
633
+ this.assertNotInProgress();
634
+ this.checkEvicted();
635
+
636
+ const config: StepConfig<T> =
637
+ typeof nameOrConfig === "string"
638
+ ? { name: nameOrConfig, run: run! }
639
+ : nameOrConfig;
640
+
641
+ this.entryInProgress = true;
642
+ try {
643
+ return await this.executeStep(config);
644
+ } finally {
645
+ this.entryInProgress = false;
646
+ }
647
+ }
648
+
649
+ async tryStep<T>(
650
+ nameOrConfig: string | TryStepConfig<T>,
651
+ run?: () => Promise<T>,
652
+ ): Promise<TryStepResult<T>> {
653
+ const config =
654
+ typeof nameOrConfig === "string"
655
+ ? ({
656
+ name: nameOrConfig,
657
+ run: run!,
658
+ } satisfies TryStepConfig<T>)
659
+ : nameOrConfig;
660
+
661
+ try {
662
+ return {
663
+ ok: true,
664
+ value: await this.step(config),
665
+ };
666
+ } catch (error) {
667
+ if (shouldRethrowTryError(error)) {
668
+ throw error;
669
+ }
670
+
671
+ const failure = readTryStepFailure(error);
672
+ if (!failure || !shouldCatchTryStepFailure(failure, config.catch)) {
673
+ throw error;
674
+ }
675
+
676
+ return {
677
+ ok: false,
678
+ failure,
679
+ };
680
+ }
681
+ }
682
+
683
+ async try<T>(
684
+ nameOrConfig: string | TryBlockConfig<T>,
685
+ run?: (ctx: WorkflowContextInterface) => Promise<T>,
686
+ ): Promise<TryBlockResult<T>> {
687
+ this.assertNotInProgress();
688
+ this.checkEvicted();
689
+
690
+ const config =
691
+ typeof nameOrConfig === "string"
692
+ ? ({
693
+ name: nameOrConfig,
694
+ run: run!,
695
+ } satisfies TryBlockConfig<T>)
696
+ : nameOrConfig;
697
+
698
+ this.entryInProgress = true;
699
+ try {
700
+ return await this.executeTry(config);
701
+ } finally {
702
+ this.entryInProgress = false;
703
+ }
704
+ }
705
+
706
+ private async executeTry<T>(
707
+ config: TryBlockConfig<T>,
708
+ ): Promise<TryBlockResult<T>> {
709
+ this.checkDuplicateName(config.name);
710
+
711
+ const location = appendName(
712
+ this.storage,
713
+ this.currentLocation,
714
+ config.name,
715
+ );
716
+ const blockCtx = this.createBranch(location);
717
+
718
+ try {
719
+ const value = await config.run(blockCtx);
720
+ blockCtx.validateComplete();
721
+ return {
722
+ ok: true,
723
+ value,
724
+ };
725
+ } catch (error) {
726
+ if (shouldRethrowTryError(error)) {
727
+ throw error;
728
+ }
729
+
730
+ const stepFailure = readTryStepFailure(error);
731
+ if (stepFailure) {
732
+ const failure: TryBlockFailure = {
733
+ source: "step",
734
+ name: stepFailure.stepName,
735
+ error: stepFailure.error,
736
+ step: stepFailure,
737
+ };
738
+ if (!shouldCatchTryBlockFailure(failure, config.catch)) {
739
+ throw error;
740
+ }
741
+ return {
742
+ ok: false,
743
+ failure,
744
+ };
745
+ }
746
+
747
+ const operationFailure = readTryBlockFailure(error);
748
+ if (operationFailure) {
749
+ const failure: TryBlockFailure = {
750
+ ...operationFailure,
751
+ error: extractErrorInfo(error),
752
+ };
753
+ if (!shouldCatchTryBlockFailure(failure, config.catch)) {
754
+ throw error;
755
+ }
756
+ return {
757
+ ok: false,
758
+ failure,
759
+ };
760
+ }
761
+
762
+ if (error instanceof RollbackError) {
763
+ const failure: TryBlockFailure = {
764
+ source: "block",
765
+ name: config.name,
766
+ error: extractErrorInfo(error),
767
+ };
768
+ if (!shouldCatchTryBlockFailure(failure, config.catch)) {
769
+ throw error;
770
+ }
771
+ return {
772
+ ok: false,
773
+ failure,
774
+ };
775
+ }
776
+
777
+ throw error;
778
+ }
779
+ }
780
+
781
+ private async executeStep<T>(config: StepConfig<T>): Promise<T> {
782
+ this.ensureRollbackCheckpoint(config);
783
+ if (this.mode === "rollback") {
784
+ return await this.executeStepRollback(config);
785
+ }
786
+
787
+ // Check for duplicate name in current execution
788
+ this.checkDuplicateName(config.name);
789
+
790
+ const location = appendName(
791
+ this.storage,
792
+ this.currentLocation,
793
+ config.name,
794
+ );
795
+ const key = locationToKey(this.storage, location);
796
+ const existing = this.storage.history.entries.get(key);
797
+
798
+ // Mark this entry as visited for validateComplete
799
+ this.markVisited(key);
800
+
801
+ if (existing) {
802
+ if (existing.kind.type !== "step") {
803
+ throw new HistoryDivergedError(
804
+ `Expected step "${config.name}" at ${key}, found ${existing.kind.type}`,
805
+ );
806
+ }
807
+
808
+ const stepData = existing.kind.data;
809
+
810
+ const metadata = await loadMetadata(
811
+ this.storage,
812
+ this.driver,
813
+ existing.id,
814
+ );
815
+
816
+ // Replay successful result (including void steps).
817
+ if (
818
+ metadata.status === "completed" ||
819
+ stepData.output !== undefined
820
+ ) {
821
+ return stepData.output as T;
822
+ }
823
+
824
+ // Check if we should retry
825
+ const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
826
+
827
+ if (metadata.attempts > maxRetries) {
828
+ // Prefer step history error, but fall back to metadata since
829
+ // driver implementations may persist metadata without the history
830
+ // entry error (e.g. partial writes/crashes between attempts).
831
+ const lastError = stepData.error ?? metadata.error;
832
+ const exhaustedError = new StepExhaustedError(
833
+ config.name,
834
+ lastError,
835
+ );
836
+ attachTryStepFailure(
837
+ exhaustedError,
838
+ getTryStepFailureFromExhaustedError(
839
+ config.name,
840
+ metadata.attempts,
841
+ exhaustedError,
842
+ ),
843
+ );
844
+ markErrorReported(exhaustedError);
845
+ if (metadata.status !== "exhausted") {
846
+ metadata.status = "exhausted";
847
+ metadata.dirty = true;
848
+ await this.flushStorage();
849
+ await this.notifyStepError(
850
+ config,
851
+ metadata.attempts,
852
+ exhaustedError,
853
+ { willRetry: false },
854
+ );
855
+ }
856
+ throw exhaustedError;
857
+ }
858
+
859
+ // Calculate backoff and yield to scheduler
860
+ // This allows the workflow to be evicted during backoff
861
+ const backoffDelay = calculateBackoff(
862
+ metadata.attempts,
863
+ config.retryBackoffBase ?? DEFAULT_RETRY_BACKOFF_BASE,
864
+ config.retryBackoffMax ?? DEFAULT_RETRY_BACKOFF_MAX,
865
+ );
866
+ const retryAt = metadata.lastAttemptAt + backoffDelay;
867
+ const now = Date.now();
868
+
869
+ if (now < retryAt) {
870
+ // Yield to scheduler - will be woken up at retryAt
871
+ throw new SleepError(retryAt);
872
+ }
873
+ }
874
+
875
+ // Execute the step
876
+ const entry =
877
+ existing ?? createEntry(location, { type: "step", data: {} });
878
+ if (!existing) {
879
+ // New entry - register name
880
+ this.log("debug", {
881
+ msg: "executing new step",
882
+ step: config.name,
883
+ key,
884
+ });
885
+ const nameIndex = registerName(this.storage, config.name);
886
+ entry.location = [...location];
887
+ entry.location[entry.location.length - 1] = nameIndex;
888
+ setEntry(this.storage, location, entry);
889
+ } else {
890
+ this.log("debug", { msg: "retrying step", step: config.name, key });
891
+ }
892
+
893
+ const metadata = getOrCreateMetadata(this.storage, entry.id);
894
+ const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
895
+ const retryBackoffBase =
896
+ config.retryBackoffBase ?? DEFAULT_RETRY_BACKOFF_BASE;
897
+ const retryBackoffMax =
898
+ config.retryBackoffMax ?? DEFAULT_RETRY_BACKOFF_MAX;
899
+ metadata.status = "running";
900
+ metadata.attempts++;
901
+ metadata.lastAttemptAt = Date.now();
902
+ metadata.dirty = true;
903
+
904
+ // Get timeout configuration
905
+ const timeout = config.timeout ?? DEFAULT_STEP_TIMEOUT;
906
+
907
+ try {
908
+ // Execute with timeout
909
+ const output = await this.executeWithTimeout(
910
+ config.run(),
911
+ timeout,
912
+ config.name,
913
+ );
914
+
915
+ if (entry.kind.type === "step") {
916
+ entry.kind.data.output = output;
917
+ }
918
+ entry.dirty = true;
919
+ metadata.status = "completed";
920
+ metadata.error = undefined;
921
+ metadata.completedAt = Date.now();
922
+
923
+ // Ephemeral steps don't trigger an immediate flush. This avoids the
924
+ // synchronous write overhead for transient operations. Note that the
925
+ // step's entry is still marked dirty and WILL be persisted on the
926
+ // next flush from a non-ephemeral operation. The purpose of ephemeral
927
+ // is to batch writes, not to avoid persistence entirely.
928
+ if (!config.ephemeral) {
929
+ this.log("debug", {
930
+ msg: "flushing step",
931
+ step: config.name,
932
+ key,
933
+ });
934
+ await this.flushStorage();
935
+ }
936
+
937
+ this.log("debug", {
938
+ msg: "step completed",
939
+ step: config.name,
940
+ key,
941
+ });
942
+ return output;
943
+ } catch (error) {
944
+ // Timeout errors are treated as critical (no retry)
945
+ if (error instanceof StepTimeoutError) {
946
+ if (entry.kind.type === "step") {
947
+ entry.kind.data.error = String(error);
948
+ }
949
+ entry.dirty = true;
950
+ metadata.status = "exhausted";
951
+ metadata.error = String(error);
952
+ await this.flushStorage();
953
+ await this.notifyStepError(config, metadata.attempts, error, {
954
+ willRetry: false,
955
+ });
956
+ throw markErrorReported(
957
+ attachTryStepFailure(
958
+ new CriticalError(error.message),
959
+ {
960
+ kind: "timeout",
961
+ stepName: config.name,
962
+ attempts: metadata.attempts,
963
+ error: extractErrorInfo(error),
964
+ },
965
+ ),
966
+ );
967
+ }
968
+
969
+ if (
970
+ error instanceof CriticalError ||
971
+ error instanceof RollbackError
972
+ ) {
973
+ if (entry.kind.type === "step") {
974
+ entry.kind.data.error = String(error);
975
+ }
976
+ entry.dirty = true;
977
+ metadata.status = "exhausted";
978
+ metadata.error = String(error);
979
+ await this.flushStorage();
980
+ await this.notifyStepError(config, metadata.attempts, error, {
981
+ willRetry: false,
982
+ });
983
+ throw markErrorReported(
984
+ attachTryStepFailure(error, {
985
+ kind:
986
+ error instanceof RollbackError
987
+ ? "rollback"
988
+ : "critical",
989
+ stepName: config.name,
990
+ attempts: metadata.attempts,
991
+ error: extractErrorInfo(error),
992
+ }),
993
+ );
994
+ }
995
+
996
+ if (entry.kind.type === "step") {
997
+ entry.kind.data.error = String(error);
998
+ }
999
+ entry.dirty = true;
1000
+ const willRetry = metadata.attempts <= maxRetries;
1001
+ metadata.status = willRetry ? "failed" : "exhausted";
1002
+ metadata.error = String(error);
1003
+
1004
+ await this.flushStorage();
1005
+ if (willRetry) {
1006
+ const retryDelay = calculateBackoff(
1007
+ metadata.attempts,
1008
+ retryBackoffBase,
1009
+ retryBackoffMax,
1010
+ );
1011
+ const retryAt = metadata.lastAttemptAt + retryDelay;
1012
+ await this.notifyStepError(config, metadata.attempts, error, {
1013
+ willRetry: true,
1014
+ retryDelay,
1015
+ retryAt,
1016
+ });
1017
+ throw new StepFailedError(
1018
+ config.name,
1019
+ error,
1020
+ metadata.attempts,
1021
+ retryAt,
1022
+ );
1023
+ }
1024
+
1025
+ const exhaustedError = markErrorReported(
1026
+ attachTryStepFailure(
1027
+ new StepExhaustedError(config.name, String(error)),
1028
+ {
1029
+ kind: "exhausted",
1030
+ stepName: config.name,
1031
+ attempts: metadata.attempts,
1032
+ error: extractErrorInfo(error),
1033
+ },
1034
+ ),
1035
+ );
1036
+ await this.notifyStepError(config, metadata.attempts, error, {
1037
+ willRetry: false,
1038
+ });
1039
+ throw exhaustedError;
1040
+ }
1041
+ }
1042
+
1043
+ /**
1044
+ * Execute a promise with timeout.
1045
+ *
1046
+ * Note: This does NOT cancel the underlying operation. JavaScript Promises
1047
+ * cannot be cancelled once started. When a timeout occurs:
1048
+ * - The step is marked as failed with StepTimeoutError
1049
+ * - The underlying async operation continues running in the background
1050
+ * - Any side effects from the operation may still occur
1051
+ *
1052
+ * For cancellable operations, pass ctx.abortSignal to APIs that support AbortSignal:
1053
+ *
1054
+ * return fetch(url, { signal: ctx.abortSignal });
1055
+
1056
+ * });
1057
+ *
1058
+ * Or check ctx.isEvicted() periodically in long-running loops.
1059
+ */
1060
+ private async executeStepRollback<T>(config: StepConfig<T>): Promise<T> {
1061
+ this.checkDuplicateName(config.name);
1062
+ this.ensureRollbackCheckpoint(config);
1063
+
1064
+ const location = appendName(
1065
+ this.storage,
1066
+ this.currentLocation,
1067
+ config.name,
1068
+ );
1069
+ const key = locationToKey(this.storage, location);
1070
+ const existing = this.storage.history.entries.get(key);
1071
+
1072
+ this.markVisited(key);
1073
+
1074
+ if (!existing || existing.kind.type !== "step") {
1075
+ this.stopRollback();
1076
+ }
1077
+
1078
+ const metadata = await loadMetadata(
1079
+ this.storage,
1080
+ this.driver,
1081
+ existing.id,
1082
+ );
1083
+ if (metadata.status !== "completed") {
1084
+ this.stopRollback();
1085
+ }
1086
+
1087
+ const output = existing.kind.data.output as T;
1088
+ this.registerRollbackAction(config, existing.id, output, metadata);
1089
+
1090
+ return output;
1091
+ }
1092
+
1093
+ private async executeWithTimeout<T>(
1094
+ promise: Promise<T>,
1095
+ timeoutMs: number,
1096
+ stepName: string,
1097
+ ): Promise<T> {
1098
+ if (timeoutMs <= 0) {
1099
+ return promise;
1100
+ }
1101
+
1102
+ let timeoutId: ReturnType<typeof setTimeout> | undefined;
1103
+ const timeoutPromise = new Promise<never>((_, reject) => {
1104
+ timeoutId = setTimeout(() => {
1105
+ reject(new StepTimeoutError(stepName, timeoutMs));
1106
+ }, timeoutMs);
1107
+ });
1108
+
1109
+ try {
1110
+ return await Promise.race([promise, timeoutPromise]);
1111
+ } finally {
1112
+ if (timeoutId !== undefined) {
1113
+ clearTimeout(timeoutId);
1114
+ }
1115
+ }
1116
+ }
1117
+
1118
+ // === Loop ===
1119
+
1120
+ async loop<S, T>(
1121
+ nameOrConfig: string | LoopConfig<S, T>,
1122
+ run?: (
1123
+ ctx: WorkflowContextInterface,
1124
+ ) => LoopIterationResult<undefined, T>,
1125
+ ): Promise<T> {
1126
+ this.assertNotInProgress();
1127
+ this.checkEvicted();
1128
+
1129
+ const config: LoopConfig<S, T> =
1130
+ typeof nameOrConfig === "string"
1131
+ ? { name: nameOrConfig, run: run as LoopConfig<S, T>["run"] }
1132
+ : nameOrConfig;
1133
+
1134
+ this.entryInProgress = true;
1135
+ try {
1136
+ return await this.executeLoop(config);
1137
+ } finally {
1138
+ this.entryInProgress = false;
1139
+ }
1140
+ }
1141
+
1142
+ private async executeLoop<S, T>(config: LoopConfig<S, T>): Promise<T> {
1143
+ // Check for duplicate name in current execution
1144
+ this.checkDuplicateName(config.name);
1145
+
1146
+ const location = appendName(
1147
+ this.storage,
1148
+ this.currentLocation,
1149
+ config.name,
1150
+ );
1151
+ const key = locationToKey(this.storage, location);
1152
+ const existing = this.storage.history.entries.get(key);
1153
+
1154
+ // Mark this entry as visited for validateComplete
1155
+ this.markVisited(key);
1156
+
1157
+ let entry: Entry;
1158
+ let metadata: EntryMetadata | undefined;
1159
+ let state: S;
1160
+ let iteration: number;
1161
+ let rollbackSingleIteration = false;
1162
+ let rollbackIterationRan = false;
1163
+ let rollbackOutput: T | undefined;
1164
+ const rollbackMode = this.mode === "rollback";
1165
+
1166
+ if (existing) {
1167
+ if (existing.kind.type !== "loop") {
1168
+ throw new HistoryDivergedError(
1169
+ `Expected loop "${config.name}" at ${key}, found ${existing.kind.type}`,
1170
+ );
1171
+ }
1172
+
1173
+ const loopData = existing.kind.data;
1174
+ metadata = await loadMetadata(
1175
+ this.storage,
1176
+ this.driver,
1177
+ existing.id,
1178
+ );
1179
+
1180
+ if (rollbackMode) {
1181
+ if (loopData.output !== undefined) {
1182
+ return loopData.output as T;
1183
+ }
1184
+ rollbackSingleIteration = true;
1185
+ rollbackIterationRan = false;
1186
+ rollbackOutput = undefined;
1187
+ }
1188
+
1189
+ if (metadata.status === "completed") {
1190
+ return loopData.output as T;
1191
+ }
1192
+
1193
+ // Loop already completed
1194
+ if (loopData.output !== undefined) {
1195
+ return loopData.output as T;
1196
+ }
1197
+
1198
+ // Resume from saved state
1199
+ entry = existing;
1200
+ state = loopData.state as S;
1201
+ iteration = loopData.iteration;
1202
+ if (rollbackMode) {
1203
+ rollbackOutput = loopData.output as T | undefined;
1204
+ rollbackIterationRan = rollbackOutput !== undefined;
1205
+ }
1206
+ } else {
1207
+ this.stopRollbackIfIncomplete(true);
1208
+
1209
+ // New loop
1210
+ state = config.state as S;
1211
+ iteration = 0;
1212
+ entry = createEntry(location, {
1213
+ type: "loop",
1214
+ data: { state, iteration },
1215
+ });
1216
+ setEntry(this.storage, location, entry);
1217
+ metadata = getOrCreateMetadata(this.storage, entry.id);
1218
+ }
1219
+
1220
+ if (metadata) {
1221
+ metadata.status = "running";
1222
+ metadata.error = undefined;
1223
+ metadata.dirty = true;
1224
+ }
1225
+
1226
+ const historyPruneInterval =
1227
+ config.historyPruneInterval ?? DEFAULT_LOOP_HISTORY_PRUNE_INTERVAL;
1228
+ const historySize = config.historySize ?? historyPruneInterval;
1229
+
1230
+ // Track the last iteration we pruned up to so we only delete
1231
+ // newly-expired iterations instead of re-scanning from 0.
1232
+ let lastPrunedUpTo = 0;
1233
+
1234
+ // Deferred flush promise from the previous prune cycle. Awaited at the
1235
+ // start of the next iteration so the flush runs in parallel with user code.
1236
+ let deferredFlush: Promise<void> | null = null;
1237
+
1238
+ // Execute loop iterations
1239
+ while (true) {
1240
+ // Await any deferred flush from the previous prune cycle
1241
+ if (deferredFlush) {
1242
+ await deferredFlush;
1243
+ deferredFlush = null;
1244
+ }
1245
+
1246
+ if (rollbackMode && rollbackSingleIteration) {
1247
+ if (rollbackIterationRan) {
1248
+ return rollbackOutput as T;
1249
+ }
1250
+ this.stopRollbackIfIncomplete(true);
1251
+ }
1252
+ this.checkEvicted();
1253
+
1254
+ // Create branch for this iteration
1255
+ const iterationLocation = appendLoopIteration(
1256
+ this.storage,
1257
+ location,
1258
+ config.name,
1259
+ iteration,
1260
+ );
1261
+ const branchCtx = this.createBranch(iterationLocation);
1262
+
1263
+ // Execute iteration
1264
+ const iterationResult = await config.run(branchCtx, state);
1265
+ if (iterationResult === undefined && state !== undefined) {
1266
+ throw new Error(
1267
+ `Loop "${config.name}" returned undefined for a stateful iteration. Return Loop.continue(state) or Loop.break(value).`,
1268
+ );
1269
+ }
1270
+ const result =
1271
+ iterationResult === undefined
1272
+ ? ({ continue: true, state } as LoopResult<S, T>)
1273
+ : iterationResult;
1274
+
1275
+ // Validate branch completed cleanly
1276
+ branchCtx.validateComplete();
1277
+
1278
+ if ("break" in result && result.break) {
1279
+ // Loop complete
1280
+ if (entry.kind.type === "loop") {
1281
+ entry.kind.data.output = result.value;
1282
+ entry.kind.data.state = state;
1283
+ entry.kind.data.iteration = iteration;
1284
+ }
1285
+ entry.dirty = true;
1286
+ if (metadata) {
1287
+ metadata.status = "completed";
1288
+ metadata.completedAt = Date.now();
1289
+ metadata.dirty = true;
1290
+ }
1291
+
1292
+ // Collect pruning deletions and flush
1293
+ const deletions = this.collectLoopPruning(
1294
+ location,
1295
+ iteration + 1,
1296
+ historySize,
1297
+ lastPrunedUpTo,
1298
+ );
1299
+ await this.flushStorageWithDeletions(deletions);
1300
+
1301
+ if (rollbackMode && rollbackSingleIteration) {
1302
+ rollbackOutput = result.value;
1303
+ rollbackIterationRan = true;
1304
+ continue;
1305
+ }
1306
+
1307
+ return result.value;
1308
+ }
1309
+
1310
+ // Continue with new state
1311
+ if ("continue" in result && result.continue) {
1312
+ state = result.state;
1313
+ }
1314
+ iteration++;
1315
+
1316
+ if (!rollbackMode) {
1317
+ if (entry.kind.type === "loop") {
1318
+ entry.kind.data.state = state;
1319
+ entry.kind.data.iteration = iteration;
1320
+ }
1321
+ entry.dirty = true;
1322
+ }
1323
+
1324
+ // Periodically defer the flush so the next iteration can overlap
1325
+ // with loop pruning and any pending dirty state writes.
1326
+ if (iteration % historyPruneInterval === 0) {
1327
+ const deletions = this.collectLoopPruning(
1328
+ location,
1329
+ iteration,
1330
+ historySize,
1331
+ lastPrunedUpTo,
1332
+ );
1333
+ lastPrunedUpTo = Math.max(0, iteration - historySize);
1334
+
1335
+ // Defer the flush to run in parallel with the next iteration
1336
+ deferredFlush = this.flushStorageWithDeletions(deletions);
1337
+ }
1338
+ }
1339
+ }
1340
+
1341
+ /**
1342
+ * Collect pending deletions for loop history pruning.
1343
+ *
1344
+ * Only deletes iterations in the range [fromIteration, keepFrom) where
1345
+ * keepFrom = currentIteration - historySize. This avoids re-scanning
1346
+ * already-deleted iterations.
1347
+ */
1348
+ private collectLoopPruning(
1349
+ loopLocation: Location,
1350
+ currentIteration: number,
1351
+ historySize: number,
1352
+ fromIteration: number,
1353
+ ): PendingDeletions | undefined {
1354
+ if (currentIteration <= historySize) {
1355
+ return undefined;
1356
+ }
1357
+
1358
+ const keepFrom = Math.max(0, currentIteration - historySize);
1359
+ if (fromIteration >= keepFrom) {
1360
+ return undefined;
1361
+ }
1362
+
1363
+ const loopSegment = loopLocation[loopLocation.length - 1];
1364
+ if (typeof loopSegment !== "number") {
1365
+ throw new Error("Expected loop location to end with a name index");
1366
+ }
1367
+
1368
+ const range = buildLoopIterationRange(
1369
+ loopLocation,
1370
+ loopSegment,
1371
+ fromIteration,
1372
+ keepFrom,
1373
+ );
1374
+ const metadataKeys: Uint8Array[] = [];
1375
+
1376
+ for (const [key, entry] of this.storage.history.entries) {
1377
+ if (!isLocationPrefix(loopLocation, entry.location)) {
1378
+ continue;
1379
+ }
1380
+
1381
+ const iterationSegment = entry.location[loopLocation.length];
1382
+ if (
1383
+ !iterationSegment ||
1384
+ typeof iterationSegment === "number" ||
1385
+ iterationSegment.loop !== loopSegment ||
1386
+ iterationSegment.iteration < fromIteration ||
1387
+ iterationSegment.iteration >= keepFrom
1388
+ ) {
1389
+ continue;
1390
+ }
1391
+
1392
+ metadataKeys.push(buildEntryMetadataKey(entry.id));
1393
+ this.storage.entryMetadata.delete(entry.id);
1394
+ this.storage.history.entries.delete(key);
1395
+ }
1396
+
1397
+ return {
1398
+ prefixes: [],
1399
+ keys: metadataKeys,
1400
+ ranges: [range],
1401
+ };
1402
+ }
1403
+
1404
+ /**
1405
+ * Flush storage with optional pending deletions so pruning
1406
+ * happens alongside the state write.
1407
+ */
1408
+ private async flushStorageWithDeletions(
1409
+ deletions?: PendingDeletions,
1410
+ ): Promise<void> {
1411
+ await flush(this.storage, this.driver, this.historyNotifier, deletions);
1412
+ }
1413
+
1414
+ // === Sleep ===
1415
+
1416
+ async sleep(name: string, durationMs: number): Promise<void> {
1417
+ const deadline = Date.now() + durationMs;
1418
+ return this.sleepUntil(name, deadline);
1419
+ }
1420
+
1421
+ async sleepUntil(name: string, timestampMs: number): Promise<void> {
1422
+ this.assertNotInProgress();
1423
+ this.checkEvicted();
1424
+
1425
+ this.entryInProgress = true;
1426
+ try {
1427
+ await this.executeSleep(name, timestampMs);
1428
+ } finally {
1429
+ this.entryInProgress = false;
1430
+ }
1431
+ }
1432
+
1433
+ private async executeSleep(name: string, deadline: number): Promise<void> {
1434
+ // Check for duplicate name in current execution
1435
+ this.checkDuplicateName(name);
1436
+
1437
+ const location = appendName(this.storage, this.currentLocation, name);
1438
+ const key = locationToKey(this.storage, location);
1439
+ const existing = this.storage.history.entries.get(key);
1440
+
1441
+ // Mark this entry as visited for validateComplete
1442
+ this.markVisited(key);
1443
+
1444
+ let entry: Entry;
1445
+
1446
+ if (existing) {
1447
+ if (existing.kind.type !== "sleep") {
1448
+ throw new HistoryDivergedError(
1449
+ `Expected sleep "${name}" at ${key}, found ${existing.kind.type}`,
1450
+ );
1451
+ }
1452
+
1453
+ const sleepData = existing.kind.data;
1454
+
1455
+ if (this.mode === "rollback") {
1456
+ this.stopRollbackIfIncomplete(sleepData.state === "pending");
1457
+ return;
1458
+ }
1459
+
1460
+ // Already completed or interrupted
1461
+ if (sleepData.state !== "pending") {
1462
+ return;
1463
+ }
1464
+
1465
+ // Use stored deadline
1466
+ deadline = sleepData.deadline;
1467
+ entry = existing;
1468
+ } else {
1469
+ this.stopRollbackIfIncomplete(true);
1470
+
1471
+ entry = createEntry(location, {
1472
+ type: "sleep",
1473
+ data: { deadline, state: "pending" },
1474
+ });
1475
+ setEntry(this.storage, location, entry);
1476
+ entry.dirty = true;
1477
+ await this.flushStorage();
1478
+ }
1479
+
1480
+ const now = Date.now();
1481
+ const remaining = deadline - now;
1482
+
1483
+ if (remaining <= 0) {
1484
+ // Deadline passed
1485
+ if (entry.kind.type === "sleep") {
1486
+ entry.kind.data.state = "completed";
1487
+ }
1488
+ entry.dirty = true;
1489
+ await this.flushStorage();
1490
+ return;
1491
+ }
1492
+
1493
+ // Short sleep: wait in memory
1494
+ if (remaining < this.driver.workerPollInterval) {
1495
+ await Promise.race([sleep(remaining), this.waitForEviction()]);
1496
+
1497
+ this.checkEvicted();
1498
+
1499
+ if (entry.kind.type === "sleep") {
1500
+ entry.kind.data.state = "completed";
1501
+ }
1502
+ entry.dirty = true;
1503
+ await this.flushStorage();
1504
+ return;
1505
+ }
1506
+
1507
+ // Long sleep: yield to scheduler
1508
+ throw new SleepError(deadline);
1509
+ }
1510
+
1511
+ // === Rollback Checkpoint ===
1512
+
1513
+ async rollbackCheckpoint(name: string): Promise<void> {
1514
+ this.assertNotInProgress();
1515
+ this.checkEvicted();
1516
+
1517
+ this.entryInProgress = true;
1518
+ try {
1519
+ await this.executeRollbackCheckpoint(name);
1520
+ } finally {
1521
+ this.entryInProgress = false;
1522
+ }
1523
+ }
1524
+
1525
+ private async executeRollbackCheckpoint(name: string): Promise<void> {
1526
+ this.checkDuplicateName(name);
1527
+
1528
+ const location = appendName(this.storage, this.currentLocation, name);
1529
+ const key = locationToKey(this.storage, location);
1530
+ const existing = this.storage.history.entries.get(key);
1531
+
1532
+ this.markVisited(key);
1533
+
1534
+ if (existing) {
1535
+ if (existing.kind.type !== "rollback_checkpoint") {
1536
+ throw new HistoryDivergedError(
1537
+ `Expected rollback checkpoint "${name}" at ${key}, found ${existing.kind.type}`,
1538
+ );
1539
+ }
1540
+ this.rollbackCheckpointSet = true;
1541
+ return;
1542
+ }
1543
+
1544
+ if (this.mode === "rollback") {
1545
+ throw new HistoryDivergedError(
1546
+ `Missing rollback checkpoint "${name}" at ${key}`,
1547
+ );
1548
+ }
1549
+
1550
+ const entry = createEntry(location, {
1551
+ type: "rollback_checkpoint",
1552
+ data: { name },
1553
+ });
1554
+ setEntry(this.storage, location, entry);
1555
+ entry.dirty = true;
1556
+ await this.flushStorage();
1557
+
1558
+ this.rollbackCheckpointSet = true;
1559
+ }
1560
+
1561
+ // === Queue ===
1562
+
1563
+ private async queueSend(name: string, body: unknown): Promise<void> {
1564
+ const message: Message = {
1565
+ id: crypto.randomUUID(),
1566
+ name,
1567
+ data: body,
1568
+ sentAt: Date.now(),
1569
+ };
1570
+ await this.messageDriver.addMessage(message);
1571
+ }
1572
+
1573
+ private async queueNext<T>(
1574
+ name: string,
1575
+ opts?: WorkflowQueueNextOptions,
1576
+ ): Promise<WorkflowQueueMessage<T>> {
1577
+ const messages = await this.queueNextBatch<T>(name, {
1578
+ ...(opts ?? {}),
1579
+ count: 1,
1580
+ });
1581
+ const message = messages[0];
1582
+ if (!message) {
1583
+ throw new Error(
1584
+ `queue.next("${name}") timed out before receiving a message. Use queue.nextBatch(...) for optional/time-limited reads.`,
1585
+ );
1586
+ }
1587
+ return message;
1588
+ }
1589
+
1590
+ private async queueNextBatch<T>(
1591
+ name: string,
1592
+ opts?: WorkflowQueueNextBatchOptions,
1593
+ ): Promise<Array<WorkflowQueueMessage<T>>> {
1594
+ this.assertNotInProgress();
1595
+ this.checkEvicted();
1596
+
1597
+ this.entryInProgress = true;
1598
+ try {
1599
+ return await this.executeQueueNextBatch<T>(name, opts);
1600
+ } finally {
1601
+ this.entryInProgress = false;
1602
+ }
1603
+ }
1604
+
1605
+ private async executeQueueNextBatch<T>(
1606
+ name: string,
1607
+ opts?: WorkflowQueueNextBatchOptions,
1608
+ ): Promise<Array<WorkflowQueueMessage<T>>> {
1609
+ if (this.pendingCompletableMessageIds.size > 0) {
1610
+ throw new Error(
1611
+ "Previous completable queue message is not completed. Call `message.complete(...)` before receiving the next message.",
1612
+ );
1613
+ }
1614
+
1615
+ const resolvedOpts = opts ?? {};
1616
+ const messageNames = this.normalizeQueueNames(resolvedOpts.names);
1617
+ const messageNameLabel = this.messageNamesLabel(messageNames);
1618
+ const count = Math.max(1, resolvedOpts.count ?? 1);
1619
+ const completable = resolvedOpts.completable === true;
1620
+
1621
+ this.checkDuplicateName(name);
1622
+
1623
+ const countLocation = appendName(
1624
+ this.storage,
1625
+ this.currentLocation,
1626
+ `${name}:count`,
1627
+ );
1628
+ const countKey = locationToKey(this.storage, countLocation);
1629
+ const existingCount = this.storage.history.entries.get(countKey);
1630
+ this.markVisited(countKey);
1631
+ this.stopRollbackIfMissing(existingCount);
1632
+
1633
+ let deadline: number | undefined;
1634
+ let deadlineEntry: Entry | undefined;
1635
+ if (resolvedOpts.timeout !== undefined) {
1636
+ const deadlineLocation = appendName(
1637
+ this.storage,
1638
+ this.currentLocation,
1639
+ `${name}:deadline`,
1640
+ );
1641
+ const deadlineKey = locationToKey(this.storage, deadlineLocation);
1642
+ deadlineEntry = this.storage.history.entries.get(deadlineKey);
1643
+ this.markVisited(deadlineKey);
1644
+ this.stopRollbackIfMissing(deadlineEntry);
1645
+
1646
+ if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
1647
+ deadline = deadlineEntry.kind.data.deadline;
1648
+ } else {
1649
+ deadline = Date.now() + resolvedOpts.timeout;
1650
+ const created = createEntry(deadlineLocation, {
1651
+ type: "sleep",
1652
+ data: { deadline, state: "pending" },
1653
+ });
1654
+ setEntry(this.storage, deadlineLocation, created);
1655
+ created.dirty = true;
1656
+ await this.flushStorage();
1657
+ deadlineEntry = created;
1658
+ }
1659
+ }
1660
+
1661
+ if (existingCount && existingCount.kind.type === "message") {
1662
+ const replayCount = existingCount.kind.data.data as number;
1663
+ return await this.readReplayQueueMessages<T>(
1664
+ name,
1665
+ replayCount,
1666
+ completable,
1667
+ );
1668
+ }
1669
+
1670
+ const now = Date.now();
1671
+ if (deadline !== undefined && now >= deadline) {
1672
+ if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
1673
+ deadlineEntry.kind.data.state = "completed";
1674
+ deadlineEntry.dirty = true;
1675
+ }
1676
+ await this.recordQueueCountEntry(
1677
+ countLocation,
1678
+ `${messageNameLabel}:count`,
1679
+ 0,
1680
+ );
1681
+ return [];
1682
+ }
1683
+
1684
+ const received = await this.receiveMessagesNow(
1685
+ messageNames,
1686
+ count,
1687
+ completable,
1688
+ );
1689
+ if (received.length > 0) {
1690
+ const historyMessages = received.map((message) =>
1691
+ this.toWorkflowQueueMessage<T>(message),
1692
+ );
1693
+ if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
1694
+ deadlineEntry.kind.data.state = "interrupted";
1695
+ deadlineEntry.dirty = true;
1696
+ }
1697
+ await this.recordQueueMessages(
1698
+ name,
1699
+ countLocation,
1700
+ messageNames,
1701
+ historyMessages,
1702
+ );
1703
+ const queueMessages = received.map((message, index) =>
1704
+ this.createQueueMessage<T>(message, completable, {
1705
+ historyLocation: appendName(
1706
+ this.storage,
1707
+ this.currentLocation,
1708
+ `${name}:${index}`,
1709
+ ),
1710
+ }),
1711
+ );
1712
+ return queueMessages;
1713
+ }
1714
+
1715
+ if (deadline === undefined) {
1716
+ throw new MessageWaitError(messageNames);
1717
+ }
1718
+ throw new SleepError(deadline, messageNames);
1719
+ }
1720
+
1721
+ private normalizeQueueNames(names?: readonly string[]): string[] {
1722
+ if (!names || names.length === 0) {
1723
+ return [];
1724
+ }
1725
+ const deduped: string[] = [];
1726
+ const seen = new Set<string>();
1727
+ for (const name of names) {
1728
+ if (seen.has(name)) {
1729
+ continue;
1730
+ }
1731
+ seen.add(name);
1732
+ deduped.push(name);
1733
+ }
1734
+ return deduped;
1735
+ }
1736
+
1737
+ private messageNamesLabel(messageNames: string[]): string {
1738
+ if (messageNames.length === 0) {
1739
+ return "*";
1740
+ }
1741
+ return messageNames.length === 1
1742
+ ? messageNames[0]
1743
+ : messageNames.join("|");
1744
+ }
1745
+
1746
+ private async receiveMessagesNow(
1747
+ messageNames: string[],
1748
+ count: number,
1749
+ completable: boolean,
1750
+ ): Promise<Message[]> {
1751
+ return await this.messageDriver.receiveMessages({
1752
+ names: messageNames.length > 0 ? messageNames : undefined,
1753
+ count,
1754
+ completable,
1755
+ });
1756
+ }
1757
+
1758
+ private async recordQueueMessages<T>(
1759
+ name: string,
1760
+ countLocation: Location,
1761
+ messageNames: string[],
1762
+ messages: Array<WorkflowQueueMessage<T>>,
1763
+ ): Promise<void> {
1764
+ for (let i = 0; i < messages.length; i++) {
1765
+ const messageLocation = appendName(
1766
+ this.storage,
1767
+ this.currentLocation,
1768
+ `${name}:${i}`,
1769
+ );
1770
+ const messageEntry = createEntry(messageLocation, {
1771
+ type: "message",
1772
+ data: {
1773
+ name: messages[i].name,
1774
+ data: this.toHistoryQueueMessage(messages[i]),
1775
+ },
1776
+ });
1777
+ setEntry(this.storage, messageLocation, messageEntry);
1778
+ this.markVisited(locationToKey(this.storage, messageLocation));
1779
+ }
1780
+ await this.recordQueueCountEntry(
1781
+ countLocation,
1782
+ `${this.messageNamesLabel(messageNames)}:count`,
1783
+ messages.length,
1784
+ );
1785
+ }
1786
+
1787
+ private async recordQueueCountEntry(
1788
+ countLocation: Location,
1789
+ countLabel: string,
1790
+ count: number,
1791
+ ): Promise<void> {
1792
+ const countEntry = createEntry(countLocation, {
1793
+ type: "message",
1794
+ data: {
1795
+ name: countLabel,
1796
+ data: count,
1797
+ },
1798
+ });
1799
+ setEntry(this.storage, countLocation, countEntry);
1800
+ await this.flushStorage();
1801
+ }
1802
+
1803
+ private async readReplayQueueMessages<T>(
1804
+ name: string,
1805
+ count: number,
1806
+ completable: boolean,
1807
+ ): Promise<Array<WorkflowQueueMessage<T>>> {
1808
+ const results: Array<WorkflowQueueMessage<T>> = [];
1809
+ for (let i = 0; i < count; i++) {
1810
+ const messageLocation = appendName(
1811
+ this.storage,
1812
+ this.currentLocation,
1813
+ `${name}:${i}`,
1814
+ );
1815
+ const messageKey = locationToKey(this.storage, messageLocation);
1816
+ this.markVisited(messageKey);
1817
+ const existingMessage =
1818
+ this.storage.history.entries.get(messageKey);
1819
+ if (!existingMessage || existingMessage.kind.type !== "message") {
1820
+ throw new HistoryDivergedError(
1821
+ `Expected queue message "${name}:${i}" in history`,
1822
+ );
1823
+ }
1824
+ const parsed = this.fromHistoryQueueMessage(
1825
+ existingMessage.kind.data.name,
1826
+ existingMessage.kind.data.data,
1827
+ );
1828
+ results.push(
1829
+ this.createQueueMessage<T>(parsed.message, completable, {
1830
+ historyLocation: messageLocation,
1831
+ completed: parsed.completed,
1832
+ replay: true,
1833
+ }),
1834
+ );
1835
+ }
1836
+ return results;
1837
+ }
1838
+
1839
+ private toWorkflowQueueMessage<T>(
1840
+ message: Message,
1841
+ ): WorkflowQueueMessage<T> {
1842
+ return {
1843
+ id: message.id,
1844
+ name: message.name,
1845
+ body: message.data as T,
1846
+ createdAt: message.sentAt,
1847
+ };
1848
+ }
1849
+
1850
+ private createQueueMessage<T>(
1851
+ message: Message,
1852
+ completable: boolean,
1853
+ opts?: {
1854
+ historyLocation?: Location;
1855
+ completed?: boolean;
1856
+ replay?: boolean;
1857
+ },
1858
+ ): WorkflowQueueMessage<T> {
1859
+ const queueMessage = this.toWorkflowQueueMessage<T>(message);
1860
+ if (!completable) {
1861
+ return queueMessage;
1862
+ }
1863
+
1864
+ if (opts?.replay && opts.completed) {
1865
+ return {
1866
+ ...queueMessage,
1867
+ complete: async () => {
1868
+ // No-op: this message was already completed in a prior run.
1869
+ },
1870
+ };
1871
+ }
1872
+
1873
+ const messageId = message.id;
1874
+ this.pendingCompletableMessageIds.add(messageId);
1875
+ let completed = false;
1876
+
1877
+ return {
1878
+ ...queueMessage,
1879
+ complete: async (response?: unknown) => {
1880
+ if (completed) {
1881
+ throw new Error("Queue message already completed");
1882
+ }
1883
+ completed = true;
1884
+ try {
1885
+ await this.completeMessage(message, response);
1886
+ await this.markQueueMessageCompleted(opts?.historyLocation);
1887
+ this.pendingCompletableMessageIds.delete(messageId);
1888
+ } catch (error) {
1889
+ completed = false;
1890
+ throw error;
1891
+ }
1892
+ },
1893
+ };
1894
+ }
1895
+
1896
+ private async markQueueMessageCompleted(
1897
+ historyLocation: Location | undefined,
1898
+ ): Promise<void> {
1899
+ if (!historyLocation) {
1900
+ return;
1901
+ }
1902
+ const key = locationToKey(this.storage, historyLocation);
1903
+ const entry = this.storage.history.entries.get(key);
1904
+ if (!entry || entry.kind.type !== "message") {
1905
+ return;
1906
+ }
1907
+ const parsed = this.fromHistoryQueueMessage(
1908
+ entry.kind.data.name,
1909
+ entry.kind.data.data,
1910
+ );
1911
+ entry.kind.data.data = this.toHistoryQueueMessage(
1912
+ this.toWorkflowQueueMessage(parsed.message),
1913
+ true,
1914
+ );
1915
+ entry.dirty = true;
1916
+ await this.flushStorage();
1917
+ }
1918
+
1919
+ private async completeMessage(
1920
+ message: Message,
1921
+ response?: unknown,
1922
+ ): Promise<void> {
1923
+ if (message.complete) {
1924
+ await message.complete(response);
1925
+ return;
1926
+ }
1927
+ await this.messageDriver.completeMessage(message.id, response);
1928
+ }
1929
+
1930
+ private toHistoryQueueMessage(
1931
+ message: WorkflowQueueMessage<unknown>,
1932
+ completed = false,
1933
+ ): unknown {
1934
+ return {
1935
+ [QUEUE_HISTORY_MESSAGE_MARKER]: 1,
1936
+ id: message.id,
1937
+ name: message.name,
1938
+ body: message.body,
1939
+ createdAt: message.createdAt,
1940
+ completed,
1941
+ };
1942
+ }
1943
+
1944
+ private fromHistoryQueueMessage(
1945
+ name: string,
1946
+ value: unknown,
1947
+ ): { message: Message; completed: boolean } {
1948
+ if (
1949
+ typeof value === "object" &&
1950
+ value !== null &&
1951
+ (value as Record<string, unknown>)[QUEUE_HISTORY_MESSAGE_MARKER] ===
1952
+ 1
1953
+ ) {
1954
+ const serialized = value as Record<string, unknown>;
1955
+ const id = typeof serialized.id === "string" ? serialized.id : "";
1956
+ const serializedName =
1957
+ typeof serialized.name === "string" ? serialized.name : name;
1958
+ const createdAt =
1959
+ typeof serialized.createdAt === "number"
1960
+ ? serialized.createdAt
1961
+ : 0;
1962
+ const completed =
1963
+ typeof serialized.completed === "boolean"
1964
+ ? serialized.completed
1965
+ : false;
1966
+ return {
1967
+ message: {
1968
+ id,
1969
+ name: serializedName,
1970
+ data: serialized.body,
1971
+ sentAt: createdAt,
1972
+ },
1973
+ completed,
1974
+ };
1975
+ }
1976
+ return {
1977
+ message: {
1978
+ id: "",
1979
+ name,
1980
+ data: value,
1981
+ sentAt: 0,
1982
+ },
1983
+ completed: false,
1984
+ };
1985
+ }
1986
+
1987
+ // === Join ===
1988
+
1989
+ async join<T extends Record<string, BranchConfig<unknown>>>(
1990
+ name: string,
1991
+ branches: T,
1992
+ ): Promise<{ [K in keyof T]: BranchOutput<T[K]> }> {
1993
+ this.assertNotInProgress();
1994
+ this.checkEvicted();
1995
+
1996
+ this.entryInProgress = true;
1997
+ try {
1998
+ return await this.executeJoin(name, branches);
1999
+ } finally {
2000
+ this.entryInProgress = false;
2001
+ }
2002
+ }
2003
+
2004
+ private async executeJoin<T extends Record<string, BranchConfig<unknown>>>(
2005
+ name: string,
2006
+ branches: T,
2007
+ ): Promise<{ [K in keyof T]: BranchOutput<T[K]> }> {
2008
+ // Check for duplicate name in current execution
2009
+ this.checkDuplicateName(name);
2010
+
2011
+ const location = appendName(this.storage, this.currentLocation, name);
2012
+ const key = locationToKey(this.storage, location);
2013
+ const existing = this.storage.history.entries.get(key);
2014
+
2015
+ // Mark this entry as visited for validateComplete
2016
+ this.markVisited(key);
2017
+
2018
+ this.stopRollbackIfMissing(existing);
2019
+
2020
+ let entry: Entry;
2021
+
2022
+ if (existing) {
2023
+ if (existing.kind.type !== "join") {
2024
+ throw new HistoryDivergedError(
2025
+ `Expected join "${name}" at ${key}, found ${existing.kind.type}`,
2026
+ );
2027
+ }
2028
+ entry = existing;
2029
+ } else {
2030
+ entry = createEntry(location, {
2031
+ type: "join",
2032
+ data: {
2033
+ branches: Object.fromEntries(
2034
+ Object.keys(branches).map((k) => [
2035
+ k,
2036
+ { status: "pending" as const },
2037
+ ]),
2038
+ ),
2039
+ },
2040
+ });
2041
+ setEntry(this.storage, location, entry);
2042
+ entry.dirty = true;
2043
+ // Flush immediately to persist entry before branches execute
2044
+ await this.flushStorage();
2045
+ }
2046
+
2047
+ if (entry.kind.type !== "join") {
2048
+ throw new HistoryDivergedError("Entry type mismatch");
2049
+ }
2050
+
2051
+ this.stopRollbackIfIncomplete(
2052
+ Object.values(entry.kind.data.branches).some(
2053
+ (branch) => branch.status !== "completed",
2054
+ ),
2055
+ );
2056
+
2057
+ const joinData = entry.kind.data;
2058
+ const results: Record<string, unknown> = {};
2059
+ const errors: Record<string, Error> = {};
2060
+ let schedulerYieldState: SchedulerYieldState | undefined;
2061
+ let propagatedError: Error | undefined;
2062
+
2063
+ for (const [branchName, branchStatus] of Object.entries(
2064
+ joinData.branches,
2065
+ )) {
2066
+ if (branchStatus.status === "completed") {
2067
+ results[branchName] = branchStatus.output;
2068
+ continue;
2069
+ }
2070
+
2071
+ if (branchStatus.status === "failed") {
2072
+ errors[branchName] = new Error(
2073
+ branchStatus.error ?? "branch failed",
2074
+ );
2075
+ }
2076
+ }
2077
+
2078
+ // Execute all branches in parallel
2079
+ const branchPromises = Object.entries(branches).map(
2080
+ async ([branchName, config]) => {
2081
+ const branchStatus = joinData.branches[branchName];
2082
+ if (!branchStatus) {
2083
+ throw new HistoryDivergedError(
2084
+ `Expected join branch "${branchName}" in "${name}"`,
2085
+ );
2086
+ }
2087
+
2088
+ // Already completed
2089
+ if (branchStatus.status === "completed") {
2090
+ results[branchName] = branchStatus.output;
2091
+ return;
2092
+ }
2093
+
2094
+ // Already failed
2095
+ if (branchStatus.status === "failed") {
2096
+ errors[branchName] = new Error(branchStatus.error);
2097
+ return;
2098
+ }
2099
+
2100
+ // Execute branch
2101
+ const branchLocation = appendName(
2102
+ this.storage,
2103
+ location,
2104
+ branchName,
2105
+ );
2106
+ const branchCtx = this.createBranch(branchLocation);
2107
+
2108
+ branchStatus.status = "running";
2109
+ branchStatus.error = undefined;
2110
+ entry.dirty = true;
2111
+
2112
+ try {
2113
+ const output = await config.run(branchCtx);
2114
+ branchCtx.validateComplete();
2115
+
2116
+ branchStatus.status = "completed";
2117
+ branchStatus.output = output;
2118
+ branchStatus.error = undefined;
2119
+ results[branchName] = output;
2120
+ } catch (error) {
2121
+ if (
2122
+ error instanceof SleepError ||
2123
+ error instanceof MessageWaitError ||
2124
+ error instanceof StepFailedError
2125
+ ) {
2126
+ schedulerYieldState = mergeSchedulerYield(
2127
+ schedulerYieldState,
2128
+ error,
2129
+ );
2130
+ branchStatus.status = "running";
2131
+ branchStatus.error = undefined;
2132
+ entry.dirty = true;
2133
+ return;
2134
+ }
2135
+
2136
+ if (
2137
+ error instanceof EvictedError ||
2138
+ error instanceof HistoryDivergedError ||
2139
+ error instanceof EntryInProgressError ||
2140
+ error instanceof RollbackCheckpointError ||
2141
+ error instanceof RollbackStopError
2142
+ ) {
2143
+ propagatedError = selectControlFlowError(
2144
+ propagatedError,
2145
+ error,
2146
+ );
2147
+ branchStatus.status = "running";
2148
+ branchStatus.error = undefined;
2149
+ entry.dirty = true;
2150
+ return;
2151
+ }
2152
+
2153
+ branchStatus.status = "failed";
2154
+ branchStatus.output = undefined;
2155
+ branchStatus.error = String(error);
2156
+ errors[branchName] = error as Error;
2157
+ }
2158
+
2159
+ entry.dirty = true;
2160
+ },
2161
+ );
2162
+
2163
+ // Wait for ALL branches (no short-circuit on error)
2164
+ await Promise.allSettled(branchPromises);
2165
+ await this.flushStorage();
2166
+
2167
+ if (propagatedError) {
2168
+ throw propagatedError;
2169
+ }
2170
+
2171
+ if (
2172
+ Object.values(joinData.branches).some(
2173
+ (branch) =>
2174
+ branch.status === "pending" || branch.status === "running",
2175
+ )
2176
+ ) {
2177
+ if (!schedulerYieldState) {
2178
+ throw new Error(
2179
+ `Join "${name}" has pending branches without a scheduler yield`,
2180
+ );
2181
+ }
2182
+ throw buildSchedulerYieldError(schedulerYieldState);
2183
+ }
2184
+
2185
+ // Throw if any branches failed
2186
+ if (Object.keys(errors).length > 0) {
2187
+ throw attachTryBlockFailure(new JoinError(errors), {
2188
+ source: "join",
2189
+ name,
2190
+ });
2191
+ }
2192
+
2193
+ return results as { [K in keyof T]: BranchOutput<T[K]> };
2194
+ }
2195
+
2196
+ // === Race ===
2197
+
2198
+ async race<T>(
2199
+ name: string,
2200
+ branches: Array<{
2201
+ name: string;
2202
+ run: (ctx: WorkflowContextInterface) => Promise<T>;
2203
+ }>,
2204
+ ): Promise<{ winner: string; value: T }> {
2205
+ this.assertNotInProgress();
2206
+ this.checkEvicted();
2207
+
2208
+ this.entryInProgress = true;
2209
+ try {
2210
+ return await this.executeRace(name, branches);
2211
+ } finally {
2212
+ this.entryInProgress = false;
2213
+ }
2214
+ }
2215
+
2216
+ private async executeRace<T>(
2217
+ name: string,
2218
+ branches: Array<{
2219
+ name: string;
2220
+ run: (ctx: WorkflowContextInterface) => Promise<T>;
2221
+ }>,
2222
+ ): Promise<{ winner: string; value: T }> {
2223
+ // Check for duplicate name in current execution
2224
+ this.checkDuplicateName(name);
2225
+
2226
+ const location = appendName(this.storage, this.currentLocation, name);
2227
+ const key = locationToKey(this.storage, location);
2228
+ const existing = this.storage.history.entries.get(key);
2229
+
2230
+ // Mark this entry as visited for validateComplete
2231
+ this.markVisited(key);
2232
+
2233
+ this.stopRollbackIfMissing(existing);
2234
+
2235
+ let entry: Entry;
2236
+
2237
+ if (existing) {
2238
+ if (existing.kind.type !== "race") {
2239
+ throw new HistoryDivergedError(
2240
+ `Expected race "${name}" at ${key}, found ${existing.kind.type}`,
2241
+ );
2242
+ }
2243
+ entry = existing;
2244
+
2245
+ // Check if we already have a winner
2246
+ const raceKind = existing.kind;
2247
+ if (raceKind.data.winner !== null) {
2248
+ const winnerStatus =
2249
+ raceKind.data.branches[raceKind.data.winner];
2250
+ return {
2251
+ winner: raceKind.data.winner,
2252
+ value: winnerStatus.output as T,
2253
+ };
2254
+ }
2255
+
2256
+ this.stopRollbackIfIncomplete(true);
2257
+ } else {
2258
+ entry = createEntry(location, {
2259
+ type: "race",
2260
+ data: {
2261
+ winner: null,
2262
+ branches: Object.fromEntries(
2263
+ branches.map((b) => [
2264
+ b.name,
2265
+ { status: "pending" as const },
2266
+ ]),
2267
+ ),
2268
+ },
2269
+ });
2270
+ setEntry(this.storage, location, entry);
2271
+ entry.dirty = true;
2272
+ // Flush immediately to persist entry before branches execute
2273
+ await this.flushStorage();
2274
+ }
2275
+
2276
+ if (entry.kind.type !== "race") {
2277
+ throw new HistoryDivergedError("Entry type mismatch");
2278
+ }
2279
+
2280
+ const raceData = entry.kind.data;
2281
+
2282
+ // Create abort controller for cancellation
2283
+ const raceAbortController = new AbortController();
2284
+
2285
+ // Track all branch promises to wait for cleanup
2286
+ const branchPromises: Promise<void>[] = [];
2287
+
2288
+ // Track winner info
2289
+ let winnerName: string | null = null;
2290
+ let winnerValue: T | undefined;
2291
+ let hasWinner = false;
2292
+ const errors: Record<string, Error> = {};
2293
+ const lateErrors: Array<{ name: string; error: string }> = [];
2294
+ let schedulerYieldState: SchedulerYieldState | undefined;
2295
+ let propagatedError: Error | undefined;
2296
+
2297
+ // Check for replay winners first
2298
+ for (const branch of branches) {
2299
+ const branchStatus = raceData.branches[branch.name];
2300
+ if (!branchStatus) {
2301
+ throw new HistoryDivergedError(
2302
+ `Expected race branch "${branch.name}" in "${name}"`,
2303
+ );
2304
+ }
2305
+ if (
2306
+ branchStatus.status !== "pending" &&
2307
+ branchStatus.status !== "running"
2308
+ ) {
2309
+ if (branchStatus.status === "failed") {
2310
+ errors[branch.name] = new Error(
2311
+ branchStatus.error ?? "branch failed",
2312
+ );
2313
+ }
2314
+ if (branchStatus.status === "completed" && !hasWinner) {
2315
+ hasWinner = true;
2316
+ winnerName = branch.name;
2317
+ winnerValue = branchStatus.output as T;
2318
+ }
2319
+ }
2320
+ }
2321
+
2322
+ // If we found a replay winner, return immediately
2323
+ if (hasWinner && winnerName !== null) {
2324
+ return { winner: winnerName, value: winnerValue as T };
2325
+ }
2326
+
2327
+ // Execute branches that need to run
2328
+ for (const branch of branches) {
2329
+ const branchStatus = raceData.branches[branch.name];
2330
+ if (!branchStatus) {
2331
+ throw new HistoryDivergedError(
2332
+ `Expected race branch "${branch.name}" in "${name}"`,
2333
+ );
2334
+ }
2335
+
2336
+ // Skip already completed/cancelled
2337
+ if (
2338
+ branchStatus.status !== "pending" &&
2339
+ branchStatus.status !== "running"
2340
+ ) {
2341
+ continue;
2342
+ }
2343
+
2344
+ const branchLocation = appendName(
2345
+ this.storage,
2346
+ location,
2347
+ branch.name,
2348
+ );
2349
+ const branchCtx = this.createBranch(
2350
+ branchLocation,
2351
+ raceAbortController,
2352
+ );
2353
+
2354
+ branchStatus.status = "running";
2355
+ branchStatus.error = undefined;
2356
+ entry.dirty = true;
2357
+
2358
+ const branchPromise = branch.run(branchCtx).then(
2359
+ async (output) => {
2360
+ if (hasWinner) {
2361
+ // This branch completed after a winner was determined
2362
+ // Still record the completion for observability
2363
+ branchStatus.status = "completed";
2364
+ branchStatus.output = output;
2365
+ branchStatus.error = undefined;
2366
+ entry.dirty = true;
2367
+ return;
2368
+ }
2369
+
2370
+ if (propagatedError) {
2371
+ branchStatus.status = "completed";
2372
+ branchStatus.output = output;
2373
+ branchStatus.error = undefined;
2374
+ entry.dirty = true;
2375
+ return;
2376
+ }
2377
+
2378
+ hasWinner = true;
2379
+ winnerName = branch.name;
2380
+ winnerValue = output;
2381
+
2382
+ branchCtx.validateComplete();
2383
+
2384
+ branchStatus.status = "completed";
2385
+ branchStatus.output = output;
2386
+ branchStatus.error = undefined;
2387
+ raceData.winner = branch.name;
2388
+ entry.dirty = true;
2389
+
2390
+ // Cancel other branches
2391
+ raceAbortController.abort();
2392
+ },
2393
+ (error) => {
2394
+ if (hasWinner) {
2395
+ if (
2396
+ error instanceof CancelledError ||
2397
+ error instanceof EvictedError
2398
+ ) {
2399
+ branchStatus.status = "cancelled";
2400
+ } else {
2401
+ lateErrors.push({
2402
+ name: branch.name,
2403
+ error: String(error),
2404
+ });
2405
+ }
2406
+ entry.dirty = true;
2407
+ return;
2408
+ }
2409
+
2410
+ if (
2411
+ error instanceof SleepError ||
2412
+ error instanceof MessageWaitError ||
2413
+ error instanceof StepFailedError
2414
+ ) {
2415
+ schedulerYieldState = mergeSchedulerYield(
2416
+ schedulerYieldState,
2417
+ error,
2418
+ );
2419
+ branchStatus.status = "running";
2420
+ branchStatus.error = undefined;
2421
+ entry.dirty = true;
2422
+ return;
2423
+ }
2424
+
2425
+ if (
2426
+ error instanceof EvictedError ||
2427
+ error instanceof HistoryDivergedError ||
2428
+ error instanceof EntryInProgressError ||
2429
+ error instanceof RollbackCheckpointError ||
2430
+ error instanceof RollbackStopError
2431
+ ) {
2432
+ propagatedError = selectControlFlowError(
2433
+ propagatedError,
2434
+ error,
2435
+ );
2436
+ branchStatus.status = "running";
2437
+ branchStatus.error = undefined;
2438
+ entry.dirty = true;
2439
+ return;
2440
+ }
2441
+
2442
+ if (error instanceof CancelledError) {
2443
+ branchStatus.status = "cancelled";
2444
+ } else {
2445
+ branchStatus.status = "failed";
2446
+ branchStatus.output = undefined;
2447
+ branchStatus.error = String(error);
2448
+ errors[branch.name] = error as Error;
2449
+ }
2450
+ entry.dirty = true;
2451
+ },
2452
+ );
2453
+
2454
+ branchPromises.push(branchPromise);
2455
+ }
2456
+
2457
+ // Wait for all branches to complete or be cancelled
2458
+ await Promise.allSettled(branchPromises);
2459
+
2460
+ if (propagatedError) {
2461
+ await this.flushStorage();
2462
+ throw propagatedError;
2463
+ }
2464
+
2465
+ if (
2466
+ !hasWinner &&
2467
+ Object.values(raceData.branches).some(
2468
+ (branch) =>
2469
+ branch.status === "pending" || branch.status === "running",
2470
+ )
2471
+ ) {
2472
+ await this.flushStorage();
2473
+ if (!schedulerYieldState) {
2474
+ throw new Error(
2475
+ `Race "${name}" has pending branches without a scheduler yield`,
2476
+ );
2477
+ }
2478
+ throw buildSchedulerYieldError(schedulerYieldState);
2479
+ }
2480
+
2481
+ // Clean up entries from non-winning branches
2482
+ if (hasWinner && winnerName !== null) {
2483
+ for (const branch of branches) {
2484
+ if (branch.name !== winnerName) {
2485
+ const branchLocation = appendName(
2486
+ this.storage,
2487
+ location,
2488
+ branch.name,
2489
+ );
2490
+ await deleteEntriesWithPrefix(
2491
+ this.storage,
2492
+ this.driver,
2493
+ branchLocation,
2494
+ this.historyNotifier,
2495
+ );
2496
+ }
2497
+ }
2498
+ }
2499
+
2500
+ // Flush final state
2501
+ await this.flushStorage();
2502
+
2503
+ // Log late errors if any (these occurred after a winner was determined)
2504
+ if (lateErrors.length > 0) {
2505
+ console.warn(
2506
+ `Race "${name}" had ${lateErrors.length} branch(es) fail after winner was determined:`,
2507
+ lateErrors,
2508
+ );
2509
+ }
2510
+
2511
+ // Return result or throw error
2512
+ if (hasWinner && winnerName !== null) {
2513
+ return { winner: winnerName, value: winnerValue as T };
2514
+ }
2515
+
2516
+ // All branches failed
2517
+ throw attachTryBlockFailure(
2518
+ new RaceError(
2519
+ "All branches failed",
2520
+ Object.entries(errors).map(([branchName, error]) => ({
2521
+ name: branchName,
2522
+ error: String(error),
2523
+ })),
2524
+ ),
2525
+ {
2526
+ source: "race",
2527
+ name,
2528
+ },
2529
+ );
2530
+ }
2531
+
2532
+ // === Removed ===
2533
+
2534
+ async removed(name: string, originalType: EntryKindType): Promise<void> {
2535
+ this.assertNotInProgress();
2536
+ this.checkEvicted();
2537
+
2538
+ this.entryInProgress = true;
2539
+ try {
2540
+ await this.executeRemoved(name, originalType);
2541
+ } finally {
2542
+ this.entryInProgress = false;
2543
+ }
2544
+ }
2545
+
2546
+ private async executeRemoved(
2547
+ name: string,
2548
+ originalType: EntryKindType,
2549
+ ): Promise<void> {
2550
+ // Check for duplicate name in current execution
2551
+ this.checkDuplicateName(name);
2552
+
2553
+ const location = appendName(this.storage, this.currentLocation, name);
2554
+ const key = locationToKey(this.storage, location);
2555
+ const existing = this.storage.history.entries.get(key);
2556
+
2557
+ // Mark this entry as visited for validateComplete
2558
+ this.markVisited(key);
2559
+
2560
+ this.stopRollbackIfMissing(existing);
2561
+
2562
+ if (existing) {
2563
+ // Validate the existing entry matches what we expect
2564
+ if (
2565
+ existing.kind.type !== "removed" &&
2566
+ existing.kind.type !== originalType
2567
+ ) {
2568
+ throw new HistoryDivergedError(
2569
+ `Expected ${originalType} or removed at ${key}, found ${existing.kind.type}`,
2570
+ );
2571
+ }
2572
+
2573
+ // If it's not already marked as removed, we just skip it
2574
+ return;
2575
+ }
2576
+
2577
+ // Create a removed entry placeholder
2578
+ const entry = createEntry(location, {
2579
+ type: "removed",
2580
+ data: { originalType, originalName: name },
2581
+ });
2582
+ setEntry(this.storage, location, entry);
2583
+ await this.flushStorage();
2584
+ }
2585
+ }