@rivetkit/workflow-engine 2.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +203 -0
- package/dist/schemas/v1.ts +781 -0
- package/dist/tsup/chunk-GJ66YE5W.cjs +3441 -0
- package/dist/tsup/chunk-GJ66YE5W.cjs.map +1 -0
- package/dist/tsup/chunk-JWHWQBZP.js +3441 -0
- package/dist/tsup/chunk-JWHWQBZP.js.map +1 -0
- package/dist/tsup/index.cjs +93 -0
- package/dist/tsup/index.cjs.map +1 -0
- package/dist/tsup/index.d.cts +884 -0
- package/dist/tsup/index.d.ts +884 -0
- package/dist/tsup/index.js +93 -0
- package/dist/tsup/index.js.map +1 -0
- package/dist/tsup/testing.cjs +316 -0
- package/dist/tsup/testing.cjs.map +1 -0
- package/dist/tsup/testing.d.cts +52 -0
- package/dist/tsup/testing.d.ts +52 -0
- package/dist/tsup/testing.js +316 -0
- package/dist/tsup/testing.js.map +1 -0
- package/package.json +70 -0
- package/schemas/serde.ts +609 -0
- package/schemas/v1.bare +203 -0
- package/schemas/versioned.ts +107 -0
- package/src/context.ts +1845 -0
- package/src/driver.ts +103 -0
- package/src/errors.ts +170 -0
- package/src/index.ts +907 -0
- package/src/keys.ts +277 -0
- package/src/location.ts +168 -0
- package/src/storage.ts +364 -0
- package/src/testing.ts +292 -0
- package/src/types.ts +508 -0
- package/src/utils.ts +48 -0
package/src/context.ts
ADDED
|
@@ -0,0 +1,1845 @@
|
|
|
1
|
+
import type { Logger } from "pino";
|
|
2
|
+
import type { EngineDriver } from "./driver.js";
|
|
3
|
+
import {
|
|
4
|
+
CancelledError,
|
|
5
|
+
CriticalError,
|
|
6
|
+
EntryInProgressError,
|
|
7
|
+
EvictedError,
|
|
8
|
+
HistoryDivergedError,
|
|
9
|
+
JoinError,
|
|
10
|
+
MessageWaitError,
|
|
11
|
+
RaceError,
|
|
12
|
+
RollbackCheckpointError,
|
|
13
|
+
RollbackError,
|
|
14
|
+
RollbackStopError,
|
|
15
|
+
SleepError,
|
|
16
|
+
StepExhaustedError,
|
|
17
|
+
StepFailedError,
|
|
18
|
+
} from "./errors.js";
|
|
19
|
+
import {
|
|
20
|
+
appendLoopIteration,
|
|
21
|
+
appendName,
|
|
22
|
+
emptyLocation,
|
|
23
|
+
locationToKey,
|
|
24
|
+
registerName,
|
|
25
|
+
} from "./location.js";
|
|
26
|
+
import {
|
|
27
|
+
createEntry,
|
|
28
|
+
deleteEntriesWithPrefix,
|
|
29
|
+
flush,
|
|
30
|
+
getOrCreateMetadata,
|
|
31
|
+
loadMetadata,
|
|
32
|
+
setEntry,
|
|
33
|
+
} from "./storage.js";
|
|
34
|
+
import type {
|
|
35
|
+
BranchConfig,
|
|
36
|
+
BranchOutput,
|
|
37
|
+
BranchStatus,
|
|
38
|
+
Entry,
|
|
39
|
+
EntryKindType,
|
|
40
|
+
EntryMetadata,
|
|
41
|
+
Location,
|
|
42
|
+
LoopConfig,
|
|
43
|
+
LoopResult,
|
|
44
|
+
Message,
|
|
45
|
+
RollbackContextInterface,
|
|
46
|
+
StepConfig,
|
|
47
|
+
Storage,
|
|
48
|
+
WorkflowContextInterface,
|
|
49
|
+
WorkflowQueue,
|
|
50
|
+
WorkflowQueueMessage,
|
|
51
|
+
WorkflowQueueNextOptions,
|
|
52
|
+
WorkflowMessageDriver,
|
|
53
|
+
} from "./types.js";
|
|
54
|
+
import { sleep } from "./utils.js";
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Default values for step configuration.
|
|
58
|
+
* These are exported so users can reference them when overriding.
|
|
59
|
+
*/
|
|
60
|
+
export const DEFAULT_MAX_RETRIES = 3;
|
|
61
|
+
export const DEFAULT_RETRY_BACKOFF_BASE = 100;
|
|
62
|
+
export const DEFAULT_RETRY_BACKOFF_MAX = 30000;
|
|
63
|
+
export const DEFAULT_LOOP_COMMIT_INTERVAL = 20;
|
|
64
|
+
export const DEFAULT_LOOP_HISTORY_EVERY = 20;
|
|
65
|
+
export const DEFAULT_LOOP_HISTORY_KEEP = 20;
|
|
66
|
+
export const DEFAULT_STEP_TIMEOUT = 30000; // 30 seconds
|
|
67
|
+
|
|
68
|
+
const QUEUE_HISTORY_MESSAGE_MARKER = "__rivetWorkflowQueueMessage";
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Calculate backoff delay with exponential backoff.
|
|
72
|
+
* Uses deterministic calculation (no jitter) for replay consistency.
|
|
73
|
+
*/
|
|
74
|
+
function calculateBackoff(attempts: number, base: number, max: number): number {
|
|
75
|
+
// Exponential backoff without jitter for determinism
|
|
76
|
+
return Math.min(max, base * 2 ** attempts);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Error thrown when a step times out.
|
|
81
|
+
*/
|
|
82
|
+
export class StepTimeoutError extends Error {
|
|
83
|
+
constructor(
|
|
84
|
+
public readonly stepName: string,
|
|
85
|
+
public readonly timeoutMs: number,
|
|
86
|
+
) {
|
|
87
|
+
super(`Step "${stepName}" timed out after ${timeoutMs}ms`);
|
|
88
|
+
this.name = "StepTimeoutError";
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Internal representation of a rollback handler.
|
|
94
|
+
*/
|
|
95
|
+
export interface RollbackAction<T = unknown> {
|
|
96
|
+
entryId: string;
|
|
97
|
+
name: string;
|
|
98
|
+
output: T;
|
|
99
|
+
rollback: (ctx: RollbackContextInterface, output: T) => Promise<void>;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Internal implementation of WorkflowContext.
|
|
104
|
+
*/
|
|
105
|
+
export class WorkflowContextImpl implements WorkflowContextInterface {
|
|
106
|
+
private entryInProgress = false;
|
|
107
|
+
private abortController: AbortController;
|
|
108
|
+
private currentLocation: Location;
|
|
109
|
+
private visitedKeys = new Set<string>();
|
|
110
|
+
private mode: "forward" | "rollback";
|
|
111
|
+
private rollbackActions?: RollbackAction[];
|
|
112
|
+
private rollbackCheckpointSet: boolean;
|
|
113
|
+
/** Track names used in current execution to detect duplicates */
|
|
114
|
+
private usedNamesInExecution = new Set<string>();
|
|
115
|
+
private pendingCompletableMessageIds = new Set<string>();
|
|
116
|
+
private historyNotifier?: () => void;
|
|
117
|
+
private logger?: Logger;
|
|
118
|
+
|
|
119
|
+
constructor(
|
|
120
|
+
public readonly workflowId: string,
|
|
121
|
+
private storage: Storage,
|
|
122
|
+
private driver: EngineDriver,
|
|
123
|
+
private messageDriver: WorkflowMessageDriver,
|
|
124
|
+
location: Location = emptyLocation(),
|
|
125
|
+
abortController?: AbortController,
|
|
126
|
+
mode: "forward" | "rollback" = "forward",
|
|
127
|
+
rollbackActions?: RollbackAction[],
|
|
128
|
+
rollbackCheckpointSet = false,
|
|
129
|
+
historyNotifier?: () => void,
|
|
130
|
+
logger?: Logger,
|
|
131
|
+
) {
|
|
132
|
+
this.currentLocation = location;
|
|
133
|
+
this.abortController = abortController ?? new AbortController();
|
|
134
|
+
this.mode = mode;
|
|
135
|
+
this.rollbackActions = rollbackActions;
|
|
136
|
+
this.rollbackCheckpointSet = rollbackCheckpointSet;
|
|
137
|
+
this.historyNotifier = historyNotifier;
|
|
138
|
+
this.logger = logger;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
get abortSignal(): AbortSignal {
|
|
142
|
+
return this.abortController.signal;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
get queue(): WorkflowQueue {
|
|
146
|
+
return {
|
|
147
|
+
next: async (name, opts) => await this.queueNext(name, opts),
|
|
148
|
+
send: async (name, body) => await this.queueSend(name, body),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
isEvicted(): boolean {
|
|
153
|
+
return this.abortSignal.aborted;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
private assertNotInProgress(): void {
|
|
157
|
+
if (this.entryInProgress) {
|
|
158
|
+
throw new EntryInProgressError();
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
private checkEvicted(): void {
|
|
163
|
+
if (this.abortSignal.aborted) {
|
|
164
|
+
throw new EvictedError();
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
private async flushStorage(): Promise<void> {
|
|
169
|
+
await flush(this.storage, this.driver, this.historyNotifier);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Create a new branch context for parallel/nested execution.
|
|
174
|
+
*/
|
|
175
|
+
createBranch(
|
|
176
|
+
location: Location,
|
|
177
|
+
abortController?: AbortController,
|
|
178
|
+
): WorkflowContextImpl {
|
|
179
|
+
return new WorkflowContextImpl(
|
|
180
|
+
this.workflowId,
|
|
181
|
+
this.storage,
|
|
182
|
+
this.driver,
|
|
183
|
+
this.messageDriver,
|
|
184
|
+
location,
|
|
185
|
+
abortController ?? this.abortController,
|
|
186
|
+
this.mode,
|
|
187
|
+
this.rollbackActions,
|
|
188
|
+
this.rollbackCheckpointSet,
|
|
189
|
+
this.historyNotifier,
|
|
190
|
+
this.logger,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Log a debug message using the configured logger.
|
|
196
|
+
*/
|
|
197
|
+
private log(level: "debug" | "info" | "warn" | "error", data: Record<string, unknown>): void {
|
|
198
|
+
if (!this.logger) return;
|
|
199
|
+
this.logger[level](data);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Mark a key as visited.
|
|
204
|
+
*/
|
|
205
|
+
private markVisited(key: string): void {
|
|
206
|
+
this.visitedKeys.add(key);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Check if a name has already been used at the current location in this execution.
|
|
211
|
+
* Throws HistoryDivergedError if duplicate detected.
|
|
212
|
+
*/
|
|
213
|
+
private checkDuplicateName(name: string): void {
|
|
214
|
+
const fullKey =
|
|
215
|
+
locationToKey(this.storage, this.currentLocation) + "/" + name;
|
|
216
|
+
if (this.usedNamesInExecution.has(fullKey)) {
|
|
217
|
+
throw new HistoryDivergedError(
|
|
218
|
+
`Duplicate entry name "${name}" at location "${locationToKey(this.storage, this.currentLocation)}". ` +
|
|
219
|
+
`Each step/loop/sleep/queue.next/join/race must have a unique name within its scope.`,
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
this.usedNamesInExecution.add(fullKey);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
private stopRollback(): never {
|
|
226
|
+
throw new RollbackStopError();
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
private stopRollbackIfMissing(entry: Entry | undefined): void {
|
|
230
|
+
if (this.mode === "rollback" && !entry) {
|
|
231
|
+
this.stopRollback();
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private stopRollbackIfIncomplete(condition: boolean): void {
|
|
236
|
+
if (this.mode === "rollback" && condition) {
|
|
237
|
+
this.stopRollback();
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
private registerRollbackAction<T>(
|
|
242
|
+
config: StepConfig<T>,
|
|
243
|
+
entryId: string,
|
|
244
|
+
output: T,
|
|
245
|
+
metadata: EntryMetadata,
|
|
246
|
+
): void {
|
|
247
|
+
if (!config.rollback) {
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
if (metadata.rollbackCompletedAt !== undefined) {
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
this.rollbackActions?.push({
|
|
254
|
+
entryId,
|
|
255
|
+
name: config.name,
|
|
256
|
+
output: output as unknown,
|
|
257
|
+
rollback: config.rollback as (
|
|
258
|
+
ctx: RollbackContextInterface,
|
|
259
|
+
output: unknown,
|
|
260
|
+
) => Promise<void>,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Ensure a rollback checkpoint exists before registering rollback handlers.
|
|
266
|
+
*/
|
|
267
|
+
private ensureRollbackCheckpoint<T>(config: StepConfig<T>): void {
|
|
268
|
+
if (!config.rollback) {
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
if (!this.rollbackCheckpointSet) {
|
|
272
|
+
throw new RollbackCheckpointError();
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Validate that all expected entries in the branch were visited.
|
|
278
|
+
* Throws HistoryDivergedError if there are unvisited entries.
|
|
279
|
+
*/
|
|
280
|
+
validateComplete(): void {
|
|
281
|
+
const prefix = locationToKey(this.storage, this.currentLocation);
|
|
282
|
+
|
|
283
|
+
for (const key of this.storage.history.entries.keys()) {
|
|
284
|
+
// Check if this key is under our current location prefix
|
|
285
|
+
// Handle root prefix (empty string) specially - all keys are under root
|
|
286
|
+
const isUnderPrefix =
|
|
287
|
+
prefix === ""
|
|
288
|
+
? true // Root: all keys are children
|
|
289
|
+
: key.startsWith(prefix + "/") || key === prefix;
|
|
290
|
+
|
|
291
|
+
if (isUnderPrefix) {
|
|
292
|
+
if (!this.visitedKeys.has(key)) {
|
|
293
|
+
// Entry exists in history but wasn't visited
|
|
294
|
+
// This means workflow code may have changed
|
|
295
|
+
throw new HistoryDivergedError(
|
|
296
|
+
`Entry "${key}" exists in history but was not visited. ` +
|
|
297
|
+
`Workflow code may have changed. Use ctx.removed() to handle migrations.`,
|
|
298
|
+
);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Evict the workflow.
|
|
306
|
+
*/
|
|
307
|
+
evict(): void {
|
|
308
|
+
this.abortController.abort(new EvictedError());
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Wait for eviction message.
|
|
313
|
+
*
|
|
314
|
+
* The event listener uses { once: true } to auto-remove after firing,
|
|
315
|
+
* preventing memory leaks if this method is called multiple times.
|
|
316
|
+
*/
|
|
317
|
+
waitForEviction(): Promise<never> {
|
|
318
|
+
return new Promise((_, reject) => {
|
|
319
|
+
if (this.abortSignal.aborted) {
|
|
320
|
+
reject(new EvictedError());
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
this.abortSignal.addEventListener(
|
|
324
|
+
"abort",
|
|
325
|
+
() => {
|
|
326
|
+
reject(new EvictedError());
|
|
327
|
+
},
|
|
328
|
+
{ once: true },
|
|
329
|
+
);
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// === Step ===
|
|
334
|
+
|
|
335
|
+
async step<T>(
|
|
336
|
+
nameOrConfig: string | StepConfig<T>,
|
|
337
|
+
run?: () => Promise<T>,
|
|
338
|
+
): Promise<T> {
|
|
339
|
+
this.assertNotInProgress();
|
|
340
|
+
this.checkEvicted();
|
|
341
|
+
|
|
342
|
+
const config: StepConfig<T> =
|
|
343
|
+
typeof nameOrConfig === "string"
|
|
344
|
+
? { name: nameOrConfig, run: run! }
|
|
345
|
+
: nameOrConfig;
|
|
346
|
+
|
|
347
|
+
this.entryInProgress = true;
|
|
348
|
+
try {
|
|
349
|
+
return await this.executeStep(config);
|
|
350
|
+
} finally {
|
|
351
|
+
this.entryInProgress = false;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
private async executeStep<T>(config: StepConfig<T>): Promise<T> {
|
|
356
|
+
this.ensureRollbackCheckpoint(config);
|
|
357
|
+
if (this.mode === "rollback") {
|
|
358
|
+
return await this.executeStepRollback(config);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Check for duplicate name in current execution
|
|
362
|
+
this.checkDuplicateName(config.name);
|
|
363
|
+
|
|
364
|
+
const location = appendName(
|
|
365
|
+
this.storage,
|
|
366
|
+
this.currentLocation,
|
|
367
|
+
config.name,
|
|
368
|
+
);
|
|
369
|
+
const key = locationToKey(this.storage, location);
|
|
370
|
+
const existing = this.storage.history.entries.get(key);
|
|
371
|
+
|
|
372
|
+
// Mark this entry as visited for validateComplete
|
|
373
|
+
this.markVisited(key);
|
|
374
|
+
|
|
375
|
+
if (existing) {
|
|
376
|
+
if (existing.kind.type !== "step") {
|
|
377
|
+
throw new HistoryDivergedError(
|
|
378
|
+
`Expected step "${config.name}" at ${key}, found ${existing.kind.type}`,
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
const stepData = existing.kind.data;
|
|
383
|
+
|
|
384
|
+
// Replay successful result
|
|
385
|
+
if (stepData.output !== undefined) {
|
|
386
|
+
this.log("debug", { msg: "replaying step from history", step: config.name, key });
|
|
387
|
+
return stepData.output as T;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// Check if we should retry
|
|
391
|
+
const metadata = await loadMetadata(
|
|
392
|
+
this.storage,
|
|
393
|
+
this.driver,
|
|
394
|
+
existing.id,
|
|
395
|
+
);
|
|
396
|
+
const maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
397
|
+
|
|
398
|
+
if (metadata.attempts >= maxRetries) {
|
|
399
|
+
// Prefer step history error, but fall back to metadata since
|
|
400
|
+
// driver implementations may persist metadata without the history
|
|
401
|
+
// entry error (e.g. partial writes/crashes between attempts).
|
|
402
|
+
const lastError = stepData.error ?? metadata.error;
|
|
403
|
+
throw new StepExhaustedError(config.name, lastError);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// Calculate backoff and yield to scheduler
|
|
407
|
+
// This allows the workflow to be evicted during backoff
|
|
408
|
+
const backoffDelay = calculateBackoff(
|
|
409
|
+
metadata.attempts,
|
|
410
|
+
config.retryBackoffBase ?? DEFAULT_RETRY_BACKOFF_BASE,
|
|
411
|
+
config.retryBackoffMax ?? DEFAULT_RETRY_BACKOFF_MAX,
|
|
412
|
+
);
|
|
413
|
+
const retryAt = metadata.lastAttemptAt + backoffDelay;
|
|
414
|
+
const now = Date.now();
|
|
415
|
+
|
|
416
|
+
if (now < retryAt) {
|
|
417
|
+
// Yield to scheduler - will be woken up at retryAt
|
|
418
|
+
throw new SleepError(retryAt);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// Execute the step
|
|
423
|
+
const entry =
|
|
424
|
+
existing ?? createEntry(location, { type: "step", data: {} });
|
|
425
|
+
if (!existing) {
|
|
426
|
+
// New entry - register name
|
|
427
|
+
this.log("debug", { msg: "executing new step", step: config.name, key });
|
|
428
|
+
const nameIndex = registerName(this.storage, config.name);
|
|
429
|
+
entry.location = [...location];
|
|
430
|
+
entry.location[entry.location.length - 1] = nameIndex;
|
|
431
|
+
setEntry(this.storage, location, entry);
|
|
432
|
+
} else {
|
|
433
|
+
this.log("debug", { msg: "retrying step", step: config.name, key });
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
const metadata = getOrCreateMetadata(this.storage, entry.id);
|
|
437
|
+
metadata.status = "running";
|
|
438
|
+
metadata.attempts++;
|
|
439
|
+
metadata.lastAttemptAt = Date.now();
|
|
440
|
+
metadata.dirty = true;
|
|
441
|
+
|
|
442
|
+
// Get timeout configuration
|
|
443
|
+
const timeout = config.timeout ?? DEFAULT_STEP_TIMEOUT;
|
|
444
|
+
|
|
445
|
+
try {
|
|
446
|
+
// Execute with timeout
|
|
447
|
+
const output = await this.executeWithTimeout(
|
|
448
|
+
config.run(),
|
|
449
|
+
timeout,
|
|
450
|
+
config.name,
|
|
451
|
+
);
|
|
452
|
+
|
|
453
|
+
if (entry.kind.type === "step") {
|
|
454
|
+
entry.kind.data.output = output;
|
|
455
|
+
}
|
|
456
|
+
entry.dirty = true;
|
|
457
|
+
metadata.status = "completed";
|
|
458
|
+
metadata.error = undefined;
|
|
459
|
+
metadata.completedAt = Date.now();
|
|
460
|
+
|
|
461
|
+
// Ephemeral steps don't trigger an immediate flush. This avoids the
|
|
462
|
+
// synchronous write overhead for transient operations. Note that the
|
|
463
|
+
// step's entry is still marked dirty and WILL be persisted on the
|
|
464
|
+
// next flush from a non-ephemeral operation. The purpose of ephemeral
|
|
465
|
+
// is to batch writes, not to avoid persistence entirely.
|
|
466
|
+
if (!config.ephemeral) {
|
|
467
|
+
this.log("debug", { msg: "flushing step", step: config.name, key });
|
|
468
|
+
await this.flushStorage();
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
this.log("debug", { msg: "step completed", step: config.name, key });
|
|
472
|
+
return output;
|
|
473
|
+
} catch (error) {
|
|
474
|
+
// Timeout errors are treated as critical (no retry)
|
|
475
|
+
if (error instanceof StepTimeoutError) {
|
|
476
|
+
if (entry.kind.type === "step") {
|
|
477
|
+
entry.kind.data.error = String(error);
|
|
478
|
+
}
|
|
479
|
+
entry.dirty = true;
|
|
480
|
+
metadata.status = "exhausted";
|
|
481
|
+
metadata.error = String(error);
|
|
482
|
+
await this.flushStorage();
|
|
483
|
+
throw new CriticalError(error.message);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
if (
|
|
487
|
+
error instanceof CriticalError ||
|
|
488
|
+
error instanceof RollbackError
|
|
489
|
+
) {
|
|
490
|
+
if (entry.kind.type === "step") {
|
|
491
|
+
entry.kind.data.error = String(error);
|
|
492
|
+
}
|
|
493
|
+
entry.dirty = true;
|
|
494
|
+
metadata.status = "exhausted";
|
|
495
|
+
metadata.error = String(error);
|
|
496
|
+
await this.flushStorage();
|
|
497
|
+
throw error;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
if (entry.kind.type === "step") {
|
|
501
|
+
entry.kind.data.error = String(error);
|
|
502
|
+
}
|
|
503
|
+
entry.dirty = true;
|
|
504
|
+
metadata.status = "failed";
|
|
505
|
+
metadata.error = String(error);
|
|
506
|
+
|
|
507
|
+
await this.flushStorage();
|
|
508
|
+
|
|
509
|
+
throw new StepFailedError(config.name, error, metadata.attempts);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/**
|
|
514
|
+
* Execute a promise with timeout.
|
|
515
|
+
*
|
|
516
|
+
* Note: This does NOT cancel the underlying operation. JavaScript Promises
|
|
517
|
+
* cannot be cancelled once started. When a timeout occurs:
|
|
518
|
+
* - The step is marked as failed with StepTimeoutError
|
|
519
|
+
* - The underlying async operation continues running in the background
|
|
520
|
+
* - Any side effects from the operation may still occur
|
|
521
|
+
*
|
|
522
|
+
* For cancellable operations, pass ctx.abortSignal to APIs that support AbortSignal:
|
|
523
|
+
*
|
|
524
|
+
* return fetch(url, { signal: ctx.abortSignal });
|
|
525
|
+
|
|
526
|
+
* });
|
|
527
|
+
*
|
|
528
|
+
* Or check ctx.isEvicted() periodically in long-running loops.
|
|
529
|
+
*/
|
|
530
|
+
private async executeStepRollback<T>(config: StepConfig<T>): Promise<T> {
|
|
531
|
+
this.checkDuplicateName(config.name);
|
|
532
|
+
this.ensureRollbackCheckpoint(config);
|
|
533
|
+
|
|
534
|
+
const location = appendName(
|
|
535
|
+
this.storage,
|
|
536
|
+
this.currentLocation,
|
|
537
|
+
config.name,
|
|
538
|
+
);
|
|
539
|
+
const key = locationToKey(this.storage, location);
|
|
540
|
+
const existing = this.storage.history.entries.get(key);
|
|
541
|
+
|
|
542
|
+
this.markVisited(key);
|
|
543
|
+
|
|
544
|
+
if (!existing || existing.kind.type !== "step") {
|
|
545
|
+
this.stopRollback();
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
const metadata = await loadMetadata(
|
|
549
|
+
this.storage,
|
|
550
|
+
this.driver,
|
|
551
|
+
existing.id,
|
|
552
|
+
);
|
|
553
|
+
if (metadata.status !== "completed") {
|
|
554
|
+
this.stopRollback();
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
const output = existing.kind.data.output as T;
|
|
558
|
+
this.registerRollbackAction(config, existing.id, output, metadata);
|
|
559
|
+
|
|
560
|
+
return output;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
private async executeWithTimeout<T>(
|
|
564
|
+
promise: Promise<T>,
|
|
565
|
+
timeoutMs: number,
|
|
566
|
+
stepName: string,
|
|
567
|
+
): Promise<T> {
|
|
568
|
+
if (timeoutMs <= 0) {
|
|
569
|
+
return promise;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
573
|
+
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
574
|
+
timeoutId = setTimeout(() => {
|
|
575
|
+
reject(new StepTimeoutError(stepName, timeoutMs));
|
|
576
|
+
}, timeoutMs);
|
|
577
|
+
});
|
|
578
|
+
|
|
579
|
+
try {
|
|
580
|
+
return await Promise.race([promise, timeoutPromise]);
|
|
581
|
+
} finally {
|
|
582
|
+
if (timeoutId !== undefined) {
|
|
583
|
+
clearTimeout(timeoutId);
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
// === Loop ===
|
|
589
|
+
|
|
590
|
+
async loop<S, T>(
|
|
591
|
+
nameOrConfig: string | LoopConfig<S, T>,
|
|
592
|
+
run?: (
|
|
593
|
+
ctx: WorkflowContextInterface,
|
|
594
|
+
) => Promise<LoopResult<undefined, T>>,
|
|
595
|
+
): Promise<T> {
|
|
596
|
+
this.assertNotInProgress();
|
|
597
|
+
this.checkEvicted();
|
|
598
|
+
|
|
599
|
+
const config: LoopConfig<S, T> =
|
|
600
|
+
typeof nameOrConfig === "string"
|
|
601
|
+
? { name: nameOrConfig, run: run as LoopConfig<S, T>["run"] }
|
|
602
|
+
: nameOrConfig;
|
|
603
|
+
|
|
604
|
+
this.entryInProgress = true;
|
|
605
|
+
try {
|
|
606
|
+
return await this.executeLoop(config);
|
|
607
|
+
} finally {
|
|
608
|
+
this.entryInProgress = false;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
private async executeLoop<S, T>(config: LoopConfig<S, T>): Promise<T> {
|
|
613
|
+
// Check for duplicate name in current execution
|
|
614
|
+
this.checkDuplicateName(config.name);
|
|
615
|
+
|
|
616
|
+
const location = appendName(
|
|
617
|
+
this.storage,
|
|
618
|
+
this.currentLocation,
|
|
619
|
+
config.name,
|
|
620
|
+
);
|
|
621
|
+
const key = locationToKey(this.storage, location);
|
|
622
|
+
const existing = this.storage.history.entries.get(key);
|
|
623
|
+
|
|
624
|
+
// Mark this entry as visited for validateComplete
|
|
625
|
+
this.markVisited(key);
|
|
626
|
+
|
|
627
|
+
let entry: Entry;
|
|
628
|
+
let state: S;
|
|
629
|
+
let iteration: number;
|
|
630
|
+
let rollbackSingleIteration = false;
|
|
631
|
+
let rollbackIterationRan = false;
|
|
632
|
+
let rollbackOutput: T | undefined;
|
|
633
|
+
const rollbackMode = this.mode === "rollback";
|
|
634
|
+
|
|
635
|
+
if (existing) {
|
|
636
|
+
if (existing.kind.type !== "loop") {
|
|
637
|
+
throw new HistoryDivergedError(
|
|
638
|
+
`Expected loop "${config.name}" at ${key}, found ${existing.kind.type}`,
|
|
639
|
+
);
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
const loopData = existing.kind.data;
|
|
643
|
+
|
|
644
|
+
if (rollbackMode) {
|
|
645
|
+
if (loopData.output !== undefined) {
|
|
646
|
+
return loopData.output as T;
|
|
647
|
+
}
|
|
648
|
+
rollbackSingleIteration = true;
|
|
649
|
+
rollbackIterationRan = false;
|
|
650
|
+
rollbackOutput = undefined;
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
// Loop already completed
|
|
654
|
+
if (loopData.output !== undefined) {
|
|
655
|
+
return loopData.output as T;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
// Resume from saved state
|
|
659
|
+
entry = existing;
|
|
660
|
+
state = loopData.state as S;
|
|
661
|
+
iteration = loopData.iteration;
|
|
662
|
+
if (rollbackMode) {
|
|
663
|
+
rollbackOutput = loopData.output as T | undefined;
|
|
664
|
+
rollbackIterationRan = rollbackOutput !== undefined;
|
|
665
|
+
}
|
|
666
|
+
} else {
|
|
667
|
+
this.stopRollbackIfIncomplete(true);
|
|
668
|
+
|
|
669
|
+
// New loop
|
|
670
|
+
state = config.state as S;
|
|
671
|
+
iteration = 0;
|
|
672
|
+
entry = createEntry(location, {
|
|
673
|
+
type: "loop",
|
|
674
|
+
data: { state, iteration },
|
|
675
|
+
});
|
|
676
|
+
setEntry(this.storage, location, entry);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// TODO: Add validation for commitInterval (must be > 0)
|
|
680
|
+
const commitInterval =
|
|
681
|
+
config.commitInterval ?? DEFAULT_LOOP_COMMIT_INTERVAL;
|
|
682
|
+
const historyEvery =
|
|
683
|
+
config.historyEvery ??
|
|
684
|
+
config.commitInterval ??
|
|
685
|
+
DEFAULT_LOOP_HISTORY_EVERY;
|
|
686
|
+
const historyKeep =
|
|
687
|
+
config.historyKeep ??
|
|
688
|
+
config.commitInterval ??
|
|
689
|
+
DEFAULT_LOOP_HISTORY_KEEP;
|
|
690
|
+
|
|
691
|
+
// Execute loop iterations
|
|
692
|
+
while (true) {
|
|
693
|
+
if (rollbackMode && rollbackSingleIteration) {
|
|
694
|
+
if (rollbackIterationRan) {
|
|
695
|
+
return rollbackOutput as T;
|
|
696
|
+
}
|
|
697
|
+
this.stopRollbackIfIncomplete(true);
|
|
698
|
+
}
|
|
699
|
+
this.checkEvicted();
|
|
700
|
+
|
|
701
|
+
// Create branch for this iteration
|
|
702
|
+
const iterationLocation = appendLoopIteration(
|
|
703
|
+
this.storage,
|
|
704
|
+
location,
|
|
705
|
+
config.name,
|
|
706
|
+
iteration,
|
|
707
|
+
);
|
|
708
|
+
const branchCtx = this.createBranch(iterationLocation);
|
|
709
|
+
|
|
710
|
+
// Execute iteration
|
|
711
|
+
const result = await config.run(branchCtx, state);
|
|
712
|
+
|
|
713
|
+
// Validate branch completed cleanly
|
|
714
|
+
branchCtx.validateComplete();
|
|
715
|
+
|
|
716
|
+
if ("break" in result && result.break) {
|
|
717
|
+
// Loop complete
|
|
718
|
+
if (entry.kind.type === "loop") {
|
|
719
|
+
entry.kind.data.output = result.value;
|
|
720
|
+
entry.kind.data.state = state;
|
|
721
|
+
entry.kind.data.iteration = iteration;
|
|
722
|
+
}
|
|
723
|
+
entry.dirty = true;
|
|
724
|
+
|
|
725
|
+
await this.flushStorage();
|
|
726
|
+
await this.forgetOldIterations(
|
|
727
|
+
location,
|
|
728
|
+
iteration + 1,
|
|
729
|
+
historyEvery,
|
|
730
|
+
historyKeep,
|
|
731
|
+
);
|
|
732
|
+
|
|
733
|
+
if (rollbackMode && rollbackSingleIteration) {
|
|
734
|
+
rollbackOutput = result.value;
|
|
735
|
+
rollbackIterationRan = true;
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
return result.value;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// Continue with new state
|
|
743
|
+
if ("continue" in result && result.continue) {
|
|
744
|
+
state = result.state;
|
|
745
|
+
}
|
|
746
|
+
iteration++;
|
|
747
|
+
|
|
748
|
+
// Periodic commit
|
|
749
|
+
if (iteration % commitInterval === 0) {
|
|
750
|
+
if (entry.kind.type === "loop") {
|
|
751
|
+
entry.kind.data.state = state;
|
|
752
|
+
entry.kind.data.iteration = iteration;
|
|
753
|
+
}
|
|
754
|
+
entry.dirty = true;
|
|
755
|
+
|
|
756
|
+
await this.flushStorage();
|
|
757
|
+
await this.forgetOldIterations(
|
|
758
|
+
location,
|
|
759
|
+
iteration,
|
|
760
|
+
historyEvery,
|
|
761
|
+
historyKeep,
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* Delete old loop iteration entries to save storage space.
|
|
769
|
+
*
|
|
770
|
+
* Loop locations always end with a NameIndex (number) because loops are
|
|
771
|
+
* created via appendName(). Even for nested loops, the innermost loop's
|
|
772
|
+
* location ends with its name index:
|
|
773
|
+
*
|
|
774
|
+
* ctx.loop("outer") → location: [outerIndex]
|
|
775
|
+
* iteration 0 → location: [{ loop: outerIndex, iteration: 0 }]
|
|
776
|
+
* ctx.loop("inner") → location: [{ loop: outerIndex, iteration: 0 }, innerIndex]
|
|
777
|
+
*
|
|
778
|
+
* This function removes iterations older than (currentIteration - historyKeep)
|
|
779
|
+
* every historyEvery iterations.
|
|
780
|
+
*/
|
|
781
|
+
private async forgetOldIterations(
|
|
782
|
+
loopLocation: Location,
|
|
783
|
+
currentIteration: number,
|
|
784
|
+
historyEvery: number,
|
|
785
|
+
historyKeep: number,
|
|
786
|
+
): Promise<void> {
|
|
787
|
+
if (historyEvery <= 0 || historyKeep <= 0) {
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
if (currentIteration === 0 || currentIteration % historyEvery !== 0) {
|
|
791
|
+
return;
|
|
792
|
+
}
|
|
793
|
+
const keepFrom = Math.max(0, currentIteration - historyKeep);
|
|
794
|
+
// Get the loop name index from the last segment of loopLocation.
|
|
795
|
+
// This is always a NameIndex (number) because loop entries are created
|
|
796
|
+
// via appendName(), not appendLoopIteration().
|
|
797
|
+
const loopSegment = loopLocation[loopLocation.length - 1];
|
|
798
|
+
if (typeof loopSegment !== "number") {
|
|
799
|
+
throw new Error("Expected loop location to end with a name index");
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
for (let i = 0; i < keepFrom; i++) {
|
|
803
|
+
const iterationLocation: Location = [
|
|
804
|
+
...loopLocation,
|
|
805
|
+
{ loop: loopSegment, iteration: i },
|
|
806
|
+
];
|
|
807
|
+
await deleteEntriesWithPrefix(
|
|
808
|
+
this.storage,
|
|
809
|
+
this.driver,
|
|
810
|
+
iterationLocation,
|
|
811
|
+
this.historyNotifier,
|
|
812
|
+
);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
// === Sleep ===
|
|
817
|
+
|
|
818
|
+
async sleep(name: string, durationMs: number): Promise<void> {
|
|
819
|
+
const deadline = Date.now() + durationMs;
|
|
820
|
+
return this.sleepUntil(name, deadline);
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
async sleepUntil(name: string, timestampMs: number): Promise<void> {
|
|
824
|
+
this.assertNotInProgress();
|
|
825
|
+
this.checkEvicted();
|
|
826
|
+
|
|
827
|
+
this.entryInProgress = true;
|
|
828
|
+
try {
|
|
829
|
+
await this.executeSleep(name, timestampMs);
|
|
830
|
+
} finally {
|
|
831
|
+
this.entryInProgress = false;
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
private async executeSleep(name: string, deadline: number): Promise<void> {
|
|
836
|
+
// Check for duplicate name in current execution
|
|
837
|
+
this.checkDuplicateName(name);
|
|
838
|
+
|
|
839
|
+
const location = appendName(this.storage, this.currentLocation, name);
|
|
840
|
+
const key = locationToKey(this.storage, location);
|
|
841
|
+
const existing = this.storage.history.entries.get(key);
|
|
842
|
+
|
|
843
|
+
// Mark this entry as visited for validateComplete
|
|
844
|
+
this.markVisited(key);
|
|
845
|
+
|
|
846
|
+
let entry: Entry;
|
|
847
|
+
|
|
848
|
+
if (existing) {
|
|
849
|
+
if (existing.kind.type !== "sleep") {
|
|
850
|
+
throw new HistoryDivergedError(
|
|
851
|
+
`Expected sleep "${name}" at ${key}, found ${existing.kind.type}`,
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
const sleepData = existing.kind.data;
|
|
856
|
+
|
|
857
|
+
if (this.mode === "rollback") {
|
|
858
|
+
this.stopRollbackIfIncomplete(sleepData.state === "pending");
|
|
859
|
+
return;
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
// Already completed or interrupted
|
|
863
|
+
if (sleepData.state !== "pending") {
|
|
864
|
+
return;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
// Use stored deadline
|
|
868
|
+
deadline = sleepData.deadline;
|
|
869
|
+
entry = existing;
|
|
870
|
+
} else {
|
|
871
|
+
this.stopRollbackIfIncomplete(true);
|
|
872
|
+
|
|
873
|
+
entry = createEntry(location, {
|
|
874
|
+
type: "sleep",
|
|
875
|
+
data: { deadline, state: "pending" },
|
|
876
|
+
});
|
|
877
|
+
setEntry(this.storage, location, entry);
|
|
878
|
+
entry.dirty = true;
|
|
879
|
+
await this.flushStorage();
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
const now = Date.now();
|
|
883
|
+
const remaining = deadline - now;
|
|
884
|
+
|
|
885
|
+
if (remaining <= 0) {
|
|
886
|
+
// Deadline passed
|
|
887
|
+
if (entry.kind.type === "sleep") {
|
|
888
|
+
entry.kind.data.state = "completed";
|
|
889
|
+
}
|
|
890
|
+
entry.dirty = true;
|
|
891
|
+
await this.flushStorage();
|
|
892
|
+
return;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// Short sleep: wait in memory
|
|
896
|
+
if (remaining < this.driver.workerPollInterval) {
|
|
897
|
+
await Promise.race([sleep(remaining), this.waitForEviction()]);
|
|
898
|
+
|
|
899
|
+
this.checkEvicted();
|
|
900
|
+
|
|
901
|
+
if (entry.kind.type === "sleep") {
|
|
902
|
+
entry.kind.data.state = "completed";
|
|
903
|
+
}
|
|
904
|
+
entry.dirty = true;
|
|
905
|
+
await this.flushStorage();
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
// Long sleep: yield to scheduler
|
|
910
|
+
throw new SleepError(deadline);
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
// === Rollback Checkpoint ===
|
|
914
|
+
|
|
915
|
+
async rollbackCheckpoint(name: string): Promise<void> {
|
|
916
|
+
this.assertNotInProgress();
|
|
917
|
+
this.checkEvicted();
|
|
918
|
+
|
|
919
|
+
this.entryInProgress = true;
|
|
920
|
+
try {
|
|
921
|
+
await this.executeRollbackCheckpoint(name);
|
|
922
|
+
} finally {
|
|
923
|
+
this.entryInProgress = false;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
private async executeRollbackCheckpoint(name: string): Promise<void> {
|
|
928
|
+
this.checkDuplicateName(name);
|
|
929
|
+
|
|
930
|
+
const location = appendName(this.storage, this.currentLocation, name);
|
|
931
|
+
const key = locationToKey(this.storage, location);
|
|
932
|
+
const existing = this.storage.history.entries.get(key);
|
|
933
|
+
|
|
934
|
+
this.markVisited(key);
|
|
935
|
+
|
|
936
|
+
if (existing) {
|
|
937
|
+
if (existing.kind.type !== "rollback_checkpoint") {
|
|
938
|
+
throw new HistoryDivergedError(
|
|
939
|
+
`Expected rollback checkpoint "${name}" at ${key}, found ${existing.kind.type}`,
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
this.rollbackCheckpointSet = true;
|
|
943
|
+
return;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
if (this.mode === "rollback") {
|
|
947
|
+
throw new HistoryDivergedError(
|
|
948
|
+
`Missing rollback checkpoint "${name}" at ${key}`,
|
|
949
|
+
);
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
const entry = createEntry(location, {
|
|
953
|
+
type: "rollback_checkpoint",
|
|
954
|
+
data: { name },
|
|
955
|
+
});
|
|
956
|
+
setEntry(this.storage, location, entry);
|
|
957
|
+
entry.dirty = true;
|
|
958
|
+
await this.flushStorage();
|
|
959
|
+
|
|
960
|
+
this.rollbackCheckpointSet = true;
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
// === Queue ===
|
|
964
|
+
|
|
965
|
+
private async queueSend(name: string, body: unknown): Promise<void> {
|
|
966
|
+
const message: Message = {
|
|
967
|
+
id: crypto.randomUUID(),
|
|
968
|
+
name,
|
|
969
|
+
data: body,
|
|
970
|
+
sentAt: Date.now(),
|
|
971
|
+
};
|
|
972
|
+
await this.messageDriver.addMessage(message);
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
private async queueNext<T>(
|
|
976
|
+
name: string,
|
|
977
|
+
opts?: WorkflowQueueNextOptions,
|
|
978
|
+
): Promise<Array<WorkflowQueueMessage<T>>> {
|
|
979
|
+
this.assertNotInProgress();
|
|
980
|
+
this.checkEvicted();
|
|
981
|
+
|
|
982
|
+
this.entryInProgress = true;
|
|
983
|
+
try {
|
|
984
|
+
return await this.executeQueueNext<T>(name, opts);
|
|
985
|
+
} finally {
|
|
986
|
+
this.entryInProgress = false;
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
private async executeQueueNext<T>(
|
|
991
|
+
name: string,
|
|
992
|
+
opts?: WorkflowQueueNextOptions,
|
|
993
|
+
): Promise<Array<WorkflowQueueMessage<T>>> {
|
|
994
|
+
if (this.pendingCompletableMessageIds.size > 0) {
|
|
995
|
+
throw new Error(
|
|
996
|
+
"Previous completable queue message is not completed. Call `message.complete(...)` before receiving the next message.",
|
|
997
|
+
);
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
const resolvedOpts = opts ?? {};
|
|
1001
|
+
const messageNames = this.normalizeQueueNames(resolvedOpts.names);
|
|
1002
|
+
const messageNameLabel = this.messageNamesLabel(messageNames);
|
|
1003
|
+
const count = Math.max(1, resolvedOpts.count ?? 1);
|
|
1004
|
+
const completable = resolvedOpts.completable === true;
|
|
1005
|
+
|
|
1006
|
+
this.checkDuplicateName(name);
|
|
1007
|
+
|
|
1008
|
+
const countLocation = appendName(
|
|
1009
|
+
this.storage,
|
|
1010
|
+
this.currentLocation,
|
|
1011
|
+
`${name}:count`,
|
|
1012
|
+
);
|
|
1013
|
+
const countKey = locationToKey(this.storage, countLocation);
|
|
1014
|
+
const existingCount = this.storage.history.entries.get(countKey);
|
|
1015
|
+
this.markVisited(countKey);
|
|
1016
|
+
this.stopRollbackIfMissing(existingCount);
|
|
1017
|
+
|
|
1018
|
+
let deadline: number | undefined;
|
|
1019
|
+
let deadlineEntry: Entry | undefined;
|
|
1020
|
+
if (resolvedOpts.timeout !== undefined) {
|
|
1021
|
+
const deadlineLocation = appendName(
|
|
1022
|
+
this.storage,
|
|
1023
|
+
this.currentLocation,
|
|
1024
|
+
`${name}:deadline`,
|
|
1025
|
+
);
|
|
1026
|
+
const deadlineKey = locationToKey(this.storage, deadlineLocation);
|
|
1027
|
+
deadlineEntry = this.storage.history.entries.get(deadlineKey);
|
|
1028
|
+
this.markVisited(deadlineKey);
|
|
1029
|
+
this.stopRollbackIfMissing(deadlineEntry);
|
|
1030
|
+
|
|
1031
|
+
if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
|
|
1032
|
+
deadline = deadlineEntry.kind.data.deadline;
|
|
1033
|
+
} else {
|
|
1034
|
+
deadline = Date.now() + resolvedOpts.timeout;
|
|
1035
|
+
const created = createEntry(deadlineLocation, {
|
|
1036
|
+
type: "sleep",
|
|
1037
|
+
data: { deadline, state: "pending" },
|
|
1038
|
+
});
|
|
1039
|
+
setEntry(this.storage, deadlineLocation, created);
|
|
1040
|
+
created.dirty = true;
|
|
1041
|
+
await this.flushStorage();
|
|
1042
|
+
deadlineEntry = created;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
if (existingCount && existingCount.kind.type === "message") {
|
|
1047
|
+
const replayCount = existingCount.kind.data.data as number;
|
|
1048
|
+
return await this.readReplayQueueMessages<T>(
|
|
1049
|
+
name,
|
|
1050
|
+
replayCount,
|
|
1051
|
+
completable,
|
|
1052
|
+
);
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
const now = Date.now();
|
|
1056
|
+
if (deadline !== undefined && now >= deadline) {
|
|
1057
|
+
if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
|
|
1058
|
+
deadlineEntry.kind.data.state = "completed";
|
|
1059
|
+
deadlineEntry.dirty = true;
|
|
1060
|
+
}
|
|
1061
|
+
await this.recordQueueCountEntry(
|
|
1062
|
+
countLocation,
|
|
1063
|
+
`${messageNameLabel}:count`,
|
|
1064
|
+
0,
|
|
1065
|
+
);
|
|
1066
|
+
return [];
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
const received = await this.receiveMessagesNow(
|
|
1070
|
+
messageNames,
|
|
1071
|
+
count,
|
|
1072
|
+
completable,
|
|
1073
|
+
);
|
|
1074
|
+
if (received.length > 0) {
|
|
1075
|
+
const historyMessages = received.map((message) =>
|
|
1076
|
+
this.toWorkflowQueueMessage<T>(message),
|
|
1077
|
+
);
|
|
1078
|
+
if (deadlineEntry && deadlineEntry.kind.type === "sleep") {
|
|
1079
|
+
deadlineEntry.kind.data.state = "interrupted";
|
|
1080
|
+
deadlineEntry.dirty = true;
|
|
1081
|
+
}
|
|
1082
|
+
await this.recordQueueMessages(
|
|
1083
|
+
name,
|
|
1084
|
+
countLocation,
|
|
1085
|
+
messageNames,
|
|
1086
|
+
historyMessages,
|
|
1087
|
+
);
|
|
1088
|
+
const queueMessages = received.map((message, index) =>
|
|
1089
|
+
this.createQueueMessage<T>(message, completable, {
|
|
1090
|
+
historyLocation: appendName(
|
|
1091
|
+
this.storage,
|
|
1092
|
+
this.currentLocation,
|
|
1093
|
+
`${name}:${index}`,
|
|
1094
|
+
),
|
|
1095
|
+
}),
|
|
1096
|
+
);
|
|
1097
|
+
return queueMessages;
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
if (deadline === undefined) {
|
|
1101
|
+
throw new MessageWaitError(messageNames);
|
|
1102
|
+
}
|
|
1103
|
+
throw new SleepError(deadline, messageNames);
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
private normalizeQueueNames(names?: readonly string[]): string[] {
|
|
1107
|
+
if (!names || names.length === 0) {
|
|
1108
|
+
return [];
|
|
1109
|
+
}
|
|
1110
|
+
const deduped: string[] = [];
|
|
1111
|
+
const seen = new Set<string>();
|
|
1112
|
+
for (const name of names) {
|
|
1113
|
+
if (seen.has(name)) {
|
|
1114
|
+
continue;
|
|
1115
|
+
}
|
|
1116
|
+
seen.add(name);
|
|
1117
|
+
deduped.push(name);
|
|
1118
|
+
}
|
|
1119
|
+
return deduped;
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
private messageNamesLabel(messageNames: string[]): string {
|
|
1123
|
+
if (messageNames.length === 0) {
|
|
1124
|
+
return "*";
|
|
1125
|
+
}
|
|
1126
|
+
return messageNames.length === 1
|
|
1127
|
+
? messageNames[0]
|
|
1128
|
+
: messageNames.join("|");
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
private async receiveMessagesNow(
|
|
1132
|
+
messageNames: string[],
|
|
1133
|
+
count: number,
|
|
1134
|
+
completable: boolean,
|
|
1135
|
+
): Promise<Message[]> {
|
|
1136
|
+
return await this.messageDriver.receiveMessages({
|
|
1137
|
+
names: messageNames.length > 0 ? messageNames : undefined,
|
|
1138
|
+
count,
|
|
1139
|
+
completable,
|
|
1140
|
+
});
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
private async recordQueueMessages<T>(
|
|
1144
|
+
name: string,
|
|
1145
|
+
countLocation: Location,
|
|
1146
|
+
messageNames: string[],
|
|
1147
|
+
messages: Array<WorkflowQueueMessage<T>>,
|
|
1148
|
+
): Promise<void> {
|
|
1149
|
+
for (let i = 0; i < messages.length; i++) {
|
|
1150
|
+
const messageLocation = appendName(
|
|
1151
|
+
this.storage,
|
|
1152
|
+
this.currentLocation,
|
|
1153
|
+
`${name}:${i}`,
|
|
1154
|
+
);
|
|
1155
|
+
const messageEntry = createEntry(messageLocation, {
|
|
1156
|
+
type: "message",
|
|
1157
|
+
data: {
|
|
1158
|
+
name: messages[i].name,
|
|
1159
|
+
data: this.toHistoryQueueMessage(messages[i]),
|
|
1160
|
+
},
|
|
1161
|
+
});
|
|
1162
|
+
setEntry(this.storage, messageLocation, messageEntry);
|
|
1163
|
+
this.markVisited(locationToKey(this.storage, messageLocation));
|
|
1164
|
+
}
|
|
1165
|
+
await this.recordQueueCountEntry(
|
|
1166
|
+
countLocation,
|
|
1167
|
+
`${this.messageNamesLabel(messageNames)}:count`,
|
|
1168
|
+
messages.length,
|
|
1169
|
+
);
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
private async recordQueueCountEntry(
|
|
1173
|
+
countLocation: Location,
|
|
1174
|
+
countLabel: string,
|
|
1175
|
+
count: number,
|
|
1176
|
+
): Promise<void> {
|
|
1177
|
+
const countEntry = createEntry(countLocation, {
|
|
1178
|
+
type: "message",
|
|
1179
|
+
data: {
|
|
1180
|
+
name: countLabel,
|
|
1181
|
+
data: count,
|
|
1182
|
+
},
|
|
1183
|
+
});
|
|
1184
|
+
setEntry(this.storage, countLocation, countEntry);
|
|
1185
|
+
await this.flushStorage();
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
private async readReplayQueueMessages<T>(
|
|
1189
|
+
name: string,
|
|
1190
|
+
count: number,
|
|
1191
|
+
completable: boolean,
|
|
1192
|
+
): Promise<Array<WorkflowQueueMessage<T>>> {
|
|
1193
|
+
const results: Array<WorkflowQueueMessage<T>> = [];
|
|
1194
|
+
for (let i = 0; i < count; i++) {
|
|
1195
|
+
const messageLocation = appendName(
|
|
1196
|
+
this.storage,
|
|
1197
|
+
this.currentLocation,
|
|
1198
|
+
`${name}:${i}`,
|
|
1199
|
+
);
|
|
1200
|
+
const messageKey = locationToKey(this.storage, messageLocation);
|
|
1201
|
+
this.markVisited(messageKey);
|
|
1202
|
+
const existingMessage = this.storage.history.entries.get(messageKey);
|
|
1203
|
+
if (!existingMessage || existingMessage.kind.type !== "message") {
|
|
1204
|
+
throw new HistoryDivergedError(
|
|
1205
|
+
`Expected queue message "${name}:${i}" in history`,
|
|
1206
|
+
);
|
|
1207
|
+
}
|
|
1208
|
+
const parsed = this.fromHistoryQueueMessage(
|
|
1209
|
+
existingMessage.kind.data.name,
|
|
1210
|
+
existingMessage.kind.data.data,
|
|
1211
|
+
);
|
|
1212
|
+
results.push(
|
|
1213
|
+
this.createQueueMessage<T>(parsed.message, completable, {
|
|
1214
|
+
historyLocation: messageLocation,
|
|
1215
|
+
completed: parsed.completed,
|
|
1216
|
+
replay: true,
|
|
1217
|
+
}),
|
|
1218
|
+
);
|
|
1219
|
+
}
|
|
1220
|
+
return results;
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
private toWorkflowQueueMessage<T>(message: Message): WorkflowQueueMessage<T> {
|
|
1224
|
+
return {
|
|
1225
|
+
id: message.id,
|
|
1226
|
+
name: message.name,
|
|
1227
|
+
body: message.data as T,
|
|
1228
|
+
createdAt: message.sentAt,
|
|
1229
|
+
};
|
|
1230
|
+
}
|
|
1231
|
+
|
|
1232
|
+
private createQueueMessage<T>(
|
|
1233
|
+
message: Message,
|
|
1234
|
+
completable: boolean,
|
|
1235
|
+
opts?: {
|
|
1236
|
+
historyLocation?: Location;
|
|
1237
|
+
completed?: boolean;
|
|
1238
|
+
replay?: boolean;
|
|
1239
|
+
},
|
|
1240
|
+
): WorkflowQueueMessage<T> {
|
|
1241
|
+
const queueMessage = this.toWorkflowQueueMessage<T>(message);
|
|
1242
|
+
if (!completable) {
|
|
1243
|
+
return queueMessage;
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
if (opts?.replay && opts.completed) {
|
|
1247
|
+
return {
|
|
1248
|
+
...queueMessage,
|
|
1249
|
+
complete: async () => {
|
|
1250
|
+
// No-op: this message was already completed in a prior run.
|
|
1251
|
+
},
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
const messageId = message.id;
|
|
1256
|
+
this.pendingCompletableMessageIds.add(messageId);
|
|
1257
|
+
let completed = false;
|
|
1258
|
+
|
|
1259
|
+
return {
|
|
1260
|
+
...queueMessage,
|
|
1261
|
+
complete: async (response?: unknown) => {
|
|
1262
|
+
if (completed) {
|
|
1263
|
+
throw new Error("Queue message already completed");
|
|
1264
|
+
}
|
|
1265
|
+
completed = true;
|
|
1266
|
+
try {
|
|
1267
|
+
await this.completeMessage(message, response);
|
|
1268
|
+
await this.markQueueMessageCompleted(opts?.historyLocation);
|
|
1269
|
+
this.pendingCompletableMessageIds.delete(messageId);
|
|
1270
|
+
} catch (error) {
|
|
1271
|
+
completed = false;
|
|
1272
|
+
throw error;
|
|
1273
|
+
}
|
|
1274
|
+
},
|
|
1275
|
+
};
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
private async markQueueMessageCompleted(
|
|
1279
|
+
historyLocation: Location | undefined,
|
|
1280
|
+
): Promise<void> {
|
|
1281
|
+
if (!historyLocation) {
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
const key = locationToKey(this.storage, historyLocation);
|
|
1285
|
+
const entry = this.storage.history.entries.get(key);
|
|
1286
|
+
if (!entry || entry.kind.type !== "message") {
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1289
|
+
const parsed = this.fromHistoryQueueMessage(
|
|
1290
|
+
entry.kind.data.name,
|
|
1291
|
+
entry.kind.data.data,
|
|
1292
|
+
);
|
|
1293
|
+
entry.kind.data.data = this.toHistoryQueueMessage(
|
|
1294
|
+
this.toWorkflowQueueMessage(parsed.message),
|
|
1295
|
+
true,
|
|
1296
|
+
);
|
|
1297
|
+
entry.dirty = true;
|
|
1298
|
+
await this.flushStorage();
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
private async completeMessage(
|
|
1302
|
+
message: Message,
|
|
1303
|
+
response?: unknown,
|
|
1304
|
+
): Promise<void> {
|
|
1305
|
+
if (message.complete) {
|
|
1306
|
+
await message.complete(response);
|
|
1307
|
+
return;
|
|
1308
|
+
}
|
|
1309
|
+
await this.messageDriver.completeMessage(message.id, response);
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
private toHistoryQueueMessage(
|
|
1313
|
+
message: WorkflowQueueMessage<unknown>,
|
|
1314
|
+
completed = false,
|
|
1315
|
+
): unknown {
|
|
1316
|
+
return {
|
|
1317
|
+
[QUEUE_HISTORY_MESSAGE_MARKER]: 1,
|
|
1318
|
+
id: message.id,
|
|
1319
|
+
name: message.name,
|
|
1320
|
+
body: message.body,
|
|
1321
|
+
createdAt: message.createdAt,
|
|
1322
|
+
completed,
|
|
1323
|
+
};
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
private fromHistoryQueueMessage(
|
|
1327
|
+
name: string,
|
|
1328
|
+
value: unknown,
|
|
1329
|
+
): { message: Message; completed: boolean } {
|
|
1330
|
+
if (
|
|
1331
|
+
typeof value === "object" &&
|
|
1332
|
+
value !== null &&
|
|
1333
|
+
(value as Record<string, unknown>)[QUEUE_HISTORY_MESSAGE_MARKER] === 1
|
|
1334
|
+
) {
|
|
1335
|
+
const serialized = value as Record<string, unknown>;
|
|
1336
|
+
const id =
|
|
1337
|
+
typeof serialized.id === "string" ? serialized.id : "";
|
|
1338
|
+
const serializedName =
|
|
1339
|
+
typeof serialized.name === "string" ? serialized.name : name;
|
|
1340
|
+
const createdAt =
|
|
1341
|
+
typeof serialized.createdAt === "number" ? serialized.createdAt : 0;
|
|
1342
|
+
const completed =
|
|
1343
|
+
typeof serialized.completed === "boolean"
|
|
1344
|
+
? serialized.completed
|
|
1345
|
+
: false;
|
|
1346
|
+
return {
|
|
1347
|
+
message: {
|
|
1348
|
+
id,
|
|
1349
|
+
name: serializedName,
|
|
1350
|
+
data: serialized.body,
|
|
1351
|
+
sentAt: createdAt,
|
|
1352
|
+
},
|
|
1353
|
+
completed,
|
|
1354
|
+
};
|
|
1355
|
+
}
|
|
1356
|
+
return {
|
|
1357
|
+
message: {
|
|
1358
|
+
id: "",
|
|
1359
|
+
name,
|
|
1360
|
+
data: value,
|
|
1361
|
+
sentAt: 0,
|
|
1362
|
+
},
|
|
1363
|
+
completed: false,
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
// === Join ===
|
|
1368
|
+
|
|
1369
|
+
async join<T extends Record<string, BranchConfig<unknown>>>(
|
|
1370
|
+
name: string,
|
|
1371
|
+
branches: T,
|
|
1372
|
+
): Promise<{ [K in keyof T]: BranchOutput<T[K]> }> {
|
|
1373
|
+
this.assertNotInProgress();
|
|
1374
|
+
this.checkEvicted();
|
|
1375
|
+
|
|
1376
|
+
this.entryInProgress = true;
|
|
1377
|
+
try {
|
|
1378
|
+
return await this.executeJoin(name, branches);
|
|
1379
|
+
} finally {
|
|
1380
|
+
this.entryInProgress = false;
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
private async executeJoin<T extends Record<string, BranchConfig<unknown>>>(
|
|
1385
|
+
name: string,
|
|
1386
|
+
branches: T,
|
|
1387
|
+
): Promise<{ [K in keyof T]: BranchOutput<T[K]> }> {
|
|
1388
|
+
// Check for duplicate name in current execution
|
|
1389
|
+
this.checkDuplicateName(name);
|
|
1390
|
+
|
|
1391
|
+
const location = appendName(this.storage, this.currentLocation, name);
|
|
1392
|
+
const key = locationToKey(this.storage, location);
|
|
1393
|
+
const existing = this.storage.history.entries.get(key);
|
|
1394
|
+
|
|
1395
|
+
// Mark this entry as visited for validateComplete
|
|
1396
|
+
this.markVisited(key);
|
|
1397
|
+
|
|
1398
|
+
this.stopRollbackIfMissing(existing);
|
|
1399
|
+
|
|
1400
|
+
let entry: Entry;
|
|
1401
|
+
|
|
1402
|
+
if (existing) {
|
|
1403
|
+
if (existing.kind.type !== "join") {
|
|
1404
|
+
throw new HistoryDivergedError(
|
|
1405
|
+
`Expected join "${name}" at ${key}, found ${existing.kind.type}`,
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
entry = existing;
|
|
1409
|
+
} else {
|
|
1410
|
+
entry = createEntry(location, {
|
|
1411
|
+
type: "join",
|
|
1412
|
+
data: {
|
|
1413
|
+
branches: Object.fromEntries(
|
|
1414
|
+
Object.keys(branches).map((k) => [
|
|
1415
|
+
k,
|
|
1416
|
+
{ status: "pending" as const },
|
|
1417
|
+
]),
|
|
1418
|
+
),
|
|
1419
|
+
},
|
|
1420
|
+
});
|
|
1421
|
+
setEntry(this.storage, location, entry);
|
|
1422
|
+
entry.dirty = true;
|
|
1423
|
+
// Flush immediately to persist entry before branches execute
|
|
1424
|
+
await this.flushStorage();
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
if (entry.kind.type !== "join") {
|
|
1428
|
+
throw new HistoryDivergedError("Entry type mismatch");
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
this.stopRollbackIfIncomplete(
|
|
1432
|
+
Object.values(entry.kind.data.branches).some(
|
|
1433
|
+
(branch) => branch.status !== "completed",
|
|
1434
|
+
),
|
|
1435
|
+
);
|
|
1436
|
+
|
|
1437
|
+
const joinData = entry.kind.data;
|
|
1438
|
+
const results: Record<string, unknown> = {};
|
|
1439
|
+
const errors: Record<string, Error> = {};
|
|
1440
|
+
|
|
1441
|
+
// Execute all branches in parallel
|
|
1442
|
+
const branchPromises = Object.entries(branches).map(
|
|
1443
|
+
async ([branchName, config]) => {
|
|
1444
|
+
const branchStatus = joinData.branches[branchName];
|
|
1445
|
+
|
|
1446
|
+
// Already completed
|
|
1447
|
+
if (branchStatus.status === "completed") {
|
|
1448
|
+
results[branchName] = branchStatus.output;
|
|
1449
|
+
return;
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
// Already failed
|
|
1453
|
+
if (branchStatus.status === "failed") {
|
|
1454
|
+
errors[branchName] = new Error(branchStatus.error);
|
|
1455
|
+
return;
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
// Execute branch
|
|
1459
|
+
const branchLocation = appendName(
|
|
1460
|
+
this.storage,
|
|
1461
|
+
location,
|
|
1462
|
+
branchName,
|
|
1463
|
+
);
|
|
1464
|
+
const branchCtx = this.createBranch(branchLocation);
|
|
1465
|
+
|
|
1466
|
+
branchStatus.status = "running";
|
|
1467
|
+
entry.dirty = true;
|
|
1468
|
+
|
|
1469
|
+
try {
|
|
1470
|
+
const output = await config.run(branchCtx);
|
|
1471
|
+
branchCtx.validateComplete();
|
|
1472
|
+
|
|
1473
|
+
branchStatus.status = "completed";
|
|
1474
|
+
branchStatus.output = output;
|
|
1475
|
+
results[branchName] = output;
|
|
1476
|
+
} catch (error) {
|
|
1477
|
+
branchStatus.status = "failed";
|
|
1478
|
+
branchStatus.error = String(error);
|
|
1479
|
+
errors[branchName] = error as Error;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
entry.dirty = true;
|
|
1483
|
+
},
|
|
1484
|
+
);
|
|
1485
|
+
|
|
1486
|
+
// Wait for ALL branches (no short-circuit on error)
|
|
1487
|
+
await Promise.allSettled(branchPromises);
|
|
1488
|
+
await this.flushStorage();
|
|
1489
|
+
|
|
1490
|
+
// Throw if any branches failed
|
|
1491
|
+
if (Object.keys(errors).length > 0) {
|
|
1492
|
+
throw new JoinError(errors);
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
return results as { [K in keyof T]: BranchOutput<T[K]> };
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
// === Race ===
|
|
1499
|
+
|
|
1500
|
+
async race<T>(
|
|
1501
|
+
name: string,
|
|
1502
|
+
branches: Array<{
|
|
1503
|
+
name: string;
|
|
1504
|
+
run: (ctx: WorkflowContextInterface) => Promise<T>;
|
|
1505
|
+
}>,
|
|
1506
|
+
): Promise<{ winner: string; value: T }> {
|
|
1507
|
+
this.assertNotInProgress();
|
|
1508
|
+
this.checkEvicted();
|
|
1509
|
+
|
|
1510
|
+
this.entryInProgress = true;
|
|
1511
|
+
try {
|
|
1512
|
+
return await this.executeRace(name, branches);
|
|
1513
|
+
} finally {
|
|
1514
|
+
this.entryInProgress = false;
|
|
1515
|
+
}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
private async executeRace<T>(
|
|
1519
|
+
name: string,
|
|
1520
|
+
branches: Array<{
|
|
1521
|
+
name: string;
|
|
1522
|
+
run: (ctx: WorkflowContextInterface) => Promise<T>;
|
|
1523
|
+
}>,
|
|
1524
|
+
): Promise<{ winner: string; value: T }> {
|
|
1525
|
+
// Check for duplicate name in current execution
|
|
1526
|
+
this.checkDuplicateName(name);
|
|
1527
|
+
|
|
1528
|
+
const location = appendName(this.storage, this.currentLocation, name);
|
|
1529
|
+
const key = locationToKey(this.storage, location);
|
|
1530
|
+
const existing = this.storage.history.entries.get(key);
|
|
1531
|
+
|
|
1532
|
+
// Mark this entry as visited for validateComplete
|
|
1533
|
+
this.markVisited(key);
|
|
1534
|
+
|
|
1535
|
+
this.stopRollbackIfMissing(existing);
|
|
1536
|
+
|
|
1537
|
+
let entry: Entry;
|
|
1538
|
+
|
|
1539
|
+
if (existing) {
|
|
1540
|
+
if (existing.kind.type !== "race") {
|
|
1541
|
+
throw new HistoryDivergedError(
|
|
1542
|
+
`Expected race "${name}" at ${key}, found ${existing.kind.type}`,
|
|
1543
|
+
);
|
|
1544
|
+
}
|
|
1545
|
+
entry = existing;
|
|
1546
|
+
|
|
1547
|
+
// Check if we already have a winner
|
|
1548
|
+
const raceKind = existing.kind;
|
|
1549
|
+
if (raceKind.data.winner !== null) {
|
|
1550
|
+
const winnerStatus =
|
|
1551
|
+
raceKind.data.branches[raceKind.data.winner];
|
|
1552
|
+
return {
|
|
1553
|
+
winner: raceKind.data.winner,
|
|
1554
|
+
value: winnerStatus.output as T,
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1557
|
+
|
|
1558
|
+
this.stopRollbackIfIncomplete(true);
|
|
1559
|
+
} else {
|
|
1560
|
+
entry = createEntry(location, {
|
|
1561
|
+
type: "race",
|
|
1562
|
+
data: {
|
|
1563
|
+
winner: null,
|
|
1564
|
+
branches: Object.fromEntries(
|
|
1565
|
+
branches.map((b) => [
|
|
1566
|
+
b.name,
|
|
1567
|
+
{ status: "pending" as const },
|
|
1568
|
+
]),
|
|
1569
|
+
),
|
|
1570
|
+
},
|
|
1571
|
+
});
|
|
1572
|
+
setEntry(this.storage, location, entry);
|
|
1573
|
+
entry.dirty = true;
|
|
1574
|
+
// Flush immediately to persist entry before branches execute
|
|
1575
|
+
await this.flushStorage();
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
if (entry.kind.type !== "race") {
|
|
1579
|
+
throw new HistoryDivergedError("Entry type mismatch");
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
const raceData = entry.kind.data;
|
|
1583
|
+
|
|
1584
|
+
// Create abort controller for cancellation
|
|
1585
|
+
const raceAbortController = new AbortController();
|
|
1586
|
+
|
|
1587
|
+
// Track all branch promises to wait for cleanup
|
|
1588
|
+
const branchPromises: Promise<void>[] = [];
|
|
1589
|
+
|
|
1590
|
+
// Track winner info
|
|
1591
|
+
let winnerName: string | null = null;
|
|
1592
|
+
let winnerValue: T | null = null;
|
|
1593
|
+
let settled = false;
|
|
1594
|
+
let pendingCount = branches.length;
|
|
1595
|
+
const errors: Record<string, Error> = {};
|
|
1596
|
+
const lateErrors: Array<{ name: string; error: string }> = [];
|
|
1597
|
+
// Track scheduler yield errors - we need to propagate these after allSettled
|
|
1598
|
+
let yieldError: SleepError | MessageWaitError | null = null;
|
|
1599
|
+
|
|
1600
|
+
// Check for replay winners first
|
|
1601
|
+
for (const branch of branches) {
|
|
1602
|
+
const branchStatus = raceData.branches[branch.name];
|
|
1603
|
+
if (
|
|
1604
|
+
branchStatus.status !== "pending" &&
|
|
1605
|
+
branchStatus.status !== "running"
|
|
1606
|
+
) {
|
|
1607
|
+
pendingCount--;
|
|
1608
|
+
if (branchStatus.status === "completed" && !settled) {
|
|
1609
|
+
settled = true;
|
|
1610
|
+
winnerName = branch.name;
|
|
1611
|
+
winnerValue = branchStatus.output as T;
|
|
1612
|
+
}
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
// If we found a replay winner, return immediately
|
|
1617
|
+
if (settled && winnerName !== null && winnerValue !== null) {
|
|
1618
|
+
return { winner: winnerName, value: winnerValue };
|
|
1619
|
+
}
|
|
1620
|
+
|
|
1621
|
+
// Execute branches that need to run
|
|
1622
|
+
for (const branch of branches) {
|
|
1623
|
+
const branchStatus = raceData.branches[branch.name];
|
|
1624
|
+
|
|
1625
|
+
// Skip already completed/cancelled
|
|
1626
|
+
if (
|
|
1627
|
+
branchStatus.status !== "pending" &&
|
|
1628
|
+
branchStatus.status !== "running"
|
|
1629
|
+
) {
|
|
1630
|
+
continue;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
const branchLocation = appendName(
|
|
1634
|
+
this.storage,
|
|
1635
|
+
location,
|
|
1636
|
+
branch.name,
|
|
1637
|
+
);
|
|
1638
|
+
const branchCtx = this.createBranch(
|
|
1639
|
+
branchLocation,
|
|
1640
|
+
raceAbortController,
|
|
1641
|
+
);
|
|
1642
|
+
|
|
1643
|
+
branchStatus.status = "running";
|
|
1644
|
+
entry.dirty = true;
|
|
1645
|
+
|
|
1646
|
+
const branchPromise = branch.run(branchCtx).then(
|
|
1647
|
+
async (output) => {
|
|
1648
|
+
if (settled) {
|
|
1649
|
+
// This branch completed after a winner was determined
|
|
1650
|
+
// Still record the completion for observability
|
|
1651
|
+
branchStatus.status = "completed";
|
|
1652
|
+
branchStatus.output = output;
|
|
1653
|
+
entry.dirty = true;
|
|
1654
|
+
return;
|
|
1655
|
+
}
|
|
1656
|
+
settled = true;
|
|
1657
|
+
winnerName = branch.name;
|
|
1658
|
+
winnerValue = output;
|
|
1659
|
+
|
|
1660
|
+
branchCtx.validateComplete();
|
|
1661
|
+
|
|
1662
|
+
branchStatus.status = "completed";
|
|
1663
|
+
branchStatus.output = output;
|
|
1664
|
+
raceData.winner = branch.name;
|
|
1665
|
+
entry.dirty = true;
|
|
1666
|
+
|
|
1667
|
+
// Cancel other branches
|
|
1668
|
+
raceAbortController.abort();
|
|
1669
|
+
},
|
|
1670
|
+
(error) => {
|
|
1671
|
+
pendingCount--;
|
|
1672
|
+
|
|
1673
|
+
// Track sleep/message errors - they need to bubble up to the scheduler
|
|
1674
|
+
// We'll re-throw after allSettled to allow cleanup
|
|
1675
|
+
if (error instanceof SleepError) {
|
|
1676
|
+
// Track the earliest deadline
|
|
1677
|
+
if (
|
|
1678
|
+
!yieldError ||
|
|
1679
|
+
!(yieldError instanceof SleepError) ||
|
|
1680
|
+
error.deadline < yieldError.deadline
|
|
1681
|
+
) {
|
|
1682
|
+
yieldError = error;
|
|
1683
|
+
}
|
|
1684
|
+
branchStatus.status = "running"; // Keep as running since we'll resume
|
|
1685
|
+
entry.dirty = true;
|
|
1686
|
+
return;
|
|
1687
|
+
}
|
|
1688
|
+
if (error instanceof MessageWaitError) {
|
|
1689
|
+
// Track message wait errors, prefer sleep errors with deadlines
|
|
1690
|
+
if (!yieldError || !(yieldError instanceof SleepError)) {
|
|
1691
|
+
if (!yieldError) {
|
|
1692
|
+
yieldError = error;
|
|
1693
|
+
} else if (yieldError instanceof MessageWaitError) {
|
|
1694
|
+
// Merge message names
|
|
1695
|
+
yieldError = new MessageWaitError([
|
|
1696
|
+
...yieldError.messageNames,
|
|
1697
|
+
...error.messageNames,
|
|
1698
|
+
]);
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
branchStatus.status = "running"; // Keep as running since we'll resume
|
|
1702
|
+
entry.dirty = true;
|
|
1703
|
+
return;
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
if (
|
|
1707
|
+
error instanceof CancelledError ||
|
|
1708
|
+
error instanceof EvictedError
|
|
1709
|
+
) {
|
|
1710
|
+
branchStatus.status = "cancelled";
|
|
1711
|
+
} else {
|
|
1712
|
+
branchStatus.status = "failed";
|
|
1713
|
+
branchStatus.error = String(error);
|
|
1714
|
+
|
|
1715
|
+
if (settled) {
|
|
1716
|
+
// Track late errors for observability
|
|
1717
|
+
lateErrors.push({
|
|
1718
|
+
name: branch.name,
|
|
1719
|
+
error: String(error),
|
|
1720
|
+
});
|
|
1721
|
+
} else {
|
|
1722
|
+
errors[branch.name] = error;
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
entry.dirty = true;
|
|
1726
|
+
|
|
1727
|
+
// All branches failed (only if no winner yet)
|
|
1728
|
+
if (pendingCount === 0 && !settled) {
|
|
1729
|
+
settled = true;
|
|
1730
|
+
}
|
|
1731
|
+
},
|
|
1732
|
+
);
|
|
1733
|
+
|
|
1734
|
+
branchPromises.push(branchPromise);
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
// Wait for all branches to complete or be cancelled
|
|
1738
|
+
await Promise.allSettled(branchPromises);
|
|
1739
|
+
|
|
1740
|
+
// If any branch needs to yield to the scheduler (sleep/message wait),
|
|
1741
|
+
// save state and re-throw the error to exit the workflow execution
|
|
1742
|
+
if (yieldError && !settled) {
|
|
1743
|
+
await this.flushStorage();
|
|
1744
|
+
throw yieldError;
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
// Clean up entries from non-winning branches
|
|
1748
|
+
if (winnerName !== null) {
|
|
1749
|
+
for (const branch of branches) {
|
|
1750
|
+
if (branch.name !== winnerName) {
|
|
1751
|
+
const branchLocation = appendName(
|
|
1752
|
+
this.storage,
|
|
1753
|
+
location,
|
|
1754
|
+
branch.name,
|
|
1755
|
+
);
|
|
1756
|
+
await deleteEntriesWithPrefix(
|
|
1757
|
+
this.storage,
|
|
1758
|
+
this.driver,
|
|
1759
|
+
branchLocation,
|
|
1760
|
+
this.historyNotifier,
|
|
1761
|
+
);
|
|
1762
|
+
}
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
// Flush final state
|
|
1767
|
+
await this.flushStorage();
|
|
1768
|
+
|
|
1769
|
+
// Log late errors if any (these occurred after a winner was determined)
|
|
1770
|
+
if (lateErrors.length > 0) {
|
|
1771
|
+
console.warn(
|
|
1772
|
+
`Race "${name}" had ${lateErrors.length} branch(es) fail after winner was determined:`,
|
|
1773
|
+
lateErrors,
|
|
1774
|
+
);
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
// Return result or throw error
|
|
1778
|
+
if (winnerName !== null && winnerValue !== null) {
|
|
1779
|
+
return { winner: winnerName, value: winnerValue };
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
// All branches failed
|
|
1783
|
+
throw new RaceError(
|
|
1784
|
+
"All branches failed",
|
|
1785
|
+
Object.entries(errors).map(([name, error]) => ({
|
|
1786
|
+
name,
|
|
1787
|
+
error: String(error),
|
|
1788
|
+
})),
|
|
1789
|
+
);
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1792
|
+
// === Removed ===
|
|
1793
|
+
|
|
1794
|
+
async removed(name: string, originalType: EntryKindType): Promise<void> {
|
|
1795
|
+
this.assertNotInProgress();
|
|
1796
|
+
this.checkEvicted();
|
|
1797
|
+
|
|
1798
|
+
this.entryInProgress = true;
|
|
1799
|
+
try {
|
|
1800
|
+
await this.executeRemoved(name, originalType);
|
|
1801
|
+
} finally {
|
|
1802
|
+
this.entryInProgress = false;
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
private async executeRemoved(
|
|
1807
|
+
name: string,
|
|
1808
|
+
originalType: EntryKindType,
|
|
1809
|
+
): Promise<void> {
|
|
1810
|
+
// Check for duplicate name in current execution
|
|
1811
|
+
this.checkDuplicateName(name);
|
|
1812
|
+
|
|
1813
|
+
const location = appendName(this.storage, this.currentLocation, name);
|
|
1814
|
+
const key = locationToKey(this.storage, location);
|
|
1815
|
+
const existing = this.storage.history.entries.get(key);
|
|
1816
|
+
|
|
1817
|
+
// Mark this entry as visited for validateComplete
|
|
1818
|
+
this.markVisited(key);
|
|
1819
|
+
|
|
1820
|
+
this.stopRollbackIfMissing(existing);
|
|
1821
|
+
|
|
1822
|
+
if (existing) {
|
|
1823
|
+
// Validate the existing entry matches what we expect
|
|
1824
|
+
if (
|
|
1825
|
+
existing.kind.type !== "removed" &&
|
|
1826
|
+
existing.kind.type !== originalType
|
|
1827
|
+
) {
|
|
1828
|
+
throw new HistoryDivergedError(
|
|
1829
|
+
`Expected ${originalType} or removed at ${key}, found ${existing.kind.type}`,
|
|
1830
|
+
);
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
// If it's not already marked as removed, we just skip it
|
|
1834
|
+
return;
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
// Create a removed entry placeholder
|
|
1838
|
+
const entry = createEntry(location, {
|
|
1839
|
+
type: "removed",
|
|
1840
|
+
data: { originalType, originalName: name },
|
|
1841
|
+
});
|
|
1842
|
+
setEntry(this.storage, location, entry);
|
|
1843
|
+
await this.flushStorage();
|
|
1844
|
+
}
|
|
1845
|
+
}
|