@redflow/client 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +280 -0
- package/package.json +1 -1
- package/src/client.ts +35 -96
- package/src/internal/keys.ts +1 -2
- package/src/types.ts +28 -36
- package/src/worker.ts +18 -37
- package/src/workflow.ts +12 -7
- package/tests/bugfixes.test.ts +21 -32
- package/tests/fixtures/worker-crash.ts +1 -1
- package/tests/fixtures/worker-recover.ts +1 -1
- package/tests/redflow.e2e.test.ts +136 -640
package/src/worker.ts
CHANGED
|
@@ -43,6 +43,8 @@ export type WorkerHandle = {
|
|
|
43
43
|
stop(): Promise<void>;
|
|
44
44
|
};
|
|
45
45
|
|
|
46
|
+
const POLL_MS = 250;
|
|
47
|
+
|
|
46
48
|
const UNLOCK_LUA = `
|
|
47
49
|
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
48
50
|
return redis.call("del", KEYS[1])
|
|
@@ -224,10 +226,6 @@ function defaultStepWorkflowIdempotencyKey(parentRunId: string, stepName: string
|
|
|
224
226
|
return `stepwf:${encodeIdempotencyPart(parentRunId)}:${encodeIdempotencyPart(stepName)}:${encodeIdempotencyPart(childWorkflowName)}`;
|
|
225
227
|
}
|
|
226
228
|
|
|
227
|
-
function defaultStepEventIdempotencyKey(parentRunId: string, stepName: string, eventName: string): string {
|
|
228
|
-
return `stepev:${encodeIdempotencyPart(parentRunId)}:${encodeIdempotencyPart(stepName)}:${encodeIdempotencyPart(eventName)}`;
|
|
229
|
-
}
|
|
230
|
-
|
|
231
229
|
async function claimQueuedRunForInlineExecution(args: {
|
|
232
230
|
redis: RedisClient;
|
|
233
231
|
prefix: string;
|
|
@@ -616,13 +614,9 @@ async function processRun(args: {
|
|
|
616
614
|
|
|
617
615
|
const waitForRunResultWithInlineAssist = async <TOutput>(
|
|
618
616
|
childRunId: string,
|
|
619
|
-
resultOptions: { timeoutMs?: number; pollMs?: number } | undefined,
|
|
620
617
|
stepSignal: AbortSignal,
|
|
621
618
|
): Promise<TOutput> => {
|
|
622
|
-
const
|
|
623
|
-
const pollMs = resultOptions?.pollMs ?? 250;
|
|
624
|
-
const deadline = nowMs() + timeoutMs;
|
|
625
|
-
const missingGraceMs = Math.max(250, Math.min(2000, pollMs * 4));
|
|
619
|
+
const missingGraceMs = Math.max(250, Math.min(2000, POLL_MS * 4));
|
|
626
620
|
|
|
627
621
|
let missingSince: number | null = null;
|
|
628
622
|
let seenState = false;
|
|
@@ -642,8 +636,7 @@ async function processRun(args: {
|
|
|
642
636
|
throw new Error(`Run not found: ${childRunId}`);
|
|
643
637
|
}
|
|
644
638
|
|
|
645
|
-
|
|
646
|
-
await sleep(pollMs);
|
|
639
|
+
await sleep(POLL_MS);
|
|
647
640
|
continue;
|
|
648
641
|
}
|
|
649
642
|
|
|
@@ -681,8 +674,7 @@ async function processRun(args: {
|
|
|
681
674
|
}
|
|
682
675
|
}
|
|
683
676
|
|
|
684
|
-
|
|
685
|
-
await sleep(pollMs);
|
|
677
|
+
await sleep(POLL_MS);
|
|
686
678
|
}
|
|
687
679
|
};
|
|
688
680
|
|
|
@@ -690,41 +682,31 @@ async function processRun(args: {
|
|
|
690
682
|
{ name: options.name, timeoutMs: options.timeoutMs },
|
|
691
683
|
async ({ signal: stepSignal }) => {
|
|
692
684
|
const handle = await workflow.run(workflowInput, {
|
|
693
|
-
|
|
685
|
+
runAt: options.runAt,
|
|
686
|
+
queueOverride: options.queueOverride,
|
|
687
|
+
idempotencyTtl: options.idempotencyTtl,
|
|
694
688
|
idempotencyKey,
|
|
695
689
|
});
|
|
696
690
|
|
|
697
|
-
return await waitForRunResultWithInlineAssist(handle.id,
|
|
691
|
+
return await waitForRunResultWithInlineAssist(handle.id, stepSignal);
|
|
698
692
|
},
|
|
699
693
|
);
|
|
700
694
|
};
|
|
701
695
|
|
|
702
|
-
const
|
|
696
|
+
const emitWorkflowStep: StepApi["emitWorkflow"] = async (options, workflow, workflowInput) => {
|
|
703
697
|
const idempotencyKey =
|
|
704
|
-
options.idempotencyKey ??
|
|
705
|
-
|
|
706
|
-
return await runStep(
|
|
707
|
-
{ name: options.name, timeoutMs: options.timeoutMs },
|
|
708
|
-
async () => {
|
|
709
|
-
return await client.emitEvent(options.event, payload, {
|
|
710
|
-
...(options.emit ?? {}),
|
|
711
|
-
idempotencyKey,
|
|
712
|
-
});
|
|
713
|
-
},
|
|
714
|
-
);
|
|
715
|
-
};
|
|
716
|
-
|
|
717
|
-
const scheduleEventStep: StepApi["scheduleEvent"] = async (options, payload) => {
|
|
718
|
-
const idempotencyKey =
|
|
719
|
-
options.idempotencyKey ?? defaultStepEventIdempotencyKey(runId, options.name, options.event);
|
|
698
|
+
options.idempotencyKey ?? defaultStepWorkflowIdempotencyKey(runId, options.name, workflow.name);
|
|
720
699
|
|
|
721
700
|
return await runStep(
|
|
722
701
|
{ name: options.name, timeoutMs: options.timeoutMs },
|
|
723
702
|
async () => {
|
|
724
|
-
|
|
725
|
-
|
|
703
|
+
const handle = await workflow.run(workflowInput, {
|
|
704
|
+
runAt: options.runAt,
|
|
705
|
+
queueOverride: options.queueOverride,
|
|
706
|
+
idempotencyTtl: options.idempotencyTtl,
|
|
726
707
|
idempotencyKey,
|
|
727
708
|
});
|
|
709
|
+
return handle.id;
|
|
728
710
|
},
|
|
729
711
|
);
|
|
730
712
|
};
|
|
@@ -732,8 +714,7 @@ async function processRun(args: {
|
|
|
732
714
|
const step: StepApi = {
|
|
733
715
|
run: runStep,
|
|
734
716
|
runWorkflow: runWorkflowStep,
|
|
735
|
-
|
|
736
|
-
scheduleEvent: scheduleEventStep,
|
|
717
|
+
emitWorkflow: emitWorkflowStep,
|
|
737
718
|
};
|
|
738
719
|
|
|
739
720
|
let outputJson: string;
|
|
@@ -1039,4 +1020,4 @@ async function invokeOnFailure(
|
|
|
1039
1020
|
} catch {
|
|
1040
1021
|
// onFailure must not interfere with the run lifecycle.
|
|
1041
1022
|
}
|
|
1042
|
-
}
|
|
1023
|
+
}
|
package/src/workflow.ts
CHANGED
|
@@ -11,15 +11,18 @@ export type Workflow<TInput, TOutput> = {
|
|
|
11
11
|
run(input: TInput, options?: RunOptions): Promise<RunHandle<TOutput>>;
|
|
12
12
|
};
|
|
13
13
|
|
|
14
|
+
export type DefineWorkflowConfig<TSchema extends ZodTypeAny | undefined> = Omit<DefineWorkflowOptions<TSchema>, "name">;
|
|
15
|
+
|
|
14
16
|
export function defineWorkflow<TSchema extends ZodTypeAny | undefined, TOutput>(
|
|
15
|
-
|
|
17
|
+
name: string,
|
|
18
|
+
options: DefineWorkflowConfig<TSchema>,
|
|
16
19
|
handler: WorkflowHandler<InferInput<TSchema>, TOutput>,
|
|
17
20
|
): Workflow<InferInput<TSchema>, TOutput> {
|
|
18
|
-
|
|
21
|
+
const fullOptions: DefineWorkflowOptions<TSchema> = { name, ...options };
|
|
22
|
+
getDefaultRegistry().register({ options: fullOptions, handler });
|
|
19
23
|
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
const configuredMaxAttempts = options.retries?.maxAttempts;
|
|
24
|
+
const queue = fullOptions.queue ?? "default";
|
|
25
|
+
const configuredMaxAttempts = fullOptions.retries?.maxAttempts;
|
|
23
26
|
const maxAttemptsOverride =
|
|
24
27
|
typeof configuredMaxAttempts === "number" && Number.isFinite(configuredMaxAttempts) && configuredMaxAttempts > 0
|
|
25
28
|
? Math.floor(configuredMaxAttempts)
|
|
@@ -28,7 +31,9 @@ export function defineWorkflow<TSchema extends ZodTypeAny | undefined, TOutput>(
|
|
|
28
31
|
return {
|
|
29
32
|
name,
|
|
30
33
|
async run(input, runOptions) {
|
|
31
|
-
const parsedInput =
|
|
34
|
+
const parsedInput = fullOptions.schema
|
|
35
|
+
? (validateInputWithSchema(fullOptions.schema, input) as InferInput<TSchema>)
|
|
36
|
+
: input;
|
|
32
37
|
const client = getDefaultClient();
|
|
33
38
|
return await client.runByName<TOutput>(name, parsedInput, {
|
|
34
39
|
...runOptions,
|
|
@@ -39,4 +44,4 @@ export function defineWorkflow<TSchema extends ZodTypeAny | undefined, TOutput>(
|
|
|
39
44
|
};
|
|
40
45
|
}
|
|
41
46
|
|
|
42
|
-
export type { WorkflowHandler, WorkflowHandlerContext };
|
|
47
|
+
export type { WorkflowHandler, WorkflowHandlerContext };
|
package/tests/bugfixes.test.ts
CHANGED
|
@@ -142,7 +142,7 @@ test("transitionRunStatus: atomically moves status and indexes", async () => {
|
|
|
142
142
|
const client = createClient({ redis, prefix });
|
|
143
143
|
setDefaultClient(client);
|
|
144
144
|
|
|
145
|
-
const wf = defineWorkflow(
|
|
145
|
+
const wf = defineWorkflow("atomic-transition-wf", {queue: "q_atomic" }, async () => ({ ok: true }));
|
|
146
146
|
|
|
147
147
|
// Enqueue a run manually
|
|
148
148
|
const handle = await wf.run({});
|
|
@@ -177,8 +177,8 @@ test("transitionRunStatus: clears availableAt when transitioning away from sched
|
|
|
177
177
|
const client = createClient({ redis, prefix });
|
|
178
178
|
setDefaultClient(client);
|
|
179
179
|
|
|
180
|
-
const wf = defineWorkflow(
|
|
181
|
-
const handle = await wf.run({}, {
|
|
180
|
+
const wf = defineWorkflow("clear-avail-wf", {queue: "q_avail" }, async () => ({ ok: true }));
|
|
181
|
+
const handle = await wf.run({}, { runAt: new Date(Date.now() + 60_000) });
|
|
182
182
|
|
|
183
183
|
// Should be scheduled
|
|
184
184
|
const state1 = await client.getRun(handle.id);
|
|
@@ -209,8 +209,7 @@ test("crash recovery: attempt is not double-incremented", async () => {
|
|
|
209
209
|
const queue = "q_no_double";
|
|
210
210
|
const countKey = `${prefix}:t:noDoubleCount`;
|
|
211
211
|
|
|
212
|
-
const wf = defineWorkflow(
|
|
213
|
-
{ name: "no-double-wf", queue, retries: { maxAttempts: 3 } },
|
|
212
|
+
const wf = defineWorkflow("no-double-wf", {queue, retries: { maxAttempts: 3 } },
|
|
214
213
|
async ({ step, run }) => {
|
|
215
214
|
await step.run({ name: "do" }, async () => {
|
|
216
215
|
await redis.incr(countKey);
|
|
@@ -229,7 +228,7 @@ test("crash recovery: attempt is not double-incremented", async () => {
|
|
|
229
228
|
});
|
|
230
229
|
|
|
231
230
|
const handle = await wf.run({});
|
|
232
|
-
const result = await handle.result({ timeoutMs: 5000
|
|
231
|
+
const result = await handle.result({ timeoutMs: 5000 });
|
|
233
232
|
expect(result.attempt).toBe(1);
|
|
234
233
|
|
|
235
234
|
const state = await client.getRun(handle.id);
|
|
@@ -249,8 +248,7 @@ test("crash recovery: re-processing a running run does not bump attempt", async
|
|
|
249
248
|
|
|
250
249
|
const queue = "q_recover_attempt";
|
|
251
250
|
|
|
252
|
-
const wf = defineWorkflow(
|
|
253
|
-
{ name: "recover-attempt-wf", queue },
|
|
251
|
+
const wf = defineWorkflow("recover-attempt-wf", {queue },
|
|
254
252
|
async ({ run }) => {
|
|
255
253
|
return { attempt: run.attempt };
|
|
256
254
|
},
|
|
@@ -318,10 +316,7 @@ test("retry: run is atomically transitioned to scheduled with queue entry", asyn
|
|
|
318
316
|
|
|
319
317
|
const queue = "q_atomic_retry";
|
|
320
318
|
|
|
321
|
-
const wf = defineWorkflow(
|
|
322
|
-
{
|
|
323
|
-
name: "atomic-retry-wf",
|
|
324
|
-
queue,
|
|
319
|
+
const wf = defineWorkflow("atomic-retry-wf", {queue,
|
|
325
320
|
retries: { maxAttempts: 2 },
|
|
326
321
|
},
|
|
327
322
|
async ({ run }) => {
|
|
@@ -338,7 +333,7 @@ test("retry: run is atomically transitioned to scheduled with queue entry", asyn
|
|
|
338
333
|
});
|
|
339
334
|
|
|
340
335
|
const handle = await wf.run({});
|
|
341
|
-
const result = await handle.result({ timeoutMs: 15_000
|
|
336
|
+
const result = await handle.result({ timeoutMs: 15_000 });
|
|
342
337
|
expect(result).toEqual({ ok: true });
|
|
343
338
|
|
|
344
339
|
const state = await client.getRun(handle.id);
|
|
@@ -431,7 +426,7 @@ test(
|
|
|
431
426
|
const queue = "q_batch_promote";
|
|
432
427
|
const countKey = `${prefix}:t:batchPromoteCount`;
|
|
433
428
|
|
|
434
|
-
const wf = defineWorkflow(
|
|
429
|
+
const wf = defineWorkflow("batch-promote-wf", {queue }, async ({ step }) => {
|
|
435
430
|
await step.run({ name: "do" }, async () => {
|
|
436
431
|
await redis.incr(countKey);
|
|
437
432
|
return true;
|
|
@@ -442,7 +437,7 @@ test(
|
|
|
442
437
|
// Schedule 5 runs for "now" so they all become due simultaneously
|
|
443
438
|
const handles = [];
|
|
444
439
|
for (let i = 0; i < 5; i++) {
|
|
445
|
-
handles.push(await wf.run({}, {
|
|
440
|
+
handles.push(await wf.run({}, { runAt: new Date(Date.now() + 200) }));
|
|
446
441
|
}
|
|
447
442
|
|
|
448
443
|
const worker = await startWorker({
|
|
@@ -489,14 +484,14 @@ test("cancelRun: cleans processing list for canceled runs", async () => {
|
|
|
489
484
|
|
|
490
485
|
const queue = "q_cancel_processing";
|
|
491
486
|
|
|
492
|
-
const wf = defineWorkflow(
|
|
487
|
+
const wf = defineWorkflow("cancel-proc-wf", {queue }, async () => {
|
|
493
488
|
// Long-running so we can cancel while queued/scheduled
|
|
494
489
|
await new Promise((r) => setTimeout(r, 60_000));
|
|
495
490
|
return { ok: true };
|
|
496
491
|
});
|
|
497
492
|
|
|
498
493
|
// Schedule a run for the near future
|
|
499
|
-
const handle = await wf.run({}, {
|
|
494
|
+
const handle = await wf.run({}, { runAt: new Date(Date.now() + 100) });
|
|
500
495
|
|
|
501
496
|
// Manually place it into processing to simulate race condition
|
|
502
497
|
const processingKey = keys.queueProcessing(prefix, queue);
|
|
@@ -525,7 +520,7 @@ test("duplicate step name: throws a clear error", async () => {
|
|
|
525
520
|
|
|
526
521
|
const queue = "q_dup_step";
|
|
527
522
|
|
|
528
|
-
const wf = defineWorkflow(
|
|
523
|
+
const wf = defineWorkflow("dup-step-wf", {queue }, async ({ step }) => {
|
|
529
524
|
await step.run({ name: "same-name" }, async () => 1);
|
|
530
525
|
// This should throw because step name is already used
|
|
531
526
|
await step.run({ name: "same-name" }, async () => 2);
|
|
@@ -565,7 +560,7 @@ test("unique step names: work normally", async () => {
|
|
|
565
560
|
|
|
566
561
|
const queue = "q_unique_steps";
|
|
567
562
|
|
|
568
|
-
const wf = defineWorkflow(
|
|
563
|
+
const wf = defineWorkflow("unique-steps-wf", {queue }, async ({ step }) => {
|
|
569
564
|
const a = await step.run({ name: "step-a" }, async () => 1);
|
|
570
565
|
const b = await step.run({ name: "step-b" }, async () => 2);
|
|
571
566
|
const c = await step.run({ name: "step-c" }, async () => 3);
|
|
@@ -580,7 +575,7 @@ test("unique step names: work normally", async () => {
|
|
|
580
575
|
});
|
|
581
576
|
|
|
582
577
|
const handle = await wf.run({});
|
|
583
|
-
const result = await handle.result({ timeoutMs: 5000
|
|
578
|
+
const result = await handle.result({ timeoutMs: 5000 });
|
|
584
579
|
expect(result).toEqual({ sum: 6 });
|
|
585
580
|
|
|
586
581
|
await worker.stop();
|
|
@@ -602,10 +597,7 @@ test(
|
|
|
602
597
|
const queue = "q_full_retry";
|
|
603
598
|
const attemptsKey = `${prefix}:t:fullRetryAttempts`;
|
|
604
599
|
|
|
605
|
-
const wf = defineWorkflow(
|
|
606
|
-
{
|
|
607
|
-
name: "full-retry-wf",
|
|
608
|
-
queue,
|
|
600
|
+
const wf = defineWorkflow("full-retry-wf", {queue,
|
|
609
601
|
retries: { maxAttempts: 3 },
|
|
610
602
|
},
|
|
611
603
|
async ({ run, step }) => {
|
|
@@ -626,7 +618,7 @@ test(
|
|
|
626
618
|
});
|
|
627
619
|
|
|
628
620
|
const handle = await wf.run({});
|
|
629
|
-
const result = await handle.result({ timeoutMs: 30_000
|
|
621
|
+
const result = await handle.result({ timeoutMs: 30_000 });
|
|
630
622
|
expect(result).toEqual({ ok: true, finalAttempt: 3 });
|
|
631
623
|
|
|
632
624
|
const state = await client.getRun(handle.id);
|
|
@@ -661,10 +653,7 @@ test(
|
|
|
661
653
|
|
|
662
654
|
const queue = "q_exhaust";
|
|
663
655
|
|
|
664
|
-
const wf = defineWorkflow(
|
|
665
|
-
{
|
|
666
|
-
name: "exhaust-wf",
|
|
667
|
-
queue,
|
|
656
|
+
const wf = defineWorkflow("exhaust-wf", {queue,
|
|
668
657
|
retries: { maxAttempts: 2 },
|
|
669
658
|
},
|
|
670
659
|
async () => {
|
|
@@ -716,7 +705,7 @@ test("status indexes: consistent through full lifecycle", async () => {
|
|
|
716
705
|
|
|
717
706
|
const queue = "q_status_idx";
|
|
718
707
|
|
|
719
|
-
const wf = defineWorkflow(
|
|
708
|
+
const wf = defineWorkflow("status-idx-wf", {queue }, async ({ step }) => {
|
|
720
709
|
await step.run({ name: "do" }, async () => {
|
|
721
710
|
await new Promise((r) => setTimeout(r, 200));
|
|
722
711
|
return true;
|
|
@@ -749,7 +738,7 @@ test("status indexes: consistent through full lifecycle", async () => {
|
|
|
749
738
|
expect(queuedMembers.includes(handle.id)).toBe(false);
|
|
750
739
|
|
|
751
740
|
// Wait for succeeded
|
|
752
|
-
await handle.result({ timeoutMs: 5000
|
|
741
|
+
await handle.result({ timeoutMs: 5000 });
|
|
753
742
|
|
|
754
743
|
const succeededMembers = await redis.zrange(`${prefix}:runs:status:succeeded`, 0, -1);
|
|
755
744
|
expect(succeededMembers.includes(handle.id)).toBe(true);
|
|
@@ -758,4 +747,4 @@ test("status indexes: consistent through full lifecycle", async () => {
|
|
|
758
747
|
|
|
759
748
|
await worker.stop();
|
|
760
749
|
redis.close();
|
|
761
|
-
});
|
|
750
|
+
});
|