@redflow/client 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +17 -0
- package/src/client.ts +1111 -0
- package/src/default.ts +13 -0
- package/src/index.ts +17 -0
- package/src/internal/errors.ts +150 -0
- package/src/internal/json.ts +32 -0
- package/src/internal/keys.ts +38 -0
- package/src/internal/sleep.ts +4 -0
- package/src/internal/time.ts +3 -0
- package/src/registry.ts +55 -0
- package/src/types.ts +181 -0
- package/src/worker.ts +1042 -0
- package/src/workflow.ts +42 -0
- package/tests/bugfixes.test.ts +761 -0
- package/tests/fixtures/worker-crash.ts +42 -0
- package/tests/fixtures/worker-recover.ts +63 -0
- package/tests/helpers/getFreePort.ts +18 -0
- package/tests/helpers/redisTestServer.ts +77 -0
- package/tests/helpers/waitFor.ts +16 -0
- package/tests/redflow.e2e.test.ts +3154 -0
- package/tsconfig.json +3 -0
package/src/client.ts
ADDED
|
@@ -0,0 +1,1111 @@
|
|
|
1
|
+
import { RedisClient, redis as defaultRedis } from "bun";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { Cron } from "croner";
|
|
4
|
+
import type { ZodTypeAny } from "zod";
|
|
5
|
+
import {
|
|
6
|
+
CanceledError,
|
|
7
|
+
InputValidationError,
|
|
8
|
+
NonRetriableError,
|
|
9
|
+
OutputSerializationError,
|
|
10
|
+
TimeoutError,
|
|
11
|
+
UnknownWorkflowError,
|
|
12
|
+
serializeError,
|
|
13
|
+
type SerializedError,
|
|
14
|
+
} from "./internal/errors";
|
|
15
|
+
import { keys } from "./internal/keys";
|
|
16
|
+
import { safeJsonParse, safeJsonStringify, safeJsonTryParse } from "./internal/json";
|
|
17
|
+
import { nowMs } from "./internal/time";
|
|
18
|
+
import { sleep } from "./internal/sleep";
|
|
19
|
+
import type {
|
|
20
|
+
EmitEventOptions,
|
|
21
|
+
ListedRun,
|
|
22
|
+
ListRunsParams,
|
|
23
|
+
RunHandle,
|
|
24
|
+
RunOptions,
|
|
25
|
+
ScheduleEventOptions,
|
|
26
|
+
RunState,
|
|
27
|
+
RunStatus,
|
|
28
|
+
StepState,
|
|
29
|
+
WorkflowMeta,
|
|
30
|
+
} from "./types";
|
|
31
|
+
import type { WorkflowRegistry } from "./registry";
|
|
32
|
+
|
|
33
|
+
export type CreateClientOptions = {
|
|
34
|
+
redis?: RedisClient;
|
|
35
|
+
url?: string;
|
|
36
|
+
prefix?: string;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export type SyncRegistryOptions = {
|
|
40
|
+
/**
|
|
41
|
+
* Workflows are pruned only when they were last synced by the same owner.
|
|
42
|
+
* Set a stable service id (for example, app name) to enable safe stale cleanup.
|
|
43
|
+
*/
|
|
44
|
+
owner?: string;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export function defaultPrefix(): string {
|
|
48
|
+
return process.env.REDFLOW_PREFIX || "redflow:v1";
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const STALE_WORKFLOW_GRACE_MS = 30_000;
|
|
52
|
+
const IDEMPOTENCY_TTL_SEC = 60 * 60 * 24 * 7;
|
|
53
|
+
|
|
54
|
+
const ENQUEUE_RUN_LUA = `
|
|
55
|
+
local runId = ARGV[1]
|
|
56
|
+
local workflowName = ARGV[2]
|
|
57
|
+
local queue = ARGV[3]
|
|
58
|
+
local status = ARGV[4]
|
|
59
|
+
local inputJson = ARGV[5]
|
|
60
|
+
local createdAt = ARGV[6]
|
|
61
|
+
local availableAt = ARGV[7]
|
|
62
|
+
local maxAttempts = ARGV[8]
|
|
63
|
+
local hasIdempotency = ARGV[9] == "1"
|
|
64
|
+
local idempotencyTtlSec = ARGV[10]
|
|
65
|
+
local runKeyPrefix = ARGV[11]
|
|
66
|
+
|
|
67
|
+
if hasIdempotency then
|
|
68
|
+
local existingRunId = redis.call("get", KEYS[7])
|
|
69
|
+
if existingRunId and existingRunId ~= "" then
|
|
70
|
+
local existingStatus = redis.call("hget", runKeyPrefix .. existingRunId, "status")
|
|
71
|
+
if existingStatus then
|
|
72
|
+
redis.call("expire", KEYS[7], idempotencyTtlSec)
|
|
73
|
+
return { "existing", existingRunId }
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
redis.call("set", KEYS[7], runId, "EX", idempotencyTtlSec)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
redis.call("hset", KEYS[1],
|
|
81
|
+
"workflow", workflowName,
|
|
82
|
+
"queue", queue,
|
|
83
|
+
"status", status,
|
|
84
|
+
"inputJson", inputJson,
|
|
85
|
+
"attempt", "0",
|
|
86
|
+
"maxAttempts", maxAttempts,
|
|
87
|
+
"createdAt", createdAt,
|
|
88
|
+
"availableAt", availableAt,
|
|
89
|
+
"cancelRequestedAt", "",
|
|
90
|
+
"cancelReason", ""
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
redis.call("zadd", KEYS[2], createdAt, runId)
|
|
94
|
+
redis.call("zadd", KEYS[3], createdAt, runId)
|
|
95
|
+
redis.call("zadd", KEYS[4], createdAt, runId)
|
|
96
|
+
|
|
97
|
+
if status == "scheduled" then
|
|
98
|
+
redis.call("zadd", KEYS[6], availableAt, runId)
|
|
99
|
+
else
|
|
100
|
+
redis.call("lpush", KEYS[5], runId)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
return { "created", runId }
|
|
104
|
+
`;
|
|
105
|
+
|
|
106
|
+
const TRANSITION_IF_CURRENT_STATUS_LUA = `
|
|
107
|
+
if redis.call("hget", KEYS[1], "status") ~= ARGV[2] then
|
|
108
|
+
return 0
|
|
109
|
+
end
|
|
110
|
+
redis.call("zrem", KEYS[2], ARGV[1])
|
|
111
|
+
redis.call("hset", KEYS[1], "status", ARGV[3])
|
|
112
|
+
if ARGV[3] ~= "scheduled" then
|
|
113
|
+
redis.call("hdel", KEYS[1], "availableAt")
|
|
114
|
+
end
|
|
115
|
+
redis.call("zadd", KEYS[3], ARGV[4], ARGV[1])
|
|
116
|
+
return 1
|
|
117
|
+
`;
|
|
118
|
+
|
|
119
|
+
// Atomic version of transitionRunStatus: reads current status, removes from
|
|
120
|
+
// old status index, sets new status, clears availableAt if not "scheduled",
|
|
121
|
+
// and adds to new status index — all in one round-trip.
|
|
122
|
+
const TRANSITION_RUN_STATUS_LUA = `
|
|
123
|
+
local runKey = KEYS[1]
|
|
124
|
+
local newStatusIndexKey = KEYS[2]
|
|
125
|
+
local runId = ARGV[1]
|
|
126
|
+
local nextStatus = ARGV[2]
|
|
127
|
+
local updatedAt = ARGV[3]
|
|
128
|
+
local statusIndexPrefix = ARGV[4]
|
|
129
|
+
|
|
130
|
+
local prev = redis.call("hget", runKey, "status")
|
|
131
|
+
if prev and prev ~= "" then
|
|
132
|
+
local oldIndexKey = statusIndexPrefix .. prev
|
|
133
|
+
redis.call("zrem", oldIndexKey, runId)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
redis.call("hset", runKey, "status", nextStatus)
|
|
137
|
+
if nextStatus ~= "scheduled" then
|
|
138
|
+
redis.call("hdel", runKey, "availableAt")
|
|
139
|
+
end
|
|
140
|
+
redis.call("zadd", newStatusIndexKey, updatedAt, runId)
|
|
141
|
+
return 1
|
|
142
|
+
`;
|
|
143
|
+
|
|
144
|
+
// Atomically: set availableAt + errorJson on run hash, transition status to
|
|
145
|
+
// "scheduled" (cleaning old status index), and zadd into the queue's scheduled
|
|
146
|
+
// sorted set. Prevents a run from being stranded if the worker crashes
|
|
147
|
+
// mid-retry between transitionRunStatus and the zadd.
|
|
148
|
+
const SCHEDULE_RETRY_LUA = `
|
|
149
|
+
local runKey = KEYS[1]
|
|
150
|
+
local newStatusIndexKey = KEYS[2]
|
|
151
|
+
local queueScheduledKey = KEYS[3]
|
|
152
|
+
local runId = ARGV[1]
|
|
153
|
+
local updatedAt = ARGV[2]
|
|
154
|
+
local nextAt = ARGV[3]
|
|
155
|
+
local errorJson = ARGV[4]
|
|
156
|
+
local statusIndexPrefix = ARGV[5]
|
|
157
|
+
|
|
158
|
+
local prev = redis.call("hget", runKey, "status")
|
|
159
|
+
if prev and prev ~= "" then
|
|
160
|
+
local oldIndexKey = statusIndexPrefix .. prev
|
|
161
|
+
redis.call("zrem", oldIndexKey, runId)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
redis.call("hset", runKey, "status", "scheduled", "availableAt", nextAt, "errorJson", errorJson)
|
|
165
|
+
redis.call("zadd", newStatusIndexKey, updatedAt, runId)
|
|
166
|
+
redis.call("zadd", queueScheduledKey, nextAt, runId)
|
|
167
|
+
return 1
|
|
168
|
+
`;
|
|
169
|
+
|
|
170
|
+
const REQUEST_CANCELLATION_LUA = `
|
|
171
|
+
local status = redis.call("hget", KEYS[1], "status")
|
|
172
|
+
if not status then
|
|
173
|
+
return "__missing__"
|
|
174
|
+
end
|
|
175
|
+
if status == "succeeded" or status == "failed" or status == "canceled" then
|
|
176
|
+
return status
|
|
177
|
+
end
|
|
178
|
+
redis.call("hset", KEYS[1], "cancelRequestedAt", ARGV[1], "cancelReason", ARGV[2])
|
|
179
|
+
return status
|
|
180
|
+
`;
|
|
181
|
+
|
|
182
|
+
const FINALIZE_TERMINAL_RUN_LUA = `
|
|
183
|
+
local status = redis.call("hget", KEYS[1], "status")
|
|
184
|
+
if not status then
|
|
185
|
+
return "__missing__"
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
local runId = ARGV[1]
|
|
189
|
+
local requestedStatus = ARGV[2]
|
|
190
|
+
local finishedAt = ARGV[3]
|
|
191
|
+
local outputJson = ARGV[4]
|
|
192
|
+
local errorJson = ARGV[5]
|
|
193
|
+
local hasOutput = ARGV[6] == "1"
|
|
194
|
+
local hasError = ARGV[7] == "1"
|
|
195
|
+
|
|
196
|
+
local nextStatus = requestedStatus
|
|
197
|
+
if requestedStatus == "succeeded" or requestedStatus == "failed" then
|
|
198
|
+
local cancelRequestedAt = redis.call("hget", KEYS[1], "cancelRequestedAt")
|
|
199
|
+
if cancelRequestedAt and cancelRequestedAt ~= "" then
|
|
200
|
+
nextStatus = "canceled"
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
for i = 2, 7 do
|
|
205
|
+
redis.call("zrem", KEYS[i], runId)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
redis.call("hdel", KEYS[1], "availableAt")
|
|
209
|
+
|
|
210
|
+
if nextStatus == "succeeded" then
|
|
211
|
+
redis.call("hdel", KEYS[1], "errorJson")
|
|
212
|
+
if hasOutput then
|
|
213
|
+
redis.call("hset", KEYS[1], "outputJson", outputJson)
|
|
214
|
+
else
|
|
215
|
+
redis.call("hdel", KEYS[1], "outputJson")
|
|
216
|
+
end
|
|
217
|
+
redis.call("hset", KEYS[1], "cancelRequestedAt", "", "cancelReason", "")
|
|
218
|
+
elseif nextStatus == "failed" then
|
|
219
|
+
redis.call("hdel", KEYS[1], "outputJson")
|
|
220
|
+
if hasError then
|
|
221
|
+
redis.call("hset", KEYS[1], "errorJson", errorJson)
|
|
222
|
+
else
|
|
223
|
+
redis.call("hdel", KEYS[1], "errorJson")
|
|
224
|
+
end
|
|
225
|
+
redis.call("hset", KEYS[1], "cancelRequestedAt", "", "cancelReason", "")
|
|
226
|
+
else
|
|
227
|
+
redis.call("hdel", KEYS[1], "outputJson")
|
|
228
|
+
if hasError then
|
|
229
|
+
redis.call("hset", KEYS[1], "errorJson", errorJson)
|
|
230
|
+
else
|
|
231
|
+
redis.call("hdel", KEYS[1], "errorJson")
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
redis.call("hset", KEYS[1], "status", nextStatus, "finishedAt", finishedAt)
|
|
236
|
+
|
|
237
|
+
if nextStatus == "succeeded" then
|
|
238
|
+
redis.call("zadd", KEYS[5], finishedAt, runId)
|
|
239
|
+
elseif nextStatus == "failed" then
|
|
240
|
+
redis.call("zadd", KEYS[6], finishedAt, runId)
|
|
241
|
+
else
|
|
242
|
+
redis.call("zadd", KEYS[7], finishedAt, runId)
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
return nextStatus
|
|
246
|
+
`;
|
|
247
|
+
|
|
248
|
+
function encodeCompositePart(value: string): string {
|
|
249
|
+
return `${value.length}:${value}`;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function eventScopedIdempotencyKey(eventName: string, idempotencyKey: string): string {
|
|
253
|
+
return `event:${encodeCompositePart(eventName)}:${encodeCompositePart(idempotencyKey)}`;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function defaultRegistryOwner(): string {
|
|
257
|
+
const envOwner = process.env.REDFLOW_SYNC_OWNER?.trim();
|
|
258
|
+
if (envOwner) return envOwner;
|
|
259
|
+
|
|
260
|
+
const argvOwner = process.argv[1]?.trim();
|
|
261
|
+
if (argvOwner) return argvOwner;
|
|
262
|
+
|
|
263
|
+
return "redflow:unknown-owner";
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function parseEnqueueScriptResult(value: unknown): { kind: "created" | "existing"; runId: string } | null {
|
|
267
|
+
if (Array.isArray(value) && value.length === 1 && Array.isArray(value[0])) {
|
|
268
|
+
return parseEnqueueScriptResult(value[0]);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (!Array.isArray(value) || value.length < 2) return null;
|
|
272
|
+
|
|
273
|
+
const kind = value[0];
|
|
274
|
+
const runId = value[1];
|
|
275
|
+
if ((kind !== "created" && kind !== "existing") || typeof runId !== "string" || runId === "") {
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return { kind, runId };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function parseScanResult(value: unknown): { cursor: string; keys: string[] } | null {
|
|
283
|
+
if (Array.isArray(value) && value.length === 1 && Array.isArray(value[0])) {
|
|
284
|
+
return parseScanResult(value[0]);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if (!Array.isArray(value) || value.length < 2) return null;
|
|
288
|
+
|
|
289
|
+
const cursorRaw = value[0];
|
|
290
|
+
const keysRaw = value[1];
|
|
291
|
+
const cursor =
|
|
292
|
+
typeof cursorRaw === "string" ? cursorRaw : typeof cursorRaw === "number" ? String(Math.floor(cursorRaw)) : null;
|
|
293
|
+
if (!cursor || !Array.isArray(keysRaw)) return null;
|
|
294
|
+
|
|
295
|
+
const keys: string[] = [];
|
|
296
|
+
for (const item of keysRaw) {
|
|
297
|
+
if (typeof item === "string" && item) keys.push(item);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return { cursor, keys };
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function parseRedisInteger(value: unknown): number {
|
|
304
|
+
if (typeof value === "number" && Number.isFinite(value)) return Math.max(0, Math.floor(value));
|
|
305
|
+
if (typeof value === "string") {
|
|
306
|
+
const parsed = Number(value);
|
|
307
|
+
if (Number.isFinite(parsed)) return Math.max(0, Math.floor(parsed));
|
|
308
|
+
}
|
|
309
|
+
return 0;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function isValidDate(value: Date): boolean {
|
|
313
|
+
return value instanceof Date && Number.isFinite(value.getTime());
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export class RedflowClient {
|
|
317
|
+
constructor(
|
|
318
|
+
public readonly redis: RedisClient,
|
|
319
|
+
public readonly prefix: string,
|
|
320
|
+
) {}
|
|
321
|
+
|
|
322
|
+
async close(): Promise<void> {
|
|
323
|
+
this.redis.close();
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
async resetState(): Promise<{ deleted: number }> {
|
|
327
|
+
const normalizedPrefix = this.prefix.endsWith(":") ? this.prefix.slice(0, -1) : this.prefix;
|
|
328
|
+
const pattern = `${normalizedPrefix}:*`;
|
|
329
|
+
let cursor = "0";
|
|
330
|
+
let deleted = 0;
|
|
331
|
+
|
|
332
|
+
while (true) {
|
|
333
|
+
const scanned = parseScanResult(await this.redis.send("SCAN", [cursor, "MATCH", pattern, "COUNT", "500"]));
|
|
334
|
+
if (!scanned) {
|
|
335
|
+
throw new Error("Unexpected Redis SCAN response while resetting state");
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
cursor = scanned.cursor;
|
|
339
|
+
if (scanned.keys.length > 0) {
|
|
340
|
+
deleted += parseRedisInteger(await this.redis.send("UNLINK", scanned.keys));
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (cursor === "0") break;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
deleted += parseRedisInteger(await this.redis.send("UNLINK", [normalizedPrefix]));
|
|
347
|
+
|
|
348
|
+
return { deleted };
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
async listWorkflows(): Promise<string[]> {
|
|
352
|
+
return await this.redis.smembers(keys.workflows(this.prefix));
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
async getWorkflowMeta(name: string): Promise<WorkflowMeta | null> {
|
|
356
|
+
const data = await this.redis.hgetall(keys.workflow(this.prefix, name));
|
|
357
|
+
if (!data || Object.keys(data).length === 0) return null;
|
|
358
|
+
|
|
359
|
+
const triggers = safeJsonTryParse<any>(data.triggersJson ?? null) as any;
|
|
360
|
+
const retries = safeJsonTryParse<any>(data.retriesJson ?? null) as any;
|
|
361
|
+
const updatedAt = Number(data.updatedAt ?? "0");
|
|
362
|
+
const queue = data.queue ?? "default";
|
|
363
|
+
return {
|
|
364
|
+
name,
|
|
365
|
+
queue,
|
|
366
|
+
triggers,
|
|
367
|
+
retries,
|
|
368
|
+
updatedAt,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
async emitEvent(name: string, payload: unknown, options?: EmitEventOptions): Promise<string[]> {
|
|
373
|
+
return await this.dispatchEvent(name, payload, {
|
|
374
|
+
idempotencyKey: options?.idempotencyKey,
|
|
375
|
+
idempotencyTtl: options?.idempotencyTtl,
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
async scheduleEvent(name: string, payload: unknown, options: ScheduleEventOptions): Promise<string[]> {
|
|
380
|
+
if (!options || !isValidDate(options.availableAt)) {
|
|
381
|
+
throw new Error("Invalid 'availableAt' for scheduleEvent");
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
return await this.dispatchEvent(name, payload, {
|
|
385
|
+
idempotencyKey: options.idempotencyKey,
|
|
386
|
+
idempotencyTtl: options.idempotencyTtl,
|
|
387
|
+
availableAt: options.availableAt,
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
private async dispatchEvent(
|
|
392
|
+
name: string,
|
|
393
|
+
payload: unknown,
|
|
394
|
+
options: { idempotencyKey?: string; idempotencyTtl?: number; availableAt?: Date },
|
|
395
|
+
): Promise<string[]> {
|
|
396
|
+
const subsKey = keys.eventWorkflows(this.prefix, name);
|
|
397
|
+
const workflowNames = await this.redis.smembers(subsKey);
|
|
398
|
+
const runIds: string[] = [];
|
|
399
|
+
|
|
400
|
+
for (const workflowName of workflowNames) {
|
|
401
|
+
const idempotencyKey = options.idempotencyKey ? eventScopedIdempotencyKey(name, options.idempotencyKey) : undefined;
|
|
402
|
+
try {
|
|
403
|
+
const handle = await this.runByName(workflowName, payload, {
|
|
404
|
+
idempotencyKey,
|
|
405
|
+
idempotencyTtl: options.idempotencyTtl,
|
|
406
|
+
availableAt: options.availableAt,
|
|
407
|
+
});
|
|
408
|
+
runIds.push(handle.id);
|
|
409
|
+
} catch (err) {
|
|
410
|
+
if (err instanceof UnknownWorkflowError) {
|
|
411
|
+
// Stale event subscriptions can remain briefly across deployments.
|
|
412
|
+
// Drop them lazily so valid subscribers still receive events.
|
|
413
|
+
await this.redis.srem(subsKey, workflowName);
|
|
414
|
+
continue;
|
|
415
|
+
}
|
|
416
|
+
throw err;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
return runIds;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async runByName<TOutput = unknown>(workflowName: string, input: unknown, options?: RunOptions): Promise<RunHandle<TOutput>> {
|
|
424
|
+
const queueFromRegistry = await this.getQueueForWorkflow(workflowName);
|
|
425
|
+
const queue = options?.queueOverride ?? queueFromRegistry;
|
|
426
|
+
if (!queue) {
|
|
427
|
+
throw new UnknownWorkflowError(workflowName);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
const maxAttemptsOverride =
|
|
431
|
+
typeof options?.__maxAttemptsOverride === "number" &&
|
|
432
|
+
Number.isFinite(options.__maxAttemptsOverride) &&
|
|
433
|
+
options.__maxAttemptsOverride > 0
|
|
434
|
+
? Math.floor(options.__maxAttemptsOverride)
|
|
435
|
+
: null;
|
|
436
|
+
|
|
437
|
+
const maxAttempts = maxAttemptsOverride ?? (await this.getMaxAttemptsForWorkflow(workflowName)) ?? 1;
|
|
438
|
+
|
|
439
|
+
return await this.enqueueRun<TOutput>({
|
|
440
|
+
workflowName,
|
|
441
|
+
queue,
|
|
442
|
+
input,
|
|
443
|
+
availableAt: options?.availableAt,
|
|
444
|
+
idempotencyKey: options?.idempotencyKey,
|
|
445
|
+
idempotencyTtl: options?.idempotencyTtl,
|
|
446
|
+
maxAttempts,
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
async cancelRun(runId: string, options?: { reason?: string }): Promise<boolean> {
|
|
451
|
+
const runKey = keys.run(this.prefix, runId);
|
|
452
|
+
const ts = nowMs();
|
|
453
|
+
const reason = options?.reason ?? "";
|
|
454
|
+
|
|
455
|
+
const requestedFrom = await this.requestCancellation(runId, ts, reason);
|
|
456
|
+
if (requestedFrom === "__missing__") {
|
|
457
|
+
return false;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
if (requestedFrom === "succeeded" || requestedFrom === "failed" || requestedFrom === "canceled") {
|
|
461
|
+
return false;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
if (requestedFrom === "running") {
|
|
465
|
+
// Cooperative cancellation for in-flight handlers.
|
|
466
|
+
return true;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const queue = (await this.redis.hget(runKey, "queue")) ?? "default";
|
|
470
|
+
|
|
471
|
+
const canceledNow = await this.transitionRunStatusIfCurrent(runId, requestedFrom, "canceled", ts);
|
|
472
|
+
if (canceledNow) {
|
|
473
|
+
await this.finalizeCanceledRunImmediately(runId, queue, ts);
|
|
474
|
+
return true;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// The status changed after we requested cancellation; retry immediate cancel if it's
|
|
478
|
+
// still not running/terminal, otherwise respect the latest terminal state.
|
|
479
|
+
const latestStatus = (await this.redis.hget(runKey, "status")) as RunStatus | null;
|
|
480
|
+
if (latestStatus === "queued" || latestStatus === "scheduled") {
|
|
481
|
+
const canceledLatest = await this.transitionRunStatusIfCurrent(runId, latestStatus, "canceled", ts);
|
|
482
|
+
if (canceledLatest) {
|
|
483
|
+
await this.finalizeCanceledRunImmediately(runId, queue, ts);
|
|
484
|
+
return true;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
const finalStatus = (await this.redis.hget(runKey, "status")) as RunStatus | null;
|
|
489
|
+
if (finalStatus === "running") return true;
|
|
490
|
+
if (finalStatus === "queued" || finalStatus === "scheduled") return true;
|
|
491
|
+
if (finalStatus === "canceled") return true;
|
|
492
|
+
return false;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
async getRun(runId: string): Promise<RunState | null> {
|
|
496
|
+
const runKey = keys.run(this.prefix, runId);
|
|
497
|
+
const data = await this.redis.hgetall(runKey);
|
|
498
|
+
if (!data || Object.keys(data).length === 0) return null;
|
|
499
|
+
|
|
500
|
+
const input = safeJsonParse(data.inputJson ?? "null");
|
|
501
|
+
const output = safeJsonTryParse(data.outputJson ?? null);
|
|
502
|
+
const error = safeJsonTryParse(data.errorJson ?? null);
|
|
503
|
+
|
|
504
|
+
return {
|
|
505
|
+
id: runId,
|
|
506
|
+
workflow: data.workflow ?? "",
|
|
507
|
+
queue: data.queue ?? "default",
|
|
508
|
+
status: (data.status as RunStatus) ?? "queued",
|
|
509
|
+
input,
|
|
510
|
+
output,
|
|
511
|
+
error,
|
|
512
|
+
attempt: Number(data.attempt ?? "0"),
|
|
513
|
+
maxAttempts: Number(data.maxAttempts ?? "1"),
|
|
514
|
+
createdAt: Number(data.createdAt ?? "0"),
|
|
515
|
+
availableAt: data.availableAt ? Number(data.availableAt) : undefined,
|
|
516
|
+
startedAt: data.startedAt ? Number(data.startedAt) : undefined,
|
|
517
|
+
finishedAt: data.finishedAt ? Number(data.finishedAt) : undefined,
|
|
518
|
+
cancelRequestedAt: data.cancelRequestedAt ? Number(data.cancelRequestedAt) : undefined,
|
|
519
|
+
cancelReason: data.cancelReason || undefined,
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
async getRunSteps(runId: string): Promise<StepState[]> {
|
|
524
|
+
const stepsKey = keys.runSteps(this.prefix, runId);
|
|
525
|
+
const data = await this.redis.hgetall(stepsKey);
|
|
526
|
+
const result: Array<{ order: number; step: StepState }> = [];
|
|
527
|
+
|
|
528
|
+
for (const [name, raw] of Object.entries(data)) {
|
|
529
|
+
const parsed = safeJsonParse<any>(raw);
|
|
530
|
+
const order = typeof parsed.seq === "number" ? parsed.seq : Number(parsed.startedAt ?? 0);
|
|
531
|
+
result.push({
|
|
532
|
+
order,
|
|
533
|
+
step: {
|
|
534
|
+
name,
|
|
535
|
+
status: parsed.status,
|
|
536
|
+
startedAt: parsed.startedAt,
|
|
537
|
+
finishedAt: parsed.finishedAt,
|
|
538
|
+
output: safeJsonTryParse(parsed.outputJson ?? null),
|
|
539
|
+
error: safeJsonTryParse(parsed.errorJson ?? null),
|
|
540
|
+
},
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
result.sort((a, b) => a.order - b.order);
|
|
545
|
+
return result.map((r) => r.step);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
async listRuns(params?: ListRunsParams): Promise<ListedRun[]> {
|
|
549
|
+
const limitRaw = params?.limit ?? 50;
|
|
550
|
+
const offsetRaw = params?.offset ?? 0;
|
|
551
|
+
const limit = Number.isFinite(limitRaw) ? Math.max(1, Math.floor(limitRaw)) : 50;
|
|
552
|
+
const offset = Number.isFinite(offsetRaw) ? Math.max(0, Math.floor(offsetRaw)) : 0;
|
|
553
|
+
const status = params?.status;
|
|
554
|
+
const workflow = params?.workflow;
|
|
555
|
+
|
|
556
|
+
if (status && workflow) {
|
|
557
|
+
const workflowIndexKey = keys.workflowRuns(this.prefix, workflow);
|
|
558
|
+
const result: ListedRun[] = [];
|
|
559
|
+
let cursor = 0;
|
|
560
|
+
let skipped = 0;
|
|
561
|
+
const batchSize = Math.max(100, limit * 2);
|
|
562
|
+
|
|
563
|
+
while (result.length < limit) {
|
|
564
|
+
const runIds = await this.redis.zrevrange(workflowIndexKey, cursor, cursor + batchSize - 1);
|
|
565
|
+
if (runIds.length === 0) break;
|
|
566
|
+
cursor += runIds.length;
|
|
567
|
+
|
|
568
|
+
for (const runId of runIds) {
|
|
569
|
+
const run = await this.getRun(runId);
|
|
570
|
+
if (!run || run.status !== status) continue;
|
|
571
|
+
|
|
572
|
+
if (skipped < offset) {
|
|
573
|
+
skipped += 1;
|
|
574
|
+
continue;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
result.push(this.toListedRun(run));
|
|
578
|
+
if (result.length >= limit) break;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
return result;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
let indexKey = keys.runsCreated(this.prefix);
|
|
586
|
+
if (status) indexKey = keys.runsStatus(this.prefix, status);
|
|
587
|
+
if (workflow) indexKey = keys.workflowRuns(this.prefix, workflow);
|
|
588
|
+
|
|
589
|
+
const runIds = await this.redis.zrevrange(indexKey, offset, offset + limit - 1);
|
|
590
|
+
const runs: ListedRun[] = [];
|
|
591
|
+
for (const runId of runIds) {
|
|
592
|
+
const run = await this.getRun(runId);
|
|
593
|
+
if (!run) continue;
|
|
594
|
+
runs.push(this.toListedRun(run));
|
|
595
|
+
}
|
|
596
|
+
return runs;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
makeRunHandle<TOutput = unknown>(runId: string): RunHandle<TOutput> {
|
|
600
|
+
return {
|
|
601
|
+
id: runId,
|
|
602
|
+
getState: () => this.getRun(runId),
|
|
603
|
+
result: (options) => this.waitForResult<TOutput>(runId, options),
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
async waitForResult<TOutput = unknown>(
|
|
608
|
+
runId: string,
|
|
609
|
+
options?: { timeoutMs?: number; pollMs?: number },
|
|
610
|
+
): Promise<TOutput> {
|
|
611
|
+
const timeoutMs = options?.timeoutMs ?? 30_000;
|
|
612
|
+
const pollMs = options?.pollMs ?? 250;
|
|
613
|
+
const deadline = nowMs() + timeoutMs;
|
|
614
|
+
const missingGraceMs = Math.max(250, Math.min(2000, pollMs * 4));
|
|
615
|
+
|
|
616
|
+
let missingSince: number | null = null;
|
|
617
|
+
let seenState = false;
|
|
618
|
+
|
|
619
|
+
while (true) {
|
|
620
|
+
const state = await this.getRun(runId);
|
|
621
|
+
if (!state) {
|
|
622
|
+
const t = nowMs();
|
|
623
|
+
if (missingSince == null) missingSince = t;
|
|
624
|
+
|
|
625
|
+
if (t - missingSince >= missingGraceMs) {
|
|
626
|
+
if (seenState) {
|
|
627
|
+
throw new Error(`Run state unavailable: ${runId}`);
|
|
628
|
+
}
|
|
629
|
+
throw new Error(`Run not found: ${runId}`);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
if (t > deadline) throw new TimeoutError(`Timed out waiting for run result (${runId})`);
|
|
633
|
+
await sleep(pollMs);
|
|
634
|
+
continue;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
seenState = true;
|
|
638
|
+
missingSince = null;
|
|
639
|
+
|
|
640
|
+
if (state.status === "succeeded") return state.output as TOutput;
|
|
641
|
+
if (state.status === "failed") throw new Error(`Run failed: ${safeJsonStringify(state.error)}`);
|
|
642
|
+
if (state.status === "canceled") throw new CanceledError(state.cancelReason ? `Run canceled: ${state.cancelReason}` : "Run canceled");
|
|
643
|
+
|
|
644
|
+
if (nowMs() > deadline) throw new TimeoutError(`Timed out waiting for run result (${runId})`);
|
|
645
|
+
await sleep(pollMs);
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
async syncRegistry(registry: WorkflowRegistry, options?: SyncRegistryOptions): Promise<void> {
|
|
650
|
+
const defs = registry.list();
|
|
651
|
+
const syncStartedAt = nowMs();
|
|
652
|
+
const owner = options?.owner?.trim() || defaultRegistryOwner();
|
|
653
|
+
const registeredNames = new Set(defs.map((def) => def.options.name));
|
|
654
|
+
|
|
655
|
+
await this.cleanupStaleWorkflows(registeredNames, syncStartedAt, owner);
|
|
656
|
+
|
|
657
|
+
for (const def of defs) {
|
|
658
|
+
const name = def.options.name;
|
|
659
|
+
const queue = def.options.queue ?? "default";
|
|
660
|
+
const triggers = def.options.triggers ?? {};
|
|
661
|
+
const retries = def.options.retries ?? {};
|
|
662
|
+
const updatedAt = nowMs();
|
|
663
|
+
|
|
664
|
+
const customCronIds = new Set<string>();
|
|
665
|
+
for (const cron of triggers.cron ?? []) {
|
|
666
|
+
if (!cron.id) continue;
|
|
667
|
+
if (customCronIds.has(cron.id)) {
|
|
668
|
+
throw new Error(`Duplicate cron trigger id '${cron.id}' in workflow '${name}'`);
|
|
669
|
+
}
|
|
670
|
+
customCronIds.add(cron.id);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
const workflowKey = keys.workflow(this.prefix, name);
|
|
674
|
+
await this.redis.sadd(keys.workflows(this.prefix), name);
|
|
675
|
+
|
|
676
|
+
const prevEvents = safeJsonTryParse<string[]>(await this.redis.hget(workflowKey, "eventsJson")) ?? [];
|
|
677
|
+
const nextEvents = Array.from(new Set(triggers.events ?? [])).sort();
|
|
678
|
+
|
|
679
|
+
for (const ev of prevEvents) {
|
|
680
|
+
if (!nextEvents.includes(ev)) {
|
|
681
|
+
await this.redis.srem(keys.eventWorkflows(this.prefix, ev), name);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
for (const ev of nextEvents) {
|
|
686
|
+
if (!prevEvents.includes(ev)) {
|
|
687
|
+
await this.redis.sadd(keys.eventWorkflows(this.prefix, ev), name);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
const prevCronIds = safeJsonTryParse<string[]>(await this.redis.hget(workflowKey, "cronIdsJson")) ?? [];
|
|
692
|
+
const nextCronIds = (triggers.cron ?? []).map((c) => this.computeCronId(name, c.id, c.expression, c.timezone, c.input));
|
|
693
|
+
|
|
694
|
+
const nextCronIdSet = new Set(nextCronIds);
|
|
695
|
+
for (const oldId of prevCronIds) {
|
|
696
|
+
if (!nextCronIdSet.has(oldId)) {
|
|
697
|
+
await this.redis.hdel(keys.cronDef(this.prefix), oldId);
|
|
698
|
+
await this.redis.zrem(keys.cronNext(this.prefix), oldId);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
for (let i = 0; i < (triggers.cron ?? []).length; i++) {
|
|
703
|
+
const cron = triggers.cron![i]!;
|
|
704
|
+
const cronId = nextCronIds[i]!;
|
|
705
|
+
const cronInput = Object.prototype.hasOwnProperty.call(cron, "input") ? cron.input : {};
|
|
706
|
+
|
|
707
|
+
const cronDef = {
|
|
708
|
+
id: cronId,
|
|
709
|
+
workflow: name,
|
|
710
|
+
queue,
|
|
711
|
+
expression: cron.expression,
|
|
712
|
+
timezone: cron.timezone,
|
|
713
|
+
inputJson: safeJsonStringify(cronInput),
|
|
714
|
+
};
|
|
715
|
+
|
|
716
|
+
await this.redis.hset(keys.cronDef(this.prefix), { [cronId]: safeJsonStringify(cronDef) });
|
|
717
|
+
|
|
718
|
+
const nextAt = this.computeNextCronAtMs(cron.expression, cron.timezone);
|
|
719
|
+
if (nextAt != null) {
|
|
720
|
+
await this.redis.zadd(keys.cronNext(this.prefix), nextAt, cronId);
|
|
721
|
+
} else {
|
|
722
|
+
await this.redis.zrem(keys.cronNext(this.prefix), cronId);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
const meta: Record<string, string> = {
|
|
727
|
+
name,
|
|
728
|
+
queue,
|
|
729
|
+
owner,
|
|
730
|
+
updatedAt: String(updatedAt),
|
|
731
|
+
triggersJson: safeJsonStringify(triggers),
|
|
732
|
+
retriesJson: safeJsonStringify(retries),
|
|
733
|
+
eventsJson: safeJsonStringify(nextEvents),
|
|
734
|
+
cronIdsJson: safeJsonStringify(nextCronIds),
|
|
735
|
+
};
|
|
736
|
+
|
|
737
|
+
await this.redis.hset(workflowKey, meta);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// ----------------------
|
|
742
|
+
// Internal run utilities
|
|
743
|
+
// ----------------------
|
|
744
|
+
|
|
745
|
+
private async getQueueForWorkflow(workflowName: string): Promise<string | null> {
|
|
746
|
+
const metaKey = keys.workflow(this.prefix, workflowName);
|
|
747
|
+
return await this.redis.hget(metaKey, "queue");
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
private async getMaxAttemptsForWorkflow(workflowName: string): Promise<number | null> {
|
|
751
|
+
const metaKey = keys.workflow(this.prefix, workflowName);
|
|
752
|
+
const retriesJson = await this.redis.hget(metaKey, "retriesJson");
|
|
753
|
+
if (!retriesJson) return null;
|
|
754
|
+
const retries = safeJsonParse<any>(retriesJson);
|
|
755
|
+
const maxAttempts = retries?.maxAttempts;
|
|
756
|
+
if (typeof maxAttempts === "number" && Number.isFinite(maxAttempts) && maxAttempts > 0) return maxAttempts;
|
|
757
|
+
return null;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
private computeCronId(
|
|
761
|
+
workflowName: string,
|
|
762
|
+
userId: string | undefined,
|
|
763
|
+
expression: string,
|
|
764
|
+
timezone: string | undefined,
|
|
765
|
+
input: unknown,
|
|
766
|
+
): string {
|
|
767
|
+
if (userId) {
|
|
768
|
+
const stable = safeJsonStringify({ workflowName, userId });
|
|
769
|
+
const hash = createHash("sha256").update(stable).digest("hex").slice(0, 24);
|
|
770
|
+
return `cron_user_${hash}`;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
const stable = safeJsonStringify({ workflowName, expression, timezone: timezone ?? null, input });
|
|
774
|
+
const hash = createHash("sha256").update(stable).digest("hex").slice(0, 24);
|
|
775
|
+
return `cron_auto_${hash}`;
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
private async cleanupStaleWorkflows(
|
|
779
|
+
registeredNames: Set<string>,
|
|
780
|
+
syncStartedAt: number,
|
|
781
|
+
owner: string,
|
|
782
|
+
): Promise<void> {
|
|
783
|
+
const existingNames = await this.redis.smembers(keys.workflows(this.prefix));
|
|
784
|
+
|
|
785
|
+
for (const existingName of existingNames) {
|
|
786
|
+
if (registeredNames.has(existingName)) continue;
|
|
787
|
+
|
|
788
|
+
const workflowKey = keys.workflow(this.prefix, existingName);
|
|
789
|
+
const workflowOwner = (await this.redis.hget(workflowKey, "owner")) ?? "";
|
|
790
|
+
if (!workflowOwner || workflowOwner !== owner) {
|
|
791
|
+
continue;
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
const updatedAt = Number((await this.redis.hget(workflowKey, "updatedAt")) ?? "0");
|
|
795
|
+
if (Number.isFinite(updatedAt) && updatedAt > 0 && syncStartedAt - updatedAt < STALE_WORKFLOW_GRACE_MS) {
|
|
796
|
+
continue;
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
await this.deleteWorkflowMetadata(existingName);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
private async deleteWorkflowMetadata(workflowName: string): Promise<void> {
|
|
804
|
+
const workflowKey = keys.workflow(this.prefix, workflowName);
|
|
805
|
+
|
|
806
|
+
const prevEvents = safeJsonTryParse<string[]>(await this.redis.hget(workflowKey, "eventsJson")) ?? [];
|
|
807
|
+
for (const eventName of prevEvents) {
|
|
808
|
+
await this.redis.srem(keys.eventWorkflows(this.prefix, eventName), workflowName);
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
const prevCronIds = safeJsonTryParse<string[]>(await this.redis.hget(workflowKey, "cronIdsJson")) ?? [];
|
|
812
|
+
for (const cronId of prevCronIds) {
|
|
813
|
+
await this.redis.hdel(keys.cronDef(this.prefix), cronId);
|
|
814
|
+
await this.redis.zrem(keys.cronNext(this.prefix), cronId);
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
await this.redis.del(workflowKey);
|
|
818
|
+
await this.redis.srem(keys.workflows(this.prefix), workflowName);
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
private toListedRun(run: RunState): ListedRun {
|
|
822
|
+
return {
|
|
823
|
+
id: run.id,
|
|
824
|
+
workflow: run.workflow,
|
|
825
|
+
queue: run.queue,
|
|
826
|
+
status: run.status,
|
|
827
|
+
error: run.error,
|
|
828
|
+
createdAt: run.createdAt,
|
|
829
|
+
availableAt: run.availableAt,
|
|
830
|
+
startedAt: run.startedAt,
|
|
831
|
+
finishedAt: run.finishedAt,
|
|
832
|
+
attempt: run.attempt,
|
|
833
|
+
maxAttempts: run.maxAttempts,
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
private computeNextCronAtMs(expression: string, timezone?: string): number | null {
|
|
838
|
+
try {
|
|
839
|
+
const job = new Cron(expression, { timezone, catch: true });
|
|
840
|
+
const next = job.nextRun();
|
|
841
|
+
job.stop();
|
|
842
|
+
if (!next) return null;
|
|
843
|
+
return next.getTime();
|
|
844
|
+
} catch {
|
|
845
|
+
return null;
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
private async repairExistingRunPlacement(runId: string): Promise<void> {
|
|
850
|
+
const runKey = keys.run(this.prefix, runId);
|
|
851
|
+
const run = await this.redis.hgetall(runKey);
|
|
852
|
+
if (!run || Object.keys(run).length === 0) return;
|
|
853
|
+
|
|
854
|
+
const status = (run.status as RunStatus | undefined) ?? undefined;
|
|
855
|
+
const queue = run.queue ?? "default";
|
|
856
|
+
const workflow = run.workflow ?? "";
|
|
857
|
+
const createdAtRaw = Number(run.createdAt ?? "0");
|
|
858
|
+
const createdAt = Number.isFinite(createdAtRaw) && createdAtRaw > 0 ? createdAtRaw : nowMs();
|
|
859
|
+
|
|
860
|
+
// Repair indexes in case a producer crashed mid-enqueue.
|
|
861
|
+
await this.redis.zadd(keys.runsCreated(this.prefix), createdAt, runId);
|
|
862
|
+
if (status) {
|
|
863
|
+
await this.redis.zadd(keys.runsStatus(this.prefix, status), createdAt, runId);
|
|
864
|
+
}
|
|
865
|
+
if (workflow) {
|
|
866
|
+
await this.redis.zadd(keys.workflowRuns(this.prefix, workflow), createdAt, runId);
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
if (status === "queued") {
|
|
870
|
+
const readyKey = keys.queueReady(this.prefix, queue);
|
|
871
|
+
const processingKey = keys.queueProcessing(this.prefix, queue);
|
|
872
|
+
const [ready, processing] = await Promise.all([
|
|
873
|
+
this.redis.lrange(readyKey, 0, -1),
|
|
874
|
+
this.redis.lrange(processingKey, 0, -1),
|
|
875
|
+
]);
|
|
876
|
+
|
|
877
|
+
if (!ready.includes(runId) && !processing.includes(runId)) {
|
|
878
|
+
await this.redis.lpush(readyKey, runId);
|
|
879
|
+
}
|
|
880
|
+
return;
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
if (status === "scheduled") {
|
|
884
|
+
const scheduledKey = keys.queueScheduled(this.prefix, queue);
|
|
885
|
+
const score = await this.redis.send("ZSCORE", [scheduledKey, runId]);
|
|
886
|
+
if (score == null) {
|
|
887
|
+
const availableAtRaw = Number(run.availableAt ?? "0");
|
|
888
|
+
const availableAt = Number.isFinite(availableAtRaw) && availableAtRaw > 0 ? availableAtRaw : nowMs();
|
|
889
|
+
await this.redis.zadd(scheduledKey, availableAt, runId);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
private async requestCancellation(
|
|
895
|
+
runId: string,
|
|
896
|
+
requestedAt: number,
|
|
897
|
+
reason: string,
|
|
898
|
+
): Promise<RunStatus | "__missing__"> {
|
|
899
|
+
const result = await this.redis.send("EVAL", [
|
|
900
|
+
REQUEST_CANCELLATION_LUA,
|
|
901
|
+
"1",
|
|
902
|
+
keys.run(this.prefix, runId),
|
|
903
|
+
String(requestedAt),
|
|
904
|
+
reason,
|
|
905
|
+
]);
|
|
906
|
+
|
|
907
|
+
if (typeof result !== "string") return "__missing__";
|
|
908
|
+
if (result === "__missing__") return "__missing__";
|
|
909
|
+
return result as RunStatus;
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
private async finalizeCanceledRunImmediately(runId: string, queue: string, finishedAt: number): Promise<void> {
|
|
913
|
+
const runKey = keys.run(this.prefix, runId);
|
|
914
|
+
await this.redis.zrem(keys.queueScheduled(this.prefix, queue), runId);
|
|
915
|
+
await this.redis.lrem(keys.queueReady(this.prefix, queue), 0, runId);
|
|
916
|
+
// Also clean processing list — the run could have been moved there
|
|
917
|
+
// between cancelRun's status check and this cleanup.
|
|
918
|
+
await this.redis.lrem(keys.queueProcessing(this.prefix, queue), 0, runId);
|
|
919
|
+
await this.redis.hdel(runKey, "availableAt");
|
|
920
|
+
await this.redis.hdel(runKey, "outputJson");
|
|
921
|
+
await this.redis.hdel(runKey, "errorJson");
|
|
922
|
+
await this.redis.hset(runKey, { finishedAt: String(finishedAt) });
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
private async enqueueRun<TOutput>(args: {
|
|
926
|
+
workflowName: string;
|
|
927
|
+
queue: string;
|
|
928
|
+
input: unknown;
|
|
929
|
+
availableAt?: Date;
|
|
930
|
+
idempotencyKey?: string;
|
|
931
|
+
idempotencyTtl?: number;
|
|
932
|
+
maxAttempts: number;
|
|
933
|
+
}): Promise<RunHandle<TOutput>> {
|
|
934
|
+
const createdAt = nowMs();
|
|
935
|
+
const availableAtMs = args.availableAt ? args.availableAt.getTime() : undefined;
|
|
936
|
+
const shouldSchedule = availableAtMs != null && availableAtMs > createdAt;
|
|
937
|
+
const runId = `run_${crypto.randomUUID()}`;
|
|
938
|
+
const status: RunStatus = shouldSchedule ? "scheduled" : "queued";
|
|
939
|
+
const runKey = keys.run(this.prefix, runId);
|
|
940
|
+
const idempotencyRedisKey = args.idempotencyKey
|
|
941
|
+
? keys.idempotency(this.prefix, args.workflowName, args.idempotencyKey)
|
|
942
|
+
: keys.idempotency(this.prefix, "__unused__", runId);
|
|
943
|
+
|
|
944
|
+
const result = await this.redis.send("EVAL", [
|
|
945
|
+
ENQUEUE_RUN_LUA,
|
|
946
|
+
"7",
|
|
947
|
+
runKey,
|
|
948
|
+
keys.runsCreated(this.prefix),
|
|
949
|
+
keys.runsStatus(this.prefix, status),
|
|
950
|
+
keys.workflowRuns(this.prefix, args.workflowName),
|
|
951
|
+
keys.queueReady(this.prefix, args.queue),
|
|
952
|
+
keys.queueScheduled(this.prefix, args.queue),
|
|
953
|
+
idempotencyRedisKey,
|
|
954
|
+
runId,
|
|
955
|
+
args.workflowName,
|
|
956
|
+
args.queue,
|
|
957
|
+
status,
|
|
958
|
+
safeJsonStringify(args.input),
|
|
959
|
+
String(createdAt),
|
|
960
|
+
shouldSchedule ? String(availableAtMs) : "",
|
|
961
|
+
String(args.maxAttempts),
|
|
962
|
+
args.idempotencyKey ? "1" : "0",
|
|
963
|
+
String(args.idempotencyTtl ?? IDEMPOTENCY_TTL_SEC),
|
|
964
|
+
keys.run(this.prefix, ""),
|
|
965
|
+
]);
|
|
966
|
+
|
|
967
|
+
const parsed = parseEnqueueScriptResult(result);
|
|
968
|
+
if (!parsed) {
|
|
969
|
+
throw new Error(`Failed to enqueue run '${args.workflowName}': unexpected Redis response`);
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
if (parsed.kind === "existing") {
|
|
973
|
+
await this.repairExistingRunPlacement(parsed.runId);
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
return this.makeRunHandle(parsed.runId);
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
async transitionRunStatus(runId: string, nextStatus: RunStatus, updatedAt: number): Promise<void> {
|
|
980
|
+
// Atomic: reads prev status, zrem from old index, hset new status,
|
|
981
|
+
// hdel availableAt if not scheduled, zadd to new index.
|
|
982
|
+
const statusIndexPrefix = keys.runsStatus(this.prefix, ""); // e.g. "redflow:v1:runs:status:"
|
|
983
|
+
await this.redis.send("EVAL", [
|
|
984
|
+
TRANSITION_RUN_STATUS_LUA,
|
|
985
|
+
"2",
|
|
986
|
+
keys.run(this.prefix, runId),
|
|
987
|
+
keys.runsStatus(this.prefix, nextStatus),
|
|
988
|
+
runId,
|
|
989
|
+
nextStatus,
|
|
990
|
+
String(updatedAt),
|
|
991
|
+
statusIndexPrefix,
|
|
992
|
+
]);
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
/**
|
|
996
|
+
* Atomically schedule a retry: sets availableAt + errorJson on the run,
|
|
997
|
+
* transitions status to "scheduled", and adds to queue's scheduled set.
|
|
998
|
+
* Prevents stranded runs if the worker crashes mid-retry.
|
|
999
|
+
*/
|
|
1000
|
+
async scheduleRetry(runId: string, args: {
|
|
1001
|
+
queue: string;
|
|
1002
|
+
nextAt: number;
|
|
1003
|
+
errorJson: string;
|
|
1004
|
+
updatedAt: number;
|
|
1005
|
+
}): Promise<void> {
|
|
1006
|
+
const statusIndexPrefix = keys.runsStatus(this.prefix, "");
|
|
1007
|
+
await this.redis.send("EVAL", [
|
|
1008
|
+
SCHEDULE_RETRY_LUA,
|
|
1009
|
+
"3",
|
|
1010
|
+
keys.run(this.prefix, runId),
|
|
1011
|
+
keys.runsStatus(this.prefix, "scheduled"),
|
|
1012
|
+
keys.queueScheduled(this.prefix, args.queue),
|
|
1013
|
+
runId,
|
|
1014
|
+
String(args.updatedAt),
|
|
1015
|
+
String(args.nextAt),
|
|
1016
|
+
args.errorJson,
|
|
1017
|
+
statusIndexPrefix,
|
|
1018
|
+
]);
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
async transitionRunStatusIfCurrent(
|
|
1022
|
+
runId: string,
|
|
1023
|
+
expectedStatus: RunStatus,
|
|
1024
|
+
nextStatus: RunStatus,
|
|
1025
|
+
updatedAt: number,
|
|
1026
|
+
): Promise<boolean> {
|
|
1027
|
+
const changed = await this.redis.send("EVAL", [
|
|
1028
|
+
TRANSITION_IF_CURRENT_STATUS_LUA,
|
|
1029
|
+
"3",
|
|
1030
|
+
keys.run(this.prefix, runId),
|
|
1031
|
+
keys.runsStatus(this.prefix, expectedStatus),
|
|
1032
|
+
keys.runsStatus(this.prefix, nextStatus),
|
|
1033
|
+
runId,
|
|
1034
|
+
expectedStatus,
|
|
1035
|
+
nextStatus,
|
|
1036
|
+
String(updatedAt),
|
|
1037
|
+
]);
|
|
1038
|
+
return changed === 1 || changed === "1";
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
async finalizeRun(runId: string, args: { status: RunStatus; outputJson?: string; errorJson?: string; finishedAt: number }): Promise<void> {
|
|
1042
|
+
const result = await this.redis.send("EVAL", [
|
|
1043
|
+
FINALIZE_TERMINAL_RUN_LUA,
|
|
1044
|
+
"7",
|
|
1045
|
+
keys.run(this.prefix, runId),
|
|
1046
|
+
keys.runsStatus(this.prefix, "scheduled"),
|
|
1047
|
+
keys.runsStatus(this.prefix, "queued"),
|
|
1048
|
+
keys.runsStatus(this.prefix, "running"),
|
|
1049
|
+
keys.runsStatus(this.prefix, "succeeded"),
|
|
1050
|
+
keys.runsStatus(this.prefix, "failed"),
|
|
1051
|
+
keys.runsStatus(this.prefix, "canceled"),
|
|
1052
|
+
runId,
|
|
1053
|
+
args.status,
|
|
1054
|
+
String(args.finishedAt),
|
|
1055
|
+
args.outputJson ?? "",
|
|
1056
|
+
args.errorJson ?? "",
|
|
1057
|
+
args.outputJson != null ? "1" : "0",
|
|
1058
|
+
args.errorJson != null ? "1" : "0",
|
|
1059
|
+
]);
|
|
1060
|
+
|
|
1061
|
+
if (result === "__missing__") {
|
|
1062
|
+
throw new Error(`Run not found: ${runId}`);
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
export function createClient(options?: CreateClientOptions): RedflowClient {
|
|
1068
|
+
const prefix = options?.prefix ?? defaultPrefix();
|
|
1069
|
+
const redis = options?.redis ?? (options?.url ? new RedisClient(options.url) : defaultRedis);
|
|
1070
|
+
return new RedflowClient(redis, prefix);
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
export function validateInputWithSchema(schema: ZodTypeAny, input: unknown): unknown {
|
|
1074
|
+
try {
|
|
1075
|
+
return schema.parse(input);
|
|
1076
|
+
} catch (err: any) {
|
|
1077
|
+
// zod v4 error format is stable enough for display, but we store it as raw.
|
|
1078
|
+
throw new InputValidationError("Workflow input validation failed", err);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
export function isTerminalStatus(status: RunStatus): boolean {
|
|
1083
|
+
return status === "succeeded" || status === "failed" || status === "canceled";
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
export function isRetryableError(err: unknown): boolean {
|
|
1087
|
+
if (err instanceof InputValidationError) return false;
|
|
1088
|
+
if (err instanceof UnknownWorkflowError) return false;
|
|
1089
|
+
if (err instanceof OutputSerializationError) return false;
|
|
1090
|
+
if (err instanceof CanceledError) return false;
|
|
1091
|
+
if (err instanceof NonRetriableError) return false;
|
|
1092
|
+
return true;
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
export function computeRetryDelayMs(attempt: number): number {
|
|
1096
|
+
// attempt is 1-based (first attempt = 1). Backoff starts on retry.
|
|
1097
|
+
const base = 250;
|
|
1098
|
+
const max = 30_000;
|
|
1099
|
+
const exp = Math.min(max, base * Math.pow(2, Math.max(0, attempt - 1)));
|
|
1100
|
+
const jitter = Math.floor(Math.random() * 100);
|
|
1101
|
+
return exp + jitter;
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
export function makeErrorJson(err: unknown): string {
|
|
1105
|
+
try {
|
|
1106
|
+
const serialized: SerializedError = serializeError(err);
|
|
1107
|
+
return safeJsonStringify(serialized);
|
|
1108
|
+
} catch {
|
|
1109
|
+
return '{"name":"Error","message":"Failed to serialize original error","kind":"error"}';
|
|
1110
|
+
}
|
|
1111
|
+
}
|