effect-mq 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +85 -17
  2. package/dist/Flow.d.ts +381 -0
  3. package/dist/Flow.d.ts.map +1 -0
  4. package/dist/Flow.js +340 -0
  5. package/dist/Flow.js.map +1 -0
  6. package/dist/Job.d.ts +31 -6
  7. package/dist/Job.d.ts.map +1 -1
  8. package/dist/Job.js +16 -2
  9. package/dist/Job.js.map +1 -1
  10. package/dist/JobStore.d.ts +353 -13
  11. package/dist/JobStore.d.ts.map +1 -1
  12. package/dist/JobStore.js +10 -0
  13. package/dist/JobStore.js.map +1 -1
  14. package/dist/MemoryJobStore.d.ts.map +1 -1
  15. package/dist/MemoryJobStore.js +361 -21
  16. package/dist/MemoryJobStore.js.map +1 -1
  17. package/dist/Metrics.d.ts +31 -0
  18. package/dist/Metrics.d.ts.map +1 -1
  19. package/dist/Metrics.js +39 -0
  20. package/dist/Metrics.js.map +1 -1
  21. package/dist/Worker.d.ts +120 -11
  22. package/dist/Worker.d.ts.map +1 -1
  23. package/dist/Worker.js +452 -26
  24. package/dist/Worker.js.map +1 -1
  25. package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
  26. package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
  27. package/dist/drizzle-postgres/DrizzleJobStore.js +678 -80
  28. package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
  29. package/dist/drizzle-postgres/schema.d.ts +293 -3
  30. package/dist/drizzle-postgres/schema.d.ts.map +1 -1
  31. package/dist/drizzle-postgres/schema.js +66 -1
  32. package/dist/drizzle-postgres/schema.js.map +1 -1
  33. package/dist/index.d.ts +7 -0
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +7 -0
  36. package/dist/index.js.map +1 -1
  37. package/dist/redis/RedisJobStore.d.ts +53 -0
  38. package/dist/redis/RedisJobStore.d.ts.map +1 -1
  39. package/dist/redis/RedisJobStore.js +402 -50
  40. package/dist/redis/RedisJobStore.js.map +1 -1
  41. package/dist/redis/scripts.d.ts +213 -35
  42. package/dist/redis/scripts.d.ts.map +1 -1
  43. package/dist/redis/scripts.js +689 -73
  44. package/dist/redis/scripts.js.map +1 -1
  45. package/dist/testing/conformance.d.ts +6 -0
  46. package/dist/testing/conformance.d.ts.map +1 -1
  47. package/dist/testing/conformance.js +855 -12
  48. package/dist/testing/conformance.js.map +1 -1
  49. package/package.json +1 -1
  50. package/src/Flow.ts +778 -0
  51. package/src/Job.ts +35 -11
  52. package/src/JobStore.ts +377 -12
  53. package/src/MemoryJobStore.ts +396 -25
  54. package/src/Metrics.ts +43 -0
  55. package/src/Worker.ts +726 -37
  56. package/src/drizzle-postgres/DrizzleJobStore.ts +844 -81
  57. package/src/drizzle-postgres/schema.ts +92 -0
  58. package/src/index.ts +8 -0
  59. package/src/redis/RedisJobStore.ts +540 -39
  60. package/src/redis/scripts.ts +751 -78
  61. package/src/testing/conformance.ts +1088 -12
@@ -21,6 +21,19 @@
21
21
  * - `p:delayed:<queue>` ZSET, score `runAt`
22
22
  * - `p:active` ZSET, score `lockExpiresAt`
23
23
  * - `p:all` ZSET, score `enqueuedAt` (list pagination)
24
+ * - `p:byname:<name>` ZSET, score `enqueuedAt` (list name index;
25
+ * optional, `RedisJobStoreOptions.indexes.name`)
26
+ * - `p:byqueue:<queue>` ZSET, score `enqueuedAt` (list queue index;
27
+ * optional, `RedisJobStoreOptions.indexes.queue`)
28
+ * - `p:index:<kind>:ready` STRING, millis timestamp of the last index
29
+ * reconcile's start for `<kind>` (name |
30
+ * queue). Absent: the next enabled boot does a
31
+ * full ZSCAN rebuild. Present: enabled boots
32
+ * heal the tail (rows enqueued since the value
33
+ * minus a safety margin) and re-stamp it —
34
+ * closing the rolling-deploy window where
35
+ * index-less writers inserted rows after the
36
+ * marker landed
24
37
  * - `p:finished:<state>` ZSET, score `finishedAt` (history TTL)
25
38
  * - `p:terminal:<name>:<state>` ZSET, score `finishedAt` (keep pruning)
26
39
  * - `p:counts` HASH `<queue>|<state>` -> integer
@@ -28,6 +41,23 @@
28
41
  * - `p:schedules` / `p:schedule:<key>` ZSET by nextRunAt + HASH per record
29
42
  * - `p:dedupe:<name>\0<key>` HASH {jobId, expiresAt} + `p:dedupes` index
30
43
  * ZSET (score = window expiry, +inf = pending)
44
+ * - `p:flowchild:<flowId>\0<childKey>` HASH of one flow dependency row
45
+ * - `p:flowchildren:<flowId>` ZSET (score 0, member = childKey; ZRANGEBYLEX
46
+ * gives child-key order + cursor pagination)
47
+ * - `p:flowpending` ZSET, member `<flowId>\0<childKey>`, score =
48
+ * sweep-eligibility timestamp (pendingSince at
49
+ * staging, re-armed on sweep return)
50
+ * - `p:flowcascade` ZSET, score 0, member `<flowId>\0<childKey>`
51
+ * (cancels still owed to child stores)
52
+ * - `p:flowoutbox` ZSET, score = seq from `p:flowoutbox:seq`,
53
+ * member `<seq>\0<report json>` — undelivered
54
+ * child-result reports (see OutboxEntry). The
55
+ * seq prefix makes every member id a peek
56
+ * cursor that survives deletions (see
57
+ * peekOutbox in RedisJobStore)
58
+ *
59
+ * `waiting-children` parents live only in the job hash, `p:all`, and
60
+ * `p:counts` — never in a pending zset, so `claim` can never return them.
31
61
  *
32
62
  * @since 0.2.0
33
63
  */
@@ -35,8 +65,12 @@ import { Redis } from "effect/unstable/persistence";
35
65
  /**
36
66
  * Shared helpers textually prepended to every script (the `Redis.script`
37
67
  * runner has no include mechanism). `ARGV[1]` is always the key prefix.
68
+ * Built once per store instance: `insertJobRow` writes only the list indexes
69
+ * enabled by `IndexConfig` (`deleteJob` clears both unconditionally — a ZREM
70
+ * on a missing key is free, and stray entries from an earlier configuration
71
+ * must never outlive their job).
38
72
  */
39
- const HELPERS = `
73
+ export const helpers = (indexes) => `
40
74
  local prefix = ARGV[1]
41
75
  local function fmt(x) return string.format("%.0f", x) end
42
76
  local function jobKey(id) return prefix .. ":job:" .. id end
@@ -44,6 +78,8 @@ local function attemptsKey(id) return prefix .. ":attempts:" .. id end
44
78
  local function waitingKey(queue) return prefix .. ":waiting:" .. queue end
45
79
  local function delayedKey(queue) return prefix .. ":delayed:" .. queue end
46
80
  local function terminalKey(name, state) return prefix .. ":terminal:" .. name .. ":" .. state end
81
+ local function bynameKey(name) return prefix .. ":byname:" .. name end
82
+ local function byqueueKey(queue) return prefix .. ":byqueue:" .. queue end
47
83
  -- Waiting order: score = -priority (higher priority first, full number range);
48
84
  -- FIFO within a priority via lexicographic members "<seq %016d>:<id>". A
49
85
  -- composite numeric score would clip either priority or seq past float53.
@@ -70,7 +106,79 @@ local function appendAttempt(id, outcome, startedAt, finishedAt, exitJson)
70
106
  '{"attempt":' .. n .. ',"startedAt":' .. started .. ',"finishedAt":' .. finishedAt ..
71
107
  ',"outcome":"' .. outcome .. '"' .. ex .. '}')
72
108
  end
73
- -- Remove a job and every index entry that references it.
109
+ -- The outbox invariant: every operation that moves a job carrying a parent
110
+ -- envelope INTO a terminal state appends its child-result report here, in
111
+ -- the same script. MUST run after the terminal fields (exit, failedReason)
112
+ -- are written — the report is built from the hash. The parent envelope and
113
+ -- exit are spliced as raw JSON (never cjson-decoded: precision, surrogates);
114
+ -- exit/failedReason keys are OMITTED when absent, like the attempts ledger.
115
+ local function appendOutbox(id, outcome)
116
+ local jk = jobKey(id)
117
+ local parentJson = redis.call("HGET", jk, "parent")
118
+ if parentJson == false or parentJson == "" then return end
119
+ local exitJson = redis.call("HGET", jk, "exit")
120
+ local failedReason = redis.call("HGET", jk, "failedReason")
121
+ local ex = (exitJson == false or exitJson == "") and "" or (',"exit":' .. exitJson)
122
+ local fr = (failedReason == false or failedReason == "") and ""
123
+ or (',"failedReason":' .. cjson.encode(failedReason))
124
+ local seq = redis.call("INCR", prefix .. ":flowoutbox:seq")
125
+ redis.call("ZADD", prefix .. ":flowoutbox", seq, fmt(seq) .. "\0" ..
126
+ '{"parent":' .. parentJson .. ',"outcome":"' .. outcome .. '"' .. ex .. fr .. '}')
127
+ end
128
+ -- Flow dependency rows live in the parent store, one hash per child plus a
129
+ -- per-flow member index (childKey order via ZRANGEBYLEX). Row keys and the
130
+ -- sweep-index members join flowId and childKey with NUL — ids and keys may
131
+ -- both contain ":", so a printable separator could alias two distinct
132
+ -- (flowId, childKey) pairs onto one row.
133
+ local function flowChildKey(flowId, childKey) return prefix .. ":flowchild:" .. flowId .. "\0" .. childKey end
134
+ local function flowIndexKey(flowId) return prefix .. ":flowchildren:" .. flowId end
135
+ local function flowMember(flowId, childKey) return flowId .. "\0" .. childKey end
136
+ -- Settle-time marking: remaining pending rows flip to cancelled (NOT
137
+ -- cascaded — the sweeper still owes the child stores real cancels), moving
138
+ -- from the pending sweep index to the cascade sweep index. Returns the
139
+ -- number of rows flipped, for the flow counters.
140
+ local function markPendingRowsCancelled(flowId)
141
+ local keys = redis.call("ZRANGE", flowIndexKey(flowId), 0, -1)
142
+ local marked = 0
143
+ for i = 1, #keys do
144
+ local rk = flowChildKey(flowId, keys[i])
145
+ if redis.call("HGET", rk, "status") == "pending" then
146
+ redis.call("HSET", rk, "status", "cancelled", "cascaded", "0")
147
+ redis.call("ZREM", prefix .. ":flowpending", flowMember(flowId, keys[i]))
148
+ redis.call("ZADD", prefix .. ":flowcascade", 0, flowMember(flowId, keys[i]))
149
+ marked = marked + 1
150
+ end
151
+ end
152
+ return marked
153
+ end
154
+ -- Settle-time marking plus the manifest counters: pending -> 0, cancelled +=
155
+ -- the rows flipped (the four counters always sum to the manifest size).
156
+ local function settleMarkRows(flowId)
157
+ local marked = markPendingRowsCancelled(flowId)
158
+ local jk = jobKey(flowId)
159
+ if redis.call("HEXISTS", jk, "flowPending") == 1 then
160
+ redis.call("HSET", jk, "flowPending", "0")
161
+ if marked > 0 then redis.call("HINCRBY", jk, "flowCancelled", marked) end
162
+ end
163
+ end
164
+ -- A settled flow parent whose rows still owe cascade cancels is exempt from
165
+ -- AUTOMATIC retention (keep policies, the history sweep): deleting it would
166
+ -- delete the only record that the child stores are still owed real cancels.
167
+ -- The explicit remove verb is the operator override and is NOT exempted.
168
+ -- Cheap for non-parents: the per-flow index only exists for fanned-out jobs.
169
+ local function owesCascades(id)
170
+ if redis.call("EXISTS", flowIndexKey(id)) == 0 then return false end
171
+ local keys = redis.call("ZRANGE", flowIndexKey(id), 0, -1)
172
+ for i = 1, #keys do
173
+ local rk = flowChildKey(id, keys[i])
174
+ if redis.call("HGET", rk, "status") == "cancelled" and redis.call("HGET", rk, "cascaded") == "0" then
175
+ return true
176
+ end
177
+ end
178
+ return false
179
+ end
180
+ -- Remove a job and every index entry that references it. A flow parent takes
181
+ -- its dependency rows and their sweep-index members with it (flowId = job id).
74
182
  local function deleteJob(id)
75
183
  local jk = jobKey(id)
76
184
  local queue = redis.call("HGET", jk, "queue")
@@ -79,11 +187,20 @@ local function deleteJob(id)
79
187
  local name = redis.call("HGET", jk, "name")
80
188
  countsAdd(queue, state, -1)
81
189
  redis.call("ZREM", prefix .. ":all", id)
190
+ redis.call("ZREM", bynameKey(name), id)
191
+ redis.call("ZREM", byqueueKey(queue), id)
82
192
  redis.call("ZREM", prefix .. ":finished:" .. state, id)
83
193
  redis.call("ZREM", prefix .. ":active", id)
84
194
  redis.call("ZREM", terminalKey(name, state), id)
85
195
  remWaiting(queue, id)
86
196
  redis.call("ZREM", delayedKey(queue), id)
197
+ local children = redis.call("ZRANGE", flowIndexKey(id), 0, -1)
198
+ for i = 1, #children do
199
+ redis.call("DEL", flowChildKey(id, children[i]))
200
+ redis.call("ZREM", prefix .. ":flowpending", flowMember(id, children[i]))
201
+ redis.call("ZREM", prefix .. ":flowcascade", flowMember(id, children[i]))
202
+ end
203
+ redis.call("DEL", flowIndexKey(id))
87
204
  redis.call("DEL", jk, attemptsKey(id))
88
205
  end
89
206
  -- Terminal retention for one name+state group. Correctness over speed: the
@@ -106,9 +223,13 @@ local function applyKeep(name, state, keepJson, now)
106
223
  end
107
224
  end
108
225
  local tkey = terminalKey(name, state)
226
+ -- Retention exemption: parents still owing cascade cancels are spared
227
+ -- (they still occupy a keep-count slot, exactly like the memory driver).
109
228
  if keep.ageMs ~= nil then
110
229
  local old = redis.call("ZRANGEBYSCORE", tkey, "-inf", now - keep.ageMs)
111
- for i = 1, #old do deleteJob(old[i]) end
230
+ for i = 1, #old do
231
+ if not owesCascades(old[i]) then deleteJob(old[i]) end
232
+ end
112
233
  end
113
234
  -- Floor + clamp: a fractional/negative count must degrade like the memory
114
235
  -- driver's slice(), never error mid-script (writes before an error stick).
@@ -127,7 +248,9 @@ local function applyKeep(name, state, keepJson, now)
127
248
  if a.fa ~= b.fa then return a.fa > b.fa end
128
249
  return a.seq > b.seq
129
250
  end)
130
- for i = count + 1, #arr do deleteJob(arr[i].id) end
251
+ for i = count + 1, #arr do
252
+ if not owesCascades(arr[i].id) then deleteJob(arr[i].id) end
253
+ end
131
254
  end
132
255
  end
133
256
  local function dedupeStoreKey(name, key) return prefix .. ":dedupe:" .. name .. "\0" .. key end
@@ -155,14 +278,30 @@ local function finishCancelled(id, queue, name, startedAt, now, nowStr)
155
278
  redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
156
279
  redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
157
280
  appendAttempt(id, "cancelled", startedAt, nowStr, "")
281
+ appendOutbox(id, "cancelled")
158
282
  releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
159
283
  applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
160
284
  end
285
+ -- Index one EXISTING job row into a list index (backfill and tail heal;
286
+ -- insertJobRow covers live writes). A missing row is skipped — nothing to
287
+ -- index, and the read path self-heals whatever stale member pointed here.
288
+ local function indexInto(kind, id)
289
+ local row = redis.call("HMGET", jobKey(id), "name", "queue", "enqueuedAt")
290
+ if row[1] then
291
+ local enqueuedAt = tonumber(row[3]) or 0
292
+ if kind == "name" then
293
+ redis.call("ZADD", bynameKey(row[1]), enqueuedAt, id)
294
+ else
295
+ redis.call("ZADD", byqueueKey(row[2]), enqueuedAt, id)
296
+ end
297
+ end
298
+ end
161
299
  -- Insert one fresh job row plus every index entry. String params are stored
162
- -- verbatim (payload/metadata/backoff/keep/trace are pre-encoded JSON, "" =
163
- -- absent); priority/delayMs/now numeric-coercible.
300
+ -- verbatim (payload/metadata/backoff/keep/trace/parent are pre-encoded JSON,
301
+ -- "" = absent); priority/delayMs/now numeric-coercible. New jobs never carry
302
+ -- flow fields — only the FanOut ack writes those.
164
303
  local function insertJobRow(id, name, queue, payloadJson, metadataJson, priority,
165
- attemptsMax, backoffJson, keepJson, timeoutMs, dedupeKey, traceJson, delayMs, now, nowStr)
304
+ attemptsMax, backoffJson, keepJson, timeoutMs, dedupeKey, traceJson, parentJson, delayMs, now, nowStr)
166
305
  local seq = redis.call("INCR", prefix .. ":seq")
167
306
  local state = delayMs > 0 and "delayed" or "waiting"
168
307
  local runAt = now + delayMs
@@ -171,11 +310,12 @@ local function insertJobRow(id, name, queue, payloadJson, metadataJson, priority
171
310
  "payload", payloadJson, "metadata", metadataJson, "state", state,
172
311
  "priority", priority, "attemptsMax", attemptsMax, "attemptsMade", "0", "stalledCount", "0",
173
312
  "backoff", backoffJson, "keep", keepJson, "timeoutMs", timeoutMs,
174
- "cancelRequested", "0", "dedupeKey", dedupeKey, "trace", traceJson, "runAt", fmt(runAt), "enqueuedAt", nowStr,
313
+ "cancelRequested", "0", "dedupeKey", dedupeKey, "trace", traceJson, "parent", parentJson,
314
+ "runAt", fmt(runAt), "enqueuedAt", nowStr,
175
315
  "processedAt", "", "finishedAt", "", "exit", "", "failedReason", "",
176
316
  "lockToken", "", "lockExpiresAt", "", "seq", fmt(seq))
177
317
  redis.call("ZADD", prefix .. ":all", now, id)
178
- if state == "waiting" then
318
+ ${indexes.name ? ` redis.call("ZADD", bynameKey(name), now, id)\n` : ""}${indexes.queue ? ` redis.call("ZADD", byqueueKey(queue), now, id)\n` : ""} if state == "waiting" then
179
319
  addWaiting(queue, tonumber(priority), seq, id)
180
320
  else
181
321
  redis.call("ZADD", delayedKey(queue), runAt, id)
@@ -185,11 +325,12 @@ end
185
325
  `;
186
326
  /**
187
327
  * enqueue(prefix, idMode, id, name, queue, payloadJson, metadataJson,
188
- * priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs, now)
328
+ * priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs,
329
+ * now, dedupe..., traceJson, parentJson)
189
330
  * idMode: "user" (dedup no-op), "generated" (collision -> retry sentinel),
190
331
  * "auto" (j-<seq>, in-script collision loop).
191
332
  */
192
- export const enqueue = Redis.script((prefix, idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs, now, dedupeKey, dedupeTtlMs, dedupeExtend, dedupeReplace, traceJson) => [
333
+ export const enqueue = (HELPERS) => Redis.script((prefix, idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs, now, dedupeKey, dedupeTtlMs, dedupeExtend, dedupeReplace, traceJson, parentJson) => [
193
334
  prefix,
194
335
  idMode,
195
336
  id,
@@ -208,7 +349,8 @@ export const enqueue = Redis.script((prefix, idMode, id, name, queue, payloadJso
208
349
  dedupeTtlMs,
209
350
  dedupeExtend,
210
351
  dedupeReplace,
211
- traceJson
352
+ traceJson,
353
+ parentJson
212
354
  ], {
213
355
  numberOfKeys: 0,
214
356
  lua: `${HELPERS}
@@ -281,7 +423,7 @@ if idMode == "auto" then
281
423
  if id == "" then return '{"error":"id"}' end
282
424
  end
283
425
  insertJobRow(id, ARGV[4], queue, ARGV[6], ARGV[7], ARGV[8], ARGV[9], ARGV[10], ARGV[11],
284
- ARGV[12], dKey, ARGV[19], delayMs, now, nowStr)
426
+ ARGV[12], dKey, ARGV[19], ARGV[20], delayMs, now, nowStr)
285
427
  if dKey ~= "" then
286
428
  local sk = dedupeStoreKey(name, dKey)
287
429
  redis.call("DEL", sk)
@@ -301,7 +443,7 @@ return '{"id":' .. cjson.encode(id) .. ',"duplicate":false,"wake":true}'
301
443
  * waiting job whose name matches. Returns the claimed record (HGETALL pairs)
302
444
  * or an Empty result with the earliest matching delayed runAt.
303
445
  */
304
- export const claim = Redis.script((prefix, queue, namesJson, token, lockDurationMs, now) => [
446
+ export const claim = (HELPERS) => Redis.script((prefix, queue, namesJson, token, lockDurationMs, now) => [
305
447
  prefix,
306
448
  queue,
307
449
  namesJson,
@@ -374,7 +516,7 @@ return cjson.encode({ empty = true, nextRunAt = nextRunAt })
374
516
  * Token-guarded. Retry on a cancel-requested job finishes it as cancelled
375
517
  * (cancellation wins over revival, mirroring release/recoverStalled).
376
518
  */
377
- export const ack = Redis.script((prefix, id, token, outcomeTag, exitJson, delayMs, now) => [
519
+ export const ack = (HELPERS) => Redis.script((prefix, id, token, outcomeTag, exitJson, delayMs, now) => [
378
520
  prefix,
379
521
  id,
380
522
  token,
@@ -413,6 +555,7 @@ local function finish(newState, storeExit, outcome, ledgerExit)
413
555
  redis.call("ZADD", prefix .. ":finished:" .. newState, now, id)
414
556
  redis.call("ZADD", terminalKey(name, newState), now, id)
415
557
  appendAttempt(id, outcome, startedAt, nowStr, ledgerExit)
558
+ appendOutbox(id, newState)
416
559
  releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
417
560
  applyKeep(name, newState, redis.call("HGET", jk, "keep"), now)
418
561
  end
@@ -449,7 +592,7 @@ return '{"ok":true}'
449
592
  * release(prefix, id, token, now) — hand the job back without consuming an
450
593
  * attempt; a pending cancel wins and finishes the job instead.
451
594
  */
452
- export const release = Redis.script((prefix, id, token, now) => [prefix, id, token, now], {
595
+ export const release = (HELPERS) => Redis.script((prefix, id, token, now) => [prefix, id, token, now], {
453
596
  numberOfKeys: 0,
454
597
  lua: `${HELPERS}
455
598
  local id = ARGV[2]
@@ -479,7 +622,7 @@ return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
479
622
  * extendLocks(prefix, locksJson, durationMs, now) -> { lost, cancel }
480
623
  * Cancel-requested locks are reported, not extended.
481
624
  */
482
- export const extendLocks = Redis.script((prefix, locksJson, durationMs, now) => [prefix, locksJson, durationMs, now], {
625
+ export const extendLocks = (HELPERS) => Redis.script((prefix, locksJson, durationMs, now) => [prefix, locksJson, durationMs, now], {
483
626
  numberOfKeys: 0,
484
627
  lua: `${HELPERS}
485
628
  local locks = cjson.decode(ARGV[2])
@@ -504,7 +647,7 @@ return cjson.encode({ lost = lost, cancel = cancel })
504
647
  * recoverStalled(prefix, maxStalledCount, now) -> recovered [{id, failed}]
505
648
  * A pending cancel finishes the job as cancelled (not reported as recovered).
506
649
  */
507
- export const recoverStalled = Redis.script((prefix, maxStalledCount, now) => [prefix, maxStalledCount, now], {
650
+ export const recoverStalled = (HELPERS) => Redis.script((prefix, maxStalledCount, now) => [prefix, maxStalledCount, now], {
508
651
  numberOfKeys: 0,
509
652
  lua: `${HELPERS}
510
653
  local maxStalledCount = tonumber(ARGV[2])
@@ -532,6 +675,7 @@ for _, id in ipairs(expired) do
532
675
  countsAdd(queue, "failed", 1)
533
676
  redis.call("ZADD", prefix .. ":finished:failed", now, id)
534
677
  redis.call("ZADD", terminalKey(name, "failed"), now, id)
678
+ appendOutbox(id, "failed")
535
679
  releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
536
680
  recovered[#recovered + 1] = { id = id, failed = true }
537
681
  else
@@ -551,7 +695,7 @@ return cjson.encode(recovered)
551
695
  `
552
696
  }).withReturnType();
553
697
  /** getJob(prefix, id) -> HGETALL pairs (empty array when missing). */
554
- export const getJob = Redis.script((prefix, id) => [prefix, id], {
698
+ export const getJob = (HELPERS) => Redis.script((prefix, id) => [prefix, id], {
555
699
  numberOfKeys: 0,
556
700
  lua: `${HELPERS}
557
701
  local record = redis.call("HGETALL", jobKey(ARGV[2]))
@@ -560,44 +704,114 @@ return cjson.encode(record)
560
704
  `
561
705
  }).withReturnType();
562
706
  /**
563
- * list(prefix, filtersJson, cursor, limit)
564
- * Keyset pagination over p:all, newest first (enqueuedAt DESC, id DESC).
707
+ * list(prefix, sourcesJson, order, filtersJson, cursor, limit)
708
+ *
709
+ * Indexed list. The DRIVER routes the query to the narrowest zset(s) — see
710
+ * `RedisJobStore`'s routing matrix — and this script merges those sorted
711
+ * sources by (score, id) in the requested direction, pages past the
712
+ * exclusive `<orderValue>:<id>` keyset cursor, loads each candidate row, and
713
+ * applies the residual predicates until `limit` matches accumulate. Sources
714
+ * must be plain-id-membered zsets whose score IS the requested order value
715
+ * (`all`/`byname:`/`byqueue:` = enqueuedAt, `delayed:<queue>` = runAt,
716
+ * `finished:`/`terminal:` = finishedAt) and must be pairwise disjoint; the
717
+ * waiting zsets (seq-prefixed members, priority scores) are never routed
718
+ * here. A member whose job hash is gone is an orphan: it is ZREM'd from the
719
+ * source being scanned (self-heal) and skipped.
565
720
  */
566
- export const list = Redis.script((prefix, filtersJson, cursor, limit) => [prefix, filtersJson, cursor, limit], {
721
+ export const list = (HELPERS) => Redis.script((prefix, sourcesJson, order, filtersJson, cursor, limit) => [
722
+ prefix,
723
+ sourcesJson,
724
+ order,
725
+ filtersJson,
726
+ cursor,
727
+ limit
728
+ ], {
567
729
  numberOfKeys: 0,
568
730
  lua: `${HELPERS}
569
- -- A filter that Redis's cjson cannot decode (e.g. lone-surrogate escapes)
570
- -- degrades to an empty page instead of a script error.
571
- local okFilters, filters = pcall(cjson.decode, ARGV[2])
572
- if not okFilters then return '{"items":[],"more":false}' end
731
+ -- Input that Redis's cjson cannot decode (e.g. lone-surrogate escapes in a
732
+ -- filter or key name) degrades to an empty page instead of a script error.
733
+ local okSources, sources = pcall(cjson.decode, ARGV[2])
734
+ local okFilters, filters = pcall(cjson.decode, ARGV[4])
735
+ if not okSources or not okFilters then return '{"items":[],"more":false}' end
736
+ local desc = ARGV[3] == "desc"
573
737
  local stateSet = nil
574
738
  if filters.states ~= nil then
575
739
  stateSet = {}
576
740
  for _, s in ipairs(filters.states) do stateSet[s] = true end
577
741
  end
578
742
  local cursorAt, cursorId = nil, nil
579
- if ARGV[3] ~= "" then
580
- local split = string.find(ARGV[3], ":", 1, true)
743
+ if ARGV[5] ~= "" then
744
+ local split = string.find(ARGV[5], ":", 1, true)
581
745
  if split ~= nil then
582
- cursorAt = tonumber(string.sub(ARGV[3], 1, split - 1))
583
- cursorId = string.sub(ARGV[3], split + 1)
746
+ cursorAt = tonumber(string.sub(ARGV[5], 1, split - 1))
747
+ cursorId = string.sub(ARGV[5], split + 1)
584
748
  end
585
749
  if cursorAt == nil then cursorId = nil end
586
750
  end
587
- local limit = tonumber(ARGV[4])
751
+ local limit = tonumber(ARGV[6])
752
+ -- One buffered iterator per source. offset counts consumed members still in
753
+ -- the zset — a self-healed orphan decrements it, because its ZREM shifts
754
+ -- every later rank down by one — so refills stay exact under in-script
755
+ -- removals. The score bound is the cursor score (inclusive; the per-item
756
+ -- check below excludes ids at or before the cursor within that score).
757
+ local iters = {}
758
+ for i = 1, #sources do
759
+ iters[i] = { key = sources[i], offset = 0, buf = {}, pos = 1, exhausted = false }
760
+ end
761
+ local function head(it)
762
+ if it.pos > #it.buf then
763
+ if it.exhausted then return nil end
764
+ if desc then
765
+ it.buf = redis.call("ZREVRANGEBYSCORE", it.key, cursorAt == nil and "+inf" or fmt(cursorAt), "-inf",
766
+ "WITHSCORES", "LIMIT", it.offset, 100)
767
+ else
768
+ it.buf = redis.call("ZRANGEBYSCORE", it.key, cursorAt == nil and "-inf" or fmt(cursorAt), "+inf",
769
+ "WITHSCORES", "LIMIT", it.offset, 100)
770
+ end
771
+ it.pos = 1
772
+ if #it.buf == 0 then
773
+ it.exhausted = true
774
+ return nil
775
+ end
776
+ end
777
+ return it.buf[it.pos], tonumber(it.buf[it.pos + 1])
778
+ end
588
779
  local items = {}
589
- local moreMatches = false
590
- local max = cursorAt == nil and "+inf" or fmt(cursorAt)
591
- local offset = 0
780
+ local more = false
592
781
  while true do
593
- local batch = redis.call("ZREVRANGEBYSCORE", prefix .. ":all", max, "-inf", "WITHSCORES", "LIMIT", offset, 100)
594
- if #batch == 0 then break end
595
- for i = 1, #batch, 2 do
596
- local id = batch[i]
597
- local at = tonumber(batch[i + 1])
598
- -- Skip up to and including the cursor position within its score.
599
- if cursorAt == nil or at < cursorAt or (at == cursorAt and id < cursorId) then
600
- local jk = jobKey(id)
782
+ -- The best head across sources: (score, id) in the requested direction (a
783
+ -- score-tied range comes back in member-lex order, so ids line up too).
784
+ local best, bestId, bestAt = nil, nil, nil
785
+ for i = 1, #iters do
786
+ local id, at = head(iters[i])
787
+ if id ~= nil then
788
+ local wins = best == nil
789
+ if not wins then
790
+ if at ~= bestAt then
791
+ wins = (desc and at > bestAt) or (not desc and at < bestAt)
792
+ else
793
+ wins = (desc and id > bestId) or (not desc and id < bestId)
794
+ end
795
+ end
796
+ if wins then
797
+ best, bestId, bestAt = iters[i], id, at
798
+ end
799
+ end
800
+ end
801
+ if best == nil then break end
802
+ best.pos = best.pos + 2
803
+ best.offset = best.offset + 1
804
+ -- Exclusive keyset cursor: skip up to and including the cursor position.
805
+ local past = cursorAt == nil
806
+ or (desc and (bestAt < cursorAt or (bestAt == cursorAt and bestId < cursorId)))
807
+ or (not desc and (bestAt > cursorAt or (bestAt == cursorAt and bestId > cursorId)))
808
+ if past then
809
+ local jk = jobKey(bestId)
810
+ if redis.call("EXISTS", jk) == 0 then
811
+ -- Orphaned index member (hash removed out of band): self-heal.
812
+ redis.call("ZREM", best.key, bestId)
813
+ best.offset = best.offset - 1
814
+ else
601
815
  local matches = true
602
816
  if filters.queue ~= nil and redis.call("HGET", jk, "queue") ~= filters.queue then matches = false end
603
817
  if matches and filters.name ~= nil and redis.call("HGET", jk, "name") ~= filters.name then matches = false end
@@ -617,22 +831,66 @@ while true do
617
831
  end
618
832
  if matches then
619
833
  if #items >= limit then
620
- moreMatches = true
834
+ more = true
621
835
  break
622
836
  end
623
837
  items[#items + 1] = redis.call("HGETALL", jk)
624
838
  end
625
839
  end
626
840
  end
627
- if moreMatches then break end
628
- offset = offset + 100
629
841
  end
630
842
  if #items == 0 then return '{"items":[],"more":false}' end
631
- return cjson.encode({ items = items, more = moreMatches })
843
+ return cjson.encode({ items = items, more = more })
844
+ `
845
+ }).withReturnType();
846
+ /**
847
+ * indexMembers(prefix, kind, idsJson) -> "1"
848
+ *
849
+ * Index one ZSCAN chunk of `p:all` members during a full rebuild. The driver
850
+ * owns the ZSCAN cursor loop (linear, tie-immune, guaranteed to terminate,
851
+ * and guaranteed to return every element present for the whole scan); this
852
+ * script only does the per-chunk work, so the single-threaded server is
853
+ * never held for the whole keyspace. Re-visited members and rows indexed
854
+ * live by insertJobRow mid-scan are idempotent ZADDs.
855
+ */
856
+ export const indexMembers = (HELPERS) => Redis.script((prefix, kind, idsJson) => [prefix, kind, idsJson], {
857
+ numberOfKeys: 0,
858
+ lua: `${HELPERS}
859
+ local kind = ARGV[2]
860
+ for _, id in ipairs(cjson.decode(ARGV[3])) do
861
+ indexInto(kind, id)
862
+ end
863
+ return "1"
864
+ `
865
+ }).withReturnType();
866
+ /**
867
+ * indexTailPage(prefix, kind, min, offset, pageSize) -> scanned count
868
+ *
869
+ * One bounded page of the boot-time tail heal: index every `p:all` member
870
+ * with score >= min (the previous marker minus a safety margin). Plain
871
+ * LIMIT offset paging — tails are small, and a rank-shift skip from a
872
+ * concurrent delete is covered by the margin plus the next boot's heal.
873
+ */
874
+ export const indexTailPage = (HELPERS) => Redis.script((prefix, kind, min, offset, pageSize) => [
875
+ prefix,
876
+ kind,
877
+ min,
878
+ offset,
879
+ pageSize
880
+ ], {
881
+ numberOfKeys: 0,
882
+ lua: `${HELPERS}
883
+ local kind = ARGV[2]
884
+ local batch = redis.call("ZRANGEBYSCORE", prefix .. ":all", ARGV[3], "+inf",
885
+ "LIMIT", tonumber(ARGV[4]), tonumber(ARGV[5]))
886
+ for _, id in ipairs(batch) do
887
+ indexInto(kind, id)
888
+ end
889
+ return tostring(#batch)
632
890
  `
633
891
  }).withReturnType();
634
892
  /** counts(prefix) -> HGETALL pairs of p:counts. */
635
- export const counts = Redis.script((prefix) => [prefix], {
893
+ export const counts = (HELPERS) => Redis.script((prefix) => [prefix], {
636
894
  numberOfKeys: 0,
637
895
  lua: `${HELPERS}
638
896
  local pairs_ = redis.call("HGETALL", prefix .. ":counts")
@@ -640,13 +898,14 @@ if #pairs_ == 0 then return "[]" end
640
898
  return cjson.encode(pairs_)
641
899
  `
642
900
  }).withReturnType();
643
- /** remove(prefix, id) -> removed boolean (active jobs are refused). */
644
- export const remove = Redis.script((prefix, id) => [prefix, id], {
901
+ /** remove(prefix, id) -> removed boolean (active/waiting-children refused). */
902
+ export const remove = (HELPERS) => Redis.script((prefix, id) => [prefix, id], {
645
903
  numberOfKeys: 0,
646
904
  lua: `${HELPERS}
647
905
  local id = ARGV[2]
648
906
  local jk = jobKey(id)
649
- if redis.call("EXISTS", jk) == 0 or redis.call("HGET", jk, "state") == "active" then
907
+ local state = redis.call("HGET", jk, "state")
908
+ if redis.call("EXISTS", jk) == 0 or state == "active" or state == "waiting-children" then
650
909
  return "0"
651
910
  end
652
911
  deleteJob(id)
@@ -654,7 +913,7 @@ return "1"
654
913
  `
655
914
  }).withReturnType();
656
915
  /** retry(prefix, id, now) — failed -> waiting with a fresh budget. */
657
- export const retry = Redis.script((prefix, id, now) => [prefix, id, now], {
916
+ export const retry = (HELPERS) => Redis.script((prefix, id, now) => [prefix, id, now], {
658
917
  numberOfKeys: 0,
659
918
  lua: `${HELPERS}
660
919
  local id = ARGV[2]
@@ -680,10 +939,12 @@ return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
680
939
  `
681
940
  }).withReturnType();
682
941
  /**
683
- * cancel(prefix, id, now) — waiting/delayed become terminal; active gets the
684
- * cancel-request flag; terminal states are refused.
942
+ * cancel(prefix, id, now) — waiting/delayed/waiting-children become terminal
943
+ * (a parked flow parent also flips its remaining pending rows to cancelled,
944
+ * handing them to the cascade sweep); active gets the cancel-request flag;
945
+ * terminal states are refused.
685
946
  */
686
- export const cancel = Redis.script((prefix, id, now) => [prefix, id, now], {
947
+ export const cancel = (HELPERS) => Redis.script((prefix, id, now) => [prefix, id, now], {
687
948
  numberOfKeys: 0,
688
949
  lua: `${HELPERS}
689
950
  local id = ARGV[2]
@@ -694,13 +955,16 @@ if state == "active" then
694
955
  redis.call("HSET", jk, "cancelRequested", "1")
695
956
  return '{"ok":true}'
696
957
  end
697
- if state ~= "waiting" and state ~= "delayed" then
958
+ if state ~= "waiting" and state ~= "delayed" and state ~= "waiting-children" then
698
959
  return '{"error":"state","state":' .. cjson.encode(state) .. '}'
699
960
  end
700
961
  local nowStr = ARGV[3]
701
962
  local now = tonumber(nowStr)
702
963
  local queue = redis.call("HGET", jk, "queue")
703
964
  local name = redis.call("HGET", jk, "name")
965
+ if state == "waiting-children" then
966
+ settleMarkRows(id)
967
+ end
704
968
  remWaiting(queue, id)
705
969
  redis.call("ZREM", delayedKey(queue), id)
706
970
  redis.call("HSET", jk, "state", "cancelled", "finishedAt", nowStr, "cancelRequested", "0")
@@ -709,13 +973,14 @@ countsAdd(queue, "cancelled", 1)
709
973
  redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
710
974
  redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
711
975
  appendAttempt(id, "cancelled", redis.call("HGET", jk, "processedAt"), nowStr, "")
976
+ appendOutbox(id, "cancelled")
712
977
  releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
713
978
  applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
714
979
  return '{"ok":true}'
715
980
  `
716
981
  }).withReturnType();
717
982
  /** promote(prefix, id, now) — delayed -> waiting now. */
718
- export const promote = Redis.script((prefix, id, now) => [prefix, id, now], {
983
+ export const promote = (HELPERS) => Redis.script((prefix, id, now) => [prefix, id, now], {
719
984
  numberOfKeys: 0,
720
985
  lua: `${HELPERS}
721
986
  local id = ARGV[2]
@@ -743,7 +1008,7 @@ return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
743
1008
  * digits) and empty arrays ({}). An unchanged cadence (cron/tz/everyMs)
744
1009
  * preserves the stored nextRunAt.
745
1010
  */
746
- export const upsertSchedule = Redis.script((prefix, key, jobName, queue, cron, tz, everyMs, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, group, nextRunAt) => [
1011
+ export const upsertSchedule = (HELPERS) => Redis.script((prefix, key, jobName, queue, cron, tz, everyMs, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, group, nextRunAt) => [
747
1012
  prefix,
748
1013
  key,
749
1014
  jobName,
@@ -787,7 +1052,7 @@ return '{"ok":true}'
787
1052
  `
788
1053
  }).withReturnType();
789
1054
  /** removeSchedule(prefix, key) -> existed boolean. */
790
- export const removeSchedule = Redis.script((prefix, key) => [prefix, key], {
1055
+ export const removeSchedule = (HELPERS) => Redis.script((prefix, key) => [prefix, key], {
791
1056
  numberOfKeys: 0,
792
1057
  lua: `${HELPERS}
793
1058
  local removed = redis.call("ZREM", prefix .. ":schedules", ARGV[2])
@@ -796,7 +1061,7 @@ return tostring(removed)
796
1061
  `
797
1062
  }).withReturnType();
798
1063
  /** listSchedules(prefix, filtersJson) ordered by nextRunAt ascending. */
799
- export const listSchedules = Redis.script((prefix, filtersJson) => [prefix, filtersJson], {
1064
+ export const listSchedules = (HELPERS) => Redis.script((prefix, filtersJson) => [prefix, filtersJson], {
800
1065
  numberOfKeys: 0,
801
1066
  lua: `${HELPERS}
802
1067
  local filters = cjson.decode(ARGV[2])
@@ -817,7 +1082,7 @@ return cjson.encode(out)
817
1082
  `
818
1083
  }).withReturnType();
819
1084
  /** dueSchedules(prefix, now) ordered by nextRunAt ascending. */
820
- export const dueSchedules = Redis.script((prefix, now) => [prefix, now], {
1085
+ export const dueSchedules = (HELPERS) => Redis.script((prefix, now) => [prefix, now], {
821
1086
  numberOfKeys: 0,
822
1087
  lua: `${HELPERS}
823
1088
  local keys = redis.call("ZRANGEBYSCORE", prefix .. ":schedules", "-inf", tonumber(ARGV[2]))
@@ -830,7 +1095,7 @@ return cjson.encode(out)
830
1095
  `
831
1096
  }).withReturnType();
832
1097
  /** advanceSchedule(prefix, key, expectedRunAt, nextRunAt) — conditional CAS. */
833
- export const advanceSchedule = Redis.script((prefix, key, expectedRunAt, nextRunAt) => [prefix, key, expectedRunAt, nextRunAt], {
1098
+ export const advanceSchedule = (HELPERS) => Redis.script((prefix, key, expectedRunAt, nextRunAt) => [prefix, key, expectedRunAt, nextRunAt], {
834
1099
  numberOfKeys: 0,
835
1100
  lua: `${HELPERS}
836
1101
  local key = ARGV[2]
@@ -845,12 +1110,12 @@ return "1"
845
1110
  /**
846
1111
  * tickSchedule(prefix, key, expectedRunAt, nextRunAt, id, name, queue,
847
1112
  * payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson,
848
- * timeoutMs, traceJson, delayMs, now) -> "1" fired | "0"
1113
+ * timeoutMs, traceJson, parentJson, delayMs, now) -> "1" fired | "0"
849
1114
  * Atomic occurrence claim: the nextRunAt CAS and the tick job's insert run
850
1115
  * in one script, so a stale sweeper can never re-fire a slot — even after
851
1116
  * retention pruned the previous slot's job row.
852
1117
  */
853
- export const tickSchedule = Redis.script((prefix, key, expectedRunAt, nextRunAt, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, traceJson, delayMs, now) => [
1118
+ export const tickSchedule = (HELPERS) => Redis.script((prefix, key, expectedRunAt, nextRunAt, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, traceJson, parentJson, delayMs, now) => [
854
1119
  prefix,
855
1120
  key,
856
1121
  expectedRunAt,
@@ -866,6 +1131,7 @@ export const tickSchedule = Redis.script((prefix, key, expectedRunAt, nextRunAt,
866
1131
  keepJson,
867
1132
  timeoutMs,
868
1133
  traceJson,
1134
+ parentJson,
869
1135
  delayMs,
870
1136
  now
871
1137
  ], {
@@ -882,18 +1148,18 @@ local id = ARGV[5]
882
1148
  -- schedule still advances, but nothing new fires.
883
1149
  if redis.call("EXISTS", jobKey(id)) == 1 then return "0" end
884
1150
  insertJobRow(id, ARGV[6], ARGV[7], ARGV[8], ARGV[9], ARGV[10], ARGV[11], ARGV[12], ARGV[13],
885
- ARGV[14], "", ARGV[15], tonumber(ARGV[16]), tonumber(ARGV[17]), ARGV[17])
1151
+ ARGV[14], "", ARGV[15], ARGV[16], tonumber(ARGV[17]), tonumber(ARGV[18]), ARGV[18])
886
1152
  return "1"
887
1153
  `
888
1154
  }).withReturnType();
889
1155
  /**
890
1156
  * enqueueMany(prefix, now, count, ...items) -> JSON array of per-item results
891
- * ({id, duplicate} | {collision} | {error}). Items are 13-ARGV strides:
1157
+ * ({id, duplicate} | {collision} | {error}). Items are 14-ARGV strides:
892
1158
  * idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax,
893
- * backoffJson, keepJson, timeoutMs, traceJson, delayMs. Plain (non-dedup)
894
- * items only — the caller routes dedup items through \`enqueue\`.
1159
+ * backoffJson, keepJson, timeoutMs, traceJson, parentJson, delayMs. Plain
1160
+ * (non-dedup) items only — the caller routes dedup items through \`enqueue\`.
895
1161
  */
896
- export const enqueueMany = Redis.script((prefix, now, count, items) => [prefix, now, count, ...items], {
1162
+ export const enqueueMany = (HELPERS) => Redis.script((prefix, now, count, items) => [prefix, now, count, ...items], {
897
1163
  numberOfKeys: 0,
898
1164
  lua: `${HELPERS}
899
1165
  local now = tonumber(ARGV[2])
@@ -901,7 +1167,7 @@ local nowStr = ARGV[2]
901
1167
  local count = tonumber(ARGV[3])
902
1168
  local out = {}
903
1169
  for i = 0, count - 1 do
904
- local base = 3 + i * 13
1170
+ local base = 3 + i * 14
905
1171
  local idMode = ARGV[base + 1]
906
1172
  local id = ARGV[base + 2]
907
1173
  local result
@@ -929,7 +1195,7 @@ for i = 0, count - 1 do
929
1195
  else
930
1196
  insertJobRow(id, ARGV[base + 3], ARGV[base + 4], ARGV[base + 5], ARGV[base + 6],
931
1197
  ARGV[base + 7], ARGV[base + 8], ARGV[base + 9], ARGV[base + 10], ARGV[base + 11],
932
- "", ARGV[base + 12], tonumber(ARGV[base + 13]), now, nowStr)
1198
+ "", ARGV[base + 12], ARGV[base + 13], tonumber(ARGV[base + 14]), now, nowStr)
933
1199
  result = '{"id":' .. cjson.encode(id) .. ',"duplicate":false}'
934
1200
  end
935
1201
  end
@@ -944,7 +1210,7 @@ return "[" .. table.concat(out, ",") .. "]"
944
1210
  * min(store ceiling, per-row keep.age). The caller advances the offset by
945
1211
  * (scanned - deleted) and stops when a page comes back short.
946
1212
  */
947
- export const sweepState = Redis.script((prefix, state, ttlMs, limit, offset, now) => [
1213
+ export const sweepState = (HELPERS) => Redis.script((prefix, state, ttlMs, limit, offset, now) => [
948
1214
  prefix,
949
1215
  state,
950
1216
  ttlMs,
@@ -991,7 +1257,9 @@ for i = 1, #batch, 2 do
991
1257
  end
992
1258
  end
993
1259
  end
994
- if cutoffAge ~= nil and finishedAt <= now - cutoffAge then
1260
+ -- Parents still owing cascade cancels are exempt from automatic
1261
+ -- retention; they count as scanned so the offset cursor walks past them.
1262
+ if cutoffAge ~= nil and finishedAt <= now - cutoffAge and not owesCascades(id) then
995
1263
  deleteJob(id)
996
1264
  deleted = deleted + 1
997
1265
  end
@@ -1005,7 +1273,7 @@ return cjson.encode({ scanned = scanned, deleted = deleted })
1005
1273
  * caller loops until 0): expired windows, then pending pointers (+inf)
1006
1274
  * whose job is gone or terminal.
1007
1275
  */
1008
- export const sweepDedupes = Redis.script((prefix, limit, now) => [prefix, limit, now], {
1276
+ export const sweepDedupes = (HELPERS) => Redis.script((prefix, limit, now) => [prefix, limit, now], {
1009
1277
  numberOfKeys: 0,
1010
1278
  lua: `${HELPERS}
1011
1279
  local limit = tonumber(ARGV[2])
@@ -1044,4 +1312,352 @@ end
1044
1312
  return tostring(migrated + #expired + removedPending)
1045
1313
  `
1046
1314
  }).withReturnType();
1315
+ /**
1316
+ * fanOut(prefix, id, token, final, clearStaged, failFast, total, now, count,
1317
+ * ...items) — the FanOut ack, chunked like enqueueMany. Items are 5-ARGV
1318
+ * strides: childKey, storeKey, childJobId, name, specJson.
1319
+ *
1320
+ * Every chunk is lock-token-guarded. Non-final chunks ONLY stage dependency
1321
+ * rows; the final chunk stages its rows, appends the "fanned-out" ledger
1322
+ * entry (no attempt consumed), persists the manifest (flowFailFast,
1323
+ * flowPending = total, zeroed outcome counters), and transitions the parent:
1324
+ * pending > 0 parks it in
1325
+ * waiting-children (no pending zset — never claimable), pending == 0 settles
1326
+ * it straight to runnable collect. A parent whose manifest already landed
1327
+ * keeps it untouched (rows are not re-created; the transition follows the
1328
+ * persisted pending count), so a double fan-out cannot duplicate children.
1329
+ * When staging starts with no manifest, the FIRST chunk clears previously
1330
+ * staged rows — a crashed earlier attempt may have staged different keys. A
1331
+ * raced cancelRequested wins: the parent settles cancelled and its pending
1332
+ * rows flip to cancelled (cascade work for the flow sweeper).
1333
+ */
1334
+ export const fanOut = (HELPERS) => Redis.script((prefix, id, token, final, clearStaged, failFast, total, now, count, items) => [prefix, id, token, final, clearStaged, failFast, total, now, count, ...items], {
1335
+ numberOfKeys: 0,
1336
+ lua: `${HELPERS}
1337
+ local id = ARGV[2]
1338
+ local jk = jobKey(id)
1339
+ if redis.call("EXISTS", jk) == 0 then return '{"error":"notfound"}' end
1340
+ if redis.call("HGET", jk, "state") ~= "active" or redis.call("HGET", jk, "lockToken") ~= ARGV[3] then
1341
+ return '{"error":"locklost"}'
1342
+ end
1343
+ local final = ARGV[4] == "1"
1344
+ local clearStaged = ARGV[5] == "1"
1345
+ local failFast = ARGV[6]
1346
+ local total = tonumber(ARGV[7])
1347
+ local nowStr = ARGV[8]
1348
+ local now = tonumber(nowStr)
1349
+ local count = tonumber(ARGV[9])
1350
+ local pendingStr = redis.call("HGET", jk, "flowPending")
1351
+ local hasManifest = pendingStr ~= false and pendingStr ~= ""
1352
+ if not hasManifest then
1353
+ if clearStaged then
1354
+ local staged = redis.call("ZRANGE", flowIndexKey(id), 0, -1)
1355
+ for i = 1, #staged do
1356
+ redis.call("DEL", flowChildKey(id, staged[i]))
1357
+ redis.call("ZREM", prefix .. ":flowpending", flowMember(id, staged[i]))
1358
+ redis.call("ZREM", prefix .. ":flowcascade", flowMember(id, staged[i]))
1359
+ end
1360
+ redis.call("DEL", flowIndexKey(id))
1361
+ end
1362
+ for i = 0, count - 1 do
1363
+ local base = 9 + i * 5
1364
+ local childKey = ARGV[base + 1]
1365
+ local rk = flowChildKey(id, childKey)
1366
+ redis.call("DEL", rk)
1367
+ redis.call("HSET", rk,
1368
+ "childKey", childKey, "storeKey", ARGV[base + 2], "childJobId", ARGV[base + 3],
1369
+ "name", ARGV[base + 4], "spec", ARGV[base + 5],
1370
+ "status", "pending", "exit", "", "failedReason", "", "cascaded", "0",
1371
+ "pendingSince", nowStr)
1372
+ redis.call("ZADD", flowIndexKey(id), 0, childKey)
1373
+ redis.call("ZADD", prefix .. ":flowpending", now, flowMember(id, childKey))
1374
+ end
1375
+ end
1376
+ if not final then return '{"ok":true}' end
1377
+ local queue = redis.call("HGET", jk, "queue")
1378
+ local name = redis.call("HGET", jk, "name")
1379
+ local startedAt = redis.call("HGET", jk, "processedAt")
1380
+ -- A fan-out is a phase transition, not a completed run: no attemptsMade.
1381
+ appendAttempt(id, "fanned-out", startedAt, nowStr, "")
1382
+ redis.call("ZREM", prefix .. ":active", id)
1383
+ local pending
1384
+ if hasManifest then
1385
+ pending = tonumber(pendingStr) or 0
1386
+ else
1387
+ redis.call("HSET", jk, "flowFailFast", failFast, "flowPending", fmt(total),
1388
+ "flowCompleted", "0", "flowFailed", "0", "flowCancelled", "0")
1389
+ pending = total
1390
+ end
1391
+ if redis.call("HGET", jk, "cancelRequested") == "1" then
1392
+ -- A cancel raced the fan-out: cancellation wins. The rows exist and get
1393
+ -- marked, so the sweeper cascades (mostly no-op cancels for
1394
+ -- never-enqueued children).
1395
+ settleMarkRows(id)
1396
+ finishCancelled(id, queue, name, startedAt, now, nowStr)
1397
+ return '{"ok":true}'
1398
+ end
1399
+ if pending > 0 then
1400
+ redis.call("HSET", jk, "state", "waiting-children", "lockToken", "", "lockExpiresAt", "")
1401
+ countsAdd(queue, "active", -1)
1402
+ countsAdd(queue, "waiting-children", 1)
1403
+ return '{"ok":true}'
1404
+ end
1405
+ -- Empty (or fully recorded) manifest: settle straight to runnable collect.
1406
+ local seq = redis.call("INCR", prefix .. ":seq")
1407
+ redis.call("HSET", jk, "state", "waiting", "runAt", nowStr, "seq", fmt(seq),
1408
+ "lockToken", "", "lockExpiresAt", "")
1409
+ local priority = tonumber(redis.call("HGET", jk, "priority")) or 0
1410
+ addWaiting(queue, priority, seq, id)
1411
+ countsAdd(queue, "active", -1)
1412
+ countsAdd(queue, "waiting", 1)
1413
+ return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
1414
+ `
1415
+ }).withReturnType();
1416
+ /**
1417
+ * recordChildResults(prefix, now, count, ...items) -> {results, wakes}.
1418
+ * Items are 5-ARGV strides: flowId, childKey, outcome, exitJson,
1419
+ * failedReason; results are positional {applied, parentSettled}; wakes names
1420
+ * the queues of parents that resumed runnable. One atomic batch; reports may
1421
+ * span flows.
1422
+ *
1423
+ * Phase 1 applies EVERY row update — idempotent, only while the row is
1424
+ * still pending; an applied report moves the child from the parent's `pending`
1425
+ * counter to its outcome counter and marks the row cascaded (the outcome
1426
+ * came FROM the child's store). Phase 2 settles each touched flow at most
1427
+ * once: the FIRST applied failed report in batch order under fail-fast
1428
+ * (terminal store-side failure; remaining rows flip to cancelled; wins the
1429
+ * tie over pending==0 — and this settle IS a nested parent's terminal
1430
+ * transition, so its own report goes to the outbox here), else pending==0
1431
+ * resumes the parent runnable at the flow's LAST applied report's index.
1432
+ */
1433
+ export const recordChildResults = (HELPERS) => Redis.script((prefix, now, count, items) => [prefix, now, count, ...items], {
1434
+ numberOfKeys: 0,
1435
+ lua: `${HELPERS}
1436
+ local nowStr = ARGV[2]
1437
+ local now = tonumber(nowStr)
1438
+ local count = tonumber(ARGV[3])
1439
+ local applied = {}
1440
+ local settled = {}
1441
+ -- Phase 1: every row update lands before any settle decision, so a
1442
+ -- completed batch-mate keeps its real outcome even when an earlier
1443
+ -- batch-mate settles the flow fail-fast.
1444
+ local touched = {}
1445
+ local touchedOrder = {}
1446
+ for i = 1, count do
1447
+ local base = 3 + (i - 1) * 5
1448
+ local flowId = ARGV[base + 1]
1449
+ local childKey = ARGV[base + 2]
1450
+ local outcome = ARGV[base + 3]
1451
+ applied[i] = false
1452
+ local rk = flowChildKey(flowId, childKey)
1453
+ if redis.call("HGET", rk, "status") == "pending" then
1454
+ redis.call("HSET", rk, "status", outcome, "exit", ARGV[base + 4],
1455
+ "failedReason", ARGV[base + 5], "cascaded", "1")
1456
+ redis.call("ZREM", prefix .. ":flowpending", flowMember(flowId, childKey))
1457
+ applied[i] = true
1458
+ local jk = jobKey(flowId)
1459
+ local pendingStr = redis.call("HGET", jk, "flowPending")
1460
+ if pendingStr ~= false and pendingStr ~= "" then
1461
+ redis.call("HSET", jk, "flowPending", fmt(math.max(0, (tonumber(pendingStr) or 0) - 1)))
1462
+ local bucket = outcome == "completed" and "flowCompleted"
1463
+ or outcome == "failed" and "flowFailed" or "flowCancelled"
1464
+ redis.call("HINCRBY", jk, bucket, 1)
1465
+ end
1466
+ local touch = touched[flowId]
1467
+ if touch == nil then
1468
+ touch = { last = i }
1469
+ touched[flowId] = touch
1470
+ touchedOrder[#touchedOrder + 1] = flowId
1471
+ end
1472
+ touch.last = i
1473
+ if outcome == "failed" and touch.firstFailed == nil then
1474
+ touch.firstFailed = i
1475
+ touch.failedKey = childKey
1476
+ end
1477
+ end
1478
+ end
1479
+ -- Phase 2: at most one settle per touched flow; fail-fast wins ties.
1480
+ local wakes = {}
1481
+ for _, flowId in ipairs(touchedOrder) do
1482
+ local touch = touched[flowId]
1483
+ local jk = jobKey(flowId)
1484
+ local pendingStr = redis.call("HGET", jk, "flowPending")
1485
+ if pendingStr ~= false and pendingStr ~= ""
1486
+ and redis.call("HGET", jk, "state") == "waiting-children" then
1487
+ local queue = redis.call("HGET", jk, "queue")
1488
+ if redis.call("HGET", jk, "flowFailFast") == "1" and touch.firstFailed ~= nil then
1489
+ -- First applied failure settles the parent terminally, store-side
1490
+ -- (failedReason, no exit — like stall exhaustion) and marks the
1491
+ -- remaining rows in the same op.
1492
+ local name = redis.call("HGET", jk, "name")
1493
+ local startedAt = redis.call("HGET", jk, "processedAt")
1494
+ settleMarkRows(flowId)
1495
+ redis.call("HSET", jk, "state", "failed", "finishedAt", nowStr, "cancelRequested", "0",
1496
+ "failedReason", 'effect-mq: flow child "' .. touch.failedKey .. '" failed')
1497
+ countsAdd(queue, "waiting-children", -1)
1498
+ countsAdd(queue, "failed", 1)
1499
+ redis.call("ZADD", prefix .. ":finished:failed", now, flowId)
1500
+ redis.call("ZADD", terminalKey(name, "failed"), now, flowId)
1501
+ appendAttempt(flowId, "failed", startedAt, nowStr, "")
1502
+ -- A nested parent reports upward: this settle IS its terminal
1503
+ -- transition, with no worker ack to hook.
1504
+ appendOutbox(flowId, "failed")
1505
+ releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), flowId, now)
1506
+ applyKeep(name, "failed", redis.call("HGET", jk, "keep"), now)
1507
+ settled[touch.firstFailed] = true
1508
+ elseif tonumber(pendingStr) == 0 then
1509
+ -- All children settled: the parent resumes runnable, phase collect.
1510
+ local seq = redis.call("INCR", prefix .. ":seq")
1511
+ redis.call("HSET", jk, "state", "waiting", "runAt", nowStr, "seq", fmt(seq))
1512
+ local priority = tonumber(redis.call("HGET", jk, "priority")) or 0
1513
+ addWaiting(queue, priority, seq, flowId)
1514
+ countsAdd(queue, "waiting-children", -1)
1515
+ countsAdd(queue, "waiting", 1)
1516
+ wakes[#wakes + 1] = queue
1517
+ settled[touch.last] = true
1518
+ end
1519
+ end
1520
+ end
1521
+ local out = {}
1522
+ for i = 1, count do
1523
+ out[i] = '{"applied":' .. (applied[i] and "true" or "false")
1524
+ .. ',"parentSettled":' .. (settled[i] and "true" or "false") .. '}'
1525
+ end
1526
+ local wakesJson = #wakes == 0 and "[]" or cjson.encode(wakes)
1527
+ return '{"results":[' .. table.concat(out, ",") .. '],"wakes":' .. wakesJson .. '}'
1528
+ `
1529
+ }).withReturnType();
1530
+ /**
1531
+ * listChildResults(prefix, flowId, cursor, limit) — child-key order via
1532
+ * ZRANGEBYLEX over the per-flow index; cursor = last childKey (exclusive).
1533
+ * Items are positional HMGET tuples (the field list must stay in lockstep
1534
+ * with the driver's `toChildRecord`) — the full spec JSON stays server-side.
1535
+ */
1536
+ export const listChildResults = (HELPERS) => Redis.script((prefix, flowId, cursor, limit) => [prefix, flowId, cursor, limit], {
1537
+ numberOfKeys: 0,
1538
+ lua: `${HELPERS}
1539
+ local flowId = ARGV[2]
1540
+ local min = ARGV[3] == "" and "-" or ("(" .. ARGV[3])
1541
+ local limit = tonumber(ARGV[4])
1542
+ local keys = redis.call("ZRANGEBYLEX", flowIndexKey(flowId), min, "+", "LIMIT", 0, limit + 1)
1543
+ local items = {}
1544
+ for i = 1, math.min(#keys, limit) do
1545
+ items[#items + 1] = redis.call("HMGET", flowChildKey(flowId, keys[i]),
1546
+ "childKey", "storeKey", "childJobId", "name", "status", "exit", "failedReason", "cascaded")
1547
+ end
1548
+ if #items == 0 then return '{"items":[],"more":false}' end
1549
+ return cjson.encode({ items = items, more = #keys > limit })
1550
+ `
1551
+ }).withReturnType();
1552
+ /**
1553
+ * flowSweepWork(prefix, pendingAgeMs, limit, now) -> {reconcile, cascade}
1554
+ * grouped by flowId. Reconcile scans the flowpending zset (score = the row's
1555
+ * sweep-eligibility timestamp) and yields rows whose parent is still parked
1556
+ * in waiting-children. Every scanned member is re-armed or purged so no
1557
+ * member can pin the head of the page:
1558
+ *
1559
+ * - parent missing, or terminal with NO manifest: a crashed fan-out's
1560
+ * staged orphan — purge the row, its index member, and the flowpending
1561
+ * member (left alone their old scores would head-pin every page forever);
1562
+ * - parent alive but not waiting-children (mid-staging): re-arm to now so
1563
+ * it rotates behind fresher work;
1564
+ * - returned rows: re-arm to now (defer-on-return, per the contract) so a
1565
+ * full page rotates across sweeps;
1566
+ * - a non-pending row's membership is stale (rows never return to pending):
1567
+ * self-heal by removing the member.
1568
+ *
1569
+ * Cascade lists flowcascade members (cancels still owed to child stores).
1570
+ * Spec JSON strings pass through untouched — the script never cjson-decodes
1571
+ * stored payloads (precision, lone surrogates).
1572
+ */
1573
+ export const flowSweepWork = (HELPERS) => Redis.script((prefix, pendingAgeMs, limit, now) => [prefix, pendingAgeMs, limit, now], {
1574
+ numberOfKeys: 0,
1575
+ lua: `${HELPERS}
1576
+ local pendingAgeMs = tonumber(ARGV[2])
1577
+ local limit = tonumber(ARGV[3])
1578
+ local now = tonumber(ARGV[4])
1579
+ local reconcile, rIndex = {}, {}
1580
+ local due = redis.call("ZRANGEBYSCORE", prefix .. ":flowpending", "-inf", now - pendingAgeMs,
1581
+ "LIMIT", 0, limit)
1582
+ for _, member in ipairs(due) do
1583
+ local sep = string.find(member, "\0", 1, true)
1584
+ local flowId = string.sub(member, 1, sep - 1)
1585
+ local childKey = string.sub(member, sep + 1)
1586
+ local jk = jobKey(flowId)
1587
+ local state = redis.call("HGET", jk, "state")
1588
+ local pendingField = redis.call("HGET", jk, "flowPending")
1589
+ local hasManifest = pendingField ~= false and pendingField ~= ""
1590
+ local terminal = state == "completed" or state == "failed" or state == "cancelled"
1591
+ if state == false or (terminal and not hasManifest) then
1592
+ -- Staged orphan (parent gone, or went terminal before a manifest ever
1593
+ -- landed): purge, or its old score head-pins every future page.
1594
+ redis.call("DEL", flowChildKey(flowId, childKey))
1595
+ redis.call("ZREM", flowIndexKey(flowId), childKey)
1596
+ redis.call("ZREM", prefix .. ":flowpending", member)
1597
+ elseif state ~= "waiting-children" then
1598
+ -- Alive but not parked (e.g. mid-staging): not this sweep's business —
1599
+ -- rotate it behind fresher work.
1600
+ redis.call("ZADD", prefix .. ":flowpending", now, member)
1601
+ elseif redis.call("HGET", flowChildKey(flowId, childKey), "status") ~= "pending" then
1602
+ -- Stale membership (rows never return to pending): self-heal.
1603
+ redis.call("ZREM", prefix .. ":flowpending", member)
1604
+ else
1605
+ local rk = flowChildKey(flowId, childKey)
1606
+ local group = rIndex[flowId]
1607
+ if group == nil then
1608
+ group = { flowId = flowId, children = {} }
1609
+ rIndex[flowId] = group
1610
+ reconcile[#reconcile + 1] = group
1611
+ end
1612
+ group.children[#group.children + 1] = {
1613
+ childKey = childKey,
1614
+ storeKey = redis.call("HGET", rk, "storeKey"),
1615
+ spec = redis.call("HGET", rk, "spec")
1616
+ }
1617
+ -- Returned work defers its own re-eligibility by one age: page rotation.
1618
+ redis.call("ZADD", prefix .. ":flowpending", now, member)
1619
+ end
1620
+ end
1621
+ local cascade, cIndex = {}, {}
1622
+ local owed = redis.call("ZRANGE", prefix .. ":flowcascade", 0, limit - 1)
1623
+ for _, member in ipairs(owed) do
1624
+ local sep = string.find(member, "\0", 1, true)
1625
+ local flowId = string.sub(member, 1, sep - 1)
1626
+ local childKey = string.sub(member, sep + 1)
1627
+ local rk = flowChildKey(flowId, childKey)
1628
+ if redis.call("EXISTS", rk) == 1 then
1629
+ local group = cIndex[flowId]
1630
+ if group == nil then
1631
+ group = { flowId = flowId, children = {} }
1632
+ cIndex[flowId] = group
1633
+ cascade[#cascade + 1] = group
1634
+ end
1635
+ group.children[#group.children + 1] = {
1636
+ childKey = childKey,
1637
+ storeKey = redis.call("HGET", rk, "storeKey"),
1638
+ childJobId = redis.call("HGET", rk, "childJobId")
1639
+ }
1640
+ end
1641
+ end
1642
+ return cjson.encode({ reconcile = reconcile, cascade = cascade })
1643
+ `
1644
+ }).withReturnType();
1645
+ /**
1646
+ * markChildrenCascaded(prefix, flowId, childKeysJson) — idempotent; unknown
1647
+ * keys are ignored (their index members are still cleared).
1648
+ */
1649
+ export const markChildrenCascaded = (HELPERS) => Redis.script((prefix, flowId, childKeysJson) => [prefix, flowId, childKeysJson], {
1650
+ numberOfKeys: 0,
1651
+ lua: `${HELPERS}
1652
+ local flowId = ARGV[2]
1653
+ for _, key in ipairs(cjson.decode(ARGV[3])) do
1654
+ local rk = flowChildKey(flowId, key)
1655
+ if redis.call("EXISTS", rk) == 1 then
1656
+ redis.call("HSET", rk, "cascaded", "1")
1657
+ end
1658
+ redis.call("ZREM", prefix .. ":flowcascade", flowMember(flowId, key))
1659
+ end
1660
+ return '{"ok":true}'
1661
+ `
1662
+ }).withReturnType();
1047
1663
  //# sourceMappingURL=scripts.js.map