effect-mq 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -17
- package/dist/Flow.d.ts +381 -0
- package/dist/Flow.d.ts.map +1 -0
- package/dist/Flow.js +340 -0
- package/dist/Flow.js.map +1 -0
- package/dist/Job.d.ts +31 -6
- package/dist/Job.d.ts.map +1 -1
- package/dist/Job.js +16 -2
- package/dist/Job.js.map +1 -1
- package/dist/JobStore.d.ts +353 -13
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js +10 -0
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +361 -21
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/Metrics.d.ts +31 -0
- package/dist/Metrics.d.ts.map +1 -1
- package/dist/Metrics.js +39 -0
- package/dist/Metrics.js.map +1 -1
- package/dist/Worker.d.ts +120 -11
- package/dist/Worker.d.ts.map +1 -1
- package/dist/Worker.js +452 -26
- package/dist/Worker.js.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.js +678 -80
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
- package/dist/drizzle-postgres/schema.d.ts +293 -3
- package/dist/drizzle-postgres/schema.d.ts.map +1 -1
- package/dist/drizzle-postgres/schema.js +66 -1
- package/dist/drizzle-postgres/schema.js.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/redis/RedisJobStore.d.ts +53 -0
- package/dist/redis/RedisJobStore.d.ts.map +1 -1
- package/dist/redis/RedisJobStore.js +402 -50
- package/dist/redis/RedisJobStore.js.map +1 -1
- package/dist/redis/scripts.d.ts +213 -35
- package/dist/redis/scripts.d.ts.map +1 -1
- package/dist/redis/scripts.js +689 -73
- package/dist/redis/scripts.js.map +1 -1
- package/dist/testing/conformance.d.ts +6 -0
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +855 -12
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +1 -1
- package/src/Flow.ts +778 -0
- package/src/Job.ts +35 -11
- package/src/JobStore.ts +377 -12
- package/src/MemoryJobStore.ts +396 -25
- package/src/Metrics.ts +43 -0
- package/src/Worker.ts +726 -37
- package/src/drizzle-postgres/DrizzleJobStore.ts +844 -81
- package/src/drizzle-postgres/schema.ts +92 -0
- package/src/index.ts +8 -0
- package/src/redis/RedisJobStore.ts +540 -39
- package/src/redis/scripts.ts +751 -78
- package/src/testing/conformance.ts +1088 -12
package/dist/redis/scripts.js
CHANGED
|
@@ -21,6 +21,19 @@
|
|
|
21
21
|
* - `p:delayed:<queue>` ZSET, score `runAt`
|
|
22
22
|
* - `p:active` ZSET, score `lockExpiresAt`
|
|
23
23
|
* - `p:all` ZSET, score `enqueuedAt` (list pagination)
|
|
24
|
+
* - `p:byname:<name>` ZSET, score `enqueuedAt` (list name index;
|
|
25
|
+
* optional, `RedisJobStoreOptions.indexes.name`)
|
|
26
|
+
* - `p:byqueue:<queue>` ZSET, score `enqueuedAt` (list queue index;
|
|
27
|
+
* optional, `RedisJobStoreOptions.indexes.queue`)
|
|
28
|
+
* - `p:index:<kind>:ready` STRING, millis timestamp of the last index
|
|
29
|
+
* reconcile's start for `<kind>` (name |
|
|
30
|
+
* queue). Absent: the next enabled boot does a
|
|
31
|
+
* full ZSCAN rebuild. Present: enabled boots
|
|
32
|
+
* heal the tail (rows enqueued since the value
|
|
33
|
+
* minus a safety margin) and re-stamp it —
|
|
34
|
+
* closing the rolling-deploy window where
|
|
35
|
+
* index-less writers inserted rows after the
|
|
36
|
+
* marker landed
|
|
24
37
|
* - `p:finished:<state>` ZSET, score `finishedAt` (history TTL)
|
|
25
38
|
* - `p:terminal:<name>:<state>` ZSET, score `finishedAt` (keep pruning)
|
|
26
39
|
* - `p:counts` HASH `<queue>|<state>` -> integer
|
|
@@ -28,6 +41,23 @@
|
|
|
28
41
|
* - `p:schedules` / `p:schedule:<key>` ZSET by nextRunAt + HASH per record
|
|
29
42
|
* - `p:dedupe:<name>\0<key>` HASH {jobId, expiresAt} + `p:dedupes` index
|
|
30
43
|
* ZSET (score = window expiry, +inf = pending)
|
|
44
|
+
* - `p:flowchild:<flowId>\0<childKey>` HASH of one flow dependency row
|
|
45
|
+
* - `p:flowchildren:<flowId>` ZSET (score 0, member = childKey; ZRANGEBYLEX
|
|
46
|
+
* gives child-key order + cursor pagination)
|
|
47
|
+
* - `p:flowpending` ZSET, member `<flowId>\0<childKey>`, score =
|
|
48
|
+
* sweep-eligibility timestamp (pendingSince at
|
|
49
|
+
* staging, re-armed on sweep return)
|
|
50
|
+
* - `p:flowcascade` ZSET, score 0, member `<flowId>\0<childKey>`
|
|
51
|
+
* (cancels still owed to child stores)
|
|
52
|
+
* - `p:flowoutbox` ZSET, score = seq from `p:flowoutbox:seq`,
|
|
53
|
+
* member `<seq>\0<report json>` — undelivered
|
|
54
|
+
* child-result reports (see OutboxEntry). The
|
|
55
|
+
* seq prefix makes every member id a peek
|
|
56
|
+
* cursor that survives deletions (see
|
|
57
|
+
* peekOutbox in RedisJobStore)
|
|
58
|
+
*
|
|
59
|
+
* `waiting-children` parents live only in the job hash, `p:all`, and
|
|
60
|
+
* `p:counts` — never in a pending zset, so `claim` can never return them.
|
|
31
61
|
*
|
|
32
62
|
* @since 0.2.0
|
|
33
63
|
*/
|
|
@@ -35,8 +65,12 @@ import { Redis } from "effect/unstable/persistence";
|
|
|
35
65
|
/**
|
|
36
66
|
* Shared helpers textually prepended to every script (the `Redis.script`
|
|
37
67
|
* runner has no include mechanism). `ARGV[1]` is always the key prefix.
|
|
68
|
+
* Built once per store instance: `insertJobRow` writes only the list indexes
|
|
69
|
+
* enabled by `IndexConfig` (`deleteJob` clears both unconditionally — a ZREM
|
|
70
|
+
* on a missing key is free, and stray entries from an earlier configuration
|
|
71
|
+
* must never outlive their job).
|
|
38
72
|
*/
|
|
39
|
-
const
|
|
73
|
+
export const helpers = (indexes) => `
|
|
40
74
|
local prefix = ARGV[1]
|
|
41
75
|
local function fmt(x) return string.format("%.0f", x) end
|
|
42
76
|
local function jobKey(id) return prefix .. ":job:" .. id end
|
|
@@ -44,6 +78,8 @@ local function attemptsKey(id) return prefix .. ":attempts:" .. id end
|
|
|
44
78
|
local function waitingKey(queue) return prefix .. ":waiting:" .. queue end
|
|
45
79
|
local function delayedKey(queue) return prefix .. ":delayed:" .. queue end
|
|
46
80
|
local function terminalKey(name, state) return prefix .. ":terminal:" .. name .. ":" .. state end
|
|
81
|
+
local function bynameKey(name) return prefix .. ":byname:" .. name end
|
|
82
|
+
local function byqueueKey(queue) return prefix .. ":byqueue:" .. queue end
|
|
47
83
|
-- Waiting order: score = -priority (higher priority first, full number range);
|
|
48
84
|
-- FIFO within a priority via lexicographic members "<seq %016d>:<id>". A
|
|
49
85
|
-- composite numeric score would clip either priority or seq past float53.
|
|
@@ -70,7 +106,79 @@ local function appendAttempt(id, outcome, startedAt, finishedAt, exitJson)
|
|
|
70
106
|
'{"attempt":' .. n .. ',"startedAt":' .. started .. ',"finishedAt":' .. finishedAt ..
|
|
71
107
|
',"outcome":"' .. outcome .. '"' .. ex .. '}')
|
|
72
108
|
end
|
|
73
|
-
--
|
|
109
|
+
-- The outbox invariant: every operation that moves a job carrying a parent
|
|
110
|
+
-- envelope INTO a terminal state appends its child-result report here, in
|
|
111
|
+
-- the same script. MUST run after the terminal fields (exit, failedReason)
|
|
112
|
+
-- are written — the report is built from the hash. The parent envelope and
|
|
113
|
+
-- exit are spliced as raw JSON (never cjson-decoded: precision, surrogates);
|
|
114
|
+
-- exit/failedReason keys are OMITTED when absent, like the attempts ledger.
|
|
115
|
+
local function appendOutbox(id, outcome)
|
|
116
|
+
local jk = jobKey(id)
|
|
117
|
+
local parentJson = redis.call("HGET", jk, "parent")
|
|
118
|
+
if parentJson == false or parentJson == "" then return end
|
|
119
|
+
local exitJson = redis.call("HGET", jk, "exit")
|
|
120
|
+
local failedReason = redis.call("HGET", jk, "failedReason")
|
|
121
|
+
local ex = (exitJson == false or exitJson == "") and "" or (',"exit":' .. exitJson)
|
|
122
|
+
local fr = (failedReason == false or failedReason == "") and ""
|
|
123
|
+
or (',"failedReason":' .. cjson.encode(failedReason))
|
|
124
|
+
local seq = redis.call("INCR", prefix .. ":flowoutbox:seq")
|
|
125
|
+
redis.call("ZADD", prefix .. ":flowoutbox", seq, fmt(seq) .. "\0" ..
|
|
126
|
+
'{"parent":' .. parentJson .. ',"outcome":"' .. outcome .. '"' .. ex .. fr .. '}')
|
|
127
|
+
end
|
|
128
|
+
-- Flow dependency rows live in the parent store, one hash per child plus a
|
|
129
|
+
-- per-flow member index (childKey order via ZRANGEBYLEX). Row keys and the
|
|
130
|
+
-- sweep-index members join flowId and childKey with NUL — ids and keys may
|
|
131
|
+
-- both contain ":", so a printable separator could alias two distinct
|
|
132
|
+
-- (flowId, childKey) pairs onto one row.
|
|
133
|
+
local function flowChildKey(flowId, childKey) return prefix .. ":flowchild:" .. flowId .. "\0" .. childKey end
|
|
134
|
+
local function flowIndexKey(flowId) return prefix .. ":flowchildren:" .. flowId end
|
|
135
|
+
local function flowMember(flowId, childKey) return flowId .. "\0" .. childKey end
|
|
136
|
+
-- Settle-time marking: remaining pending rows flip to cancelled (NOT
|
|
137
|
+
-- cascaded — the sweeper still owes the child stores real cancels), moving
|
|
138
|
+
-- from the pending sweep index to the cascade sweep index. Returns the
|
|
139
|
+
-- number of rows flipped, for the flow counters.
|
|
140
|
+
local function markPendingRowsCancelled(flowId)
|
|
141
|
+
local keys = redis.call("ZRANGE", flowIndexKey(flowId), 0, -1)
|
|
142
|
+
local marked = 0
|
|
143
|
+
for i = 1, #keys do
|
|
144
|
+
local rk = flowChildKey(flowId, keys[i])
|
|
145
|
+
if redis.call("HGET", rk, "status") == "pending" then
|
|
146
|
+
redis.call("HSET", rk, "status", "cancelled", "cascaded", "0")
|
|
147
|
+
redis.call("ZREM", prefix .. ":flowpending", flowMember(flowId, keys[i]))
|
|
148
|
+
redis.call("ZADD", prefix .. ":flowcascade", 0, flowMember(flowId, keys[i]))
|
|
149
|
+
marked = marked + 1
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
return marked
|
|
153
|
+
end
|
|
154
|
+
-- Settle-time marking plus the manifest counters: pending -> 0, cancelled +=
|
|
155
|
+
-- the rows flipped (the four counters always sum to the manifest size).
|
|
156
|
+
local function settleMarkRows(flowId)
|
|
157
|
+
local marked = markPendingRowsCancelled(flowId)
|
|
158
|
+
local jk = jobKey(flowId)
|
|
159
|
+
if redis.call("HEXISTS", jk, "flowPending") == 1 then
|
|
160
|
+
redis.call("HSET", jk, "flowPending", "0")
|
|
161
|
+
if marked > 0 then redis.call("HINCRBY", jk, "flowCancelled", marked) end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
-- A settled flow parent whose rows still owe cascade cancels is exempt from
|
|
165
|
+
-- AUTOMATIC retention (keep policies, the history sweep): deleting it would
|
|
166
|
+
-- delete the only record that the child stores are still owed real cancels.
|
|
167
|
+
-- The explicit remove verb is the operator override and is NOT exempted.
|
|
168
|
+
-- Cheap for non-parents: the per-flow index only exists for fanned-out jobs.
|
|
169
|
+
local function owesCascades(id)
|
|
170
|
+
if redis.call("EXISTS", flowIndexKey(id)) == 0 then return false end
|
|
171
|
+
local keys = redis.call("ZRANGE", flowIndexKey(id), 0, -1)
|
|
172
|
+
for i = 1, #keys do
|
|
173
|
+
local rk = flowChildKey(id, keys[i])
|
|
174
|
+
if redis.call("HGET", rk, "status") == "cancelled" and redis.call("HGET", rk, "cascaded") == "0" then
|
|
175
|
+
return true
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
return false
|
|
179
|
+
end
|
|
180
|
+
-- Remove a job and every index entry that references it. A flow parent takes
|
|
181
|
+
-- its dependency rows and their sweep-index members with it (flowId = job id).
|
|
74
182
|
local function deleteJob(id)
|
|
75
183
|
local jk = jobKey(id)
|
|
76
184
|
local queue = redis.call("HGET", jk, "queue")
|
|
@@ -79,11 +187,20 @@ local function deleteJob(id)
|
|
|
79
187
|
local name = redis.call("HGET", jk, "name")
|
|
80
188
|
countsAdd(queue, state, -1)
|
|
81
189
|
redis.call("ZREM", prefix .. ":all", id)
|
|
190
|
+
redis.call("ZREM", bynameKey(name), id)
|
|
191
|
+
redis.call("ZREM", byqueueKey(queue), id)
|
|
82
192
|
redis.call("ZREM", prefix .. ":finished:" .. state, id)
|
|
83
193
|
redis.call("ZREM", prefix .. ":active", id)
|
|
84
194
|
redis.call("ZREM", terminalKey(name, state), id)
|
|
85
195
|
remWaiting(queue, id)
|
|
86
196
|
redis.call("ZREM", delayedKey(queue), id)
|
|
197
|
+
local children = redis.call("ZRANGE", flowIndexKey(id), 0, -1)
|
|
198
|
+
for i = 1, #children do
|
|
199
|
+
redis.call("DEL", flowChildKey(id, children[i]))
|
|
200
|
+
redis.call("ZREM", prefix .. ":flowpending", flowMember(id, children[i]))
|
|
201
|
+
redis.call("ZREM", prefix .. ":flowcascade", flowMember(id, children[i]))
|
|
202
|
+
end
|
|
203
|
+
redis.call("DEL", flowIndexKey(id))
|
|
87
204
|
redis.call("DEL", jk, attemptsKey(id))
|
|
88
205
|
end
|
|
89
206
|
-- Terminal retention for one name+state group. Correctness over speed: the
|
|
@@ -106,9 +223,13 @@ local function applyKeep(name, state, keepJson, now)
|
|
|
106
223
|
end
|
|
107
224
|
end
|
|
108
225
|
local tkey = terminalKey(name, state)
|
|
226
|
+
-- Retention exemption: parents still owing cascade cancels are spared
|
|
227
|
+
-- (they still occupy a keep-count slot, exactly like the memory driver).
|
|
109
228
|
if keep.ageMs ~= nil then
|
|
110
229
|
local old = redis.call("ZRANGEBYSCORE", tkey, "-inf", now - keep.ageMs)
|
|
111
|
-
for i = 1, #old do
|
|
230
|
+
for i = 1, #old do
|
|
231
|
+
if not owesCascades(old[i]) then deleteJob(old[i]) end
|
|
232
|
+
end
|
|
112
233
|
end
|
|
113
234
|
-- Floor + clamp: a fractional/negative count must degrade like the memory
|
|
114
235
|
-- driver's slice(), never error mid-script (writes before an error stick).
|
|
@@ -127,7 +248,9 @@ local function applyKeep(name, state, keepJson, now)
|
|
|
127
248
|
if a.fa ~= b.fa then return a.fa > b.fa end
|
|
128
249
|
return a.seq > b.seq
|
|
129
250
|
end)
|
|
130
|
-
for i = count + 1, #arr do
|
|
251
|
+
for i = count + 1, #arr do
|
|
252
|
+
if not owesCascades(arr[i].id) then deleteJob(arr[i].id) end
|
|
253
|
+
end
|
|
131
254
|
end
|
|
132
255
|
end
|
|
133
256
|
local function dedupeStoreKey(name, key) return prefix .. ":dedupe:" .. name .. "\0" .. key end
|
|
@@ -155,14 +278,30 @@ local function finishCancelled(id, queue, name, startedAt, now, nowStr)
|
|
|
155
278
|
redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
|
|
156
279
|
redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
|
|
157
280
|
appendAttempt(id, "cancelled", startedAt, nowStr, "")
|
|
281
|
+
appendOutbox(id, "cancelled")
|
|
158
282
|
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
159
283
|
applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
|
|
160
284
|
end
|
|
285
|
+
-- Index one EXISTING job row into a list index (backfill and tail heal;
|
|
286
|
+
-- insertJobRow covers live writes). A missing row is skipped — nothing to
|
|
287
|
+
-- index, and the read path self-heals whatever stale member pointed here.
|
|
288
|
+
local function indexInto(kind, id)
|
|
289
|
+
local row = redis.call("HMGET", jobKey(id), "name", "queue", "enqueuedAt")
|
|
290
|
+
if row[1] then
|
|
291
|
+
local enqueuedAt = tonumber(row[3]) or 0
|
|
292
|
+
if kind == "name" then
|
|
293
|
+
redis.call("ZADD", bynameKey(row[1]), enqueuedAt, id)
|
|
294
|
+
else
|
|
295
|
+
redis.call("ZADD", byqueueKey(row[2]), enqueuedAt, id)
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
end
|
|
161
299
|
-- Insert one fresh job row plus every index entry. String params are stored
|
|
162
|
-
-- verbatim (payload/metadata/backoff/keep/trace are pre-encoded JSON,
|
|
163
|
-
-- absent); priority/delayMs/now numeric-coercible.
|
|
300
|
+
-- verbatim (payload/metadata/backoff/keep/trace/parent are pre-encoded JSON,
|
|
301
|
+
-- "" = absent); priority/delayMs/now numeric-coercible. New jobs never carry
|
|
302
|
+
-- flow fields — only the FanOut ack writes those.
|
|
164
303
|
local function insertJobRow(id, name, queue, payloadJson, metadataJson, priority,
|
|
165
|
-
attemptsMax, backoffJson, keepJson, timeoutMs, dedupeKey, traceJson, delayMs, now, nowStr)
|
|
304
|
+
attemptsMax, backoffJson, keepJson, timeoutMs, dedupeKey, traceJson, parentJson, delayMs, now, nowStr)
|
|
166
305
|
local seq = redis.call("INCR", prefix .. ":seq")
|
|
167
306
|
local state = delayMs > 0 and "delayed" or "waiting"
|
|
168
307
|
local runAt = now + delayMs
|
|
@@ -171,11 +310,12 @@ local function insertJobRow(id, name, queue, payloadJson, metadataJson, priority
|
|
|
171
310
|
"payload", payloadJson, "metadata", metadataJson, "state", state,
|
|
172
311
|
"priority", priority, "attemptsMax", attemptsMax, "attemptsMade", "0", "stalledCount", "0",
|
|
173
312
|
"backoff", backoffJson, "keep", keepJson, "timeoutMs", timeoutMs,
|
|
174
|
-
"cancelRequested", "0", "dedupeKey", dedupeKey, "trace", traceJson, "
|
|
313
|
+
"cancelRequested", "0", "dedupeKey", dedupeKey, "trace", traceJson, "parent", parentJson,
|
|
314
|
+
"runAt", fmt(runAt), "enqueuedAt", nowStr,
|
|
175
315
|
"processedAt", "", "finishedAt", "", "exit", "", "failedReason", "",
|
|
176
316
|
"lockToken", "", "lockExpiresAt", "", "seq", fmt(seq))
|
|
177
317
|
redis.call("ZADD", prefix .. ":all", now, id)
|
|
178
|
-
if state == "waiting" then
|
|
318
|
+
${indexes.name ? ` redis.call("ZADD", bynameKey(name), now, id)\n` : ""}${indexes.queue ? ` redis.call("ZADD", byqueueKey(queue), now, id)\n` : ""} if state == "waiting" then
|
|
179
319
|
addWaiting(queue, tonumber(priority), seq, id)
|
|
180
320
|
else
|
|
181
321
|
redis.call("ZADD", delayedKey(queue), runAt, id)
|
|
@@ -185,11 +325,12 @@ end
|
|
|
185
325
|
`;
|
|
186
326
|
/**
|
|
187
327
|
* enqueue(prefix, idMode, id, name, queue, payloadJson, metadataJson,
|
|
188
|
-
* priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs,
|
|
328
|
+
* priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs,
|
|
329
|
+
* now, dedupe..., traceJson, parentJson)
|
|
189
330
|
* idMode: "user" (dedup no-op), "generated" (collision -> retry sentinel),
|
|
190
331
|
* "auto" (j-<seq>, in-script collision loop).
|
|
191
332
|
*/
|
|
192
|
-
export const enqueue = Redis.script((prefix, idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs, now, dedupeKey, dedupeTtlMs, dedupeExtend, dedupeReplace, traceJson) => [
|
|
333
|
+
export const enqueue = (HELPERS) => Redis.script((prefix, idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs, now, dedupeKey, dedupeTtlMs, dedupeExtend, dedupeReplace, traceJson, parentJson) => [
|
|
193
334
|
prefix,
|
|
194
335
|
idMode,
|
|
195
336
|
id,
|
|
@@ -208,7 +349,8 @@ export const enqueue = Redis.script((prefix, idMode, id, name, queue, payloadJso
|
|
|
208
349
|
dedupeTtlMs,
|
|
209
350
|
dedupeExtend,
|
|
210
351
|
dedupeReplace,
|
|
211
|
-
traceJson
|
|
352
|
+
traceJson,
|
|
353
|
+
parentJson
|
|
212
354
|
], {
|
|
213
355
|
numberOfKeys: 0,
|
|
214
356
|
lua: `${HELPERS}
|
|
@@ -281,7 +423,7 @@ if idMode == "auto" then
|
|
|
281
423
|
if id == "" then return '{"error":"id"}' end
|
|
282
424
|
end
|
|
283
425
|
insertJobRow(id, ARGV[4], queue, ARGV[6], ARGV[7], ARGV[8], ARGV[9], ARGV[10], ARGV[11],
|
|
284
|
-
ARGV[12], dKey, ARGV[19], delayMs, now, nowStr)
|
|
426
|
+
ARGV[12], dKey, ARGV[19], ARGV[20], delayMs, now, nowStr)
|
|
285
427
|
if dKey ~= "" then
|
|
286
428
|
local sk = dedupeStoreKey(name, dKey)
|
|
287
429
|
redis.call("DEL", sk)
|
|
@@ -301,7 +443,7 @@ return '{"id":' .. cjson.encode(id) .. ',"duplicate":false,"wake":true}'
|
|
|
301
443
|
* waiting job whose name matches. Returns the claimed record (HGETALL pairs)
|
|
302
444
|
* or an Empty result with the earliest matching delayed runAt.
|
|
303
445
|
*/
|
|
304
|
-
export const claim = Redis.script((prefix, queue, namesJson, token, lockDurationMs, now) => [
|
|
446
|
+
export const claim = (HELPERS) => Redis.script((prefix, queue, namesJson, token, lockDurationMs, now) => [
|
|
305
447
|
prefix,
|
|
306
448
|
queue,
|
|
307
449
|
namesJson,
|
|
@@ -374,7 +516,7 @@ return cjson.encode({ empty = true, nextRunAt = nextRunAt })
|
|
|
374
516
|
* Token-guarded. Retry on a cancel-requested job finishes it as cancelled
|
|
375
517
|
* (cancellation wins over revival, mirroring release/recoverStalled).
|
|
376
518
|
*/
|
|
377
|
-
export const ack = Redis.script((prefix, id, token, outcomeTag, exitJson, delayMs, now) => [
|
|
519
|
+
export const ack = (HELPERS) => Redis.script((prefix, id, token, outcomeTag, exitJson, delayMs, now) => [
|
|
378
520
|
prefix,
|
|
379
521
|
id,
|
|
380
522
|
token,
|
|
@@ -413,6 +555,7 @@ local function finish(newState, storeExit, outcome, ledgerExit)
|
|
|
413
555
|
redis.call("ZADD", prefix .. ":finished:" .. newState, now, id)
|
|
414
556
|
redis.call("ZADD", terminalKey(name, newState), now, id)
|
|
415
557
|
appendAttempt(id, outcome, startedAt, nowStr, ledgerExit)
|
|
558
|
+
appendOutbox(id, newState)
|
|
416
559
|
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
417
560
|
applyKeep(name, newState, redis.call("HGET", jk, "keep"), now)
|
|
418
561
|
end
|
|
@@ -449,7 +592,7 @@ return '{"ok":true}'
|
|
|
449
592
|
* release(prefix, id, token, now) — hand the job back without consuming an
|
|
450
593
|
* attempt; a pending cancel wins and finishes the job instead.
|
|
451
594
|
*/
|
|
452
|
-
export const release = Redis.script((prefix, id, token, now) => [prefix, id, token, now], {
|
|
595
|
+
export const release = (HELPERS) => Redis.script((prefix, id, token, now) => [prefix, id, token, now], {
|
|
453
596
|
numberOfKeys: 0,
|
|
454
597
|
lua: `${HELPERS}
|
|
455
598
|
local id = ARGV[2]
|
|
@@ -479,7 +622,7 @@ return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
|
479
622
|
* extendLocks(prefix, locksJson, durationMs, now) -> { lost, cancel }
|
|
480
623
|
* Cancel-requested locks are reported, not extended.
|
|
481
624
|
*/
|
|
482
|
-
export const extendLocks = Redis.script((prefix, locksJson, durationMs, now) => [prefix, locksJson, durationMs, now], {
|
|
625
|
+
export const extendLocks = (HELPERS) => Redis.script((prefix, locksJson, durationMs, now) => [prefix, locksJson, durationMs, now], {
|
|
483
626
|
numberOfKeys: 0,
|
|
484
627
|
lua: `${HELPERS}
|
|
485
628
|
local locks = cjson.decode(ARGV[2])
|
|
@@ -504,7 +647,7 @@ return cjson.encode({ lost = lost, cancel = cancel })
|
|
|
504
647
|
* recoverStalled(prefix, maxStalledCount, now) -> recovered [{id, failed}]
|
|
505
648
|
* A pending cancel finishes the job as cancelled (not reported as recovered).
|
|
506
649
|
*/
|
|
507
|
-
export const recoverStalled = Redis.script((prefix, maxStalledCount, now) => [prefix, maxStalledCount, now], {
|
|
650
|
+
export const recoverStalled = (HELPERS) => Redis.script((prefix, maxStalledCount, now) => [prefix, maxStalledCount, now], {
|
|
508
651
|
numberOfKeys: 0,
|
|
509
652
|
lua: `${HELPERS}
|
|
510
653
|
local maxStalledCount = tonumber(ARGV[2])
|
|
@@ -532,6 +675,7 @@ for _, id in ipairs(expired) do
|
|
|
532
675
|
countsAdd(queue, "failed", 1)
|
|
533
676
|
redis.call("ZADD", prefix .. ":finished:failed", now, id)
|
|
534
677
|
redis.call("ZADD", terminalKey(name, "failed"), now, id)
|
|
678
|
+
appendOutbox(id, "failed")
|
|
535
679
|
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
536
680
|
recovered[#recovered + 1] = { id = id, failed = true }
|
|
537
681
|
else
|
|
@@ -551,7 +695,7 @@ return cjson.encode(recovered)
|
|
|
551
695
|
`
|
|
552
696
|
}).withReturnType();
|
|
553
697
|
/** getJob(prefix, id) -> HGETALL pairs (empty array when missing). */
|
|
554
|
-
export const getJob = Redis.script((prefix, id) => [prefix, id], {
|
|
698
|
+
export const getJob = (HELPERS) => Redis.script((prefix, id) => [prefix, id], {
|
|
555
699
|
numberOfKeys: 0,
|
|
556
700
|
lua: `${HELPERS}
|
|
557
701
|
local record = redis.call("HGETALL", jobKey(ARGV[2]))
|
|
@@ -560,44 +704,114 @@ return cjson.encode(record)
|
|
|
560
704
|
`
|
|
561
705
|
}).withReturnType();
|
|
562
706
|
/**
|
|
563
|
-
* list(prefix, filtersJson, cursor, limit)
|
|
564
|
-
*
|
|
707
|
+
* list(prefix, sourcesJson, order, filtersJson, cursor, limit)
|
|
708
|
+
*
|
|
709
|
+
* Indexed list. The DRIVER routes the query to the narrowest zset(s) — see
|
|
710
|
+
* `RedisJobStore`'s routing matrix — and this script merges those sorted
|
|
711
|
+
* sources by (score, id) in the requested direction, pages past the
|
|
712
|
+
* exclusive `<orderValue>:<id>` keyset cursor, loads each candidate row, and
|
|
713
|
+
* applies the residual predicates until `limit` matches accumulate. Sources
|
|
714
|
+
* must be plain-id-membered zsets whose score IS the requested order value
|
|
715
|
+
* (`all`/`byname:`/`byqueue:` = enqueuedAt, `delayed:<queue>` = runAt,
|
|
716
|
+
* `finished:`/`terminal:` = finishedAt) and must be pairwise disjoint; the
|
|
717
|
+
* waiting zsets (seq-prefixed members, priority scores) are never routed
|
|
718
|
+
* here. A member whose job hash is gone is an orphan: it is ZREM'd from the
|
|
719
|
+
* source being scanned (self-heal) and skipped.
|
|
565
720
|
*/
|
|
566
|
-
export const list = Redis.script((prefix, filtersJson, cursor, limit) => [
|
|
721
|
+
export const list = (HELPERS) => Redis.script((prefix, sourcesJson, order, filtersJson, cursor, limit) => [
|
|
722
|
+
prefix,
|
|
723
|
+
sourcesJson,
|
|
724
|
+
order,
|
|
725
|
+
filtersJson,
|
|
726
|
+
cursor,
|
|
727
|
+
limit
|
|
728
|
+
], {
|
|
567
729
|
numberOfKeys: 0,
|
|
568
730
|
lua: `${HELPERS}
|
|
569
|
-
--
|
|
570
|
-
-- degrades to an empty page instead of a script error.
|
|
571
|
-
local
|
|
572
|
-
|
|
731
|
+
-- Input that Redis's cjson cannot decode (e.g. lone-surrogate escapes in a
|
|
732
|
+
-- filter or key name) degrades to an empty page instead of a script error.
|
|
733
|
+
local okSources, sources = pcall(cjson.decode, ARGV[2])
|
|
734
|
+
local okFilters, filters = pcall(cjson.decode, ARGV[4])
|
|
735
|
+
if not okSources or not okFilters then return '{"items":[],"more":false}' end
|
|
736
|
+
local desc = ARGV[3] == "desc"
|
|
573
737
|
local stateSet = nil
|
|
574
738
|
if filters.states ~= nil then
|
|
575
739
|
stateSet = {}
|
|
576
740
|
for _, s in ipairs(filters.states) do stateSet[s] = true end
|
|
577
741
|
end
|
|
578
742
|
local cursorAt, cursorId = nil, nil
|
|
579
|
-
if ARGV[
|
|
580
|
-
local split = string.find(ARGV[
|
|
743
|
+
if ARGV[5] ~= "" then
|
|
744
|
+
local split = string.find(ARGV[5], ":", 1, true)
|
|
581
745
|
if split ~= nil then
|
|
582
|
-
cursorAt = tonumber(string.sub(ARGV[
|
|
583
|
-
cursorId = string.sub(ARGV[
|
|
746
|
+
cursorAt = tonumber(string.sub(ARGV[5], 1, split - 1))
|
|
747
|
+
cursorId = string.sub(ARGV[5], split + 1)
|
|
584
748
|
end
|
|
585
749
|
if cursorAt == nil then cursorId = nil end
|
|
586
750
|
end
|
|
587
|
-
local limit = tonumber(ARGV[
|
|
751
|
+
local limit = tonumber(ARGV[6])
|
|
752
|
+
-- One buffered iterator per source. offset counts consumed members still in
|
|
753
|
+
-- the zset — a self-healed orphan decrements it, because its ZREM shifts
|
|
754
|
+
-- every later rank down by one — so refills stay exact under in-script
|
|
755
|
+
-- removals. The score bound is the cursor score (inclusive; the per-item
|
|
756
|
+
-- check below excludes ids at or before the cursor within that score).
|
|
757
|
+
local iters = {}
|
|
758
|
+
for i = 1, #sources do
|
|
759
|
+
iters[i] = { key = sources[i], offset = 0, buf = {}, pos = 1, exhausted = false }
|
|
760
|
+
end
|
|
761
|
+
local function head(it)
|
|
762
|
+
if it.pos > #it.buf then
|
|
763
|
+
if it.exhausted then return nil end
|
|
764
|
+
if desc then
|
|
765
|
+
it.buf = redis.call("ZREVRANGEBYSCORE", it.key, cursorAt == nil and "+inf" or fmt(cursorAt), "-inf",
|
|
766
|
+
"WITHSCORES", "LIMIT", it.offset, 100)
|
|
767
|
+
else
|
|
768
|
+
it.buf = redis.call("ZRANGEBYSCORE", it.key, cursorAt == nil and "-inf" or fmt(cursorAt), "+inf",
|
|
769
|
+
"WITHSCORES", "LIMIT", it.offset, 100)
|
|
770
|
+
end
|
|
771
|
+
it.pos = 1
|
|
772
|
+
if #it.buf == 0 then
|
|
773
|
+
it.exhausted = true
|
|
774
|
+
return nil
|
|
775
|
+
end
|
|
776
|
+
end
|
|
777
|
+
return it.buf[it.pos], tonumber(it.buf[it.pos + 1])
|
|
778
|
+
end
|
|
588
779
|
local items = {}
|
|
589
|
-
local
|
|
590
|
-
local max = cursorAt == nil and "+inf" or fmt(cursorAt)
|
|
591
|
-
local offset = 0
|
|
780
|
+
local more = false
|
|
592
781
|
while true do
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
local at =
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
782
|
+
-- The best head across sources: (score, id) in the requested direction (a
|
|
783
|
+
-- score-tied range comes back in member-lex order, so ids line up too).
|
|
784
|
+
local best, bestId, bestAt = nil, nil, nil
|
|
785
|
+
for i = 1, #iters do
|
|
786
|
+
local id, at = head(iters[i])
|
|
787
|
+
if id ~= nil then
|
|
788
|
+
local wins = best == nil
|
|
789
|
+
if not wins then
|
|
790
|
+
if at ~= bestAt then
|
|
791
|
+
wins = (desc and at > bestAt) or (not desc and at < bestAt)
|
|
792
|
+
else
|
|
793
|
+
wins = (desc and id > bestId) or (not desc and id < bestId)
|
|
794
|
+
end
|
|
795
|
+
end
|
|
796
|
+
if wins then
|
|
797
|
+
best, bestId, bestAt = iters[i], id, at
|
|
798
|
+
end
|
|
799
|
+
end
|
|
800
|
+
end
|
|
801
|
+
if best == nil then break end
|
|
802
|
+
best.pos = best.pos + 2
|
|
803
|
+
best.offset = best.offset + 1
|
|
804
|
+
-- Exclusive keyset cursor: skip up to and including the cursor position.
|
|
805
|
+
local past = cursorAt == nil
|
|
806
|
+
or (desc and (bestAt < cursorAt or (bestAt == cursorAt and bestId < cursorId)))
|
|
807
|
+
or (not desc and (bestAt > cursorAt or (bestAt == cursorAt and bestId > cursorId)))
|
|
808
|
+
if past then
|
|
809
|
+
local jk = jobKey(bestId)
|
|
810
|
+
if redis.call("EXISTS", jk) == 0 then
|
|
811
|
+
-- Orphaned index member (hash removed out of band): self-heal.
|
|
812
|
+
redis.call("ZREM", best.key, bestId)
|
|
813
|
+
best.offset = best.offset - 1
|
|
814
|
+
else
|
|
601
815
|
local matches = true
|
|
602
816
|
if filters.queue ~= nil and redis.call("HGET", jk, "queue") ~= filters.queue then matches = false end
|
|
603
817
|
if matches and filters.name ~= nil and redis.call("HGET", jk, "name") ~= filters.name then matches = false end
|
|
@@ -617,22 +831,66 @@ while true do
|
|
|
617
831
|
end
|
|
618
832
|
if matches then
|
|
619
833
|
if #items >= limit then
|
|
620
|
-
|
|
834
|
+
more = true
|
|
621
835
|
break
|
|
622
836
|
end
|
|
623
837
|
items[#items + 1] = redis.call("HGETALL", jk)
|
|
624
838
|
end
|
|
625
839
|
end
|
|
626
840
|
end
|
|
627
|
-
if moreMatches then break end
|
|
628
|
-
offset = offset + 100
|
|
629
841
|
end
|
|
630
842
|
if #items == 0 then return '{"items":[],"more":false}' end
|
|
631
|
-
return cjson.encode({ items = items, more =
|
|
843
|
+
return cjson.encode({ items = items, more = more })
|
|
844
|
+
`
|
|
845
|
+
}).withReturnType();
|
|
846
|
+
/**
|
|
847
|
+
* indexMembers(prefix, kind, idsJson) -> "1"
|
|
848
|
+
*
|
|
849
|
+
* Index one ZSCAN chunk of `p:all` members during a full rebuild. The driver
|
|
850
|
+
* owns the ZSCAN cursor loop (linear, tie-immune, guaranteed to terminate,
|
|
851
|
+
* and guaranteed to return every element present for the whole scan); this
|
|
852
|
+
* script only does the per-chunk work, so the single-threaded server is
|
|
853
|
+
* never held for the whole keyspace. Re-visited members and rows indexed
|
|
854
|
+
* live by insertJobRow mid-scan are idempotent ZADDs.
|
|
855
|
+
*/
|
|
856
|
+
export const indexMembers = (HELPERS) => Redis.script((prefix, kind, idsJson) => [prefix, kind, idsJson], {
|
|
857
|
+
numberOfKeys: 0,
|
|
858
|
+
lua: `${HELPERS}
|
|
859
|
+
local kind = ARGV[2]
|
|
860
|
+
for _, id in ipairs(cjson.decode(ARGV[3])) do
|
|
861
|
+
indexInto(kind, id)
|
|
862
|
+
end
|
|
863
|
+
return "1"
|
|
864
|
+
`
|
|
865
|
+
}).withReturnType();
|
|
866
|
+
/**
|
|
867
|
+
* indexTailPage(prefix, kind, min, offset, pageSize) -> scanned count
|
|
868
|
+
*
|
|
869
|
+
* One bounded page of the boot-time tail heal: index every `p:all` member
|
|
870
|
+
* with score >= min (the previous marker minus a safety margin). Plain
|
|
871
|
+
* LIMIT offset paging — tails are small, and a rank-shift skip from a
|
|
872
|
+
* concurrent delete is covered by the margin plus the next boot's heal.
|
|
873
|
+
*/
|
|
874
|
+
export const indexTailPage = (HELPERS) => Redis.script((prefix, kind, min, offset, pageSize) => [
|
|
875
|
+
prefix,
|
|
876
|
+
kind,
|
|
877
|
+
min,
|
|
878
|
+
offset,
|
|
879
|
+
pageSize
|
|
880
|
+
], {
|
|
881
|
+
numberOfKeys: 0,
|
|
882
|
+
lua: `${HELPERS}
|
|
883
|
+
local kind = ARGV[2]
|
|
884
|
+
local batch = redis.call("ZRANGEBYSCORE", prefix .. ":all", ARGV[3], "+inf",
|
|
885
|
+
"LIMIT", tonumber(ARGV[4]), tonumber(ARGV[5]))
|
|
886
|
+
for _, id in ipairs(batch) do
|
|
887
|
+
indexInto(kind, id)
|
|
888
|
+
end
|
|
889
|
+
return tostring(#batch)
|
|
632
890
|
`
|
|
633
891
|
}).withReturnType();
|
|
634
892
|
/** counts(prefix) -> HGETALL pairs of p:counts. */
|
|
635
|
-
export const counts = Redis.script((prefix) => [prefix], {
|
|
893
|
+
export const counts = (HELPERS) => Redis.script((prefix) => [prefix], {
|
|
636
894
|
numberOfKeys: 0,
|
|
637
895
|
lua: `${HELPERS}
|
|
638
896
|
local pairs_ = redis.call("HGETALL", prefix .. ":counts")
|
|
@@ -640,13 +898,14 @@ if #pairs_ == 0 then return "[]" end
|
|
|
640
898
|
return cjson.encode(pairs_)
|
|
641
899
|
`
|
|
642
900
|
}).withReturnType();
|
|
643
|
-
/** remove(prefix, id) -> removed boolean (active
|
|
644
|
-
export const remove = Redis.script((prefix, id) => [prefix, id], {
|
|
901
|
+
/** remove(prefix, id) -> removed boolean (active/waiting-children refused). */
|
|
902
|
+
export const remove = (HELPERS) => Redis.script((prefix, id) => [prefix, id], {
|
|
645
903
|
numberOfKeys: 0,
|
|
646
904
|
lua: `${HELPERS}
|
|
647
905
|
local id = ARGV[2]
|
|
648
906
|
local jk = jobKey(id)
|
|
649
|
-
|
|
907
|
+
local state = redis.call("HGET", jk, "state")
|
|
908
|
+
if redis.call("EXISTS", jk) == 0 or state == "active" or state == "waiting-children" then
|
|
650
909
|
return "0"
|
|
651
910
|
end
|
|
652
911
|
deleteJob(id)
|
|
@@ -654,7 +913,7 @@ return "1"
|
|
|
654
913
|
`
|
|
655
914
|
}).withReturnType();
|
|
656
915
|
/** retry(prefix, id, now) — failed -> waiting with a fresh budget. */
|
|
657
|
-
export const retry = Redis.script((prefix, id, now) => [prefix, id, now], {
|
|
916
|
+
export const retry = (HELPERS) => Redis.script((prefix, id, now) => [prefix, id, now], {
|
|
658
917
|
numberOfKeys: 0,
|
|
659
918
|
lua: `${HELPERS}
|
|
660
919
|
local id = ARGV[2]
|
|
@@ -680,10 +939,12 @@ return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
|
680
939
|
`
|
|
681
940
|
}).withReturnType();
|
|
682
941
|
/**
|
|
683
|
-
* cancel(prefix, id, now) — waiting/delayed become terminal
|
|
684
|
-
*
|
|
942
|
+
* cancel(prefix, id, now) — waiting/delayed/waiting-children become terminal
|
|
943
|
+
* (a parked flow parent also flips its remaining pending rows to cancelled,
|
|
944
|
+
* handing them to the cascade sweep); active gets the cancel-request flag;
|
|
945
|
+
* terminal states are refused.
|
|
685
946
|
*/
|
|
686
|
-
export const cancel = Redis.script((prefix, id, now) => [prefix, id, now], {
|
|
947
|
+
export const cancel = (HELPERS) => Redis.script((prefix, id, now) => [prefix, id, now], {
|
|
687
948
|
numberOfKeys: 0,
|
|
688
949
|
lua: `${HELPERS}
|
|
689
950
|
local id = ARGV[2]
|
|
@@ -694,13 +955,16 @@ if state == "active" then
|
|
|
694
955
|
redis.call("HSET", jk, "cancelRequested", "1")
|
|
695
956
|
return '{"ok":true}'
|
|
696
957
|
end
|
|
697
|
-
if state ~= "waiting" and state ~= "delayed" then
|
|
958
|
+
if state ~= "waiting" and state ~= "delayed" and state ~= "waiting-children" then
|
|
698
959
|
return '{"error":"state","state":' .. cjson.encode(state) .. '}'
|
|
699
960
|
end
|
|
700
961
|
local nowStr = ARGV[3]
|
|
701
962
|
local now = tonumber(nowStr)
|
|
702
963
|
local queue = redis.call("HGET", jk, "queue")
|
|
703
964
|
local name = redis.call("HGET", jk, "name")
|
|
965
|
+
if state == "waiting-children" then
|
|
966
|
+
settleMarkRows(id)
|
|
967
|
+
end
|
|
704
968
|
remWaiting(queue, id)
|
|
705
969
|
redis.call("ZREM", delayedKey(queue), id)
|
|
706
970
|
redis.call("HSET", jk, "state", "cancelled", "finishedAt", nowStr, "cancelRequested", "0")
|
|
@@ -709,13 +973,14 @@ countsAdd(queue, "cancelled", 1)
|
|
|
709
973
|
redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
|
|
710
974
|
redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
|
|
711
975
|
appendAttempt(id, "cancelled", redis.call("HGET", jk, "processedAt"), nowStr, "")
|
|
976
|
+
appendOutbox(id, "cancelled")
|
|
712
977
|
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
713
978
|
applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
|
|
714
979
|
return '{"ok":true}'
|
|
715
980
|
`
|
|
716
981
|
}).withReturnType();
|
|
717
982
|
/** promote(prefix, id, now) — delayed -> waiting now. */
|
|
718
|
-
export const promote = Redis.script((prefix, id, now) => [prefix, id, now], {
|
|
983
|
+
export const promote = (HELPERS) => Redis.script((prefix, id, now) => [prefix, id, now], {
|
|
719
984
|
numberOfKeys: 0,
|
|
720
985
|
lua: `${HELPERS}
|
|
721
986
|
local id = ARGV[2]
|
|
@@ -743,7 +1008,7 @@ return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
|
743
1008
|
* digits) and empty arrays ({}). An unchanged cadence (cron/tz/everyMs)
|
|
744
1009
|
* preserves the stored nextRunAt.
|
|
745
1010
|
*/
|
|
746
|
-
export const upsertSchedule = Redis.script((prefix, key, jobName, queue, cron, tz, everyMs, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, group, nextRunAt) => [
|
|
1011
|
+
export const upsertSchedule = (HELPERS) => Redis.script((prefix, key, jobName, queue, cron, tz, everyMs, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, group, nextRunAt) => [
|
|
747
1012
|
prefix,
|
|
748
1013
|
key,
|
|
749
1014
|
jobName,
|
|
@@ -787,7 +1052,7 @@ return '{"ok":true}'
|
|
|
787
1052
|
`
|
|
788
1053
|
}).withReturnType();
|
|
789
1054
|
/** removeSchedule(prefix, key) -> existed boolean. */
|
|
790
|
-
export const removeSchedule = Redis.script((prefix, key) => [prefix, key], {
|
|
1055
|
+
export const removeSchedule = (HELPERS) => Redis.script((prefix, key) => [prefix, key], {
|
|
791
1056
|
numberOfKeys: 0,
|
|
792
1057
|
lua: `${HELPERS}
|
|
793
1058
|
local removed = redis.call("ZREM", prefix .. ":schedules", ARGV[2])
|
|
@@ -796,7 +1061,7 @@ return tostring(removed)
|
|
|
796
1061
|
`
|
|
797
1062
|
}).withReturnType();
|
|
798
1063
|
/** listSchedules(prefix, filtersJson) ordered by nextRunAt ascending. */
|
|
799
|
-
export const listSchedules = Redis.script((prefix, filtersJson) => [prefix, filtersJson], {
|
|
1064
|
+
export const listSchedules = (HELPERS) => Redis.script((prefix, filtersJson) => [prefix, filtersJson], {
|
|
800
1065
|
numberOfKeys: 0,
|
|
801
1066
|
lua: `${HELPERS}
|
|
802
1067
|
local filters = cjson.decode(ARGV[2])
|
|
@@ -817,7 +1082,7 @@ return cjson.encode(out)
|
|
|
817
1082
|
`
|
|
818
1083
|
}).withReturnType();
|
|
819
1084
|
/** dueSchedules(prefix, now) ordered by nextRunAt ascending. */
|
|
820
|
-
export const dueSchedules = Redis.script((prefix, now) => [prefix, now], {
|
|
1085
|
+
export const dueSchedules = (HELPERS) => Redis.script((prefix, now) => [prefix, now], {
|
|
821
1086
|
numberOfKeys: 0,
|
|
822
1087
|
lua: `${HELPERS}
|
|
823
1088
|
local keys = redis.call("ZRANGEBYSCORE", prefix .. ":schedules", "-inf", tonumber(ARGV[2]))
|
|
@@ -830,7 +1095,7 @@ return cjson.encode(out)
|
|
|
830
1095
|
`
|
|
831
1096
|
}).withReturnType();
|
|
832
1097
|
/** advanceSchedule(prefix, key, expectedRunAt, nextRunAt) — conditional CAS. */
|
|
833
|
-
export const advanceSchedule = Redis.script((prefix, key, expectedRunAt, nextRunAt) => [prefix, key, expectedRunAt, nextRunAt], {
|
|
1098
|
+
export const advanceSchedule = (HELPERS) => Redis.script((prefix, key, expectedRunAt, nextRunAt) => [prefix, key, expectedRunAt, nextRunAt], {
|
|
834
1099
|
numberOfKeys: 0,
|
|
835
1100
|
lua: `${HELPERS}
|
|
836
1101
|
local key = ARGV[2]
|
|
@@ -845,12 +1110,12 @@ return "1"
|
|
|
845
1110
|
/**
|
|
846
1111
|
* tickSchedule(prefix, key, expectedRunAt, nextRunAt, id, name, queue,
|
|
847
1112
|
* payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson,
|
|
848
|
-
* timeoutMs, traceJson, delayMs, now) -> "1" fired | "0"
|
|
1113
|
+
* timeoutMs, traceJson, parentJson, delayMs, now) -> "1" fired | "0"
|
|
849
1114
|
* Atomic occurrence claim: the nextRunAt CAS and the tick job's insert run
|
|
850
1115
|
* in one script, so a stale sweeper can never re-fire a slot — even after
|
|
851
1116
|
* retention pruned the previous slot's job row.
|
|
852
1117
|
*/
|
|
853
|
-
export const tickSchedule = Redis.script((prefix, key, expectedRunAt, nextRunAt, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, traceJson, delayMs, now) => [
|
|
1118
|
+
export const tickSchedule = (HELPERS) => Redis.script((prefix, key, expectedRunAt, nextRunAt, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, traceJson, parentJson, delayMs, now) => [
|
|
854
1119
|
prefix,
|
|
855
1120
|
key,
|
|
856
1121
|
expectedRunAt,
|
|
@@ -866,6 +1131,7 @@ export const tickSchedule = Redis.script((prefix, key, expectedRunAt, nextRunAt,
|
|
|
866
1131
|
keepJson,
|
|
867
1132
|
timeoutMs,
|
|
868
1133
|
traceJson,
|
|
1134
|
+
parentJson,
|
|
869
1135
|
delayMs,
|
|
870
1136
|
now
|
|
871
1137
|
], {
|
|
@@ -882,18 +1148,18 @@ local id = ARGV[5]
|
|
|
882
1148
|
-- schedule still advances, but nothing new fires.
|
|
883
1149
|
if redis.call("EXISTS", jobKey(id)) == 1 then return "0" end
|
|
884
1150
|
insertJobRow(id, ARGV[6], ARGV[7], ARGV[8], ARGV[9], ARGV[10], ARGV[11], ARGV[12], ARGV[13],
|
|
885
|
-
ARGV[14], "", ARGV[15], tonumber(ARGV[
|
|
1151
|
+
ARGV[14], "", ARGV[15], ARGV[16], tonumber(ARGV[17]), tonumber(ARGV[18]), ARGV[18])
|
|
886
1152
|
return "1"
|
|
887
1153
|
`
|
|
888
1154
|
}).withReturnType();
|
|
889
1155
|
/**
|
|
890
1156
|
* enqueueMany(prefix, now, count, ...items) -> JSON array of per-item results
|
|
891
|
-
* ({id, duplicate} | {collision} | {error}). Items are
|
|
1157
|
+
* ({id, duplicate} | {collision} | {error}). Items are 14-ARGV strides:
|
|
892
1158
|
* idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax,
|
|
893
|
-
* backoffJson, keepJson, timeoutMs, traceJson, delayMs. Plain
|
|
894
|
-
* items only — the caller routes dedup items through \`enqueue\`.
|
|
1159
|
+
* backoffJson, keepJson, timeoutMs, traceJson, parentJson, delayMs. Plain
|
|
1160
|
+
* (non-dedup) items only — the caller routes dedup items through \`enqueue\`.
|
|
895
1161
|
*/
|
|
896
|
-
export const enqueueMany = Redis.script((prefix, now, count, items) => [prefix, now, count, ...items], {
|
|
1162
|
+
export const enqueueMany = (HELPERS) => Redis.script((prefix, now, count, items) => [prefix, now, count, ...items], {
|
|
897
1163
|
numberOfKeys: 0,
|
|
898
1164
|
lua: `${HELPERS}
|
|
899
1165
|
local now = tonumber(ARGV[2])
|
|
@@ -901,7 +1167,7 @@ local nowStr = ARGV[2]
|
|
|
901
1167
|
local count = tonumber(ARGV[3])
|
|
902
1168
|
local out = {}
|
|
903
1169
|
for i = 0, count - 1 do
|
|
904
|
-
local base = 3 + i *
|
|
1170
|
+
local base = 3 + i * 14
|
|
905
1171
|
local idMode = ARGV[base + 1]
|
|
906
1172
|
local id = ARGV[base + 2]
|
|
907
1173
|
local result
|
|
@@ -929,7 +1195,7 @@ for i = 0, count - 1 do
|
|
|
929
1195
|
else
|
|
930
1196
|
insertJobRow(id, ARGV[base + 3], ARGV[base + 4], ARGV[base + 5], ARGV[base + 6],
|
|
931
1197
|
ARGV[base + 7], ARGV[base + 8], ARGV[base + 9], ARGV[base + 10], ARGV[base + 11],
|
|
932
|
-
"", ARGV[base + 12], tonumber(ARGV[base +
|
|
1198
|
+
"", ARGV[base + 12], ARGV[base + 13], tonumber(ARGV[base + 14]), now, nowStr)
|
|
933
1199
|
result = '{"id":' .. cjson.encode(id) .. ',"duplicate":false}'
|
|
934
1200
|
end
|
|
935
1201
|
end
|
|
@@ -944,7 +1210,7 @@ return "[" .. table.concat(out, ",") .. "]"
|
|
|
944
1210
|
* min(store ceiling, per-row keep.age). The caller advances the offset by
|
|
945
1211
|
* (scanned - deleted) and stops when a page comes back short.
|
|
946
1212
|
*/
|
|
947
|
-
export const sweepState = Redis.script((prefix, state, ttlMs, limit, offset, now) => [
|
|
1213
|
+
export const sweepState = (HELPERS) => Redis.script((prefix, state, ttlMs, limit, offset, now) => [
|
|
948
1214
|
prefix,
|
|
949
1215
|
state,
|
|
950
1216
|
ttlMs,
|
|
@@ -991,7 +1257,9 @@ for i = 1, #batch, 2 do
|
|
|
991
1257
|
end
|
|
992
1258
|
end
|
|
993
1259
|
end
|
|
994
|
-
|
|
1260
|
+
-- Parents still owing cascade cancels are exempt from automatic
|
|
1261
|
+
-- retention; they count as scanned so the offset cursor walks past them.
|
|
1262
|
+
if cutoffAge ~= nil and finishedAt <= now - cutoffAge and not owesCascades(id) then
|
|
995
1263
|
deleteJob(id)
|
|
996
1264
|
deleted = deleted + 1
|
|
997
1265
|
end
|
|
@@ -1005,7 +1273,7 @@ return cjson.encode({ scanned = scanned, deleted = deleted })
|
|
|
1005
1273
|
* caller loops until 0): expired windows, then pending pointers (+inf)
|
|
1006
1274
|
* whose job is gone or terminal.
|
|
1007
1275
|
*/
|
|
1008
|
-
export const sweepDedupes = Redis.script((prefix, limit, now) => [prefix, limit, now], {
|
|
1276
|
+
export const sweepDedupes = (HELPERS) => Redis.script((prefix, limit, now) => [prefix, limit, now], {
|
|
1009
1277
|
numberOfKeys: 0,
|
|
1010
1278
|
lua: `${HELPERS}
|
|
1011
1279
|
local limit = tonumber(ARGV[2])
|
|
@@ -1044,4 +1312,352 @@ end
|
|
|
1044
1312
|
return tostring(migrated + #expired + removedPending)
|
|
1045
1313
|
`
|
|
1046
1314
|
}).withReturnType();
|
|
1315
|
+
/**
|
|
1316
|
+
* fanOut(prefix, id, token, final, clearStaged, failFast, total, now, count,
|
|
1317
|
+
* ...items) — the FanOut ack, chunked like enqueueMany. Items are 5-ARGV
|
|
1318
|
+
* strides: childKey, storeKey, childJobId, name, specJson.
|
|
1319
|
+
*
|
|
1320
|
+
* Every chunk is lock-token-guarded. Non-final chunks ONLY stage dependency
|
|
1321
|
+
* rows; the final chunk stages its rows, appends the "fanned-out" ledger
|
|
1322
|
+
* entry (no attempt consumed), persists the manifest (flowFailFast,
|
|
1323
|
+
* flowPending = total, zeroed outcome counters), and transitions the parent:
|
|
1324
|
+
* pending > 0 parks it in
|
|
1325
|
+
* waiting-children (no pending zset — never claimable), pending == 0 settles
|
|
1326
|
+
* it straight to runnable collect. A parent whose manifest already landed
|
|
1327
|
+
* keeps it untouched (rows are not re-created; the transition follows the
|
|
1328
|
+
* persisted pending count), so a double fan-out cannot duplicate children.
|
|
1329
|
+
* When staging starts with no manifest, the FIRST chunk clears previously
|
|
1330
|
+
* staged rows — a crashed earlier attempt may have staged different keys. A
|
|
1331
|
+
* raced cancelRequested wins: the parent settles cancelled and its pending
|
|
1332
|
+
* rows flip to cancelled (cascade work for the flow sweeper).
|
|
1333
|
+
*/
|
|
1334
|
+
export const fanOut = (HELPERS) => Redis.script((prefix, id, token, final, clearStaged, failFast, total, now, count, items) => [prefix, id, token, final, clearStaged, failFast, total, now, count, ...items], {
|
|
1335
|
+
numberOfKeys: 0,
|
|
1336
|
+
lua: `${HELPERS}
|
|
1337
|
+
local id = ARGV[2]
|
|
1338
|
+
local jk = jobKey(id)
|
|
1339
|
+
if redis.call("EXISTS", jk) == 0 then return '{"error":"notfound"}' end
|
|
1340
|
+
if redis.call("HGET", jk, "state") ~= "active" or redis.call("HGET", jk, "lockToken") ~= ARGV[3] then
|
|
1341
|
+
return '{"error":"locklost"}'
|
|
1342
|
+
end
|
|
1343
|
+
local final = ARGV[4] == "1"
|
|
1344
|
+
local clearStaged = ARGV[5] == "1"
|
|
1345
|
+
local failFast = ARGV[6]
|
|
1346
|
+
local total = tonumber(ARGV[7])
|
|
1347
|
+
local nowStr = ARGV[8]
|
|
1348
|
+
local now = tonumber(nowStr)
|
|
1349
|
+
local count = tonumber(ARGV[9])
|
|
1350
|
+
local pendingStr = redis.call("HGET", jk, "flowPending")
|
|
1351
|
+
local hasManifest = pendingStr ~= false and pendingStr ~= ""
|
|
1352
|
+
if not hasManifest then
|
|
1353
|
+
if clearStaged then
|
|
1354
|
+
local staged = redis.call("ZRANGE", flowIndexKey(id), 0, -1)
|
|
1355
|
+
for i = 1, #staged do
|
|
1356
|
+
redis.call("DEL", flowChildKey(id, staged[i]))
|
|
1357
|
+
redis.call("ZREM", prefix .. ":flowpending", flowMember(id, staged[i]))
|
|
1358
|
+
redis.call("ZREM", prefix .. ":flowcascade", flowMember(id, staged[i]))
|
|
1359
|
+
end
|
|
1360
|
+
redis.call("DEL", flowIndexKey(id))
|
|
1361
|
+
end
|
|
1362
|
+
for i = 0, count - 1 do
|
|
1363
|
+
local base = 9 + i * 5
|
|
1364
|
+
local childKey = ARGV[base + 1]
|
|
1365
|
+
local rk = flowChildKey(id, childKey)
|
|
1366
|
+
redis.call("DEL", rk)
|
|
1367
|
+
redis.call("HSET", rk,
|
|
1368
|
+
"childKey", childKey, "storeKey", ARGV[base + 2], "childJobId", ARGV[base + 3],
|
|
1369
|
+
"name", ARGV[base + 4], "spec", ARGV[base + 5],
|
|
1370
|
+
"status", "pending", "exit", "", "failedReason", "", "cascaded", "0",
|
|
1371
|
+
"pendingSince", nowStr)
|
|
1372
|
+
redis.call("ZADD", flowIndexKey(id), 0, childKey)
|
|
1373
|
+
redis.call("ZADD", prefix .. ":flowpending", now, flowMember(id, childKey))
|
|
1374
|
+
end
|
|
1375
|
+
end
|
|
1376
|
+
if not final then return '{"ok":true}' end
|
|
1377
|
+
local queue = redis.call("HGET", jk, "queue")
|
|
1378
|
+
local name = redis.call("HGET", jk, "name")
|
|
1379
|
+
local startedAt = redis.call("HGET", jk, "processedAt")
|
|
1380
|
+
-- A fan-out is a phase transition, not a completed run: no attemptsMade.
|
|
1381
|
+
appendAttempt(id, "fanned-out", startedAt, nowStr, "")
|
|
1382
|
+
redis.call("ZREM", prefix .. ":active", id)
|
|
1383
|
+
local pending
|
|
1384
|
+
if hasManifest then
|
|
1385
|
+
pending = tonumber(pendingStr) or 0
|
|
1386
|
+
else
|
|
1387
|
+
redis.call("HSET", jk, "flowFailFast", failFast, "flowPending", fmt(total),
|
|
1388
|
+
"flowCompleted", "0", "flowFailed", "0", "flowCancelled", "0")
|
|
1389
|
+
pending = total
|
|
1390
|
+
end
|
|
1391
|
+
if redis.call("HGET", jk, "cancelRequested") == "1" then
|
|
1392
|
+
-- A cancel raced the fan-out: cancellation wins. The rows exist and get
|
|
1393
|
+
-- marked, so the sweeper cascades (mostly no-op cancels for
|
|
1394
|
+
-- never-enqueued children).
|
|
1395
|
+
settleMarkRows(id)
|
|
1396
|
+
finishCancelled(id, queue, name, startedAt, now, nowStr)
|
|
1397
|
+
return '{"ok":true}'
|
|
1398
|
+
end
|
|
1399
|
+
if pending > 0 then
|
|
1400
|
+
redis.call("HSET", jk, "state", "waiting-children", "lockToken", "", "lockExpiresAt", "")
|
|
1401
|
+
countsAdd(queue, "active", -1)
|
|
1402
|
+
countsAdd(queue, "waiting-children", 1)
|
|
1403
|
+
return '{"ok":true}'
|
|
1404
|
+
end
|
|
1405
|
+
-- Empty (or fully recorded) manifest: settle straight to runnable collect.
|
|
1406
|
+
local seq = redis.call("INCR", prefix .. ":seq")
|
|
1407
|
+
redis.call("HSET", jk, "state", "waiting", "runAt", nowStr, "seq", fmt(seq),
|
|
1408
|
+
"lockToken", "", "lockExpiresAt", "")
|
|
1409
|
+
local priority = tonumber(redis.call("HGET", jk, "priority")) or 0
|
|
1410
|
+
addWaiting(queue, priority, seq, id)
|
|
1411
|
+
countsAdd(queue, "active", -1)
|
|
1412
|
+
countsAdd(queue, "waiting", 1)
|
|
1413
|
+
return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
1414
|
+
`
|
|
1415
|
+
}).withReturnType();
|
|
1416
|
+
/**
|
|
1417
|
+
* recordChildResults(prefix, now, count, ...items) -> {results, wakes}.
|
|
1418
|
+
* Items are 5-ARGV strides: flowId, childKey, outcome, exitJson,
|
|
1419
|
+
* failedReason; results are positional {applied, parentSettled}; wakes names
|
|
1420
|
+
* the queues of parents that resumed runnable. One atomic batch; reports may
|
|
1421
|
+
* span flows.
|
|
1422
|
+
*
|
|
1423
|
+
* Phase 1 applies EVERY row update — idempotent, only while the row is
|
|
1424
|
+
* still pending; an applied report moves the child from the parent's `pending`
|
|
1425
|
+
* counter to its outcome counter and marks the row cascaded (the outcome
|
|
1426
|
+
* came FROM the child's store). Phase 2 settles each touched flow at most
|
|
1427
|
+
* once: the FIRST applied failed report in batch order under fail-fast
|
|
1428
|
+
* (terminal store-side failure; remaining rows flip to cancelled; wins the
|
|
1429
|
+
* tie over pending==0 — and this settle IS a nested parent's terminal
|
|
1430
|
+
* transition, so its own report goes to the outbox here), else pending==0
|
|
1431
|
+
* resumes the parent runnable at the flow's LAST applied report's index.
|
|
1432
|
+
*/
|
|
1433
|
+
export const recordChildResults = (HELPERS) => Redis.script((prefix, now, count, items) => [prefix, now, count, ...items], {
|
|
1434
|
+
numberOfKeys: 0,
|
|
1435
|
+
lua: `${HELPERS}
|
|
1436
|
+
local nowStr = ARGV[2]
|
|
1437
|
+
local now = tonumber(nowStr)
|
|
1438
|
+
local count = tonumber(ARGV[3])
|
|
1439
|
+
local applied = {}
|
|
1440
|
+
local settled = {}
|
|
1441
|
+
-- Phase 1: every row update lands before any settle decision, so a
|
|
1442
|
+
-- completed batch-mate keeps its real outcome even when an earlier
|
|
1443
|
+
-- batch-mate settles the flow fail-fast.
|
|
1444
|
+
local touched = {}
|
|
1445
|
+
local touchedOrder = {}
|
|
1446
|
+
for i = 1, count do
|
|
1447
|
+
local base = 3 + (i - 1) * 5
|
|
1448
|
+
local flowId = ARGV[base + 1]
|
|
1449
|
+
local childKey = ARGV[base + 2]
|
|
1450
|
+
local outcome = ARGV[base + 3]
|
|
1451
|
+
applied[i] = false
|
|
1452
|
+
local rk = flowChildKey(flowId, childKey)
|
|
1453
|
+
if redis.call("HGET", rk, "status") == "pending" then
|
|
1454
|
+
redis.call("HSET", rk, "status", outcome, "exit", ARGV[base + 4],
|
|
1455
|
+
"failedReason", ARGV[base + 5], "cascaded", "1")
|
|
1456
|
+
redis.call("ZREM", prefix .. ":flowpending", flowMember(flowId, childKey))
|
|
1457
|
+
applied[i] = true
|
|
1458
|
+
local jk = jobKey(flowId)
|
|
1459
|
+
local pendingStr = redis.call("HGET", jk, "flowPending")
|
|
1460
|
+
if pendingStr ~= false and pendingStr ~= "" then
|
|
1461
|
+
redis.call("HSET", jk, "flowPending", fmt(math.max(0, (tonumber(pendingStr) or 0) - 1)))
|
|
1462
|
+
local bucket = outcome == "completed" and "flowCompleted"
|
|
1463
|
+
or outcome == "failed" and "flowFailed" or "flowCancelled"
|
|
1464
|
+
redis.call("HINCRBY", jk, bucket, 1)
|
|
1465
|
+
end
|
|
1466
|
+
local touch = touched[flowId]
|
|
1467
|
+
if touch == nil then
|
|
1468
|
+
touch = { last = i }
|
|
1469
|
+
touched[flowId] = touch
|
|
1470
|
+
touchedOrder[#touchedOrder + 1] = flowId
|
|
1471
|
+
end
|
|
1472
|
+
touch.last = i
|
|
1473
|
+
if outcome == "failed" and touch.firstFailed == nil then
|
|
1474
|
+
touch.firstFailed = i
|
|
1475
|
+
touch.failedKey = childKey
|
|
1476
|
+
end
|
|
1477
|
+
end
|
|
1478
|
+
end
|
|
1479
|
+
-- Phase 2: at most one settle per touched flow; fail-fast wins ties.
|
|
1480
|
+
local wakes = {}
|
|
1481
|
+
for _, flowId in ipairs(touchedOrder) do
|
|
1482
|
+
local touch = touched[flowId]
|
|
1483
|
+
local jk = jobKey(flowId)
|
|
1484
|
+
local pendingStr = redis.call("HGET", jk, "flowPending")
|
|
1485
|
+
if pendingStr ~= false and pendingStr ~= ""
|
|
1486
|
+
and redis.call("HGET", jk, "state") == "waiting-children" then
|
|
1487
|
+
local queue = redis.call("HGET", jk, "queue")
|
|
1488
|
+
if redis.call("HGET", jk, "flowFailFast") == "1" and touch.firstFailed ~= nil then
|
|
1489
|
+
-- First applied failure settles the parent terminally, store-side
|
|
1490
|
+
-- (failedReason, no exit — like stall exhaustion) and marks the
|
|
1491
|
+
-- remaining rows in the same op.
|
|
1492
|
+
local name = redis.call("HGET", jk, "name")
|
|
1493
|
+
local startedAt = redis.call("HGET", jk, "processedAt")
|
|
1494
|
+
settleMarkRows(flowId)
|
|
1495
|
+
redis.call("HSET", jk, "state", "failed", "finishedAt", nowStr, "cancelRequested", "0",
|
|
1496
|
+
"failedReason", 'effect-mq: flow child "' .. touch.failedKey .. '" failed')
|
|
1497
|
+
countsAdd(queue, "waiting-children", -1)
|
|
1498
|
+
countsAdd(queue, "failed", 1)
|
|
1499
|
+
redis.call("ZADD", prefix .. ":finished:failed", now, flowId)
|
|
1500
|
+
redis.call("ZADD", terminalKey(name, "failed"), now, flowId)
|
|
1501
|
+
appendAttempt(flowId, "failed", startedAt, nowStr, "")
|
|
1502
|
+
-- A nested parent reports upward: this settle IS its terminal
|
|
1503
|
+
-- transition, with no worker ack to hook.
|
|
1504
|
+
appendOutbox(flowId, "failed")
|
|
1505
|
+
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), flowId, now)
|
|
1506
|
+
applyKeep(name, "failed", redis.call("HGET", jk, "keep"), now)
|
|
1507
|
+
settled[touch.firstFailed] = true
|
|
1508
|
+
elseif tonumber(pendingStr) == 0 then
|
|
1509
|
+
-- All children settled: the parent resumes runnable, phase collect.
|
|
1510
|
+
local seq = redis.call("INCR", prefix .. ":seq")
|
|
1511
|
+
redis.call("HSET", jk, "state", "waiting", "runAt", nowStr, "seq", fmt(seq))
|
|
1512
|
+
local priority = tonumber(redis.call("HGET", jk, "priority")) or 0
|
|
1513
|
+
addWaiting(queue, priority, seq, flowId)
|
|
1514
|
+
countsAdd(queue, "waiting-children", -1)
|
|
1515
|
+
countsAdd(queue, "waiting", 1)
|
|
1516
|
+
wakes[#wakes + 1] = queue
|
|
1517
|
+
settled[touch.last] = true
|
|
1518
|
+
end
|
|
1519
|
+
end
|
|
1520
|
+
end
|
|
1521
|
+
local out = {}
|
|
1522
|
+
for i = 1, count do
|
|
1523
|
+
out[i] = '{"applied":' .. (applied[i] and "true" or "false")
|
|
1524
|
+
.. ',"parentSettled":' .. (settled[i] and "true" or "false") .. '}'
|
|
1525
|
+
end
|
|
1526
|
+
local wakesJson = #wakes == 0 and "[]" or cjson.encode(wakes)
|
|
1527
|
+
return '{"results":[' .. table.concat(out, ",") .. '],"wakes":' .. wakesJson .. '}'
|
|
1528
|
+
`
|
|
1529
|
+
}).withReturnType();
|
|
1530
|
+
/**
|
|
1531
|
+
* listChildResults(prefix, flowId, cursor, limit) — child-key order via
|
|
1532
|
+
* ZRANGEBYLEX over the per-flow index; cursor = last childKey (exclusive).
|
|
1533
|
+
* Items are positional HMGET tuples (the field list must stay in lockstep
|
|
1534
|
+
* with the driver's `toChildRecord`) — the full spec JSON stays server-side.
|
|
1535
|
+
*/
|
|
1536
|
+
export const listChildResults = (HELPERS) => Redis.script((prefix, flowId, cursor, limit) => [prefix, flowId, cursor, limit], {
|
|
1537
|
+
numberOfKeys: 0,
|
|
1538
|
+
lua: `${HELPERS}
|
|
1539
|
+
local flowId = ARGV[2]
|
|
1540
|
+
local min = ARGV[3] == "" and "-" or ("(" .. ARGV[3])
|
|
1541
|
+
local limit = tonumber(ARGV[4])
|
|
1542
|
+
local keys = redis.call("ZRANGEBYLEX", flowIndexKey(flowId), min, "+", "LIMIT", 0, limit + 1)
|
|
1543
|
+
local items = {}
|
|
1544
|
+
for i = 1, math.min(#keys, limit) do
|
|
1545
|
+
items[#items + 1] = redis.call("HMGET", flowChildKey(flowId, keys[i]),
|
|
1546
|
+
"childKey", "storeKey", "childJobId", "name", "status", "exit", "failedReason", "cascaded")
|
|
1547
|
+
end
|
|
1548
|
+
if #items == 0 then return '{"items":[],"more":false}' end
|
|
1549
|
+
return cjson.encode({ items = items, more = #keys > limit })
|
|
1550
|
+
`
|
|
1551
|
+
}).withReturnType();
|
|
1552
|
+
/**
|
|
1553
|
+
* flowSweepWork(prefix, pendingAgeMs, limit, now) -> {reconcile, cascade}
|
|
1554
|
+
* grouped by flowId. Reconcile scans the flowpending zset (score = the row's
|
|
1555
|
+
* sweep-eligibility timestamp) and yields rows whose parent is still parked
|
|
1556
|
+
* in waiting-children. Every scanned member is re-armed or purged so no
|
|
1557
|
+
* member can pin the head of the page:
|
|
1558
|
+
*
|
|
1559
|
+
* - parent missing, or terminal with NO manifest: a crashed fan-out's
|
|
1560
|
+
* staged orphan — purge the row, its index member, and the flowpending
|
|
1561
|
+
* member (left alone their old scores would head-pin every page forever);
|
|
1562
|
+
* - parent alive but not waiting-children (mid-staging): re-arm to now so
|
|
1563
|
+
* it rotates behind fresher work;
|
|
1564
|
+
* - returned rows: re-arm to now (defer-on-return, per the contract) so a
|
|
1565
|
+
* full page rotates across sweeps;
|
|
1566
|
+
* - a non-pending row's membership is stale (rows never return to pending):
|
|
1567
|
+
* self-heal by removing the member.
|
|
1568
|
+
*
|
|
1569
|
+
* Cascade lists flowcascade members (cancels still owed to child stores).
|
|
1570
|
+
* Spec JSON strings pass through untouched — the script never cjson-decodes
|
|
1571
|
+
* stored payloads (precision, lone surrogates).
|
|
1572
|
+
*/
|
|
1573
|
+
export const flowSweepWork = (HELPERS) => Redis.script((prefix, pendingAgeMs, limit, now) => [prefix, pendingAgeMs, limit, now], {
|
|
1574
|
+
numberOfKeys: 0,
|
|
1575
|
+
lua: `${HELPERS}
|
|
1576
|
+
local pendingAgeMs = tonumber(ARGV[2])
|
|
1577
|
+
local limit = tonumber(ARGV[3])
|
|
1578
|
+
local now = tonumber(ARGV[4])
|
|
1579
|
+
local reconcile, rIndex = {}, {}
|
|
1580
|
+
local due = redis.call("ZRANGEBYSCORE", prefix .. ":flowpending", "-inf", now - pendingAgeMs,
|
|
1581
|
+
"LIMIT", 0, limit)
|
|
1582
|
+
for _, member in ipairs(due) do
|
|
1583
|
+
local sep = string.find(member, "\0", 1, true)
|
|
1584
|
+
local flowId = string.sub(member, 1, sep - 1)
|
|
1585
|
+
local childKey = string.sub(member, sep + 1)
|
|
1586
|
+
local jk = jobKey(flowId)
|
|
1587
|
+
local state = redis.call("HGET", jk, "state")
|
|
1588
|
+
local pendingField = redis.call("HGET", jk, "flowPending")
|
|
1589
|
+
local hasManifest = pendingField ~= false and pendingField ~= ""
|
|
1590
|
+
local terminal = state == "completed" or state == "failed" or state == "cancelled"
|
|
1591
|
+
if state == false or (terminal and not hasManifest) then
|
|
1592
|
+
-- Staged orphan (parent gone, or went terminal before a manifest ever
|
|
1593
|
+
-- landed): purge, or its old score head-pins every future page.
|
|
1594
|
+
redis.call("DEL", flowChildKey(flowId, childKey))
|
|
1595
|
+
redis.call("ZREM", flowIndexKey(flowId), childKey)
|
|
1596
|
+
redis.call("ZREM", prefix .. ":flowpending", member)
|
|
1597
|
+
elseif state ~= "waiting-children" then
|
|
1598
|
+
-- Alive but not parked (e.g. mid-staging): not this sweep's business —
|
|
1599
|
+
-- rotate it behind fresher work.
|
|
1600
|
+
redis.call("ZADD", prefix .. ":flowpending", now, member)
|
|
1601
|
+
elseif redis.call("HGET", flowChildKey(flowId, childKey), "status") ~= "pending" then
|
|
1602
|
+
-- Stale membership (rows never return to pending): self-heal.
|
|
1603
|
+
redis.call("ZREM", prefix .. ":flowpending", member)
|
|
1604
|
+
else
|
|
1605
|
+
local rk = flowChildKey(flowId, childKey)
|
|
1606
|
+
local group = rIndex[flowId]
|
|
1607
|
+
if group == nil then
|
|
1608
|
+
group = { flowId = flowId, children = {} }
|
|
1609
|
+
rIndex[flowId] = group
|
|
1610
|
+
reconcile[#reconcile + 1] = group
|
|
1611
|
+
end
|
|
1612
|
+
group.children[#group.children + 1] = {
|
|
1613
|
+
childKey = childKey,
|
|
1614
|
+
storeKey = redis.call("HGET", rk, "storeKey"),
|
|
1615
|
+
spec = redis.call("HGET", rk, "spec")
|
|
1616
|
+
}
|
|
1617
|
+
-- Returned work defers its own re-eligibility by one age: page rotation.
|
|
1618
|
+
redis.call("ZADD", prefix .. ":flowpending", now, member)
|
|
1619
|
+
end
|
|
1620
|
+
end
|
|
1621
|
+
local cascade, cIndex = {}, {}
|
|
1622
|
+
local owed = redis.call("ZRANGE", prefix .. ":flowcascade", 0, limit - 1)
|
|
1623
|
+
for _, member in ipairs(owed) do
|
|
1624
|
+
local sep = string.find(member, "\0", 1, true)
|
|
1625
|
+
local flowId = string.sub(member, 1, sep - 1)
|
|
1626
|
+
local childKey = string.sub(member, sep + 1)
|
|
1627
|
+
local rk = flowChildKey(flowId, childKey)
|
|
1628
|
+
if redis.call("EXISTS", rk) == 1 then
|
|
1629
|
+
local group = cIndex[flowId]
|
|
1630
|
+
if group == nil then
|
|
1631
|
+
group = { flowId = flowId, children = {} }
|
|
1632
|
+
cIndex[flowId] = group
|
|
1633
|
+
cascade[#cascade + 1] = group
|
|
1634
|
+
end
|
|
1635
|
+
group.children[#group.children + 1] = {
|
|
1636
|
+
childKey = childKey,
|
|
1637
|
+
storeKey = redis.call("HGET", rk, "storeKey"),
|
|
1638
|
+
childJobId = redis.call("HGET", rk, "childJobId")
|
|
1639
|
+
}
|
|
1640
|
+
end
|
|
1641
|
+
end
|
|
1642
|
+
return cjson.encode({ reconcile = reconcile, cascade = cascade })
|
|
1643
|
+
`
|
|
1644
|
+
}).withReturnType();
|
|
1645
|
+
/**
|
|
1646
|
+
* markChildrenCascaded(prefix, flowId, childKeysJson) — idempotent; unknown
|
|
1647
|
+
* keys are ignored (their index members are still cleared).
|
|
1648
|
+
*/
|
|
1649
|
+
export const markChildrenCascaded = (HELPERS) => Redis.script((prefix, flowId, childKeysJson) => [prefix, flowId, childKeysJson], {
|
|
1650
|
+
numberOfKeys: 0,
|
|
1651
|
+
lua: `${HELPERS}
|
|
1652
|
+
local flowId = ARGV[2]
|
|
1653
|
+
for _, key in ipairs(cjson.decode(ARGV[3])) do
|
|
1654
|
+
local rk = flowChildKey(flowId, key)
|
|
1655
|
+
if redis.call("EXISTS", rk) == 1 then
|
|
1656
|
+
redis.call("HSET", rk, "cascaded", "1")
|
|
1657
|
+
end
|
|
1658
|
+
redis.call("ZREM", prefix .. ":flowcascade", flowMember(flowId, key))
|
|
1659
|
+
end
|
|
1660
|
+
return '{"ok":true}'
|
|
1661
|
+
`
|
|
1662
|
+
}).withReturnType();
|
|
1047
1663
|
//# sourceMappingURL=scripts.js.map
|