effect-mq 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +153 -18
- package/dist/Job.d.ts +51 -1
- package/dist/Job.d.ts.map +1 -1
- package/dist/Job.js +44 -2
- package/dist/Job.js.map +1 -1
- package/dist/JobStore.d.ts +86 -6
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js +27 -1
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts +6 -5
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +160 -32
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/Worker.d.ts.map +1 -1
- package/dist/Worker.js +1 -0
- package/dist/Worker.js.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts +20 -3
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.js +352 -84
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
- package/dist/drizzle-postgres/schema.d.ts +255 -351
- package/dist/drizzle-postgres/schema.d.ts.map +1 -1
- package/dist/drizzle-postgres/schema.js +75 -27
- package/dist/drizzle-postgres/schema.js.map +1 -1
- package/dist/redis/RedisJobStore.d.ts +4 -2
- package/dist/redis/RedisJobStore.d.ts.map +1 -1
- package/dist/redis/RedisJobStore.js +67 -28
- package/dist/redis/RedisJobStore.js.map +1 -1
- package/dist/redis/scripts.d.ts +19 -6
- package/dist/redis/scripts.d.ts.map +1 -1
- package/dist/redis/scripts.js +208 -23
- package/dist/redis/scripts.js.map +1 -1
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +172 -7
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +1 -1
- package/src/Job.ts +113 -6
- package/src/JobStore.ts +122 -6
- package/src/MemoryJobStore.ts +184 -44
- package/src/Worker.ts +1 -0
- package/src/drizzle-postgres/DrizzleJobStore.ts +431 -91
- package/src/drizzle-postgres/schema.ts +177 -63
- package/src/redis/RedisJobStore.ts +90 -35
- package/src/redis/scripts.ts +217 -24
- package/src/testing/conformance.ts +246 -7
package/dist/redis/scripts.js
CHANGED
|
@@ -21,11 +21,13 @@
|
|
|
21
21
|
* - `p:delayed:<queue>` ZSET, score `runAt`
|
|
22
22
|
* - `p:active` ZSET, score `lockExpiresAt`
|
|
23
23
|
* - `p:all` ZSET, score `enqueuedAt` (list pagination)
|
|
24
|
-
* - `p:finished
|
|
24
|
+
* - `p:finished:<state>` ZSET, score `finishedAt` (history TTL)
|
|
25
25
|
* - `p:terminal:<name>:<state>` ZSET, score `finishedAt` (keep pruning)
|
|
26
26
|
* - `p:counts` HASH `<queue>|<state>` -> integer
|
|
27
27
|
* - `p:paused` SET of paused queues
|
|
28
28
|
* - `p:schedules` / `p:schedule:<key>` ZSET by nextRunAt + HASH per record
|
|
29
|
+
* - `p:dedupe:<name>\0<key>` HASH {jobId, expiresAt} + `p:dedupes` index
|
|
30
|
+
* ZSET (score = window expiry, +inf = pending)
|
|
29
31
|
*
|
|
30
32
|
* @since 0.2.0
|
|
31
33
|
*/
|
|
@@ -77,7 +79,7 @@ local function deleteJob(id)
|
|
|
77
79
|
local name = redis.call("HGET", jk, "name")
|
|
78
80
|
countsAdd(queue, state, -1)
|
|
79
81
|
redis.call("ZREM", prefix .. ":all", id)
|
|
80
|
-
redis.call("ZREM", prefix .. ":finished", id)
|
|
82
|
+
redis.call("ZREM", prefix .. ":finished:" .. state, id)
|
|
81
83
|
redis.call("ZREM", prefix .. ":active", id)
|
|
82
84
|
redis.call("ZREM", terminalKey(name, state), id)
|
|
83
85
|
remWaiting(queue, id)
|
|
@@ -89,8 +91,20 @@ end
|
|
|
89
91
|
-- zset score alone cannot carry the seq tie-break exactly.
|
|
90
92
|
local function applyKeep(name, state, keepJson, now)
|
|
91
93
|
if keepJson == nil or keepJson == "" then return end
|
|
92
|
-
local ok,
|
|
93
|
-
if not ok or type(
|
|
94
|
+
local ok, decoded = pcall(cjson.decode, keepJson)
|
|
95
|
+
if not ok or type(decoded) ~= "table" then return end
|
|
96
|
+
-- Policies are split per terminal state; rows persisted by 0.2.x carry the
|
|
97
|
+
-- flat {count, ageMs} shape and apply to every state.
|
|
98
|
+
local keep = decoded[state]
|
|
99
|
+
if type(keep) ~= "table" then
|
|
100
|
+
if decoded.completed == nil and decoded.failed == nil and decoded.cancelled == nil
|
|
101
|
+
and (decoded.count ~= nil or decoded.ageMs ~= nil)
|
|
102
|
+
then
|
|
103
|
+
keep = decoded
|
|
104
|
+
else
|
|
105
|
+
return
|
|
106
|
+
end
|
|
107
|
+
end
|
|
94
108
|
local tkey = terminalKey(name, state)
|
|
95
109
|
if keep.ageMs ~= nil then
|
|
96
110
|
local old = redis.call("ZRANGEBYSCORE", tkey, "-inf", now - keep.ageMs)
|
|
@@ -116,6 +130,20 @@ local function applyKeep(name, state, keepJson, now)
|
|
|
116
130
|
for i = count + 1, #arr do deleteJob(arr[i].id) end
|
|
117
131
|
end
|
|
118
132
|
end
|
|
133
|
+
local function dedupeStoreKey(name, key) return prefix .. ":dedupe:" .. name .. "\0" .. key end
|
|
134
|
+
-- A job leaving the pending states frees its pending-mode dedup entry; live
|
|
135
|
+
-- throttle windows deliberately outlast the job.
|
|
136
|
+
local function releaseDedupe(name, dkey, jobId, now)
|
|
137
|
+
if dkey == nil or dkey == false or dkey == "" then return end
|
|
138
|
+
local sk = dedupeStoreKey(name, dkey)
|
|
139
|
+
if redis.call("HGET", sk, "jobId") == jobId then
|
|
140
|
+
local exp = redis.call("HGET", sk, "expiresAt")
|
|
141
|
+
if exp == "" or tonumber(exp) <= now then
|
|
142
|
+
redis.call("DEL", sk)
|
|
143
|
+
redis.call("ZREM", prefix .. ":dedupes", name .. "\0" .. dkey)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
119
147
|
-- Move an active job (whose lock bookkeeping was already cleared by the
|
|
120
148
|
-- caller) to the terminal cancelled state.
|
|
121
149
|
local function finishCancelled(id, queue, name, startedAt, now, nowStr)
|
|
@@ -124,9 +152,10 @@ local function finishCancelled(id, queue, name, startedAt, now, nowStr)
|
|
|
124
152
|
"lockToken", "", "lockExpiresAt", "")
|
|
125
153
|
countsAdd(queue, "active", -1)
|
|
126
154
|
countsAdd(queue, "cancelled", 1)
|
|
127
|
-
redis.call("ZADD", prefix .. ":finished", now, id)
|
|
155
|
+
redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
|
|
128
156
|
redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
|
|
129
157
|
appendAttempt(id, "cancelled", startedAt, nowStr, "")
|
|
158
|
+
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
130
159
|
applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
|
|
131
160
|
end
|
|
132
161
|
`;
|
|
@@ -136,7 +165,7 @@ end
|
|
|
136
165
|
* idMode: "user" (dedup no-op), "generated" (collision -> retry sentinel),
|
|
137
166
|
* "auto" (j-<seq>, in-script collision loop).
|
|
138
167
|
*/
|
|
139
|
-
export const enqueue = Redis.script((prefix, idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs, now) => [
|
|
168
|
+
export const enqueue = Redis.script((prefix, idMode, id, name, queue, payloadJson, metadataJson, priority, attemptsMax, backoffJson, keepJson, timeoutMs, delayMs, now, dedupeKey, dedupeTtlMs, dedupeExtend, dedupeReplace) => [
|
|
140
169
|
prefix,
|
|
141
170
|
idMode,
|
|
142
171
|
id,
|
|
@@ -150,7 +179,11 @@ export const enqueue = Redis.script((prefix, idMode, id, name, queue, payloadJso
|
|
|
150
179
|
keepJson,
|
|
151
180
|
timeoutMs,
|
|
152
181
|
delayMs,
|
|
153
|
-
now
|
|
182
|
+
now,
|
|
183
|
+
dedupeKey,
|
|
184
|
+
dedupeTtlMs,
|
|
185
|
+
dedupeExtend,
|
|
186
|
+
dedupeReplace
|
|
154
187
|
], {
|
|
155
188
|
numberOfKeys: 0,
|
|
156
189
|
lua: `${HELPERS}
|
|
@@ -164,7 +197,54 @@ if idMode == "user" or idMode == "generated" then
|
|
|
164
197
|
if idMode == "user" then return '{"duplicate":true,"id":' .. cjson.encode(id) .. '}' end
|
|
165
198
|
return '{"collision":true}'
|
|
166
199
|
end
|
|
167
|
-
|
|
200
|
+
end
|
|
201
|
+
-- Dedup decision tree: replace-while-delayed, throttle window, pending dedup.
|
|
202
|
+
local dKey = ARGV[15]
|
|
203
|
+
local name = ARGV[4]
|
|
204
|
+
if dKey ~= "" then
|
|
205
|
+
local sk = dedupeStoreKey(name, dKey)
|
|
206
|
+
local entryJob = redis.call("HGET", sk, "jobId")
|
|
207
|
+
if entryJob then
|
|
208
|
+
local expStr = redis.call("HGET", sk, "expiresAt")
|
|
209
|
+
local windowLive = expStr ~= "" and tonumber(expStr) > now
|
|
210
|
+
local keyedState = redis.call("HGET", jobKey(entryJob), "state")
|
|
211
|
+
local function bumpWindow()
|
|
212
|
+
if ARGV[17] == "1" and ARGV[16] ~= "" then
|
|
213
|
+
local windowEnd = now + tonumber(ARGV[16])
|
|
214
|
+
redis.call("HSET", sk, "expiresAt", fmt(windowEnd))
|
|
215
|
+
redis.call("ZADD", prefix .. ":dedupes", windowEnd, name .. "\0" .. dKey)
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
-- Latest-wins while the keyed job is still delayed.
|
|
219
|
+
if ARGV[18] == "1" and keyedState == "delayed" then
|
|
220
|
+
local kjk = jobKey(entryJob)
|
|
221
|
+
local newRunAt = now + delayMs
|
|
222
|
+
redis.call("HSET", kjk, "payload", ARGV[6], "metadata", ARGV[7], "priority", ARGV[8],
|
|
223
|
+
"attemptsMax", ARGV[9], "backoff", ARGV[10], "keep", ARGV[11], "timeoutMs", ARGV[12],
|
|
224
|
+
"runAt", fmt(newRunAt))
|
|
225
|
+
local keyedQueue = redis.call("HGET", kjk, "queue")
|
|
226
|
+
redis.call("ZADD", delayedKey(keyedQueue), newRunAt, entryJob)
|
|
227
|
+
-- A landed replace re-arms the ttl window.
|
|
228
|
+
if ARGV[16] ~= "" then
|
|
229
|
+
local windowEnd = now + tonumber(ARGV[16])
|
|
230
|
+
redis.call("HSET", sk, "expiresAt", fmt(windowEnd))
|
|
231
|
+
redis.call("ZADD", prefix .. ":dedupes", windowEnd, name .. "\0" .. dKey)
|
|
232
|
+
end
|
|
233
|
+
return '{"id":' .. cjson.encode(entryJob) .. ',"duplicate":true,"wake":true,"queue":' .. cjson.encode(keyedQueue) .. '}'
|
|
234
|
+
end
|
|
235
|
+
if windowLive then
|
|
236
|
+
bumpWindow()
|
|
237
|
+
return '{"id":' .. cjson.encode(entryJob) .. ',"duplicate":true}'
|
|
238
|
+
end
|
|
239
|
+
local pending = keyedState ~= false and keyedState ~= "completed"
|
|
240
|
+
and keyedState ~= "failed" and keyedState ~= "cancelled"
|
|
241
|
+
if expStr == "" and pending then
|
|
242
|
+
return '{"id":' .. cjson.encode(entryJob) .. ',"duplicate":true}'
|
|
243
|
+
end
|
|
244
|
+
-- Dead entry: the new job takes over the key below.
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
if idMode == "auto" then
|
|
168
248
|
id = ""
|
|
169
249
|
for i = 1, 5 do
|
|
170
250
|
local candidate = "j-" .. fmt(redis.call("INCR", prefix .. ":seq"))
|
|
@@ -183,7 +263,7 @@ redis.call("HSET", jobKey(id),
|
|
|
183
263
|
"payload", ARGV[6], "metadata", ARGV[7], "state", state,
|
|
184
264
|
"priority", ARGV[8], "attemptsMax", ARGV[9], "attemptsMade", "0", "stalledCount", "0",
|
|
185
265
|
"backoff", ARGV[10], "keep", ARGV[11], "timeoutMs", ARGV[12],
|
|
186
|
-
"cancelRequested", "0", "runAt", fmt(runAt), "enqueuedAt", nowStr,
|
|
266
|
+
"cancelRequested", "0", "dedupeKey", dKey, "runAt", fmt(runAt), "enqueuedAt", nowStr,
|
|
187
267
|
"processedAt", "", "finishedAt", "", "exit", "", "failedReason", "",
|
|
188
268
|
"lockToken", "", "lockExpiresAt", "", "seq", fmt(seq))
|
|
189
269
|
redis.call("ZADD", prefix .. ":all", now, id)
|
|
@@ -193,6 +273,14 @@ else
|
|
|
193
273
|
redis.call("ZADD", delayedKey(queue), runAt, id)
|
|
194
274
|
end
|
|
195
275
|
countsAdd(queue, state, 1)
|
|
276
|
+
if dKey ~= "" then
|
|
277
|
+
local sk = dedupeStoreKey(name, dKey)
|
|
278
|
+
redis.call("DEL", sk)
|
|
279
|
+
redis.call("HSET", sk, "jobId", id,
|
|
280
|
+
"expiresAt", ARGV[16] == "" and "" or fmt(now + tonumber(ARGV[16])))
|
|
281
|
+
redis.call("ZADD", prefix .. ":dedupes",
|
|
282
|
+
ARGV[16] == "" and "inf" or tostring(now + tonumber(ARGV[16])), name .. "\0" .. dKey)
|
|
283
|
+
end
|
|
196
284
|
-- Wake on EVERY insert (delayed too): idle workers must re-claim to learn
|
|
197
285
|
-- the new nextRunAt, exactly like the memory and Postgres drivers.
|
|
198
286
|
return '{"id":' .. cjson.encode(id) .. ',"duplicate":false,"wake":true}'
|
|
@@ -313,9 +401,10 @@ local function finish(newState, storeExit, outcome, ledgerExit)
|
|
|
313
401
|
if storeExit ~= nil then redis.call("HSET", jk, "exit", storeExit) end
|
|
314
402
|
countsAdd(queue, "active", -1)
|
|
315
403
|
countsAdd(queue, newState, 1)
|
|
316
|
-
redis.call("ZADD", prefix .. ":finished", now, id)
|
|
404
|
+
redis.call("ZADD", prefix .. ":finished:" .. newState, now, id)
|
|
317
405
|
redis.call("ZADD", terminalKey(name, newState), now, id)
|
|
318
406
|
appendAttempt(id, outcome, startedAt, nowStr, ledgerExit)
|
|
407
|
+
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
319
408
|
applyKeep(name, newState, redis.call("HGET", jk, "keep"), now)
|
|
320
409
|
end
|
|
321
410
|
|
|
@@ -342,7 +431,7 @@ else
|
|
|
342
431
|
else
|
|
343
432
|
redis.call("ZADD", delayedKey(queue), runAt, id)
|
|
344
433
|
end
|
|
345
|
-
return '{"ok":true,"wake":true}'
|
|
434
|
+
return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
346
435
|
end
|
|
347
436
|
return '{"ok":true}'
|
|
348
437
|
`
|
|
@@ -374,7 +463,7 @@ redis.call("HSET", jk, "state", "waiting", "lockToken", "", "lockExpiresAt", "")
|
|
|
374
463
|
addWaiting(queue, priority, seq, id)
|
|
375
464
|
countsAdd(queue, "active", -1)
|
|
376
465
|
countsAdd(queue, "waiting", 1)
|
|
377
|
-
return '{"ok":true,"wake":true}'
|
|
466
|
+
return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
378
467
|
`
|
|
379
468
|
}).withReturnType();
|
|
380
469
|
/**
|
|
@@ -432,8 +521,9 @@ for _, id in ipairs(expired) do
|
|
|
432
521
|
"failedReason", "job stalled more than allowable limit")
|
|
433
522
|
countsAdd(queue, "active", -1)
|
|
434
523
|
countsAdd(queue, "failed", 1)
|
|
435
|
-
redis.call("ZADD", prefix .. ":finished", now, id)
|
|
524
|
+
redis.call("ZADD", prefix .. ":finished:failed", now, id)
|
|
436
525
|
redis.call("ZADD", terminalKey(name, "failed"), now, id)
|
|
526
|
+
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
437
527
|
recovered[#recovered + 1] = { id = id, failed = true }
|
|
438
528
|
else
|
|
439
529
|
local seq = tonumber(redis.call("HGET", jk, "seq")) or 0
|
|
@@ -571,13 +661,13 @@ local seq = redis.call("INCR", prefix .. ":seq")
|
|
|
571
661
|
redis.call("HSET", jk, "state", "waiting", "attemptsMade", "0", "stalledCount", "0",
|
|
572
662
|
"cancelRequested", "0", "exit", "", "failedReason", "", "finishedAt", "",
|
|
573
663
|
"processedAt", "", "runAt", nowStr, "seq", fmt(seq))
|
|
574
|
-
redis.call("ZREM", prefix .. ":finished", id)
|
|
664
|
+
redis.call("ZREM", prefix .. ":finished:failed", id)
|
|
575
665
|
redis.call("ZREM", terminalKey(name, "failed"), id)
|
|
576
666
|
local priority = tonumber(redis.call("HGET", jk, "priority")) or 0
|
|
577
667
|
addWaiting(queue, priority, seq, id)
|
|
578
668
|
countsAdd(queue, "failed", -1)
|
|
579
669
|
countsAdd(queue, "waiting", 1)
|
|
580
|
-
return '{"ok":true}'
|
|
670
|
+
return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
581
671
|
`
|
|
582
672
|
}).withReturnType();
|
|
583
673
|
/**
|
|
@@ -607,9 +697,10 @@ redis.call("ZREM", delayedKey(queue), id)
|
|
|
607
697
|
redis.call("HSET", jk, "state", "cancelled", "finishedAt", nowStr, "cancelRequested", "0")
|
|
608
698
|
countsAdd(queue, state, -1)
|
|
609
699
|
countsAdd(queue, "cancelled", 1)
|
|
610
|
-
redis.call("ZADD", prefix .. ":finished", now, id)
|
|
700
|
+
redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
|
|
611
701
|
redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
|
|
612
702
|
appendAttempt(id, "cancelled", redis.call("HGET", jk, "processedAt"), nowStr, "")
|
|
703
|
+
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
613
704
|
applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
|
|
614
705
|
return '{"ok":true}'
|
|
615
706
|
`
|
|
@@ -631,7 +722,7 @@ redis.call("HSET", jk, "state", "waiting", "runAt", ARGV[3])
|
|
|
631
722
|
addWaiting(queue, priority, seq, id)
|
|
632
723
|
countsAdd(queue, "delayed", -1)
|
|
633
724
|
countsAdd(queue, "waiting", 1)
|
|
634
|
-
return '{"ok":true}'
|
|
725
|
+
return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
635
726
|
`
|
|
636
727
|
}).withReturnType();
|
|
637
728
|
/**
|
|
@@ -741,15 +832,109 @@ return "1"
|
|
|
741
832
|
`
|
|
742
833
|
}).withReturnType();
|
|
743
834
|
/**
|
|
744
|
-
*
|
|
745
|
-
*
|
|
835
|
+
* sweepState(prefix, state, ttlMs, limit, offset, now) -> {scanned, deleted}
|
|
836
|
+
* One bounded page over a terminal state's finished zset, deleting rows past
|
|
837
|
+
* min(store ceiling, per-row keep.age). The caller advances the offset by
|
|
838
|
+
* (scanned - deleted) and stops when a page comes back short.
|
|
746
839
|
*/
|
|
747
|
-
export const
|
|
840
|
+
export const sweepState = Redis.script((prefix, state, ttlMs, limit, offset, now) => [
|
|
841
|
+
prefix,
|
|
842
|
+
state,
|
|
843
|
+
ttlMs,
|
|
844
|
+
limit,
|
|
845
|
+
offset,
|
|
846
|
+
now
|
|
847
|
+
], {
|
|
848
|
+
numberOfKeys: 0,
|
|
849
|
+
lua: `${HELPERS}
|
|
850
|
+
local state = ARGV[2]
|
|
851
|
+
local ttl = ARGV[3] ~= "" and tonumber(ARGV[3]) or nil
|
|
852
|
+
local limit = tonumber(ARGV[4])
|
|
853
|
+
local offset = tonumber(ARGV[5])
|
|
854
|
+
local now = tonumber(ARGV[6])
|
|
855
|
+
local batch = redis.call("ZRANGEBYSCORE", prefix .. ":finished:" .. state, "-inf", now,
|
|
856
|
+
"WITHSCORES", "LIMIT", offset, limit)
|
|
857
|
+
local scanned = 0
|
|
858
|
+
local deleted = 0
|
|
859
|
+
for i = 1, #batch, 2 do
|
|
860
|
+
scanned = scanned + 1
|
|
861
|
+
local id = batch[i]
|
|
862
|
+
local finishedAt = tonumber(batch[i + 1])
|
|
863
|
+
if redis.call("EXISTS", jobKey(id)) == 0 then
|
|
864
|
+
-- Orphaned member (hash evicted/removed out of band): self-heal so the
|
|
865
|
+
-- cursor math stays honest and the member never loops the sweep.
|
|
866
|
+
redis.call("ZREM", prefix .. ":finished:" .. state, id)
|
|
867
|
+
deleted = deleted + 1
|
|
868
|
+
else
|
|
869
|
+
local cutoffAge = ttl
|
|
870
|
+
local keepJson = redis.call("HGET", jobKey(id), "keep")
|
|
871
|
+
if keepJson and keepJson ~= "" then
|
|
872
|
+
local ok, keep = pcall(cjson.decode, keepJson)
|
|
873
|
+
if ok and type(keep) == "table" then
|
|
874
|
+
local policy = keep[state]
|
|
875
|
+
if type(policy) ~= "table"
|
|
876
|
+
and keep.completed == nil and keep.failed == nil and keep.cancelled == nil
|
|
877
|
+
and (keep.count ~= nil or keep.ageMs ~= nil)
|
|
878
|
+
then
|
|
879
|
+
policy = keep
|
|
880
|
+
end
|
|
881
|
+
if type(policy) == "table" and policy.ageMs ~= nil then
|
|
882
|
+
local age = tonumber(policy.ageMs)
|
|
883
|
+
if age ~= nil and (cutoffAge == nil or age < cutoffAge) then cutoffAge = age end
|
|
884
|
+
end
|
|
885
|
+
end
|
|
886
|
+
end
|
|
887
|
+
if cutoffAge ~= nil and finishedAt <= now - cutoffAge then
|
|
888
|
+
deleteJob(id)
|
|
889
|
+
deleted = deleted + 1
|
|
890
|
+
end
|
|
891
|
+
end
|
|
892
|
+
end
|
|
893
|
+
return cjson.encode({ scanned = scanned, deleted = deleted })
|
|
894
|
+
`
|
|
895
|
+
}).withReturnType();
|
|
896
|
+
/**
|
|
897
|
+
* sweepDedupes(prefix, limit, now) -> number pruned (bounded batch; the
|
|
898
|
+
* caller loops until 0): expired windows, then pending pointers (+inf)
|
|
899
|
+
* whose job is gone or terminal.
|
|
900
|
+
*/
|
|
901
|
+
export const sweepDedupes = Redis.script((prefix, limit, now) => [prefix, limit, now], {
|
|
748
902
|
numberOfKeys: 0,
|
|
749
903
|
lua: `${HELPERS}
|
|
750
|
-
local
|
|
751
|
-
|
|
752
|
-
|
|
904
|
+
local limit = tonumber(ARGV[2])
|
|
905
|
+
local now = tonumber(ARGV[3])
|
|
906
|
+
-- Lazy migration: drain the pre-0.3 unsplit finished zset into the per-state
|
|
907
|
+
-- keys (or drop orphans) so old history keeps getting swept.
|
|
908
|
+
local migrated = 0
|
|
909
|
+
local legacy = redis.call("ZRANGE", prefix .. ":finished", 0, limit - 1, "WITHSCORES")
|
|
910
|
+
for i = 1, #legacy, 2 do
|
|
911
|
+
local id = legacy[i]
|
|
912
|
+
local state = redis.call("HGET", jobKey(id), "state")
|
|
913
|
+
if state == "completed" or state == "failed" or state == "cancelled" then
|
|
914
|
+
redis.call("ZADD", prefix .. ":finished:" .. state, tonumber(legacy[i + 1]), id)
|
|
915
|
+
end
|
|
916
|
+
redis.call("ZREM", prefix .. ":finished", id)
|
|
917
|
+
migrated = migrated + 1
|
|
918
|
+
end
|
|
919
|
+
local index = prefix .. ":dedupes"
|
|
920
|
+
local expired = redis.call("ZRANGEBYSCORE", index, "-inf", now, "LIMIT", 0, limit)
|
|
921
|
+
for _, member in ipairs(expired) do
|
|
922
|
+
redis.call("DEL", prefix .. ":dedupe:" .. member)
|
|
923
|
+
redis.call("ZREM", index, member)
|
|
924
|
+
end
|
|
925
|
+
local removedPending = 0
|
|
926
|
+
local pendings = redis.call("ZRANGEBYSCORE", index, "inf", "inf", "LIMIT", 0, limit)
|
|
927
|
+
for _, member in ipairs(pendings) do
|
|
928
|
+
local sk = prefix .. ":dedupe:" .. member
|
|
929
|
+
local jobId = redis.call("HGET", sk, "jobId")
|
|
930
|
+
local state = jobId and redis.call("HGET", jobKey(jobId), "state")
|
|
931
|
+
if not state or state == "completed" or state == "failed" or state == "cancelled" then
|
|
932
|
+
redis.call("DEL", sk)
|
|
933
|
+
redis.call("ZREM", index, member)
|
|
934
|
+
removedPending = removedPending + 1
|
|
935
|
+
end
|
|
936
|
+
end
|
|
937
|
+
return tostring(migrated + #expired + removedPending)
|
|
753
938
|
`
|
|
754
939
|
}).withReturnType();
|
|
755
940
|
//# sourceMappingURL=scripts.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scripts.js","sourceRoot":"","sources":["../../src/redis/scripts.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"scripts.js","sourceRoot":"","sources":["../../src/redis/scripts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AAEnD;;;GAGG;AACH,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Hf,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CACjC,CACE,MAAc,EACd,MAAc,EACd,EAAU,EACV,IAAY,EACZ,KAAa,EACb,WAAmB,EACnB,YAAoB,EACpB,QAAgB,EAChB,WAAmB,EACnB,WAAmB,EACnB,QAAgB,EAChB,SAAiB,EACjB,OAAe,EACf,GAAW,EACX,SAAiB,EACjB,WAAmB,EACnB,YAAoB,EACpB,aAAqB,EACrB,EAAE,CAAC;IACH,MAAM;IACN,MAAM;IACN,EAAE;IACF,IAAI;IACJ,KAAK;IACL,WAAW;IACX,YAAY;IACZ,QAAQ;IACR,WAAW;IACX,WAAW;IACX,QAAQ;IACR,SAAS;IACT,OAAO;IACP,GAAG;IACH,SAAS;IACT,WAAW;IACX,YAAY;IACZ,aAAa;CACd,EACD;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkGlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,MAAc,EAAE,KAAa,EAAE,SAAiB,EAAE,KAAa,EAAE,cAAsB,EAAE,GAAW,EAAE,EAAE,CAAC;IACxG,MAAM;IACN,KAAK;IACL,SAAS;IACT,KAAK;IACL,cAAc;IACd,GAAG;CACJ,EACD;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyDlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;;GAIG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAC7B,CAAC,MAAc,EAAE,EAAU,EAAE,KAAa,EAAE,UAAkB,EAAE,QAAgB,EAAE,OAAe,EAAE,GAAW,EAAE,EAAE,CAAC;IACjH,MAAM;IACN,EAAE;IACF,KAAK;IACL,UAAU;IACV,QAAQ;IACR,OAAO;IACP,GAAG;CACJ,EACD;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2DlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,MAAc,EAAE,EAAU,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,EACpF;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;CAsBlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CACrC,CAAC,MAAc,EAAE,SAAiB,EAAE,UAAkB,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,CAAC,EAC5G;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;CAiBlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CACxC,CAAC,MAAc,EAAE,eAAuB,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,eAAe,EAAE,GAAG,CAAC,EACxF;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0ClB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,sEAAsE;AACtE,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,MAAc,EAAE,EAAU,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAC5C;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;CAIlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAC9B,CAAC,MAAc,EAAE,WAAmB,EAAE,MAAc,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,CAAC,EAC5G;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgElB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,mDAAmD;AACnD,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAC5B;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;CAIlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,uEAAuE;AACvE,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,MAAc,EAAE,EAAU,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAC5C;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;CAQlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,sEAAsE;AACtE,MAAM,CAAC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAC/B,CAAC,MAAc,EAAE,EAAU,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,EAC9D;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;CAqBlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,MAAc,EAAE,EAAU,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,EAC9D;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2BlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,yDAAyD;AACzD,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CACjC,CAAC,MAAc,EAAE,EAAU,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,EAC9D;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;CAelB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CACxC,CACE,MAAc,EACd,GAAW,EACX,OAAe,EACf,KAAa,EACb,IAAY,EACZ,EAAU,EACV,OAAe,EACf,WAAmB,EACnB,YAAoB,EACpB,QAAgB,EAChB,WAAmB,EACnB,WAAmB,EACnB,QAAgB,EAChB,SAAiB,EACjB,SAAiB,EACjB,EAAE,CAAC;IACH,MAAM;IACN,GAAG;IACH,OAAO;IACP,KAAK;IACL,IAAI;IACJ,EAAE;IACF,OAAO;IACP,WAAW;IACX,YAAY;IACZ,QAAQ;IACR,WAAW;IACX,WAAW;IACX,QAAQ;IACR,SAAS;IACT,SAAS;CACV,EACD;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;CAsBlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,sDAAsD;AACtD,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CACxC,CAAC,MAAc,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9C;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;CAIlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,yEAAyE;AACzE,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CACvC,CAAC,MAAc,EAAE,WAAmB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,EAC9D;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;CAelB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,gEAAgE;AAChE,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CACtC,CAAC,MAAc,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,EAC9C;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;CAQlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,CAAC,MAAM,CACzC,CAAC,MAAc,EAAE,GAAW,EAAE,aAAqB,EAAE,SAAiB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,SAAS,CAAC,EAClH;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;CAQlB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CACpC,CAAC,MAAc,EAAE,KAAa,EAAE,KAAa,EAAE,KAAa,EAAE,MAAc,EAAE,GAAW,EAAE,EAAE,CAAC;IAC5F,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,GAAG;CACJ,EACD;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6ClB;CACE,CACF,CAAC,cAAc,EAAU,CAAA;AAE1B;;;;GAIG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CACtC,CAAC,MAAc,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,EACpE;IACE,YAAY,EAAE,CAAC;IACf,GAAG,EAAE,GAAG,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmClB;CACE,CACF,CAAC,cAAc,EAAU,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../src/testing/conformance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAA;AAE1C,OAAO,EAAuB,KAAK,KAAK,EAAU,MAAM,QAAQ,CAAA;
|
|
1
|
+
{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../src/testing/conformance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAA;AAE1C,OAAO,EAAuB,KAAK,KAAK,EAAU,MAAM,QAAQ,CAAA;AAiChE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,SACxB,MAAM,cACA,MAAM,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAC/C,IAiuCF,CAAA"}
|
|
@@ -32,6 +32,7 @@ const baseRequest = (overrides) => ({
|
|
|
32
32
|
backoff: undefined,
|
|
33
33
|
keep: undefined,
|
|
34
34
|
timeoutMs: undefined,
|
|
35
|
+
dedupe: undefined,
|
|
35
36
|
delayMs: 0,
|
|
36
37
|
...overrides
|
|
37
38
|
});
|
|
@@ -319,7 +320,7 @@ export const jobStoreConformance = (name, storeLayer) => {
|
|
|
319
320
|
})));
|
|
320
321
|
it.effect("keep count prunes older terminal records of the same name and state", () => withStore((store) => Effect.gen(function* () {
|
|
321
322
|
for (let i = 0; i < 4; i++) {
|
|
322
|
-
const { id } = yield* store.enqueue(baseRequest({ payload: { n: i }, keep: { count: 2, ageMs: undefined } }));
|
|
323
|
+
const { id } = yield* store.enqueue(baseRequest({ payload: { n: i }, keep: { completed: { count: 2, ageMs: undefined } } }));
|
|
323
324
|
const claim = yield* store.claim(claimOptions({ token: `t-${i}` }));
|
|
324
325
|
assert(claim._tag === "Claimed");
|
|
325
326
|
yield* store.ack(id, `t-${i}`, { _tag: "Complete", exit: null });
|
|
@@ -330,7 +331,7 @@ export const jobStoreConformance = (name, storeLayer) => {
|
|
|
330
331
|
expect(listed.items.map((job) => job.payload)).toEqual([{ n: 3 }, { n: 2 }]);
|
|
331
332
|
})));
|
|
332
333
|
it.effect("keep age prunes terminal records older than the window", () => withStore((store) => Effect.gen(function* () {
|
|
333
|
-
const keep = { count: undefined, ageMs: 10_000 };
|
|
334
|
+
const keep = { completed: { count: undefined, ageMs: 10_000 } };
|
|
334
335
|
const first = yield* store.enqueue(baseRequest({ payload: { n: 1 }, keep }));
|
|
335
336
|
const claim1 = yield* store.claim(claimOptions());
|
|
336
337
|
assert(claim1._tag === "Claimed");
|
|
@@ -370,8 +371,8 @@ export const jobStoreConformance = (name, storeLayer) => {
|
|
|
370
371
|
it.effect("keep count ties on finishedAt keep the most recently acked records", () => withStore((store) => Effect.gen(function* () {
|
|
371
372
|
// Two jobs acked at the SAME TestClock instant: the tie must break
|
|
372
373
|
// on enqueue/seq order identically in every driver.
|
|
373
|
-
const first = yield* store.enqueue(baseRequest({ payload: { n: 1 }, keep: { count: 1, ageMs: undefined } }));
|
|
374
|
-
const second = yield* store.enqueue(baseRequest({ payload: { n: 2 }, keep: { count: 1, ageMs: undefined } }));
|
|
374
|
+
const first = yield* store.enqueue(baseRequest({ payload: { n: 1 }, keep: { completed: { count: 1, ageMs: undefined } } }));
|
|
375
|
+
const second = yield* store.enqueue(baseRequest({ payload: { n: 2 }, keep: { completed: { count: 1, ageMs: undefined } } }));
|
|
375
376
|
const claimA = yield* store.claim(claimOptions({ token: "t-a" }));
|
|
376
377
|
const claimB = yield* store.claim(claimOptions({ token: "t-b" }));
|
|
377
378
|
assert(claimA._tag === "Claimed" && claimB._tag === "Claimed");
|
|
@@ -381,7 +382,7 @@ export const jobStoreConformance = (name, storeLayer) => {
|
|
|
381
382
|
expect(Option.isSome(yield* store.getJob(second.id))).toBe(true);
|
|
382
383
|
})));
|
|
383
384
|
it.effect("keep applies count and age together", () => withStore((store) => Effect.gen(function* () {
|
|
384
|
-
const keep = { count: 2, ageMs: 10_000 };
|
|
385
|
+
const keep = { completed: { count: 2, ageMs: 10_000 } };
|
|
385
386
|
const ids = [];
|
|
386
387
|
for (let i = 0; i < 3; i++) {
|
|
387
388
|
const { id } = yield* store.enqueue(baseRequest({ payload: { n: i }, keep }));
|
|
@@ -626,7 +627,7 @@ export const jobStoreConformance = (name, storeLayer) => {
|
|
|
626
627
|
expect(job.value.cancelRequested).toBe(true);
|
|
627
628
|
})));
|
|
628
629
|
it.effect("cancel applies the keep retention policy", () => withStore((store) => Effect.gen(function* () {
|
|
629
|
-
const keep = { count: 1 };
|
|
630
|
+
const keep = { cancelled: { count: 1 } };
|
|
630
631
|
const ids = [];
|
|
631
632
|
for (let i = 0; i < 3; i++) {
|
|
632
633
|
const { id } = yield* store.enqueue(baseRequest({ payload: { n: i }, keep }));
|
|
@@ -711,7 +712,7 @@ export const jobStoreConformance = (name, storeLayer) => {
|
|
|
711
712
|
it.effect("counts stay consistent across cancel, retry, keep pruning, and remove", () => withStore((store) => Effect.gen(function* () {
|
|
712
713
|
// Three completions with keep {count: 1}: two get pruned.
|
|
713
714
|
for (let i = 0; i < 3; i++) {
|
|
714
|
-
const { id } = yield* store.enqueue(baseRequest({ payload: { n: i }, keep: { count: 1 } }));
|
|
715
|
+
const { id } = yield* store.enqueue(baseRequest({ payload: { n: i }, keep: { completed: { count: 1 } } }));
|
|
715
716
|
const claim = yield* store.claim(claimOptions({ token: `t-${i}` }));
|
|
716
717
|
assert(claim._tag === "Claimed");
|
|
717
718
|
yield* store.ack(id, `t-${i}`, { _tag: "Complete", exit: undefined });
|
|
@@ -765,6 +766,170 @@ export const jobStoreConformance = (name, storeLayer) => {
|
|
|
765
766
|
const stored = (yield* store.listSchedules())[0];
|
|
766
767
|
expect(stored?.payload).toEqual(payload);
|
|
767
768
|
})));
|
|
769
|
+
it.effect("dedupe never changes the job id, and keys are scoped per name", () => withStore((store) => Effect.gen(function* () {
|
|
770
|
+
const dedupe = { key: "emp-1", ttlMs: undefined, extend: false, replace: false };
|
|
771
|
+
const first = yield* store.enqueue(baseRequest({ id: JobId("my-ulid-1"), dedupe }));
|
|
772
|
+
expect(first).toEqual({ id: "my-ulid-1", duplicate: false });
|
|
773
|
+
const job = yield* store.getJob(first.id);
|
|
774
|
+
assert(Option.isSome(job));
|
|
775
|
+
expect(job.value.dedupeKey).toBe("emp-1");
|
|
776
|
+
// Same key, same name: deduplicated to the FIRST job's id.
|
|
777
|
+
const second = yield* store.enqueue(baseRequest({ id: JobId("my-ulid-2"), dedupe }));
|
|
778
|
+
expect(second).toEqual({ id: "my-ulid-1", duplicate: true });
|
|
779
|
+
expect(Option.isNone(yield* store.getJob(JobId("my-ulid-2")))).toBe(true);
|
|
780
|
+
// Same key, different name: no interference.
|
|
781
|
+
const other = yield* store.enqueue(baseRequest({ name: "OtherJob", dedupe }));
|
|
782
|
+
expect(other.duplicate).toBe(false);
|
|
783
|
+
})));
|
|
784
|
+
it.effect("pending dedupe holds while the keyed job is unfinished and frees on completion", () => withStore((store) => Effect.gen(function* () {
|
|
785
|
+
const dedupe = { key: "k", ttlMs: undefined, extend: false, replace: false };
|
|
786
|
+
const first = yield* store.enqueue(baseRequest({ dedupe }));
|
|
787
|
+
expect((yield* store.enqueue(baseRequest({ dedupe }))).duplicate).toBe(true);
|
|
788
|
+
// Still deduped while active.
|
|
789
|
+
const claim = yield* store.claim(claimOptions());
|
|
790
|
+
assert(claim._tag === "Claimed");
|
|
791
|
+
expect((yield* store.enqueue(baseRequest({ dedupe }))).duplicate).toBe(true);
|
|
792
|
+
// A terminal ack frees the key immediately.
|
|
793
|
+
yield* store.ack(first.id, "t-1", { _tag: "Complete", exit: undefined });
|
|
794
|
+
const fresh = yield* store.enqueue(baseRequest({ dedupe }));
|
|
795
|
+
expect(fresh.duplicate).toBe(false);
|
|
796
|
+
expect(fresh.id).not.toBe(first.id);
|
|
797
|
+
})));
|
|
798
|
+
it.effect("cancellation also frees a pending dedupe key", () => withStore((store) => Effect.gen(function* () {
|
|
799
|
+
const dedupe = { key: "k", ttlMs: undefined, extend: false, replace: false };
|
|
800
|
+
const first = yield* store.enqueue(baseRequest({ dedupe }));
|
|
801
|
+
yield* store.cancel(first.id);
|
|
802
|
+
expect((yield* store.enqueue(baseRequest({ dedupe }))).duplicate).toBe(false);
|
|
803
|
+
})));
|
|
804
|
+
it.effect("a ttl dedupe window throttles even past completion, then expires", () => withStore((store) => Effect.gen(function* () {
|
|
805
|
+
const dedupe = { key: "k", ttlMs: 60_000, extend: false, replace: false };
|
|
806
|
+
const first = yield* store.enqueue(baseRequest({ dedupe }));
|
|
807
|
+
const claim = yield* store.claim(claimOptions());
|
|
808
|
+
assert(claim._tag === "Claimed");
|
|
809
|
+
yield* store.ack(first.id, "t-1", { _tag: "Complete", exit: undefined });
|
|
810
|
+
// Completed, but the window still throttles...
|
|
811
|
+
yield* TestClock.adjust(30_000);
|
|
812
|
+
expect((yield* store.enqueue(baseRequest({ dedupe }))).duplicate).toBe(true);
|
|
813
|
+
// ...and a plain (non-extend) window is NOT pushed out by drops.
|
|
814
|
+
yield* TestClock.adjust(30_000);
|
|
815
|
+
const fresh = yield* store.enqueue(baseRequest({ dedupe }));
|
|
816
|
+
expect(fresh.duplicate).toBe(false);
|
|
817
|
+
expect(fresh.id).not.toBe(first.id);
|
|
818
|
+
})));
|
|
819
|
+
it.effect("an extend dedupe window is pushed out by each deduplicated enqueue", () => withStore((store) => Effect.gen(function* () {
|
|
820
|
+
const dedupe = { key: "k", ttlMs: 60_000, extend: true, replace: false };
|
|
821
|
+
yield* store.enqueue(baseRequest({ dedupe }));
|
|
822
|
+
yield* TestClock.adjust(45_000);
|
|
823
|
+
expect((yield* store.enqueue(baseRequest({ dedupe }))).duplicate).toBe(true);
|
|
824
|
+
// 75s after the FIRST enqueue — past the original window, inside
|
|
825
|
+
// the extended one.
|
|
826
|
+
yield* TestClock.adjust(30_000);
|
|
827
|
+
expect((yield* store.enqueue(baseRequest({ dedupe }))).duplicate).toBe(true);
|
|
828
|
+
// Past the latest extension: free again.
|
|
829
|
+
yield* TestClock.adjust(60_001);
|
|
830
|
+
expect((yield* store.enqueue(baseRequest({ dedupe }))).duplicate).toBe(false);
|
|
831
|
+
})));
|
|
832
|
+
it.effect("replace dedupe rewrites a still-delayed job in place, latest content wins", () => withStore((store) => Effect.gen(function* () {
|
|
833
|
+
const dedupe = { key: "k", ttlMs: undefined, extend: false, replace: true };
|
|
834
|
+
const first = yield* store.enqueue(baseRequest({ dedupe, payload: { n: 1 }, priority: 1, delayMs: 60_000 }));
|
|
835
|
+
const replaced = yield* store.enqueue(baseRequest({ dedupe, payload: { n: 2 }, priority: 7, delayMs: 5_000 }));
|
|
836
|
+
expect(replaced).toEqual({ id: first.id, duplicate: true });
|
|
837
|
+
const job = yield* store.getJob(first.id);
|
|
838
|
+
assert(Option.isSome(job));
|
|
839
|
+
expect(job.value.payload).toEqual({ n: 2 });
|
|
840
|
+
expect(job.value.priority).toBe(7);
|
|
841
|
+
expect(job.value.state).toBe("delayed");
|
|
842
|
+
// The rewritten delay is live: due after 5s, not the original 60s.
|
|
843
|
+
yield* TestClock.adjust(5_000);
|
|
844
|
+
const claim = yield* store.claim(claimOptions());
|
|
845
|
+
assert(claim._tag === "Claimed");
|
|
846
|
+
expect(claim.job.id).toBe(first.id);
|
|
847
|
+
expect(claim.job.payload).toEqual({ n: 2 });
|
|
848
|
+
// Once claimed (active), replace degrades to plain dedup.
|
|
849
|
+
const during = yield* store.enqueue(baseRequest({ dedupe, payload: { n: 3 } }));
|
|
850
|
+
expect(during).toEqual({ id: first.id, duplicate: true });
|
|
851
|
+
const active = yield* store.getJob(first.id);
|
|
852
|
+
assert(Option.isSome(active));
|
|
853
|
+
expect(active.value.payload).toEqual({ n: 2 });
|
|
854
|
+
})));
|
|
855
|
+
it.effect("an existing explicit id wins over the dedup tree in every driver", () => withStore((store) => Effect.gen(function* () {
|
|
856
|
+
const key = { key: "k", ttlMs: undefined, extend: false, replace: false };
|
|
857
|
+
// X runs under key k and completes, freeing the key but staying in
|
|
858
|
+
// history.
|
|
859
|
+
const x = yield* store.enqueue(baseRequest({ id: JobId("X"), dedupe: key }));
|
|
860
|
+
const claim = yield* store.claim(claimOptions());
|
|
861
|
+
assert(claim._tag === "Claimed");
|
|
862
|
+
yield* store.ack(x.id, "t-1", { _tag: "Complete", exit: undefined });
|
|
863
|
+
// J takes over the key, delayed.
|
|
864
|
+
const j = yield* store.enqueue(baseRequest({ dedupe: key, payload: { n: 9 }, delayMs: 60_000 }));
|
|
865
|
+
// A retried enqueue of X (id exists) must return X untouched — the
|
|
866
|
+
// id check precedes the dedup tree, so J is neither returned nor
|
|
867
|
+
// replaced.
|
|
868
|
+
const retried = yield* store.enqueue(baseRequest({
|
|
869
|
+
id: JobId("X"),
|
|
870
|
+
payload: { n: 1 },
|
|
871
|
+
dedupe: { ...key, replace: true }
|
|
872
|
+
}));
|
|
873
|
+
expect(retried).toEqual({ id: "X", duplicate: true });
|
|
874
|
+
const job = yield* store.getJob(j.id);
|
|
875
|
+
assert(Option.isSome(job));
|
|
876
|
+
expect(job.value.payload).toEqual({ n: 9 });
|
|
877
|
+
expect(job.value.state).toBe("delayed");
|
|
878
|
+
})));
|
|
879
|
+
it.effect("a landed replace re-arms the ttl window", () => withStore((store) => Effect.gen(function* () {
|
|
880
|
+
const dedupe = { key: "k", ttlMs: 60_000, extend: false, replace: true };
|
|
881
|
+
const first = yield* store.enqueue(baseRequest({ dedupe, delayMs: 300_000 }));
|
|
882
|
+
// 50s in: replace lands; the window restarts from here.
|
|
883
|
+
yield* TestClock.adjust(50_000);
|
|
884
|
+
const replaced = yield* store.enqueue(baseRequest({ dedupe, payload: { n: 2 }, delayMs: 300_000 }));
|
|
885
|
+
expect(replaced).toEqual({ id: first.id, duplicate: true });
|
|
886
|
+
// 100s after the FIRST enqueue — past the original window, inside
|
|
887
|
+
// the re-armed one: still the same job.
|
|
888
|
+
yield* TestClock.adjust(50_000);
|
|
889
|
+
const again = yield* store.enqueue(baseRequest({ dedupe, payload: { n: 3 }, delayMs: 300_000 }));
|
|
890
|
+
expect(again).toEqual({ id: first.id, duplicate: true });
|
|
891
|
+
})));
|
|
892
|
+
it.effect("keep policies are independent per terminal state", () => withStore((store) => Effect.gen(function* () {
|
|
893
|
+
// Completed records keep only the newest 1; failed keep everything.
|
|
894
|
+
const keep = { completed: { count: 1 } };
|
|
895
|
+
const completed = [];
|
|
896
|
+
const failed = [];
|
|
897
|
+
for (let i = 0; i < 2; i++) {
|
|
898
|
+
const done = yield* store.enqueue(baseRequest({ payload: { n: i }, keep }));
|
|
899
|
+
const claimA = yield* store.claim(claimOptions({ token: `tc-${i}` }));
|
|
900
|
+
assert(claimA._tag === "Claimed");
|
|
901
|
+
yield* store.ack(done.id, `tc-${i}`, { _tag: "Complete", exit: undefined });
|
|
902
|
+
completed.push(done.id);
|
|
903
|
+
const bad = yield* store.enqueue(baseRequest({ payload: { n: 10 + i }, keep }));
|
|
904
|
+
const claimB = yield* store.claim(claimOptions({ token: `tf-${i}` }));
|
|
905
|
+
assert(claimB._tag === "Claimed");
|
|
906
|
+
yield* store.ack(bad.id, `tf-${i}`, { _tag: "Fail", exit: undefined });
|
|
907
|
+
failed.push(bad.id);
|
|
908
|
+
yield* TestClock.adjust(10);
|
|
909
|
+
}
|
|
910
|
+
const counts = yield* store.counts();
|
|
911
|
+
expect(counts.completed).toBe(1);
|
|
912
|
+
expect(counts.failed).toBe(2);
|
|
913
|
+
expect(Option.isNone(yield* store.getJob(completed[0] ?? JobId("?")))).toBe(true);
|
|
914
|
+
expect(Option.isSome(yield* store.getJob(failed[0] ?? JobId("?")))).toBe(true);
|
|
915
|
+
})));
|
|
916
|
+
it.effect("an enqueue wakes only waiters watching its queue", () => withStore((store) => Effect.gen(function* () {
|
|
917
|
+
const empty = yield* store.claim(claimOptions());
|
|
918
|
+
assert(empty._tag === "Empty");
|
|
919
|
+
let woke = false;
|
|
920
|
+
const waiter = yield* Effect.forkChild(store.awaitWake([QueueName("default")], empty.wakeToken).pipe(Effect.tap(() => Effect.sync(() => void (woke = true)))));
|
|
921
|
+
yield* Effect.yieldNow;
|
|
922
|
+
// Work on ANOTHER queue must not wake the default-queue waiter.
|
|
923
|
+
yield* store.enqueue(baseRequest({ queue: QueueName("other") }));
|
|
924
|
+
for (let i = 0; i < 10; i++) {
|
|
925
|
+
yield* Effect.yieldNow;
|
|
926
|
+
}
|
|
927
|
+
expect(woke).toBe(false);
|
|
928
|
+
// Matching-queue work does.
|
|
929
|
+
yield* store.enqueue(baseRequest());
|
|
930
|
+
yield* Fiber.join(waiter);
|
|
931
|
+
expect(woke).toBe(true);
|
|
932
|
+
})));
|
|
768
933
|
it.effect("delayed jobs still promote to waiting while their queue is paused", () => withStore((store) => Effect.gen(function* () {
|
|
769
934
|
const { id } = yield* store.enqueue(baseRequest({ delayMs: 1_000 }));
|
|
770
935
|
yield* store.pause(QueueName("default"));
|