effect-mq 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/JobStore.d.ts +41 -3
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js +10 -0
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +27 -14
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.js +25 -3
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
- package/dist/redis/RedisJobStore.d.ts +53 -0
- package/dist/redis/RedisJobStore.d.ts.map +1 -1
- package/dist/redis/RedisJobStore.js +188 -37
- package/dist/redis/RedisJobStore.js.map +1 -1
- package/dist/redis/scripts.d.ts +101 -30
- package/dist/redis/scripts.d.ts.map +1 -1
- package/dist/redis/scripts.js +204 -55
- package/dist/redis/scripts.js.map +1 -1
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +127 -11
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +1 -1
- package/src/JobStore.ts +38 -3
- package/src/MemoryJobStore.ts +26 -18
- package/src/drizzle-postgres/DrizzleJobStore.ts +27 -3
- package/src/redis/RedisJobStore.ts +256 -36
- package/src/redis/scripts.ts +232 -59
- package/src/testing/conformance.ts +143 -11
package/src/redis/scripts.ts
CHANGED
|
@@ -21,6 +21,19 @@
|
|
|
21
21
|
* - `p:delayed:<queue>` ZSET, score `runAt`
|
|
22
22
|
* - `p:active` ZSET, score `lockExpiresAt`
|
|
23
23
|
* - `p:all` ZSET, score `enqueuedAt` (list pagination)
|
|
24
|
+
* - `p:byname:<name>` ZSET, score `enqueuedAt` (list name index;
|
|
25
|
+
* optional, `RedisJobStoreOptions.indexes.name`)
|
|
26
|
+
* - `p:byqueue:<queue>` ZSET, score `enqueuedAt` (list queue index;
|
|
27
|
+
* optional, `RedisJobStoreOptions.indexes.queue`)
|
|
28
|
+
* - `p:index:<kind>:ready` STRING, millis timestamp of the last index
|
|
29
|
+
* reconcile's start for `<kind>` (name |
|
|
30
|
+
* queue). Absent: the next enabled boot does a
|
|
31
|
+
* full ZSCAN rebuild. Present: enabled boots
|
|
32
|
+
* heal the tail (rows enqueued since the value
|
|
33
|
+
* minus a safety margin) and re-stamp it —
|
|
34
|
+
* closing the rolling-deploy window where
|
|
35
|
+
* index-less writers inserted rows after the
|
|
36
|
+
* marker landed
|
|
24
37
|
* - `p:finished:<state>` ZSET, score `finishedAt` (history TTL)
|
|
25
38
|
* - `p:terminal:<name>:<state>` ZSET, score `finishedAt` (keep pruning)
|
|
26
39
|
* - `p:counts` HASH `<queue>|<state>` -> integer
|
|
@@ -50,11 +63,28 @@
|
|
|
50
63
|
*/
|
|
51
64
|
import { Redis } from "effect/unstable/persistence"
|
|
52
65
|
|
|
66
|
+
/**
|
|
67
|
+
* Which optional list indexes (`p:byname:<name>` / `p:byqueue:<queue>`) this
|
|
68
|
+
* store maintains. The flags are baked into the script text (the helpers are
|
|
69
|
+
* shared by every script), so a store instance only ever runs scripts that
|
|
70
|
+
* match its configuration — disabled indexes cost no writes at all.
|
|
71
|
+
*
|
|
72
|
+
* @since 0.7.0
|
|
73
|
+
*/
|
|
74
|
+
export interface IndexConfig {
|
|
75
|
+
readonly name: boolean
|
|
76
|
+
readonly queue: boolean
|
|
77
|
+
}
|
|
78
|
+
|
|
53
79
|
/**
|
|
54
80
|
* Shared helpers textually prepended to every script (the `Redis.script`
|
|
55
81
|
* runner has no include mechanism). `ARGV[1]` is always the key prefix.
|
|
82
|
+
* Built once per store instance: `insertJobRow` writes only the list indexes
|
|
83
|
+
* enabled by `IndexConfig` (`deleteJob` clears both unconditionally — a ZREM
|
|
84
|
+
* on a missing key is free, and stray entries from an earlier configuration
|
|
85
|
+
* must never outlive their job).
|
|
56
86
|
*/
|
|
57
|
-
const
|
|
87
|
+
export const helpers = (indexes: IndexConfig): string => `
|
|
58
88
|
local prefix = ARGV[1]
|
|
59
89
|
local function fmt(x) return string.format("%.0f", x) end
|
|
60
90
|
local function jobKey(id) return prefix .. ":job:" .. id end
|
|
@@ -62,6 +92,8 @@ local function attemptsKey(id) return prefix .. ":attempts:" .. id end
|
|
|
62
92
|
local function waitingKey(queue) return prefix .. ":waiting:" .. queue end
|
|
63
93
|
local function delayedKey(queue) return prefix .. ":delayed:" .. queue end
|
|
64
94
|
local function terminalKey(name, state) return prefix .. ":terminal:" .. name .. ":" .. state end
|
|
95
|
+
local function bynameKey(name) return prefix .. ":byname:" .. name end
|
|
96
|
+
local function byqueueKey(queue) return prefix .. ":byqueue:" .. queue end
|
|
65
97
|
-- Waiting order: score = -priority (higher priority first, full number range);
|
|
66
98
|
-- FIFO within a priority via lexicographic members "<seq %016d>:<id>". A
|
|
67
99
|
-- composite numeric score would clip either priority or seq past float53.
|
|
@@ -169,6 +201,8 @@ local function deleteJob(id)
|
|
|
169
201
|
local name = redis.call("HGET", jk, "name")
|
|
170
202
|
countsAdd(queue, state, -1)
|
|
171
203
|
redis.call("ZREM", prefix .. ":all", id)
|
|
204
|
+
redis.call("ZREM", bynameKey(name), id)
|
|
205
|
+
redis.call("ZREM", byqueueKey(queue), id)
|
|
172
206
|
redis.call("ZREM", prefix .. ":finished:" .. state, id)
|
|
173
207
|
redis.call("ZREM", prefix .. ":active", id)
|
|
174
208
|
redis.call("ZREM", terminalKey(name, state), id)
|
|
@@ -262,6 +296,20 @@ local function finishCancelled(id, queue, name, startedAt, now, nowStr)
|
|
|
262
296
|
releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
|
|
263
297
|
applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
|
|
264
298
|
end
|
|
299
|
+
-- Index one EXISTING job row into a list index (backfill and tail heal;
|
|
300
|
+
-- insertJobRow covers live writes). A missing row is skipped — nothing to
|
|
301
|
+
-- index, and the read path self-heals whatever stale member pointed here.
|
|
302
|
+
local function indexInto(kind, id)
|
|
303
|
+
local row = redis.call("HMGET", jobKey(id), "name", "queue", "enqueuedAt")
|
|
304
|
+
if row[1] then
|
|
305
|
+
local enqueuedAt = tonumber(row[3]) or 0
|
|
306
|
+
if kind == "name" then
|
|
307
|
+
redis.call("ZADD", bynameKey(row[1]), enqueuedAt, id)
|
|
308
|
+
else
|
|
309
|
+
redis.call("ZADD", byqueueKey(row[2]), enqueuedAt, id)
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
end
|
|
265
313
|
-- Insert one fresh job row plus every index entry. String params are stored
|
|
266
314
|
-- verbatim (payload/metadata/backoff/keep/trace/parent are pre-encoded JSON,
|
|
267
315
|
-- "" = absent); priority/delayMs/now numeric-coercible. New jobs never carry
|
|
@@ -281,7 +329,9 @@ local function insertJobRow(id, name, queue, payloadJson, metadataJson, priority
|
|
|
281
329
|
"processedAt", "", "finishedAt", "", "exit", "", "failedReason", "",
|
|
282
330
|
"lockToken", "", "lockExpiresAt", "", "seq", fmt(seq))
|
|
283
331
|
redis.call("ZADD", prefix .. ":all", now, id)
|
|
284
|
-
|
|
332
|
+
${indexes.name ? ` redis.call("ZADD", bynameKey(name), now, id)\n` : ""}${
|
|
333
|
+
indexes.queue ? ` redis.call("ZADD", byqueueKey(queue), now, id)\n` : ""
|
|
334
|
+
} if state == "waiting" then
|
|
285
335
|
addWaiting(queue, tonumber(priority), seq, id)
|
|
286
336
|
else
|
|
287
337
|
redis.call("ZADD", delayedKey(queue), runAt, id)
|
|
@@ -297,7 +347,7 @@ end
|
|
|
297
347
|
* idMode: "user" (dedup no-op), "generated" (collision -> retry sentinel),
|
|
298
348
|
* "auto" (j-<seq>, in-script collision loop).
|
|
299
349
|
*/
|
|
300
|
-
export const enqueue = Redis.script(
|
|
350
|
+
export const enqueue = (HELPERS: string) => Redis.script(
|
|
301
351
|
(
|
|
302
352
|
prefix: string,
|
|
303
353
|
idMode: string,
|
|
@@ -435,7 +485,7 @@ return '{"id":' .. cjson.encode(id) .. ',"duplicate":false,"wake":true}'
|
|
|
435
485
|
* waiting job whose name matches. Returns the claimed record (HGETALL pairs)
|
|
436
486
|
* or an Empty result with the earliest matching delayed runAt.
|
|
437
487
|
*/
|
|
438
|
-
export const claim = Redis.script(
|
|
488
|
+
export const claim = (HELPERS: string) => Redis.script(
|
|
439
489
|
(prefix: string, queue: string, namesJson: string, token: string, lockDurationMs: number, now: number) => [
|
|
440
490
|
prefix,
|
|
441
491
|
queue,
|
|
@@ -512,7 +562,7 @@ return cjson.encode({ empty = true, nextRunAt = nextRunAt })
|
|
|
512
562
|
* Token-guarded. Retry on a cancel-requested job finishes it as cancelled
|
|
513
563
|
* (cancellation wins over revival, mirroring release/recoverStalled).
|
|
514
564
|
*/
|
|
515
|
-
export const ack = Redis.script(
|
|
565
|
+
export const ack = (HELPERS: string) => Redis.script(
|
|
516
566
|
(prefix: string, id: string, token: string, outcomeTag: string, exitJson: string, delayMs: number, now: number) => [
|
|
517
567
|
prefix,
|
|
518
568
|
id,
|
|
@@ -592,7 +642,7 @@ return '{"ok":true}'
|
|
|
592
642
|
* release(prefix, id, token, now) — hand the job back without consuming an
|
|
593
643
|
* attempt; a pending cancel wins and finishes the job instead.
|
|
594
644
|
*/
|
|
595
|
-
export const release = Redis.script(
|
|
645
|
+
export const release = (HELPERS: string) => Redis.script(
|
|
596
646
|
(prefix: string, id: string, token: string, now: number) => [prefix, id, token, now],
|
|
597
647
|
{
|
|
598
648
|
numberOfKeys: 0,
|
|
@@ -626,7 +676,7 @@ return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
|
626
676
|
* extendLocks(prefix, locksJson, durationMs, now) -> { lost, cancel }
|
|
627
677
|
* Cancel-requested locks are reported, not extended.
|
|
628
678
|
*/
|
|
629
|
-
export const extendLocks = Redis.script(
|
|
679
|
+
export const extendLocks = (HELPERS: string) => Redis.script(
|
|
630
680
|
(prefix: string, locksJson: string, durationMs: number, now: number) => [prefix, locksJson, durationMs, now],
|
|
631
681
|
{
|
|
632
682
|
numberOfKeys: 0,
|
|
@@ -655,7 +705,7 @@ return cjson.encode({ lost = lost, cancel = cancel })
|
|
|
655
705
|
* recoverStalled(prefix, maxStalledCount, now) -> recovered [{id, failed}]
|
|
656
706
|
* A pending cancel finishes the job as cancelled (not reported as recovered).
|
|
657
707
|
*/
|
|
658
|
-
export const recoverStalled = Redis.script(
|
|
708
|
+
export const recoverStalled = (HELPERS: string) => Redis.script(
|
|
659
709
|
(prefix: string, maxStalledCount: number, now: number) => [prefix, maxStalledCount, now],
|
|
660
710
|
{
|
|
661
711
|
numberOfKeys: 0,
|
|
@@ -707,7 +757,7 @@ return cjson.encode(recovered)
|
|
|
707
757
|
).withReturnType<string>()
|
|
708
758
|
|
|
709
759
|
/** getJob(prefix, id) -> HGETALL pairs (empty array when missing). */
|
|
710
|
-
export const getJob = Redis.script(
|
|
760
|
+
export const getJob = (HELPERS: string) => Redis.script(
|
|
711
761
|
(prefix: string, id: string) => [prefix, id],
|
|
712
762
|
{
|
|
713
763
|
numberOfKeys: 0,
|
|
@@ -720,46 +770,117 @@ return cjson.encode(record)
|
|
|
720
770
|
).withReturnType<string>()
|
|
721
771
|
|
|
722
772
|
/**
|
|
723
|
-
* list(prefix, filtersJson, cursor, limit)
|
|
724
|
-
*
|
|
773
|
+
* list(prefix, sourcesJson, order, filtersJson, cursor, limit)
|
|
774
|
+
*
|
|
775
|
+
* Indexed list. The DRIVER routes the query to the narrowest zset(s) — see
|
|
776
|
+
* `RedisJobStore`'s routing matrix — and this script merges those sorted
|
|
777
|
+
* sources by (score, id) in the requested direction, pages past the
|
|
778
|
+
* exclusive `<orderValue>:<id>` keyset cursor, loads each candidate row, and
|
|
779
|
+
* applies the residual predicates until `limit` matches accumulate. Sources
|
|
780
|
+
* must be plain-id-membered zsets whose score IS the requested order value
|
|
781
|
+
* (`all`/`byname:`/`byqueue:` = enqueuedAt, `delayed:<queue>` = runAt,
|
|
782
|
+
* `finished:`/`terminal:` = finishedAt) and must be pairwise disjoint; the
|
|
783
|
+
* waiting zsets (seq-prefixed members, priority scores) are never routed
|
|
784
|
+
* here. A member whose job hash is gone is an orphan: it is ZREM'd from the
|
|
785
|
+
* source being scanned (self-heal) and skipped.
|
|
725
786
|
*/
|
|
726
|
-
export const list =
|
|
727
|
-
(
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
787
|
+
export const list = (HELPERS: string) =>
|
|
788
|
+
Redis.script(
|
|
789
|
+
(prefix: string, sourcesJson: string, order: string, filtersJson: string, cursor: string, limit: number) => [
|
|
790
|
+
prefix,
|
|
791
|
+
sourcesJson,
|
|
792
|
+
order,
|
|
793
|
+
filtersJson,
|
|
794
|
+
cursor,
|
|
795
|
+
limit
|
|
796
|
+
],
|
|
797
|
+
{
|
|
798
|
+
numberOfKeys: 0,
|
|
799
|
+
lua: `${HELPERS}
|
|
800
|
+
-- Input that Redis's cjson cannot decode (e.g. lone-surrogate escapes in a
|
|
801
|
+
-- filter or key name) degrades to an empty page instead of a script error.
|
|
802
|
+
local okSources, sources = pcall(cjson.decode, ARGV[2])
|
|
803
|
+
local okFilters, filters = pcall(cjson.decode, ARGV[4])
|
|
804
|
+
if not okSources or not okFilters then return '{"items":[],"more":false}' end
|
|
805
|
+
local desc = ARGV[3] == "desc"
|
|
735
806
|
local stateSet = nil
|
|
736
807
|
if filters.states ~= nil then
|
|
737
808
|
stateSet = {}
|
|
738
809
|
for _, s in ipairs(filters.states) do stateSet[s] = true end
|
|
739
810
|
end
|
|
740
811
|
local cursorAt, cursorId = nil, nil
|
|
741
|
-
if ARGV[
|
|
742
|
-
local split = string.find(ARGV[
|
|
812
|
+
if ARGV[5] ~= "" then
|
|
813
|
+
local split = string.find(ARGV[5], ":", 1, true)
|
|
743
814
|
if split ~= nil then
|
|
744
|
-
cursorAt = tonumber(string.sub(ARGV[
|
|
745
|
-
cursorId = string.sub(ARGV[
|
|
815
|
+
cursorAt = tonumber(string.sub(ARGV[5], 1, split - 1))
|
|
816
|
+
cursorId = string.sub(ARGV[5], split + 1)
|
|
746
817
|
end
|
|
747
818
|
if cursorAt == nil then cursorId = nil end
|
|
748
819
|
end
|
|
749
|
-
local limit = tonumber(ARGV[
|
|
820
|
+
local limit = tonumber(ARGV[6])
|
|
821
|
+
-- One buffered iterator per source. offset counts consumed members still in
|
|
822
|
+
-- the zset — a self-healed orphan decrements it, because its ZREM shifts
|
|
823
|
+
-- every later rank down by one — so refills stay exact under in-script
|
|
824
|
+
-- removals. The score bound is the cursor score (inclusive; the per-item
|
|
825
|
+
-- check below excludes ids at or before the cursor within that score).
|
|
826
|
+
local iters = {}
|
|
827
|
+
for i = 1, #sources do
|
|
828
|
+
iters[i] = { key = sources[i], offset = 0, buf = {}, pos = 1, exhausted = false }
|
|
829
|
+
end
|
|
830
|
+
local function head(it)
|
|
831
|
+
if it.pos > #it.buf then
|
|
832
|
+
if it.exhausted then return nil end
|
|
833
|
+
if desc then
|
|
834
|
+
it.buf = redis.call("ZREVRANGEBYSCORE", it.key, cursorAt == nil and "+inf" or fmt(cursorAt), "-inf",
|
|
835
|
+
"WITHSCORES", "LIMIT", it.offset, 100)
|
|
836
|
+
else
|
|
837
|
+
it.buf = redis.call("ZRANGEBYSCORE", it.key, cursorAt == nil and "-inf" or fmt(cursorAt), "+inf",
|
|
838
|
+
"WITHSCORES", "LIMIT", it.offset, 100)
|
|
839
|
+
end
|
|
840
|
+
it.pos = 1
|
|
841
|
+
if #it.buf == 0 then
|
|
842
|
+
it.exhausted = true
|
|
843
|
+
return nil
|
|
844
|
+
end
|
|
845
|
+
end
|
|
846
|
+
return it.buf[it.pos], tonumber(it.buf[it.pos + 1])
|
|
847
|
+
end
|
|
750
848
|
local items = {}
|
|
751
|
-
local
|
|
752
|
-
local max = cursorAt == nil and "+inf" or fmt(cursorAt)
|
|
753
|
-
local offset = 0
|
|
849
|
+
local more = false
|
|
754
850
|
while true do
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
local at =
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
851
|
+
-- The best head across sources: (score, id) in the requested direction (a
|
|
852
|
+
-- score-tied range comes back in member-lex order, so ids line up too).
|
|
853
|
+
local best, bestId, bestAt = nil, nil, nil
|
|
854
|
+
for i = 1, #iters do
|
|
855
|
+
local id, at = head(iters[i])
|
|
856
|
+
if id ~= nil then
|
|
857
|
+
local wins = best == nil
|
|
858
|
+
if not wins then
|
|
859
|
+
if at ~= bestAt then
|
|
860
|
+
wins = (desc and at > bestAt) or (not desc and at < bestAt)
|
|
861
|
+
else
|
|
862
|
+
wins = (desc and id > bestId) or (not desc and id < bestId)
|
|
863
|
+
end
|
|
864
|
+
end
|
|
865
|
+
if wins then
|
|
866
|
+
best, bestId, bestAt = iters[i], id, at
|
|
867
|
+
end
|
|
868
|
+
end
|
|
869
|
+
end
|
|
870
|
+
if best == nil then break end
|
|
871
|
+
best.pos = best.pos + 2
|
|
872
|
+
best.offset = best.offset + 1
|
|
873
|
+
-- Exclusive keyset cursor: skip up to and including the cursor position.
|
|
874
|
+
local past = cursorAt == nil
|
|
875
|
+
or (desc and (bestAt < cursorAt or (bestAt == cursorAt and bestId < cursorId)))
|
|
876
|
+
or (not desc and (bestAt > cursorAt or (bestAt == cursorAt and bestId > cursorId)))
|
|
877
|
+
if past then
|
|
878
|
+
local jk = jobKey(bestId)
|
|
879
|
+
if redis.call("EXISTS", jk) == 0 then
|
|
880
|
+
-- Orphaned index member (hash removed out of band): self-heal.
|
|
881
|
+
redis.call("ZREM", best.key, bestId)
|
|
882
|
+
best.offset = best.offset - 1
|
|
883
|
+
else
|
|
763
884
|
local matches = true
|
|
764
885
|
if filters.queue ~= nil and redis.call("HGET", jk, "queue") ~= filters.queue then matches = false end
|
|
765
886
|
if matches and filters.name ~= nil and redis.call("HGET", jk, "name") ~= filters.name then matches = false end
|
|
@@ -779,24 +900,76 @@ while true do
|
|
|
779
900
|
end
|
|
780
901
|
if matches then
|
|
781
902
|
if #items >= limit then
|
|
782
|
-
|
|
903
|
+
more = true
|
|
783
904
|
break
|
|
784
905
|
end
|
|
785
906
|
items[#items + 1] = redis.call("HGETALL", jk)
|
|
786
907
|
end
|
|
787
908
|
end
|
|
788
909
|
end
|
|
789
|
-
if moreMatches then break end
|
|
790
|
-
offset = offset + 100
|
|
791
910
|
end
|
|
792
911
|
if #items == 0 then return '{"items":[],"more":false}' end
|
|
793
|
-
return cjson.encode({ items = items, more =
|
|
912
|
+
return cjson.encode({ items = items, more = more })
|
|
913
|
+
`
|
|
914
|
+
}
|
|
915
|
+
).withReturnType<string>()
|
|
916
|
+
|
|
917
|
+
/**
|
|
918
|
+
* indexMembers(prefix, kind, idsJson) -> "1"
|
|
919
|
+
*
|
|
920
|
+
* Index one ZSCAN chunk of `p:all` members during a full rebuild. The driver
|
|
921
|
+
* owns the ZSCAN cursor loop (linear, tie-immune, guaranteed to terminate,
|
|
922
|
+
* and guaranteed to return every element present for the whole scan); this
|
|
923
|
+
* script only does the per-chunk work, so the single-threaded server is
|
|
924
|
+
* never held for the whole keyspace. Re-visited members and rows indexed
|
|
925
|
+
* live by insertJobRow mid-scan are idempotent ZADDs.
|
|
926
|
+
*/
|
|
927
|
+
export const indexMembers = (HELPERS: string) => Redis.script(
|
|
928
|
+
(prefix: string, kind: string, idsJson: string) => [prefix, kind, idsJson],
|
|
929
|
+
{
|
|
930
|
+
numberOfKeys: 0,
|
|
931
|
+
lua: `${HELPERS}
|
|
932
|
+
local kind = ARGV[2]
|
|
933
|
+
for _, id in ipairs(cjson.decode(ARGV[3])) do
|
|
934
|
+
indexInto(kind, id)
|
|
935
|
+
end
|
|
936
|
+
return "1"
|
|
937
|
+
`
|
|
938
|
+
}
|
|
939
|
+
).withReturnType<string>()
|
|
940
|
+
|
|
941
|
+
/**
|
|
942
|
+
* indexTailPage(prefix, kind, min, offset, pageSize) -> scanned count
|
|
943
|
+
*
|
|
944
|
+
* One bounded page of the boot-time tail heal: index every `p:all` member
|
|
945
|
+
* with score >= min (the previous marker minus a safety margin). Plain
|
|
946
|
+
* LIMIT offset paging — tails are small, and a rank-shift skip from a
|
|
947
|
+
* concurrent delete is covered by the margin plus the next boot's heal.
|
|
948
|
+
*/
|
|
949
|
+
export const indexTailPage = (HELPERS: string) => Redis.script(
|
|
950
|
+
(prefix: string, kind: string, min: string, offset: number, pageSize: number) => [
|
|
951
|
+
prefix,
|
|
952
|
+
kind,
|
|
953
|
+
min,
|
|
954
|
+
offset,
|
|
955
|
+
pageSize
|
|
956
|
+
],
|
|
957
|
+
{
|
|
958
|
+
numberOfKeys: 0,
|
|
959
|
+
lua: `${HELPERS}
|
|
960
|
+
local kind = ARGV[2]
|
|
961
|
+
local batch = redis.call("ZRANGEBYSCORE", prefix .. ":all", ARGV[3], "+inf",
|
|
962
|
+
"LIMIT", tonumber(ARGV[4]), tonumber(ARGV[5]))
|
|
963
|
+
for _, id in ipairs(batch) do
|
|
964
|
+
indexInto(kind, id)
|
|
965
|
+
end
|
|
966
|
+
return tostring(#batch)
|
|
794
967
|
`
|
|
795
968
|
}
|
|
796
969
|
).withReturnType<string>()
|
|
797
970
|
|
|
798
971
|
/** counts(prefix) -> HGETALL pairs of p:counts. */
|
|
799
|
-
export const counts = Redis.script(
|
|
972
|
+
export const counts = (HELPERS: string) => Redis.script(
|
|
800
973
|
(prefix: string) => [prefix],
|
|
801
974
|
{
|
|
802
975
|
numberOfKeys: 0,
|
|
@@ -809,7 +982,7 @@ return cjson.encode(pairs_)
|
|
|
809
982
|
).withReturnType<string>()
|
|
810
983
|
|
|
811
984
|
/** remove(prefix, id) -> removed boolean (active/waiting-children refused). */
|
|
812
|
-
export const remove = Redis.script(
|
|
985
|
+
export const remove = (HELPERS: string) => Redis.script(
|
|
813
986
|
(prefix: string, id: string) => [prefix, id],
|
|
814
987
|
{
|
|
815
988
|
numberOfKeys: 0,
|
|
@@ -827,7 +1000,7 @@ return "1"
|
|
|
827
1000
|
).withReturnType<string>()
|
|
828
1001
|
|
|
829
1002
|
/** retry(prefix, id, now) — failed -> waiting with a fresh budget. */
|
|
830
|
-
export const retry = Redis.script(
|
|
1003
|
+
export const retry = (HELPERS: string) => Redis.script(
|
|
831
1004
|
(prefix: string, id: string, now: number) => [prefix, id, now],
|
|
832
1005
|
{
|
|
833
1006
|
numberOfKeys: 0,
|
|
@@ -862,7 +1035,7 @@ return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
|
862
1035
|
* handing them to the cascade sweep); active gets the cancel-request flag;
|
|
863
1036
|
* terminal states are refused.
|
|
864
1037
|
*/
|
|
865
|
-
export const cancel = Redis.script(
|
|
1038
|
+
export const cancel = (HELPERS: string) => Redis.script(
|
|
866
1039
|
(prefix: string, id: string, now: number) => [prefix, id, now],
|
|
867
1040
|
{
|
|
868
1041
|
numberOfKeys: 0,
|
|
@@ -902,7 +1075,7 @@ return '{"ok":true}'
|
|
|
902
1075
|
).withReturnType<string>()
|
|
903
1076
|
|
|
904
1077
|
/** promote(prefix, id, now) — delayed -> waiting now. */
|
|
905
|
-
export const promote = Redis.script(
|
|
1078
|
+
export const promote = (HELPERS: string) => Redis.script(
|
|
906
1079
|
(prefix: string, id: string, now: number) => [prefix, id, now],
|
|
907
1080
|
{
|
|
908
1081
|
numberOfKeys: 0,
|
|
@@ -934,7 +1107,7 @@ return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
|
934
1107
|
* digits) and empty arrays ({}). An unchanged cadence (cron/tz/everyMs)
|
|
935
1108
|
* preserves the stored nextRunAt.
|
|
936
1109
|
*/
|
|
937
|
-
export const upsertSchedule = Redis.script(
|
|
1110
|
+
export const upsertSchedule = (HELPERS: string) => Redis.script(
|
|
938
1111
|
(
|
|
939
1112
|
prefix: string,
|
|
940
1113
|
key: string,
|
|
@@ -999,7 +1172,7 @@ return '{"ok":true}'
|
|
|
999
1172
|
).withReturnType<string>()
|
|
1000
1173
|
|
|
1001
1174
|
/** removeSchedule(prefix, key) -> existed boolean. */
|
|
1002
|
-
export const removeSchedule = Redis.script(
|
|
1175
|
+
export const removeSchedule = (HELPERS: string) => Redis.script(
|
|
1003
1176
|
(prefix: string, key: string) => [prefix, key],
|
|
1004
1177
|
{
|
|
1005
1178
|
numberOfKeys: 0,
|
|
@@ -1012,7 +1185,7 @@ return tostring(removed)
|
|
|
1012
1185
|
).withReturnType<string>()
|
|
1013
1186
|
|
|
1014
1187
|
/** listSchedules(prefix, filtersJson) ordered by nextRunAt ascending. */
|
|
1015
|
-
export const listSchedules = Redis.script(
|
|
1188
|
+
export const listSchedules = (HELPERS: string) => Redis.script(
|
|
1016
1189
|
(prefix: string, filtersJson: string) => [prefix, filtersJson],
|
|
1017
1190
|
{
|
|
1018
1191
|
numberOfKeys: 0,
|
|
@@ -1037,7 +1210,7 @@ return cjson.encode(out)
|
|
|
1037
1210
|
).withReturnType<string>()
|
|
1038
1211
|
|
|
1039
1212
|
/** dueSchedules(prefix, now) ordered by nextRunAt ascending. */
|
|
1040
|
-
export const dueSchedules = Redis.script(
|
|
1213
|
+
export const dueSchedules = (HELPERS: string) => Redis.script(
|
|
1041
1214
|
(prefix: string, now: number) => [prefix, now],
|
|
1042
1215
|
{
|
|
1043
1216
|
numberOfKeys: 0,
|
|
@@ -1054,7 +1227,7 @@ return cjson.encode(out)
|
|
|
1054
1227
|
).withReturnType<string>()
|
|
1055
1228
|
|
|
1056
1229
|
/** advanceSchedule(prefix, key, expectedRunAt, nextRunAt) — conditional CAS. */
|
|
1057
|
-
export const advanceSchedule = Redis.script(
|
|
1230
|
+
export const advanceSchedule = (HELPERS: string) => Redis.script(
|
|
1058
1231
|
(prefix: string, key: string, expectedRunAt: number, nextRunAt: number) => [prefix, key, expectedRunAt, nextRunAt],
|
|
1059
1232
|
{
|
|
1060
1233
|
numberOfKeys: 0,
|
|
@@ -1078,7 +1251,7 @@ return "1"
|
|
|
1078
1251
|
* in one script, so a stale sweeper can never re-fire a slot — even after
|
|
1079
1252
|
* retention pruned the previous slot's job row.
|
|
1080
1253
|
*/
|
|
1081
|
-
export const tickSchedule = Redis.script(
|
|
1254
|
+
export const tickSchedule = (HELPERS: string) => Redis.script(
|
|
1082
1255
|
(
|
|
1083
1256
|
prefix: string,
|
|
1084
1257
|
key: string,
|
|
@@ -1145,7 +1318,7 @@ return "1"
|
|
|
1145
1318
|
* backoffJson, keepJson, timeoutMs, traceJson, parentJson, delayMs. Plain
|
|
1146
1319
|
* (non-dedup) items only — the caller routes dedup items through \`enqueue\`.
|
|
1147
1320
|
*/
|
|
1148
|
-
export const enqueueMany = Redis.script(
|
|
1321
|
+
export const enqueueMany = (HELPERS: string) => Redis.script(
|
|
1149
1322
|
(prefix: string, now: number, count: number, items: ReadonlyArray<string>) => [prefix, now, count, ...items],
|
|
1150
1323
|
{
|
|
1151
1324
|
numberOfKeys: 0,
|
|
@@ -1200,7 +1373,7 @@ return "[" .. table.concat(out, ",") .. "]"
|
|
|
1200
1373
|
* min(store ceiling, per-row keep.age). The caller advances the offset by
|
|
1201
1374
|
* (scanned - deleted) and stops when a page comes back short.
|
|
1202
1375
|
*/
|
|
1203
|
-
export const sweepState = Redis.script(
|
|
1376
|
+
export const sweepState = (HELPERS: string) => Redis.script(
|
|
1204
1377
|
(prefix: string, state: string, ttlMs: string, limit: number, offset: number, now: number) => [
|
|
1205
1378
|
prefix,
|
|
1206
1379
|
state,
|
|
@@ -1267,7 +1440,7 @@ return cjson.encode({ scanned = scanned, deleted = deleted })
|
|
|
1267
1440
|
* caller loops until 0): expired windows, then pending pointers (+inf)
|
|
1268
1441
|
* whose job is gone or terminal.
|
|
1269
1442
|
*/
|
|
1270
|
-
export const sweepDedupes = Redis.script(
|
|
1443
|
+
export const sweepDedupes = (HELPERS: string) => Redis.script(
|
|
1271
1444
|
(prefix: string, limit: number, now: number) => [prefix, limit, now],
|
|
1272
1445
|
{
|
|
1273
1446
|
numberOfKeys: 0,
|
|
@@ -1329,7 +1502,7 @@ return tostring(migrated + #expired + removedPending)
|
|
|
1329
1502
|
* raced cancelRequested wins: the parent settles cancelled and its pending
|
|
1330
1503
|
* rows flip to cancelled (cascade work for the flow sweeper).
|
|
1331
1504
|
*/
|
|
1332
|
-
export const fanOut = Redis.script(
|
|
1505
|
+
export const fanOut = (HELPERS: string) => Redis.script(
|
|
1333
1506
|
(
|
|
1334
1507
|
prefix: string,
|
|
1335
1508
|
id: string,
|
|
@@ -1443,7 +1616,7 @@ return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
|
|
|
1443
1616
|
* transition, so its own report goes to the outbox here), else pending==0
|
|
1444
1617
|
* resumes the parent runnable at the flow's LAST applied report's index.
|
|
1445
1618
|
*/
|
|
1446
|
-
export const recordChildResults = Redis.script(
|
|
1619
|
+
export const recordChildResults = (HELPERS: string) => Redis.script(
|
|
1447
1620
|
(prefix: string, now: number, count: number, items: ReadonlyArray<string>) => [prefix, now, count, ...items],
|
|
1448
1621
|
{
|
|
1449
1622
|
numberOfKeys: 0,
|
|
@@ -1550,7 +1723,7 @@ return '{"results":[' .. table.concat(out, ",") .. '],"wakes":' .. wakesJson ..
|
|
|
1550
1723
|
* Items are positional HMGET tuples (the field list must stay in lockstep
|
|
1551
1724
|
* with the driver's `toChildRecord`) — the full spec JSON stays server-side.
|
|
1552
1725
|
*/
|
|
1553
|
-
export const listChildResults = Redis.script(
|
|
1726
|
+
export const listChildResults = (HELPERS: string) => Redis.script(
|
|
1554
1727
|
(prefix: string, flowId: string, cursor: string, limit: number) => [prefix, flowId, cursor, limit],
|
|
1555
1728
|
{
|
|
1556
1729
|
numberOfKeys: 0,
|
|
@@ -1591,7 +1764,7 @@ return cjson.encode({ items = items, more = #keys > limit })
|
|
|
1591
1764
|
* Spec JSON strings pass through untouched — the script never cjson-decodes
|
|
1592
1765
|
* stored payloads (precision, lone surrogates).
|
|
1593
1766
|
*/
|
|
1594
|
-
export const flowSweepWork = Redis.script(
|
|
1767
|
+
export const flowSweepWork = (HELPERS: string) => Redis.script(
|
|
1595
1768
|
(prefix: string, pendingAgeMs: number, limit: number, now: number) => [prefix, pendingAgeMs, limit, now],
|
|
1596
1769
|
{
|
|
1597
1770
|
numberOfKeys: 0,
|
|
@@ -1671,7 +1844,7 @@ return cjson.encode({ reconcile = reconcile, cascade = cascade })
|
|
|
1671
1844
|
* markChildrenCascaded(prefix, flowId, childKeysJson) — idempotent; unknown
|
|
1672
1845
|
* keys are ignored (their index members are still cleared).
|
|
1673
1846
|
*/
|
|
1674
|
-
export const markChildrenCascaded = Redis.script(
|
|
1847
|
+
export const markChildrenCascaded = (HELPERS: string) => Redis.script(
|
|
1675
1848
|
(prefix: string, flowId: string, childKeysJson: string) => [prefix, flowId, childKeysJson],
|
|
1676
1849
|
{
|
|
1677
1850
|
numberOfKeys: 0,
|