effect-mq 0.2.0 → 0.3.0

package/src/redis/scripts.ts

@@ -21,11 +21,13 @@
  * - `p:delayed:<queue>`         ZSET, score `runAt`
  * - `p:active`                  ZSET, score `lockExpiresAt`
  * - `p:all`                     ZSET, score `enqueuedAt` (list pagination)
- * - `p:finished`                ZSET, score `finishedAt` (history TTL)
+ * - `p:finished:<state>`        ZSET, score `finishedAt` (history TTL)
  * - `p:terminal:<name>:<state>` ZSET, score `finishedAt` (keep pruning)
  * - `p:counts`                  HASH `<queue>|<state>` -> integer
  * - `p:paused`                  SET of paused queues
  * - `p:schedules` / `p:schedule:<key>`  ZSET by nextRunAt + HASH per record
+ * - `p:dedupe:<name>\0<key>`   HASH {jobId, expiresAt} + `p:dedupes` index
+ *                               ZSET (score = window expiry, +inf = pending)
  *
  * @since 0.2.0
  */
@@ -78,7 +80,7 @@ local function deleteJob(id)
   local name = redis.call("HGET", jk, "name")
   countsAdd(queue, state, -1)
   redis.call("ZREM", prefix .. ":all", id)
-  redis.call("ZREM", prefix .. ":finished", id)
+  redis.call("ZREM", prefix .. ":finished:" .. state, id)
   redis.call("ZREM", prefix .. ":active", id)
   redis.call("ZREM", terminalKey(name, state), id)
   remWaiting(queue, id)
@@ -90,8 +92,20 @@ end
 -- zset score alone cannot carry the seq tie-break exactly.
 local function applyKeep(name, state, keepJson, now)
   if keepJson == nil or keepJson == "" then return end
-  local ok, keep = pcall(cjson.decode, keepJson)
-  if not ok or type(keep) ~= "table" then return end
+  local ok, decoded = pcall(cjson.decode, keepJson)
+  if not ok or type(decoded) ~= "table" then return end
+  -- Policies are split per terminal state; rows persisted by 0.2.x carry the
+  -- flat {count, ageMs} shape and apply to every state.
+  local keep = decoded[state]
+  if type(keep) ~= "table" then
+    if decoded.completed == nil and decoded.failed == nil and decoded.cancelled == nil
+      and (decoded.count ~= nil or decoded.ageMs ~= nil)
+    then
+      keep = decoded
+    else
+      return
+    end
+  end
   local tkey = terminalKey(name, state)
   if keep.ageMs ~= nil then
     local old = redis.call("ZRANGEBYSCORE", tkey, "-inf", now - keep.ageMs)
@@ -117,6 +131,20 @@ local function applyKeep(name, state, keepJson, now)
     for i = count + 1, #arr do deleteJob(arr[i].id) end
   end
 end
+local function dedupeStoreKey(name, key) return prefix .. ":dedupe:" .. name .. "\0" .. key end
+-- A job leaving the pending states frees its pending-mode dedup entry; live
+-- throttle windows deliberately outlast the job.
+local function releaseDedupe(name, dkey, jobId, now)
+  if dkey == nil or dkey == false or dkey == "" then return end
+  local sk = dedupeStoreKey(name, dkey)
+  if redis.call("HGET", sk, "jobId") == jobId then
+    local exp = redis.call("HGET", sk, "expiresAt")
+    if exp == "" or tonumber(exp) <= now then
+      redis.call("DEL", sk)
+      redis.call("ZREM", prefix .. ":dedupes", name .. "\0" .. dkey)
+    end
+  end
+end
 -- Move an active job (whose lock bookkeeping was already cleared by the
 -- caller) to the terminal cancelled state.
 local function finishCancelled(id, queue, name, startedAt, now, nowStr)
@@ -125,9 +153,10 @@ local function finishCancelled(id, queue, name, startedAt, now, nowStr)
     "lockToken", "", "lockExpiresAt", "")
   countsAdd(queue, "active", -1)
   countsAdd(queue, "cancelled", 1)
-  redis.call("ZADD", prefix .. ":finished", now, id)
+  redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
   redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
   appendAttempt(id, "cancelled", startedAt, nowStr, "")
+  releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
   applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
 end
 `
@@ -153,7 +182,11 @@ export const enqueue = Redis.script(
     keepJson: string,
     timeoutMs: string,
     delayMs: number,
-    now: number
+    now: number,
+    dedupeKey: string,
+    dedupeTtlMs: string,
+    dedupeExtend: string,
+    dedupeReplace: string
   ) => [
     prefix,
     idMode,
@@ -168,7 +201,11 @@ export const enqueue = Redis.script(
     keepJson,
     timeoutMs,
     delayMs,
-    now
+    now,
+    dedupeKey,
+    dedupeTtlMs,
+    dedupeExtend,
+    dedupeReplace
   ],
   {
     numberOfKeys: 0,
@@ -183,7 +220,54 @@ if idMode == "user" or idMode == "generated" then
     if idMode == "user" then return '{"duplicate":true,"id":' .. cjson.encode(id) .. '}' end
     return '{"collision":true}'
   end
-else
+end
+-- Dedup decision tree: replace-while-delayed, throttle window, pending dedup.
+local dKey = ARGV[15]
+local name = ARGV[4]
+if dKey ~= "" then
+  local sk = dedupeStoreKey(name, dKey)
+  local entryJob = redis.call("HGET", sk, "jobId")
+  if entryJob then
+    local expStr = redis.call("HGET", sk, "expiresAt")
+    local windowLive = expStr ~= "" and tonumber(expStr) > now
+    local keyedState = redis.call("HGET", jobKey(entryJob), "state")
+    local function bumpWindow()
+      if ARGV[17] == "1" and ARGV[16] ~= "" then
+        local windowEnd = now + tonumber(ARGV[16])
+        redis.call("HSET", sk, "expiresAt", fmt(windowEnd))
+        redis.call("ZADD", prefix .. ":dedupes", windowEnd, name .. "\0" .. dKey)
+      end
+    end
+    -- Latest-wins while the keyed job is still delayed.
+    if ARGV[18] == "1" and keyedState == "delayed" then
+      local kjk = jobKey(entryJob)
+      local newRunAt = now + delayMs
+      redis.call("HSET", kjk, "payload", ARGV[6], "metadata", ARGV[7], "priority", ARGV[8],
+        "attemptsMax", ARGV[9], "backoff", ARGV[10], "keep", ARGV[11], "timeoutMs", ARGV[12],
+        "runAt", fmt(newRunAt))
+      local keyedQueue = redis.call("HGET", kjk, "queue")
+      redis.call("ZADD", delayedKey(keyedQueue), newRunAt, entryJob)
+      -- A landed replace re-arms the ttl window.
+      if ARGV[16] ~= "" then
+        local windowEnd = now + tonumber(ARGV[16])
+        redis.call("HSET", sk, "expiresAt", fmt(windowEnd))
+        redis.call("ZADD", prefix .. ":dedupes", windowEnd, name .. "\0" .. dKey)
+      end
+      return '{"id":' .. cjson.encode(entryJob) .. ',"duplicate":true,"wake":true,"queue":' .. cjson.encode(keyedQueue) .. '}'
+    end
+    if windowLive then
+      bumpWindow()
+      return '{"id":' .. cjson.encode(entryJob) .. ',"duplicate":true}'
+    end
+    local pending = keyedState ~= false and keyedState ~= "completed"
+      and keyedState ~= "failed" and keyedState ~= "cancelled"
+    if expStr == "" and pending then
+      return '{"id":' .. cjson.encode(entryJob) .. ',"duplicate":true}'
+    end
+    -- Dead entry: the new job takes over the key below.
+  end
+end
+if idMode == "auto" then
   id = ""
   for i = 1, 5 do
     local candidate = "j-" .. fmt(redis.call("INCR", prefix .. ":seq"))
@@ -202,7 +286,7 @@ redis.call("HSET", jobKey(id),
   "payload", ARGV[6], "metadata", ARGV[7], "state", state,
   "priority", ARGV[8], "attemptsMax", ARGV[9], "attemptsMade", "0", "stalledCount", "0",
   "backoff", ARGV[10], "keep", ARGV[11], "timeoutMs", ARGV[12],
-  "cancelRequested", "0", "runAt", fmt(runAt), "enqueuedAt", nowStr,
+  "cancelRequested", "0", "dedupeKey", dKey, "runAt", fmt(runAt), "enqueuedAt", nowStr,
   "processedAt", "", "finishedAt", "", "exit", "", "failedReason", "",
   "lockToken", "", "lockExpiresAt", "", "seq", fmt(seq))
 redis.call("ZADD", prefix .. ":all", now, id)
@@ -212,6 +296,14 @@ else
   redis.call("ZADD", delayedKey(queue), runAt, id)
 end
 countsAdd(queue, state, 1)
+if dKey ~= "" then
+  local sk = dedupeStoreKey(name, dKey)
+  redis.call("DEL", sk)
+  redis.call("HSET", sk, "jobId", id,
+    "expiresAt", ARGV[16] == "" and "" or fmt(now + tonumber(ARGV[16])))
+  redis.call("ZADD", prefix .. ":dedupes",
+    ARGV[16] == "" and "inf" or tostring(now + tonumber(ARGV[16])), name .. "\0" .. dKey)
+end
 -- Wake on EVERY insert (delayed too): idle workers must re-claim to learn
 -- the new nextRunAt, exactly like the memory and Postgres drivers.
 return '{"id":' .. cjson.encode(id) .. ',"duplicate":false,"wake":true}'
@@ -340,9 +432,10 @@ local function finish(newState, storeExit, outcome, ledgerExit)
   if storeExit ~= nil then redis.call("HSET", jk, "exit", storeExit) end
   countsAdd(queue, "active", -1)
   countsAdd(queue, newState, 1)
-  redis.call("ZADD", prefix .. ":finished", now, id)
+  redis.call("ZADD", prefix .. ":finished:" .. newState, now, id)
   redis.call("ZADD", terminalKey(name, newState), now, id)
   appendAttempt(id, outcome, startedAt, nowStr, ledgerExit)
+  releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
   applyKeep(name, newState, redis.call("HGET", jk, "keep"), now)
 end
 
@@ -369,7 +462,7 @@ else
   else
     redis.call("ZADD", delayedKey(queue), runAt, id)
   end
-  return '{"ok":true,"wake":true}'
+  return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
 end
 return '{"ok":true}'
 `
@@ -405,7 +498,7 @@ redis.call("HSET", jk, "state", "waiting", "lockToken", "", "lockExpiresAt", "")
 addWaiting(queue, priority, seq, id)
 countsAdd(queue, "active", -1)
 countsAdd(queue, "waiting", 1)
-return '{"ok":true,"wake":true}'
+return '{"ok":true,"wake":true,"queue":' .. cjson.encode(queue) .. '}'
 `
   }
 ).withReturnType<string>()
@@ -471,8 +564,9 @@ for _, id in ipairs(expired) do
           "failedReason", "job stalled more than allowable limit")
         countsAdd(queue, "active", -1)
         countsAdd(queue, "failed", 1)
-        redis.call("ZADD", prefix .. ":finished", now, id)
+        redis.call("ZADD", prefix .. ":finished:failed", now, id)
         redis.call("ZADD", terminalKey(name, "failed"), now, id)
+        releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
         recovered[#recovered + 1] = { id = id, failed = true }
       else
         local seq = tonumber(redis.call("HGET", jk, "seq")) or 0
@@ -630,13 +724,13 @@ local seq = redis.call("INCR", prefix .. ":seq")
 redis.call("HSET", jk, "state", "waiting", "attemptsMade", "0", "stalledCount", "0",
   "cancelRequested", "0", "exit", "", "failedReason", "", "finishedAt", "",
   "processedAt", "", "runAt", nowStr, "seq", fmt(seq))
-redis.call("ZREM", prefix .. ":finished", id)
+redis.call("ZREM", prefix .. ":finished:failed", id)
 redis.call("ZREM", terminalKey(name, "failed"), id)
 local priority = tonumber(redis.call("HGET", jk, "priority")) or 0
 addWaiting(queue, priority, seq, id)
 countsAdd(queue, "failed", -1)
 countsAdd(queue, "waiting", 1)
-return '{"ok":true}'
+return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
 `
   }
 ).withReturnType<string>()
@@ -670,9 +764,10 @@ redis.call("ZREM", delayedKey(queue), id)
 redis.call("HSET", jk, "state", "cancelled", "finishedAt", nowStr, "cancelRequested", "0")
 countsAdd(queue, state, -1)
 countsAdd(queue, "cancelled", 1)
-redis.call("ZADD", prefix .. ":finished", now, id)
+redis.call("ZADD", prefix .. ":finished:cancelled", now, id)
 redis.call("ZADD", terminalKey(name, "cancelled"), now, id)
 appendAttempt(id, "cancelled", redis.call("HGET", jk, "processedAt"), nowStr, "")
+releaseDedupe(name, redis.call("HGET", jk, "dedupeKey"), id, now)
 applyKeep(name, "cancelled", redis.call("HGET", jk, "keep"), now)
 return '{"ok":true}'
 `
@@ -698,7 +793,7 @@ redis.call("HSET", jk, "state", "waiting", "runAt", ARGV[3])
 addWaiting(queue, priority, seq, id)
 countsAdd(queue, "delayed", -1)
 countsAdd(queue, "waiting", 1)
-return '{"ok":true}'
+return '{"ok":true,"queue":' .. cjson.encode(queue) .. '}'
 `
   }
 ).withReturnType<string>()
@@ -846,17 +941,115 @@ return "1"
 ).withReturnType<string>()
 
 /**
- * sweepHistory(prefix, cutoff, limit) -> number swept (bounded batch; the
- * caller loops until 0).
+ * sweepState(prefix, state, ttlMs, limit, offset, now) -> {scanned, deleted}
+ * One bounded page over a terminal state's finished zset, deleting rows past
+ * min(store ceiling, per-row keep.age). The caller advances the offset by
+ * (scanned - deleted) and stops when a page comes back short.
+ */
+export const sweepState = Redis.script(
+  (prefix: string, state: string, ttlMs: string, limit: number, offset: number, now: number) => [
+    prefix,
+    state,
+    ttlMs,
+    limit,
+    offset,
+    now
+  ],
+  {
+    numberOfKeys: 0,
+    lua: `${HELPERS}
+local state = ARGV[2]
+local ttl = ARGV[3] ~= "" and tonumber(ARGV[3]) or nil
+local limit = tonumber(ARGV[4])
+local offset = tonumber(ARGV[5])
+local now = tonumber(ARGV[6])
+local batch = redis.call("ZRANGEBYSCORE", prefix .. ":finished:" .. state, "-inf", now,
+  "WITHSCORES", "LIMIT", offset, limit)
+local scanned = 0
+local deleted = 0
+for i = 1, #batch, 2 do
+  scanned = scanned + 1
+  local id = batch[i]
+  local finishedAt = tonumber(batch[i + 1])
+  if redis.call("EXISTS", jobKey(id)) == 0 then
+    -- Orphaned member (hash evicted/removed out of band): self-heal so the
+    -- cursor math stays honest and the member never loops the sweep.
+    redis.call("ZREM", prefix .. ":finished:" .. state, id)
+    deleted = deleted + 1
+  else
+    local cutoffAge = ttl
+    local keepJson = redis.call("HGET", jobKey(id), "keep")
+    if keepJson and keepJson ~= "" then
+      local ok, keep = pcall(cjson.decode, keepJson)
+      if ok and type(keep) == "table" then
+        local policy = keep[state]
+        if type(policy) ~= "table"
+          and keep.completed == nil and keep.failed == nil and keep.cancelled == nil
+          and (keep.count ~= nil or keep.ageMs ~= nil)
+        then
+          policy = keep
+        end
+        if type(policy) == "table" and policy.ageMs ~= nil then
+          local age = tonumber(policy.ageMs)
+          if age ~= nil and (cutoffAge == nil or age < cutoffAge) then cutoffAge = age end
+        end
+      end
+    end
+    if cutoffAge ~= nil and finishedAt <= now - cutoffAge then
+      deleteJob(id)
+      deleted = deleted + 1
+    end
+  end
+end
+return cjson.encode({ scanned = scanned, deleted = deleted })
+`
+  }
+).withReturnType<string>()
+
+/**
+ * sweepDedupes(prefix, limit, now) -> number pruned (bounded batch; the
+ * caller loops until 0): expired windows, then pending pointers (+inf)
+ * whose job is gone or terminal.
  */
-export const sweepHistory = Redis.script(
-  (prefix: string, cutoff: number, limit: number) => [prefix, cutoff, limit],
+export const sweepDedupes = Redis.script(
+  (prefix: string, limit: number, now: number) => [prefix, limit, now],
   {
     numberOfKeys: 0,
     lua: `${HELPERS}
-local old = redis.call("ZRANGEBYSCORE", prefix .. ":finished", "-inf", tonumber(ARGV[2]), "LIMIT", 0, tonumber(ARGV[3]))
-for _, id in ipairs(old) do deleteJob(id) end
-return tostring(#old)
+local limit = tonumber(ARGV[2])
+local now = tonumber(ARGV[3])
+-- Lazy migration: drain the pre-0.3 unsplit finished zset into the per-state
+-- keys (or drop orphans) so old history keeps getting swept.
+local migrated = 0
+local legacy = redis.call("ZRANGE", prefix .. ":finished", 0, limit - 1, "WITHSCORES")
+for i = 1, #legacy, 2 do
+  local id = legacy[i]
+  local state = redis.call("HGET", jobKey(id), "state")
+  if state == "completed" or state == "failed" or state == "cancelled" then
+    redis.call("ZADD", prefix .. ":finished:" .. state, tonumber(legacy[i + 1]), id)
+  end
+  redis.call("ZREM", prefix .. ":finished", id)
+  migrated = migrated + 1
+end
+local index = prefix .. ":dedupes"
+local expired = redis.call("ZRANGEBYSCORE", index, "-inf", now, "LIMIT", 0, limit)
+for _, member in ipairs(expired) do
+  redis.call("DEL", prefix .. ":dedupe:" .. member)
+  redis.call("ZREM", index, member)
+end
+local removedPending = 0
+local pendings = redis.call("ZRANGEBYSCORE", index, "inf", "inf", "LIMIT", 0, limit)
+for _, member in ipairs(pendings) do
+  local sk = prefix .. ":dedupe:" .. member
+  local jobId = redis.call("HGET", sk, "jobId")
+  local state = jobId and redis.call("HGET", jobKey(jobId), "state")
+  if not state or state == "completed" or state == "failed" or state == "cancelled" then
+    redis.call("DEL", sk)
+    redis.call("ZREM", index, member)
+    removedPending = removedPending + 1
+  end
+end
+return tostring(migrated + #expired + removedPending)
 `
   }
 ).withReturnType<string>()