experimental-a2 0.17.0 → 0.18.1

package/src/store-redis-core.ts

@@ -123,6 +123,23 @@ const READ_CONCURRENCY = 16
 
 const META_CHUNK_SIZE = 1000
 
+/** The metadata columns of a full stored row. */
+const STORED_EVENT_META_SUFFIXES = [
+  'p',
+  'pa',
+  'ri',
+  'ch',
+  'ce',
+  'f',
+  'fc',
+  'lc',
+  'lf',
+  'lfa',
+  'd',
+  'a',
+  'e',
+] as const
+
 /**
  * Upstash bills every call a script makes, so scripts read the counter and
  * split with one MGET, read metadata with one HMGET per hash (`read_fields`),
@@ -278,6 +295,33 @@ local function flush_queue()
 end
 `
 
+/**
+ * Settles one unfinished event for scripts that include ENQUEUE_LUA: it
+ * leaves the pending and ready sets, and a lane head hands its lane to the
+ * next unfinished event. A dead-lettered successor stays out of the ready set.
+ */
+const SETTLE_LUA = `
+local function settle(field, lane, next_field)
+  redis.call('ZREM', PENDING, field)
+  redis.call('ZREM', READY, field)
+  if lane == '' then return end
+  local head_key = lane_field(lane, 'head')
+  if redis.call('HGET', META, head_key) ~= field then return end
+  while next_field do
+    local row = read_fields({ next_field }, { 'p', 'f', 'next' })[next_field]
+    if not row[1] then
+      redis.call('HSET', META, head_key, next_field)
+      if not row[2] then
+        redis.call('ZADD', READY, tonumber(next_field), next_field)
+      end
+      return
+    end
+    next_field = row[3]
+  end
+  clear_lane(lane)
+end
+`
+
 /**
  * Event-id ownership for scripts that add events. Each script reads the
  * counter, split, clear boundary, and per-id keys with one MGET and applies
@@ -450,7 +494,7 @@ return { 'ok', tostring(base + 1), pending_after }
 /**
  * KEYS: counter, meta, pending, ready, event snapshot pins, snapshot pin counts,
  * metadata split.
- * ARGV: idx, attempt, error, maxFailures, nowMs.
+ * ARGV: idx, attempt, error, maxFailures, nowMs, retryAtMs ('' for none).
  */
 const FAIL_ATTEMPT_LUA = `
 local COUNTER, META, SPLIT = KEYS[1], KEYS[2], KEYS[7]
@@ -482,8 +526,14 @@ put(meta_key, field .. ':e', ARGV[3])
 put(meta_key, field .. ':lf', ARGV[5])
 put(meta_key, field .. ':lfa', ARGV[2])
 if dead then put(meta_key, field .. ':f', ARGV[5]) end
+local hold = not dead and ARGV[6] ~= ''
+if hold then put(meta_key, field .. ':ce', ARGV[6]) end
 flush()
-redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
+if hold then
+  redis.call('HDEL', meta_key, field .. ':ch')
+else
+  redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
+end
 if dead then
   redis.call('ZREM', KEYS[4], field)
   release_snapshot_pins(KEYS[5], KEYS[6])
@@ -493,23 +543,29 @@ return { 'failed', tostring(failures) }
 `
 
 /**
- * KEYS: store, counter, meta, pending, ready, metadata split.
- * ARGV: holder, nowMs, expiresAtMs, excludedCount, excluded indexes.
+ * KEYS: store, counter, meta, pending, ready, metadata split, snapshot pin
+ * counts.
+ * ARGV: holder, nowMs, expiresAtMs, excludedCount, excluded indexes, then
+ * maxFailures ('' for none) and the event snapshot-pins key prefix.
  * The ready set is a subset of the pending set, so an empty ready set is
  * enough to report 'settled'. A claimed result ends with the clear boundary.
  */
 const CLAIM_AVAILABLE_LUA = `
 local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[6]
 ${META_LUA}
+${RELEASE_PINS_LUA}
 local ready = redis.call('ZRANGE', KEYS[5], 0, -1)
 local excluded = {}
+local tail = 5 + tonumber(ARGV[4])
 for i = 1, tonumber(ARGV[4]) do excluded[ARGV[4 + i]] = true end
+local max_failures = tonumber(ARGV[tail])
 local candidates = {}
 for _, field in ipairs(ready) do
   if not excluded[field] then candidates[#candidates + 1] = field end
 end
 local rows = read_fields(candidates, { 'ce', 'd', 'fc', 'a', 'e', 'lf', 'lfa' })
 local claimed = {}
+local dead = {}
 local retry_at = nil
 for _, field in ipairs(candidates) do
   local row = rows[field]
@@ -519,29 +575,63 @@ for _, field in ipairs(candidates) do
   else
     local entries = redis.call('XRANGE', KEYS[1], field .. '-0', field .. '-0')
     if #entries == 0 then return { 'missing', field } end
-    local attempt = tostring(tonumber(row[2] or '0') + 1)
-    local first_claimed = row[3] or ARGV[2]
     local meta_key = mkey(field)
+    local attempt = tostring(tonumber(row[2] or '0') + 1)
+    -- An expiry that is not the current attempt's retry hold is a lapsed
+    -- claim: its attempt never finished.
+    local lapsed = row[1] and row[7] ~= row[2]
+    local failures = tonumber(row[4] or '0')
+    if lapsed then
+      failures = failures + 1
+      put(meta_key, field .. ':a', tostring(failures))
+    end
     put(meta_key, field .. ':d', attempt)
-    put(meta_key, field .. ':fc', first_claimed)
-    put(meta_key, field .. ':lc', ARGV[2])
-    put(meta_key, field .. ':ch', ARGV[1])
-    put(meta_key, field .. ':ce', ARGV[3])
-    claimed[#claimed + 1] = {
-      attempt,
-      row[4] or '0',
-      row[5] or false,
-      first_claimed,
-      ARGV[2],
-      row[6] or false,
-      row[7] or false,
-      ARGV[1],
-      ARGV[3],
-      entries[1]
-    }
+    if lapsed and max_failures and failures >= max_failures then
+      -- That takeover spent the last failed attempt: dead-letter, unclaimed.
+      -- Advancing the attempt fences the lapsed one; nothing runs.
+      put(meta_key, field .. ':f', ARGV[2])
+      dead[#dead + 1] = field
+      claimed[#claimed + 1] = {
+        attempt,
+        tostring(failures),
+        row[5] or false,
+        row[3] or false,
+        read_fields({ field }, { 'lc' })[field][1] or false,
+        row[6] or false,
+        row[7] or false,
+        false,
+        false,
+        entries[1],
+        ARGV[2]
+      }
+    else
+      local first_claimed = row[3] or ARGV[2]
+      put(meta_key, field .. ':fc', first_claimed)
+      put(meta_key, field .. ':lc', ARGV[2])
+      put(meta_key, field .. ':ch', ARGV[1])
+      put(meta_key, field .. ':ce', ARGV[3])
+      claimed[#claimed + 1] = {
+        attempt,
+        tostring(failures),
+        row[5] or false,
+        first_claimed,
+        ARGV[2],
+        row[6] or false,
+        row[7] or false,
+        ARGV[1],
+        ARGV[3],
+        entries[1],
+        false
+      }
+    end
   end
 end
 flush()
+for _, field in ipairs(dead) do
+  redis.call('HDEL', mkey(field), field .. ':ch', field .. ':ce')
+  redis.call('ZREM', KEYS[5], field)
+  release_snapshot_pins(ARGV[tail + 1] .. field, KEYS[7])
+end
 if #claimed > 0 then
   local result = { 'claimed', tostring(#claimed) }
   for _, claim in ipairs(claimed) do
@@ -611,25 +701,7 @@ ${ENQUEUE_LUA}
 ${ID_OWNERS_LUA}
 ${RELEASE_PINS_LUA}
 ${WAKE_LUA}
-local function settle(field, lane, next_field)
-  redis.call('ZREM', PENDING, field)
-  redis.call('ZREM', READY, field)
-  if lane == '' then return end
-  local head_key = lane_field(lane, 'head')
-  if redis.call('HGET', META, head_key) ~= field then return end
-  while next_field do
-    local row = read_fields({ next_field }, { 'p', 'f', 'next' })[next_field]
-    if not row[1] then
-      redis.call('HSET', META, head_key, next_field)
-      if not row[2] then
-        redis.call('ZADD', READY, tonumber(next_field), next_field)
-      end
-      return
-    end
-    next_field = row[3]
-  end
-  clear_lane(lane)
-end
+${SETTLE_LUA}
 local parent = ARGV[3]
 local attempt = tonumber(ARGV[4])
 local n = tonumber(ARGV[5])
@@ -715,6 +787,78 @@ if n > 0 then wake(ARGV[6 + n * 5], tostring(total + n), ARGV[2]) end
 return { 'ok', tostring(total + 1) }
 `
 
+/**
+ * KEYS: counter, meta, pending, ready, metadata split.
+ * ARGV: idx, action ('retry' | 'skip'), nowMs.
+ * Returns [redriven, hasPending]. Acts only on an unfinished, dead-lettered
+ * event above the clear boundary. A retry keeps the event pending and makes
+ * it ready again when it is unlaned or its lane's head.
+ */
+const REDRIVE_LUA = `
+local COUNTER, META, SPLIT, PENDING, READY = KEYS[1], KEYS[2], KEYS[5], KEYS[3], KEYS[4]
+${META_LUA}
+${ENQUEUE_LUA}
+${SETTLE_LUA}
+local function result(redriven)
+  return { redriven, redis.call('ZCARD', PENDING) > 0 and '1' or '0' }
+end
+local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
+local total = tonumber(counts[1] or '0')
+use_split(counts[2], counts[1], counts[3])
+local idx = tonumber(ARGV[1])
+if idx <= boundary or idx > total then return result('0') end
+local field = tostring(idx)
+local row = read_fields({ field }, { 'p', 'f', 'l', 'next' })[field]
+if row[1] or not row[2] then return result('0') end
+local meta_key = mkey(field)
+local lane = row[3] or ''
+if ARGV[2] == 'skip' then
+  put(meta_key, field .. ':p', ARGV[3])
+  settle(field, lane, row[4])
+  flush_queue()
+  flush()
+else
+  redis.call('HDEL', meta_key, field .. ':f', field .. ':a')
+  if lane == '' or redis.call('HGET', META, lane_field(lane, 'head')) == field then
+    redis.call('ZADD', READY, idx, field)
+  end
+end
+return result('1')
+`
+
+/**
+ * KEYS: store, counter, meta, pending, metadata split.
+ * Returns the dead-lettered events above the clear boundary, oldest first,
+ * as pairs of a stream entry and its STORED_EVENT_META_SUFFIXES columns.
+ * Every unfinished event is in the pending set, so the scan is bounded by
+ * unfinished work, not history; only dead letters read their full columns.
+ */
+const DEAD_LETTERS_LUA = `
+local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[5]
+${META_LUA}
+local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
+use_split(counts[2], counts[1], counts[3])
+local candidates = {}
+for _, field in ipairs(redis.call('ZRANGE', KEYS[4], 0, -1)) do
+  if tonumber(field) > boundary then candidates[#candidates + 1] = field end
+end
+local status = read_fields(candidates, { 'p', 'f' })
+local dead = {}
+for _, field in ipairs(candidates) do
+  if status[field][2] and not status[field][1] then dead[#dead + 1] = field end
+end
+local rows = read_fields(dead, { ${STORED_EVENT_META_SUFFIXES.map((suffix) => `'${suffix}'`).join(', ')} })
+local result = {}
+for _, field in ipairs(dead) do
+  local entries = redis.call('XRANGE', KEYS[1], field .. '-0', field .. '-0')
+  if #entries > 0 then
+    result[#result + 1] = entries[1]
+    result[#result + 1] = rows[field]
+  end
+end
+return result
+`
+
 /**
  * KEYS: head, history, event counter, event meta, pin counts, metadata split.
  * ARGV: nowMs, event pin key prefix, write count, then per write:
@@ -1117,22 +1261,6 @@ const fieldMap = (fields: string[]): Record<string, string> => {
   return map
 }
 
-const STORED_EVENT_META_SUFFIXES = [
-  'p',
-  'pa',
-  'ri',
-  'ch',
-  'ce',
-  'f',
-  'fc',
-  'lc',
-  'lf',
-  'lfa',
-  'd',
-  'a',
-  'e',
-] as const
-
 const entryIndex = (entryId: string): number =>
   Number(entryId.slice(0, entryId.indexOf('-')))
 
@@ -1240,6 +1368,46 @@ const toCause = (raw: string | undefined): EventCause | null => {
   }
 }
 
+type MetaSuffix = (typeof STORED_EVENT_META_SUFFIXES)[number]
+
+/** A stream entry joined with its metadata columns; absent columns are unset. */
+const toStoredEvent = (
+  sessionId: string,
+  entry: StreamEntry,
+  column: (suffix: MetaSuffix) => string | undefined,
+): StoredEvent => {
+  const fields = fieldMap(entry[1])
+  const date = (suffix: MetaSuffix): Date | null => {
+    const value = column(suffix)
+    return value === undefined ? null : new Date(Number(value))
+  }
+  const count = (suffix: MetaSuffix): number | null => {
+    const value = column(suffix)
+    return value === undefined ? null : Number(value)
+  }
+  const returnedEventIds = column('ri')
+  return Object.assign(toEvent(sessionId, entry), {
+    cause: toCause(fields['cause']),
+    lane: fields['lane'] === '' ? null : fields['lane']!,
+    processedAt: date('p'),
+    processedByAttempt: count('pa'),
+    returnedEventIds:
+      returnedEventIds === undefined
+        ? null
+        : decodeReturnedEventIds(returnedEventIds),
+    firstClaimedAt: date('fc'),
+    lastClaimedAt: date('lc'),
+    attemptCount: count('d') ?? 0,
+    claimHolder: column('ch') ?? null,
+    claimExpiresAt: date('ce'),
+    failureCount: count('a') ?? 0,
+    lastFailedAt: date('lf'),
+    lastFailedAttempt: count('lfa'),
+    lastError: column('e') ?? null,
+    failedAt: date('f'),
+  })
+}
+
 const toClaimAvailableResult = (
   sessionId: string,
   reply: unknown[],
@@ -1249,7 +1417,7 @@ const toClaimAvailableResult = (
     const count = Number(reply[1])
     const events: StoredEvent[] = []
     for (let i = 0; i < count; i++) {
-      const offset = 2 + i * 10
+      const offset = 2 + i * 11
       const entry = reply[offset + 9] as StreamEntry
       const event = toEvent(sessionId, entry)
       const fields = fieldMap(entry[1])
@@ -1264,10 +1432,17 @@ const toClaimAvailableResult = (
             reply[offset + 3] === null
               ? null
               : new Date(Number(reply[offset + 3])),
-          lastClaimedAt: new Date(Number(reply[offset + 4])),
+          lastClaimedAt:
+            reply[offset + 4] === null
+              ? null
+              : new Date(Number(reply[offset + 4])),
           attemptCount: Number(reply[offset]),
-          claimHolder: String(reply[offset + 7]),
-          claimExpiresAt: new Date(Number(reply[offset + 8])),
+          claimHolder:
+            reply[offset + 7] === null ? null : String(reply[offset + 7]),
+          claimExpiresAt:
+            reply[offset + 8] === null
+              ? null
+              : new Date(Number(reply[offset + 8])),
           failureCount: Number(reply[offset + 1]),
           lastFailedAt:
             reply[offset + 5] === null
@@ -1277,11 +1452,14 @@ const toClaimAvailableResult = (
             reply[offset + 6] === null ? null : Number(reply[offset + 6]),
           lastError:
             reply[offset + 2] === null ? null : String(reply[offset + 2]),
-          failedAt: null,
+          failedAt:
+            reply[offset + 10] === null
+              ? null
+              : new Date(Number(reply[offset + 10])),
         }),
       )
     }
-    const clearIndex = Number(reply[2 + count * 10] ?? 0)
+    const clearIndex = Number(reply[2 + count * 11] ?? 0)
     if (clearIndex > 0)
       for (const event of events) event.clearIndex = clearIndex
     return {
@@ -1471,50 +1649,12 @@ export function createRedisStoreCore(
       }
     }
     return entries.map((entry) => {
-      const event = toEvent(sessionId, entry)
-      const idx = event.index
-      const fields = fieldMap(entry[1])
-      const processedAt = meta[`${idx}:p`]
-      const processedByAttempt = meta[`${idx}:pa`]
-      const returnedEventIds = meta[`${idx}:ri`]
-      const claimHolder = meta[`${idx}:ch`]
-      const claimExpiresAt = meta[`${idx}:ce`]
-      const failedAt = meta[`${idx}:f`]
-      const firstClaimedAt = meta[`${idx}:fc`]
-      const lastClaimedAt = meta[`${idx}:lc`]
-      const lastFailedAt = meta[`${idx}:lf`]
-      const lastFailedAttempt = meta[`${idx}:lfa`]
-      return Object.assign(event, {
-        cause: toCause(fields['cause']),
-        lane: fields['lane'] === '' ? null : fields['lane']!,
-        processedAt:
-          processedAt === undefined ? null : new Date(Number(processedAt)),
-        processedByAttempt:
-          processedByAttempt === undefined ? null : Number(processedByAttempt),
-        returnedEventIds:
-          returnedEventIds === undefined
-            ? null
-            : decodeReturnedEventIds(returnedEventIds),
-        firstClaimedAt:
-          firstClaimedAt === undefined
-            ? null
-            : new Date(Number(firstClaimedAt)),
-        lastClaimedAt:
-          lastClaimedAt === undefined ? null : new Date(Number(lastClaimedAt)),
-        attemptCount: Number(meta[`${idx}:d`] ?? 0),
-        claimHolder: claimHolder ?? null,
-        claimExpiresAt:
-          claimExpiresAt === undefined
-            ? null
-            : new Date(Number(claimExpiresAt)),
-        failureCount: Number(meta[`${idx}:a`] ?? 0),
-        lastFailedAt:
-          lastFailedAt === undefined ? null : new Date(Number(lastFailedAt)),
-        lastFailedAttempt:
-          lastFailedAttempt === undefined ? null : Number(lastFailedAttempt),
-        lastError: meta[`${idx}:e`] ?? null,
-        failedAt: failedAt === undefined ? null : new Date(Number(failedAt)),
-      })
+      const idx = entryIndex(entry[0])
+      return toStoredEvent(
+        sessionId,
+        entry,
+        (suffix) => meta[`${idx}:${suffix}`],
+      )
     })
   }
 
@@ -1858,6 +1998,7 @@ export function createRedisStoreCore(
       ttlMs,
       expiresAtMs,
       excludeIndexes,
+      maxFailures,
     }) {
       return wrap(async () => {
         const now = clock.now().getTime()
@@ -1872,8 +2013,17 @@ export function createRedisStoreCore(
             pendingKey(sessionId),
             readyKey(sessionId),
             metaSplitKey(sessionId),
+            snapshotPinCountsKey(sessionId),
+          ],
+          [
+            holder,
+            now,
+            expiresAt,
+            excluded.length,
+            ...excluded,
+            maxFailures ?? '',
+            snapshotEventPinsPrefix(sessionId),
           ],
-          [holder, now, expiresAt, excluded.length, ...excluded],
         )) as unknown[]
         return toClaimAvailableResult(sessionId, reply)
       })
@@ -2009,8 +2159,16 @@ export function createRedisStoreCore(
       })
     },
 
-    async failAttempt({ sessionId, index, attempt, error, maxFailures }) {
+    async failAttempt({
+      sessionId,
+      index,
+      attempt,
+      error,
+      maxFailures,
+      retryAfterMs,
+    }) {
       return wrap(async () => {
+        const now = clock.now().getTime()
         const reply = (await evalScript(
           FAIL_ATTEMPT_LUA,
           [
@@ -2022,7 +2180,14 @@ export function createRedisStoreCore(
             snapshotPinCountsKey(sessionId),
             metaSplitKey(sessionId),
           ],
-          [index, attempt, error, maxFailures, clock.now().getTime()],
+          [
+            index,
+            attempt,
+            error,
+            maxFailures,
+            now,
+            retryAfterMs === undefined ? '' : now + retryAfterMs,
+          ],
         )) as string[]
         const outcome = reply[0]
         if (outcome === 'cleared')
@@ -2039,6 +2204,53 @@ export function createRedisStoreCore(
       })
     },
 
+    async deadLetters(sessionId) {
+      return wrap(async () => {
+        const reply = (await evalScript(
+          DEAD_LETTERS_LUA,
+          [
+            storeKey(sessionId),
+            countKey(sessionId),
+            metaKey(sessionId),
+            pendingKey(sessionId),
+            metaSplitKey(sessionId),
+          ],
+          [],
+        )) as unknown[]
+        const events: StoredEvent[] = []
+        for (let i = 0; i + 1 < reply.length; i += 2) {
+          const columns = reply[i + 1] as Array<string | null>
+          events.push(
+            toStoredEvent(
+              sessionId,
+              reply[i] as StreamEntry,
+              (suffix) =>
+                columns[STORED_EVENT_META_SUFFIXES.indexOf(suffix)] ??
+                undefined,
+            ),
+          )
+        }
+        return events
+      })
+    },
+
+    async redrive({ sessionId, index, action }) {
+      return wrap(async () => {
+        const [redriven, hasPending] = (await evalScript(
+          REDRIVE_LUA,
+          [
+            countKey(sessionId),
+            metaKey(sessionId),
+            pendingKey(sessionId),
+            readyKey(sessionId),
+            metaSplitKey(sessionId),
+          ],
+          [index, action, clock.now().getTime()],
+        )) as [string, string]
+        return { redriven: redriven === '1', hasPending: hasPending === '1' }
+      })
+    },
+
     async readState(sessionId, reducerName, stateOptions) {
       return wrap(() =>
         retryAfterGap(async () => {

package/src/store-redis-notify.ts

@@ -292,6 +292,8 @@ export function createRedisNotifyStore(options: {
     putSnapshots: core.putSnapshots,
     clear: core.clear!,
     reclaim: core.reclaim!,
+    deadLetters: core.deadLetters!,
+    redrive: core.redrive!,
 
     presence: {
       async set(ns, participant, values, meta) {