experimental-a2 0.18.0 → 0.18.1

package/src/store-postgres.ts

@@ -350,6 +350,34 @@ const createListener = ({
     return listenerConnection
   })
 
+/** A dead-lettered event's snapshot pins, and history no pin keeps. */
+const releasePins = async ({
+  query,
+  sessionId,
+  index,
+}: {
+  query: PostgresConnection['query']
+  sessionId: string
+  index: number
+}): Promise<void> => {
+  await query(
+    `delete from a2_snapshot_pins
+      where session_id = $1 and event_index = $2`,
+    [sessionId, index],
+  )
+  await query(
+    `delete from a2_snapshot_history history
+      where history.session_id = $1
+        and not exists (
+          select 1 from a2_snapshot_pins pin
+           where pin.session_id = history.session_id
+             and pin.reducer_name = history.reducer_name
+             and pin.up_to_index = history.up_to_index
+        )`,
+    [sessionId],
+  )
+}
+
 export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
   const clock = options.clock ?? SYSTEM_CLOCK
   const generateId = options.ids ?? RANDOM_IDS
@@ -942,11 +970,16 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
       ttlMs,
       expiresAtMs,
       excludeIndexes = [],
+      maxFailures,
     }) {
       return wrap(() =>
         withTx(async (query) => {
           const now = clock.now().getTime()
           const expiresAt = expiresAtMs ?? now + ttlMs
+          // A lapsed claim's takeover that spends the last failed attempt
+          // dead-letters the event instead of claiming it.
+          const exhausted = `(claimed.claim_holder is not null
+               and claimed.failure_count + 1 >= $6::int)`
           const { rows } = await query(
             `with eligible as materialized (
                select event.idx
@@ -962,15 +995,36 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
              )
              update a2_events claimed set
                attempt_count = claimed.attempt_count + 1,
-               first_claimed_at = coalesce(claimed.first_claimed_at, $2),
-               last_claimed_at = $2,
-               claim_holder = $3,
-               claim_expires_at = $4
+               failure_count = claimed.failure_count
+                 + case when claimed.claim_holder is null then 0 else 1 end,
+               failed_at = case when ${exhausted} then $2::bigint else claimed.failed_at end,
+               first_claimed_at = case when ${exhausted}
+                 then claimed.first_claimed_at
+                 else coalesce(claimed.first_claimed_at, $2::bigint) end,
+               last_claimed_at = case when ${exhausted}
+                 then claimed.last_claimed_at else $2::bigint end,
+               claim_holder = case when ${exhausted} then null else $3::text end,
+               claim_expires_at = case when ${exhausted} then null else $4::bigint end
              from eligible
              where claimed.session_id = $1 and claimed.idx = eligible.idx
              returning claimed.*, ${clearIndexOf('$1')} as clear_index`,
-            [sessionId, now, holder, expiresAt, [...excludeIndexes]],
+            [
+              sessionId,
+              now,
+              holder,
+              expiresAt,
+              [...excludeIndexes],
+              maxFailures ?? null,
+            ],
           )
+          for (const row of rows) {
+            if (row['failed_at'] !== null)
+              await releasePins({
+                query,
+                sessionId,
+                index: asNumber(row['idx']),
+              })
+          }
           if (rows.length > 0) {
             const events = rows
               .map((row) => {
@@ -990,8 +1044,9 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
                 and failed_at is null
                 and lane_ready
                 and idx > ${clearIndexOf('$1')}
-                and claim_expires_at > $2`,
-            [sessionId, now],
+                and claim_expires_at > $2
+                and not (idx = any($3::bigint[]))`,
+            [sessionId, now, [...excludeIndexes]],
           )
           const retryAt = activeRows[0]?.['retry_at']
           return retryAt === null || retryAt === undefined
@@ -1262,7 +1317,15 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
       )
     },
 
-    async failAttempt({ sessionId, index, attempt, error, maxFailures }) {
+    async failAttempt({
+      sessionId,
+      index,
+      attempt,
+      error,
+      maxFailures,
+      retryAfterMs,
+    }) {
+      const now = clock.now().getTime()
       return wrap(() =>
         withTx(async (query) => {
           const { rows } = await query(
@@ -1282,7 +1345,10 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
                  else e.failed_at
                end,
                claim_holder = null,
-               claim_expires_at = null
+               claim_expires_at = case
+                 when e.failure_count + 1 >= $5 then null
+                 else $7::bigint
+               end
              from current
              where e.session_id = $1
                and e.idx = $2
@@ -1322,7 +1388,8 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
               attempt,
               error,
               maxFailures,
-              clock.now().getTime(),
+              now,
+              retryAfterMs === undefined ? null : now + retryAfterMs,
             ],
           )
           const row = rows[0]
@@ -1337,24 +1404,8 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
               `no event at index ${index} in session '${sessionId}'`,
             )
           }
-          if (row['outcome'] === 'dead_lettered') {
-            await query(
-              `delete from a2_snapshot_pins
-                where session_id = $1 and event_index = $2`,
-              [sessionId, index],
-            )
-            await query(
-              `delete from a2_snapshot_history history
-                where history.session_id = $1
-                  and not exists (
-                    select 1 from a2_snapshot_pins pin
-                     where pin.session_id = history.session_id
-                       and pin.reducer_name = history.reducer_name
-                       and pin.up_to_index = history.up_to_index
-                  )`,
-              [sessionId],
-            )
-          }
+          if (row['outcome'] === 'dead_lettered')
+            await releasePins({ query, sessionId, index })
           return {
             outcome: row['outcome'] as
               'failed' | 'dead_lettered' | 'superseded',

package/src/store-redis-core.ts

@@ -494,7 +494,7 @@ return { 'ok', tostring(base + 1), pending_after }
 /**
  * KEYS: counter, meta, pending, ready, event snapshot pins, snapshot pin counts,
  * metadata split.
- * ARGV: idx, attempt, error, maxFailures, nowMs.
+ * ARGV: idx, attempt, error, maxFailures, nowMs, retryAtMs ('' for none).
  */
 const FAIL_ATTEMPT_LUA = `
 local COUNTER, META, SPLIT = KEYS[1], KEYS[2], KEYS[7]
@@ -526,8 +526,14 @@ put(meta_key, field .. ':e', ARGV[3])
 put(meta_key, field .. ':lf', ARGV[5])
 put(meta_key, field .. ':lfa', ARGV[2])
 if dead then put(meta_key, field .. ':f', ARGV[5]) end
+local hold = not dead and ARGV[6] ~= ''
+if hold then put(meta_key, field .. ':ce', ARGV[6]) end
 flush()
-redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
+if hold then
+  redis.call('HDEL', meta_key, field .. ':ch')
+else
+  redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
+end
 if dead then
   redis.call('ZREM', KEYS[4], field)
   release_snapshot_pins(KEYS[5], KEYS[6])
@@ -537,23 +543,29 @@ return { 'failed', tostring(failures) }
 `
 
 /**
- * KEYS: store, counter, meta, pending, ready, metadata split.
- * ARGV: holder, nowMs, expiresAtMs, excludedCount, excluded indexes.
+ * KEYS: store, counter, meta, pending, ready, metadata split, snapshot pin
+ * counts.
+ * ARGV: holder, nowMs, expiresAtMs, excludedCount, excluded indexes, then
+ * maxFailures ('' for none) and the event snapshot-pins key prefix.
  * The ready set is a subset of the pending set, so an empty ready set is
  * enough to report 'settled'. A claimed result ends with the clear boundary.
  */
 const CLAIM_AVAILABLE_LUA = `
 local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[6]
 ${META_LUA}
+${RELEASE_PINS_LUA}
 local ready = redis.call('ZRANGE', KEYS[5], 0, -1)
 local excluded = {}
+local tail = 5 + tonumber(ARGV[4])
 for i = 1, tonumber(ARGV[4]) do excluded[ARGV[4 + i]] = true end
+local max_failures = tonumber(ARGV[tail])
 local candidates = {}
 for _, field in ipairs(ready) do
   if not excluded[field] then candidates[#candidates + 1] = field end
 end
 local rows = read_fields(candidates, { 'ce', 'd', 'fc', 'a', 'e', 'lf', 'lfa' })
 local claimed = {}
+local dead = {}
 local retry_at = nil
 for _, field in ipairs(candidates) do
   local row = rows[field]
@@ -563,29 +575,63 @@ for _, field in ipairs(candidates) do
   else
     local entries = redis.call('XRANGE', KEYS[1], field .. '-0', field .. '-0')
     if #entries == 0 then return { 'missing', field } end
-    local attempt = tostring(tonumber(row[2] or '0') + 1)
-    local first_claimed = row[3] or ARGV[2]
     local meta_key = mkey(field)
+    local attempt = tostring(tonumber(row[2] or '0') + 1)
+    -- An expiry that is not the current attempt's retry hold is a lapsed
+    -- claim: its attempt never finished.
+    local lapsed = row[1] and row[7] ~= row[2]
+    local failures = tonumber(row[4] or '0')
+    if lapsed then
+      failures = failures + 1
+      put(meta_key, field .. ':a', tostring(failures))
+    end
     put(meta_key, field .. ':d', attempt)
-    put(meta_key, field .. ':fc', first_claimed)
-    put(meta_key, field .. ':lc', ARGV[2])
-    put(meta_key, field .. ':ch', ARGV[1])
-    put(meta_key, field .. ':ce', ARGV[3])
-    claimed[#claimed + 1] = {
-      attempt,
-      row[4] or '0',
-      row[5] or false,
-      first_claimed,
-      ARGV[2],
-      row[6] or false,
-      row[7] or false,
-      ARGV[1],
-      ARGV[3],
-      entries[1]
-    }
+    if lapsed and max_failures and failures >= max_failures then
+      -- That takeover spent the last failed attempt: dead-letter, unclaimed.
+      -- Advancing the attempt fences the lapsed one; nothing runs.
+      put(meta_key, field .. ':f', ARGV[2])
+      dead[#dead + 1] = field
+      claimed[#claimed + 1] = {
+        attempt,
+        tostring(failures),
+        row[5] or false,
+        row[3] or false,
+        read_fields({ field }, { 'lc' })[field][1] or false,
+        row[6] or false,
+        row[7] or false,
+        false,
+        false,
+        entries[1],
+        ARGV[2]
+      }
+    else
+      local first_claimed = row[3] or ARGV[2]
+      put(meta_key, field .. ':fc', first_claimed)
+      put(meta_key, field .. ':lc', ARGV[2])
+      put(meta_key, field .. ':ch', ARGV[1])
+      put(meta_key, field .. ':ce', ARGV[3])
+      claimed[#claimed + 1] = {
+        attempt,
+        tostring(failures),
+        row[5] or false,
+        first_claimed,
+        ARGV[2],
+        row[6] or false,
+        row[7] or false,
+        ARGV[1],
+        ARGV[3],
+        entries[1],
+        false
+      }
+    end
   end
 end
 flush()
+for _, field in ipairs(dead) do
+  redis.call('HDEL', mkey(field), field .. ':ch', field .. ':ce')
+  redis.call('ZREM', KEYS[5], field)
+  release_snapshot_pins(ARGV[tail + 1] .. field, KEYS[7])
+end
 if #claimed > 0 then
   local result = { 'claimed', tostring(#claimed) }
   for _, claim in ipairs(claimed) do
@@ -1371,7 +1417,7 @@ const toClaimAvailableResult = (
     const count = Number(reply[1])
     const events: StoredEvent[] = []
     for (let i = 0; i < count; i++) {
-      const offset = 2 + i * 10
+      const offset = 2 + i * 11
       const entry = reply[offset + 9] as StreamEntry
       const event = toEvent(sessionId, entry)
       const fields = fieldMap(entry[1])
@@ -1386,10 +1432,17 @@ const toClaimAvailableResult = (
             reply[offset + 3] === null
               ? null
               : new Date(Number(reply[offset + 3])),
-          lastClaimedAt: new Date(Number(reply[offset + 4])),
+          lastClaimedAt:
+            reply[offset + 4] === null
+              ? null
+              : new Date(Number(reply[offset + 4])),
           attemptCount: Number(reply[offset]),
-          claimHolder: String(reply[offset + 7]),
-          claimExpiresAt: new Date(Number(reply[offset + 8])),
+          claimHolder:
+            reply[offset + 7] === null ? null : String(reply[offset + 7]),
+          claimExpiresAt:
+            reply[offset + 8] === null
+              ? null
+              : new Date(Number(reply[offset + 8])),
           failureCount: Number(reply[offset + 1]),
           lastFailedAt:
             reply[offset + 5] === null
@@ -1399,11 +1452,14 @@ const toClaimAvailableResult = (
             reply[offset + 6] === null ? null : Number(reply[offset + 6]),
           lastError:
             reply[offset + 2] === null ? null : String(reply[offset + 2]),
-          failedAt: null,
+          failedAt:
+            reply[offset + 10] === null
+              ? null
+              : new Date(Number(reply[offset + 10])),
         }),
       )
     }
-    const clearIndex = Number(reply[2 + count * 10] ?? 0)
+    const clearIndex = Number(reply[2 + count * 11] ?? 0)
     if (clearIndex > 0)
       for (const event of events) event.clearIndex = clearIndex
     return {
@@ -1942,6 +1998,7 @@ export function createRedisStoreCore(
       ttlMs,
       expiresAtMs,
       excludeIndexes,
+      maxFailures,
     }) {
       return wrap(async () => {
         const now = clock.now().getTime()
@@ -1956,8 +2013,17 @@ export function createRedisStoreCore(
             pendingKey(sessionId),
             readyKey(sessionId),
             metaSplitKey(sessionId),
+            snapshotPinCountsKey(sessionId),
+          ],
+          [
+            holder,
+            now,
+            expiresAt,
+            excluded.length,
+            ...excluded,
+            maxFailures ?? '',
+            snapshotEventPinsPrefix(sessionId),
           ],
-          [holder, now, expiresAt, excluded.length, ...excluded],
         )) as unknown[]
         return toClaimAvailableResult(sessionId, reply)
       })
@@ -2093,8 +2159,16 @@ export function createRedisStoreCore(
       })
     },
 
-    async failAttempt({ sessionId, index, attempt, error, maxFailures }) {
+    async failAttempt({
+      sessionId,
+      index,
+      attempt,
+      error,
+      maxFailures,
+      retryAfterMs,
+    }) {
       return wrap(async () => {
+        const now = clock.now().getTime()
         const reply = (await evalScript(
           FAIL_ATTEMPT_LUA,
           [
@@ -2106,7 +2180,14 @@ export function createRedisStoreCore(
             snapshotPinCountsKey(sessionId),
             metaSplitKey(sessionId),
           ],
-          [index, attempt, error, maxFailures, clock.now().getTime()],
+          [
+            index,
+            attempt,
+            error,
+            maxFailures,
+            now,
+            retryAfterMs === undefined ? '' : now + retryAfterMs,
+          ],
         )) as string[]
         const outcome = reply[0]
         if (outcome === 'cleared')

package/src/store-sqlite.ts

@@ -470,6 +470,22 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
     }
   }
 
+  /** A dead-lettered event's snapshot pins, and history no pin keeps. */
+  const releasePins = (sessionId: string, index: number): void => {
+    db.prepare(
+      'delete from a2_snapshot_pins where session_id = ? and event_index = ?',
+    ).run(sessionId, index)
+    db.prepare(
+      `delete from a2_snapshot_history
+        where session_id = ?
+          and not exists (
+            select 1 from a2_snapshot_pins pin
+             where pin.session_id = a2_snapshot_history.session_id
+               and pin.reducer_name = a2_snapshot_history.reducer_name
+               and pin.up_to_index = a2_snapshot_history.up_to_index
+          )`,
+    ).run(sessionId)
+  }
   return {
     async append(sessionId, events, appendOptions) {
       if (events.length === 0) {
@@ -626,6 +642,7 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
       ttlMs,
       expiresAtMs,
       excludeIndexes = [],
+      maxFailures,
     }) {
       return wrap(() =>
         tx(() => {
@@ -638,7 +655,7 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
               : `and event.idx not in (${excludeIndexes.map(() => '?').join(', ')})`
           const eligible = db
             .prepare(
-              `select event.idx
+              `select event.idx, event.claim_holder, event.failure_count
                  from a2_events event
                 where event.session_id = ?
                   and event.processed_at is null
@@ -656,11 +673,15 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
               ...excludeIndexes,
             ) as unknown as Array<{
             idx: number | bigint
+            claim_holder: string | null
+            failure_count: number | bigint
           }>
           if (eligible.length > 0) {
             const update = db.prepare(
               `update a2_events set
                  attempt_count = attempt_count + 1,
+                 failure_count = failure_count
+                   + case when claim_holder is null then 0 else 1 end,
                  first_claimed_at = coalesce(first_claimed_at, ?),
                  last_claimed_at = ?,
                  claim_holder = ?,
@@ -668,15 +689,30 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
                where session_id = ? and idx = ?
                returning *`,
             )
+            // Advancing the attempt fences the lapsed one; nothing runs.
+            const deadLetter = () =>
+              db.prepare(
+                `update a2_events set
+                   attempt_count = attempt_count + 1,
+                   failure_count = failure_count + 1,
+                   failed_at = ?,
+                   claim_holder = null,
+                   claim_expires_at = null
+                 where session_id = ? and idx = ?
+                 returning *`,
+              )
             const events = eligible.map((row) => {
-              const claimed = update.get(
-                now,
-                now,
-                holder,
-                expiresAt,
-                sessionId,
-                row.idx,
+              // A lapsed claim's takeover that spends the last failed attempt.
+              const exhausted =
+                row.claim_holder !== null &&
+                maxFailures !== undefined &&
+                Number(row.failure_count) + 1 >= maxFailures
+              const claimed = (
+                exhausted
+                  ? deadLetter().get(now, sessionId, row.idx)
+                  : update.get(now, now, holder, expiresAt, sessionId, row.idx)
               ) as EventRow
+              if (exhausted) releasePins(sessionId, Number(row.idx))
               const stored = toStored(claimed)
               if (clearIndex > 0) stored.clearIndex = clearIndex
               return stored
@@ -687,15 +723,16 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
           const active = db
             .prepare(
               `select min(claim_expires_at) as retry_at
-                 from a2_events
+                 from a2_events event
                 where session_id = ?
                   and processed_at is null
                   and failed_at is null
                   and lane_ready = 1
                   and idx > ?
-                  and claim_expires_at > ?`,
+                  and claim_expires_at > ?
+                  ${exclusion}`,
             )
-            .get(sessionId, clearIndex, now) as {
+            .get(sessionId, clearIndex, now, ...excludeIndexes) as {
             retry_at: number | bigint | null
           }
           return active.retry_at === null
@@ -908,7 +945,14 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
       )
     },
 
-    async failAttempt({ sessionId, index, attempt, error, maxFailures }) {
+    async failAttempt({
+      sessionId,
+      index,
+      attempt,
+      error,
+      maxFailures,
+      retryAfterMs,
+    }) {
       return wrap(() =>
         tx(() => {
           if (index <= clearIndexOf(sessionId)) {
@@ -954,7 +998,7 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
                last_failed_attempt = ?,
                failed_at = ?,
                claim_holder = null,
-               claim_expires_at = null
+               claim_expires_at = ?
              where session_id = ? and idx = ?`,
           ).run(
             nextFailureCount,
@@ -962,24 +1006,13 @@ export function sqlite(options: SqliteStoreOptions = {}): SqliteStore {
             now,
             attempt,
             deadLettered ? now : null,
+            deadLettered || retryAfterMs === undefined
+              ? null
+              : now + retryAfterMs,
             sessionId,
             index,
           )
-          if (deadLettered) {
-            db.prepare(
-              'delete from a2_snapshot_pins where session_id = ? and event_index = ?',
-            ).run(sessionId, index)
-            db.prepare(
-              `delete from a2_snapshot_history
-                where session_id = ?
-                  and not exists (
-                    select 1 from a2_snapshot_pins pin
-                     where pin.session_id = a2_snapshot_history.session_id
-                       and pin.reducer_name = a2_snapshot_history.reducer_name
-                       and pin.up_to_index = a2_snapshot_history.up_to_index
-                  )`,
-            ).run(sessionId)
-          }
+          if (deadLettered) releasePins(sessionId, index)
           return {
             outcome: deadLettered ? 'dead_lettered' : 'failed',
             failureCount: nextFailureCount,

package/src/store.ts

@@ -53,9 +53,15 @@ export type StoredEvent = Event & {
   attemptCount: number
   /** Current dispatch holder; null when the event is not claimed. */
   claimHolder: string | null
-  /** Adapter clock expiry for the current dispatch claim. */
+  /**
+   * Adapter clock expiry for the current dispatch claim, or, with no holder,
+   * the end of a failed attempt's retry hold.
+   */
   claimExpiresAt: Date | null
-  /** Caught handler failures. This alone drives dead-lettering. */
+  /**
+   * Failed attempts: caught handler failures, and lapsed claims a later claim
+   * took over. This alone drives dead-lettering.
+   */
   failureCount: number
   /** Adapter clock time recorded by the most recent caught handler failure. */
   lastFailedAt: Date | null
@@ -299,6 +305,15 @@ export interface A2Store {
     ttlMs: number
     expiresAtMs?: number
     excludeIndexes?: readonly number[]
+    /**
+     * Taking over a lapsed claim (one whose holder is still recorded, never
+     * completed or failed) adds one to `failureCount`. When that reaches
+     * `maxFailures`, the event dead-letters instead of being claimed: it
+     * comes back in `events` with `failedAt` set and no claim. Its
+     * `attemptCount` still advances, fencing the lapsed attempt, and its
+     * `lastFailedAttempt` is unchanged.
+     */
+    maxFailures?: number
   }): Promise<StoreClaimAvailableResult>
 
   /**
@@ -333,7 +348,10 @@ export interface A2Store {
   /**
    * Atomically records a caught failure for one claimed attempt. A stale
    * attempt cannot poison a processed event or a newer dispatch. Accepted
-   * failures record the operation's clock time.
+   * failures record the operation's clock time. A failure that does not
+   * dead-letter releases the claim and, with `retryAfterMs`, keeps the event
+   * unclaimable for that long: its claim expiry is set with no holder, so
+   * `claimAvailable` reports it `busy` until then.
    */
   failAttempt(options: {
     sessionId: string
@@ -341,6 +359,7 @@ export interface A2Store {
     attempt: number
     error: string
     maxFailures: number
+    retryAfterMs?: number
   }): Promise<FailAttemptResult>
 
   /**