experimental-a2 0.17.0 → 0.18.1

package/src/store-memory.ts

@@ -259,16 +259,18 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
     return row
   }
 
-  const appendResult = (sessionId: string, events: StoredEvent[]) => {
+  const hasPending = (sessionId: string): boolean => {
     const log = sessions.get(sessionId)
-    return {
-      events,
-      hasPending: (log?.rows ?? []).some(
-        (row) => row.index > log!.clearIndex && row.processedAt === null,
-      ),
-    }
+    return (log?.rows ?? []).some(
+      (row) => row.index > log!.clearIndex && row.processedAt === null,
+    )
   }
 
+  const appendResult = (sessionId: string, events: StoredEvent[]) => ({
+    events,
+    hasPending: hasPending(sessionId),
+  })
+
   /** The attempt-currency fence shared by handler appends and clears. */
   const assertCurrentCause = (sessionId: string, cause: EventCause): void => {
     if (cause.index <= clearIndexOf(sessionId)) {
@@ -280,12 +282,35 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
     }
   }
 
-  const claim = (
-    row: Row,
-    holder: string,
-    now: Date,
-    expiresAt: Date,
-  ): StoredEvent => {
+  const claim = ({
+    sessionId,
+    row,
+    holder,
+    now,
+    expiresAt,
+    maxFailures,
+  }: {
+    sessionId: string
+    row: Row
+    holder: string
+    now: Date
+    expiresAt: Date
+    maxFailures: number | undefined
+  }): StoredEvent => {
+    // A claim still held here lapsed: its attempt never finished.
+    if (row.claimHolder !== null) {
+      row.failureCount += 1
+      if (maxFailures !== undefined && row.failureCount >= maxFailures) {
+        // Advancing the attempt fences the lapsed one; nothing runs.
+        row.attemptCount += 1
+        row.failedAt = new Date(now)
+        row.claimHolder = null
+        row.claimExpiresAt = null
+        dispatchOf(sessionId).ready.delete(row.index)
+        releaseSnapshotPins(sessionId, row.index)
+        return toStored(row)
+      }
+    }
     row.attemptCount += 1
     row.firstClaimedAt ??= new Date(now)
     row.lastClaimedAt = new Date(now)
@@ -545,6 +570,7 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
       ttlMs,
       expiresAtMs,
       excludeIndexes,
+      maxFailures,
     }) {
       const claimedAt = clock.now()
       const now = claimedAt.getTime()
@@ -572,7 +598,14 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
       return {
         outcome: 'claimed',
         events: eligible.map((row) => {
-          const claimed = claim(row, holder, claimedAt, expiresAt)
+          const claimed = claim({
+            sessionId,
+            row,
+            holder,
+            now: claimedAt,
+            expiresAt,
+            maxFailures,
+          })
           if (clearIndex > 0) claimed.clearIndex = clearIndex
           return claimed
         }),
@@ -679,7 +712,14 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
       return { outcome: 'completed', events: inserted.map(toStored) }
     },
 
-    async failAttempt({ sessionId, index, attempt, error, maxFailures }) {
+    async failAttempt({
+      sessionId,
+      index,
+      attempt,
+      error,
+      maxFailures,
+      retryAfterMs,
+    }) {
       if (index <= clearIndexOf(sessionId)) {
         return { outcome: 'superseded', failureCount: 0 }
       }
@@ -706,7 +746,10 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
       row.lastFailedAt = new Date(failedAt)
       row.lastFailedAttempt = attempt
       row.claimHolder = null
-      row.claimExpiresAt = null
+      row.claimExpiresAt =
+        retryAfterMs === undefined || row.failureCount >= maxFailures
+          ? null
+          : new Date(failedAt.getTime() + retryAfterMs)
       if (row.failureCount >= maxFailures) {
         row.failedAt = new Date(failedAt)
         dispatchOf(sessionId).ready.delete(row.index)
@@ -719,6 +762,39 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
       return { outcome: 'failed', failureCount: row.failureCount }
     },
 
+    async deadLetters(sessionId) {
+      const log = sessions.get(sessionId)
+      return (log?.rows ?? [])
+        .filter(
+          (row) =>
+            row.index > log!.clearIndex &&
+            row.processedAt === null &&
+            row.failedAt !== null,
+        )
+        .map(toStored)
+    },
+
+    async redrive({ sessionId, index, action }) {
+      const row =
+        index > clearIndexOf(sessionId) ? rowAt(sessionId, index) : undefined
+      if (!row || row.processedAt !== null || row.failedAt === null) {
+        return { redriven: false, hasPending: hasPending(sessionId) }
+      }
+      if (action === 'skip') {
+        row.processedAt = clock.now()
+        settle(sessionId, row)
+      } else {
+        row.failedAt = null
+        row.failureCount = 0
+        const state = dispatchOf(sessionId)
+        const queue = row.lane === null ? undefined : state.lanes.get(row.lane)
+        if (row.lane === null || queue?.rows[queue.head] === row) {
+          state.ready.set(row.index, row)
+        }
+      }
+      return { redriven: true, hasPending: hasPending(sessionId) }
+    },
+
     async readState(sessionId, reducerName, stateOptions) {
       return readStateOf(
         sessionId,

package/src/store-postgres.ts

@@ -350,6 +350,34 @@ const createListener = ({
     return listenerConnection
   })
 
+/** A dead-lettered event's snapshot pins, and history no pin keeps. */
+const releasePins = async ({
+  query,
+  sessionId,
+  index,
+}: {
+  query: PostgresConnection['query']
+  sessionId: string
+  index: number
+}): Promise<void> => {
+  await query(
+    `delete from a2_snapshot_pins
+      where session_id = $1 and event_index = $2`,
+    [sessionId, index],
+  )
+  await query(
+    `delete from a2_snapshot_history history
+      where history.session_id = $1
+        and not exists (
+          select 1 from a2_snapshot_pins pin
+           where pin.session_id = history.session_id
+             and pin.reducer_name = history.reducer_name
+             and pin.up_to_index = history.up_to_index
+        )`,
+    [sessionId],
+  )
+}
+
 export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
   const clock = options.clock ?? SYSTEM_CLOCK
   const generateId = options.ids ?? RANDOM_IDS
@@ -942,11 +970,16 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
       ttlMs,
       expiresAtMs,
       excludeIndexes = [],
+      maxFailures,
     }) {
       return wrap(() =>
         withTx(async (query) => {
           const now = clock.now().getTime()
           const expiresAt = expiresAtMs ?? now + ttlMs
+          // A lapsed claim's takeover that spends the last failed attempt
+          // dead-letters the event instead of claiming it.
+          const exhausted = `(claimed.claim_holder is not null
+               and claimed.failure_count + 1 >= $6::int)`
           const { rows } = await query(
             `with eligible as materialized (
                select event.idx
@@ -962,15 +995,36 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
              )
              update a2_events claimed set
                attempt_count = claimed.attempt_count + 1,
-               first_claimed_at = coalesce(claimed.first_claimed_at, $2),
-               last_claimed_at = $2,
-               claim_holder = $3,
-               claim_expires_at = $4
+               failure_count = claimed.failure_count
+                 + case when claimed.claim_holder is null then 0 else 1 end,
+               failed_at = case when ${exhausted} then $2::bigint else claimed.failed_at end,
+               first_claimed_at = case when ${exhausted}
+                 then claimed.first_claimed_at
+                 else coalesce(claimed.first_claimed_at, $2::bigint) end,
+               last_claimed_at = case when ${exhausted}
+                 then claimed.last_claimed_at else $2::bigint end,
+               claim_holder = case when ${exhausted} then null else $3::text end,
+               claim_expires_at = case when ${exhausted} then null else $4::bigint end
              from eligible
              where claimed.session_id = $1 and claimed.idx = eligible.idx
              returning claimed.*, ${clearIndexOf('$1')} as clear_index`,
-            [sessionId, now, holder, expiresAt, [...excludeIndexes]],
+            [
+              sessionId,
+              now,
+              holder,
+              expiresAt,
+              [...excludeIndexes],
+              maxFailures ?? null,
+            ],
           )
+          for (const row of rows) {
+            if (row['failed_at'] !== null)
+              await releasePins({
+                query,
+                sessionId,
+                index: asNumber(row['idx']),
+              })
+          }
           if (rows.length > 0) {
             const events = rows
               .map((row) => {
@@ -990,8 +1044,9 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
                 and failed_at is null
                 and lane_ready
                 and idx > ${clearIndexOf('$1')}
-                and claim_expires_at > $2`,
-            [sessionId, now],
+                and claim_expires_at > $2
+                and not (idx = any($3::bigint[]))`,
+            [sessionId, now, [...excludeIndexes]],
           )
           const retryAt = activeRows[0]?.['retry_at']
           return retryAt === null || retryAt === undefined
@@ -1262,7 +1317,15 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
       )
     },
 
-    async failAttempt({ sessionId, index, attempt, error, maxFailures }) {
+    async failAttempt({
+      sessionId,
+      index,
+      attempt,
+      error,
+      maxFailures,
+      retryAfterMs,
+    }) {
+      const now = clock.now().getTime()
       return wrap(() =>
         withTx(async (query) => {
           const { rows } = await query(
@@ -1282,7 +1345,10 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
                  else e.failed_at
                end,
                claim_holder = null,
-               claim_expires_at = null
+               claim_expires_at = case
+                 when e.failure_count + 1 >= $5 then null
+                 else $7::bigint
+               end
              from current
              where e.session_id = $1
                and e.idx = $2
@@ -1322,7 +1388,8 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
               attempt,
               error,
               maxFailures,
-              clock.now().getTime(),
+              now,
+              retryAfterMs === undefined ? null : now + retryAfterMs,
             ],
           )
           const row = rows[0]
@@ -1337,28 +1404,92 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
               `no event at index ${index} in session '${sessionId}'`,
             )
           }
-          if (row['outcome'] === 'dead_lettered') {
+          if (row['outcome'] === 'dead_lettered')
+            await releasePins({ query, sessionId, index })
+          return {
+            outcome: row['outcome'] as
+              'failed' | 'dead_lettered' | 'superseded',
+            failureCount: asNumber(row['failure_count']),
+          }
+        }),
+      )
+    },
+
+    async deadLetters(sessionId) {
+      return wrap(async () => {
+        const c = await client()
+        const { rows } = await c.query(
+          `select * from a2_events
+            where session_id = $1
+              and processed_at is null
+              and failed_at is not null
+              and idx > ${clearIndexOf('$1')}
+            order by idx`,
+          [sessionId],
+        )
+        return rows.map(toStored)
+      })
+    },
+
+    async redrive({ sessionId, index, action }) {
+      return wrap(() =>
+        withTx(async (query) => {
+          // Lane promotion must see every append that committed before it,
+          // so a skip takes the session lock first, in append's order. The
+          // row lock serializes both actions with claims and handler appends.
+          if (action === 'skip') {
+            await query('select pg_advisory_xact_lock(hashtext($1))', [
+              sessionId,
+            ])
+          }
+          const { rows } = await query(
+            `select lane from a2_events
+              where session_id = $1 and idx = $2
+                and idx > ${clearIndexOf('$1')}
+                and processed_at is null
+                and failed_at is not null
+              for update`,
+            [sessionId, index],
+          )
+          const row = rows[0]
+          if (row && action === 'retry') {
             await query(
-              `delete from a2_snapshot_pins
-                where session_id = $1 and event_index = $2`,
+              `update a2_events set failed_at = null, failure_count = 0
+                where session_id = $1 and idx = $2`,
               [sessionId, index],
             )
+          } else if (row) {
             await query(
-              `delete from a2_snapshot_history history
-                where history.session_id = $1
-                  and not exists (
-                    select 1 from a2_snapshot_pins pin
-                     where pin.session_id = history.session_id
-                       and pin.reducer_name = history.reducer_name
-                       and pin.up_to_index = history.up_to_index
-                  )`,
-              [sessionId],
+              `update a2_events set processed_at = $3, lane_ready = false
+                where session_id = $1 and idx = $2`,
+              [sessionId, index, clock.now().getTime()],
             )
+            const lane = (row['lane'] as string | null) ?? null
+            if (lane !== null) {
+              await query(
+                `update a2_events set lane_ready = true
+                  where session_id = $1
+                    and idx = (
+                      select idx from a2_events
+                       where session_id = $1 and lane = $2 and processed_at is null
+                         and idx > ${clearIndexOf('$1')}
+                       order by idx limit 1
+                    )`,
+                [sessionId, lane],
+              )
+            }
           }
+          const { rows: status } = await query(
+            `select exists(
+               select 1 from a2_events
+               where session_id = $1 and processed_at is null
+                 and idx > ${clearIndexOf('$1')}
+             ) as has_pending`,
+            [sessionId],
+          )
           return {
-            outcome: row['outcome'] as
-              'failed' | 'dead_lettered' | 'superseded',
-            failureCount: asNumber(row['failure_count']),
+            redriven: row !== undefined,
+            hasPending: status[0]?.['has_pending'] === true,
           }
         }),
       )