experimental-a2 0.17.0 → 0.18.0

package/src/ai-sdk-step.ts

@@ -8,7 +8,6 @@ import type {
   FinishReason,
   InferUIMessageChunk,
   Instructions,
-  LanguageModel,
   LanguageModelUsage,
   ModelMessage,
   TextStreamPart,
@@ -125,7 +124,8 @@ export type AISDKStepSettings<T extends ToolSet> = Partial<
 }
 
 export type AISDKStepInput<M extends UIMessage, T extends ToolSet> = {
-  model: LanguageModel
+  /** An AI Gateway model ID, resolved through the AI SDK's global provider. */
+  model: string
   tools: T
   messages: M[]
   modelMessages?: ModelMessage[]

package/src/ai-server.ts

@@ -6,6 +6,7 @@
 
 export { agent, runWithHandlerContext } from './ai-agent.ts'
 export type {
+  AISDKHarness,
   Agent,
   AgentFromContractOptions,
   AgentGenerate,
@@ -19,8 +20,9 @@ export type {
   CompactionOptions,
   CompactionPolicy,
   Instructions,
-  LanguageModel,
   ModelSettings,
   Resolvable,
+  ToolRecovery,
   ToolSet,
 } from './ai-agent.ts'
+export type { Harness } from './ai-harness.ts'

package/src/internal.ts

@@ -73,6 +73,13 @@ export type ServerInternals = {
 
 export const serverInternals: WeakMap<object, ServerInternals> = new WeakMap()
 
+/**
+ * Handler contexts whose previous attempt ended in a failure the handler
+ * threw, as opposed to a claim that lapsed (a crash, a timeout, a frozen
+ * process). Only the attempt's own outcome is known, not how far it got.
+ */
+export const failedPreviousAttempt: WeakSet<object> = new WeakSet()
+
 const objectLike = (value: unknown): value is object =>
   (typeof value === 'object' && value !== null) || typeof value === 'function'
 

package/src/libfx.d.ts

@@ -0,0 +1,34 @@
+// The subset of libfx's agent API that experimental-a2/ai/fx uses; libfx
+// ships no type declarations.
+declare module 'libfx' {
+  export type FxHostTool = {
+    name: string
+    description: string
+    inputSchema: object
+    execute(input: unknown, context: { signal: AbortSignal }): unknown
+  }
+
+  export type FxTurn = AsyncIterable<{ type: string }> & {
+    result: Promise<{ stopReason: string }>
+    steer(input: string): Promise<void>
+    cancel(): void
+  }
+
+  export type FxAgent = {
+    prompt(input: string, options?: { signal?: AbortSignal }): FxTurn
+    checkpoint(): Promise<Uint8Array>
+    close(): Promise<void>
+  }
+
+  export function createFxAgent(options: {
+    apiKey: string
+    model?: string | { id: string; effort?: string; fast?: boolean }
+    instructions?: string | string[]
+    tools?: FxHostTool[]
+    checkpoint?: Uint8Array
+    fetch?: (
+      input: string | URL | Request,
+      init?: RequestInit,
+    ) => Promise<Response>
+  }): Promise<FxAgent>
+}

package/src/reducer.ts

@@ -62,7 +62,7 @@ export function foldEvent<D extends EventDefs, S>(
  * the browser cache) are keyed by it, which makes it the invalidation
  * knob too: changed the fold's logic? Change the name. Snapshots under
  * the old name are ignored and the next read refolds from raw events.
- * See docs/concepts/04-state.mdx.
+ * See docs/contract/02-state.mdx.
  */
 export type Reducer<
   D extends EventDefs = EventDefs,

package/src/server.ts

@@ -25,6 +25,7 @@ import {
   DRAIN_TIMINGS,
   POLL_TIMINGS,
   MAX_DATE_MS,
+  failedPreviousAttempt,
   RESERVED_PARTICIPANT_IDS,
   STATE_CACHE,
   markSchedulerSendFailure,
@@ -55,6 +56,7 @@ import type {
   StoreClaimAvailableResult,
   StoreClearResult,
   StoreReclaimResult,
+  StoreRedriveResult,
   StoreSnapshotWrite,
   StoreStateRead,
   PresenceRow,
@@ -188,6 +190,42 @@ export type SessionClearOptions = {
 
 export type SessionClearResult = { cleared: boolean; index: number }
 
+export type SessionRedriveOptions = {
+  /** `retry` runs the handler again with a fresh failure budget; `skip` settles the event without running it. */
+  action: 'retry' | 'skip'
+}
+
+export type SessionRedriveResult = {
+  /** Whether the event was dead-lettered and is now resolved. */
+  redriven: boolean
+}
+
+/** A dead-lettered event, with what an operator needs to resolve it. */
+export type DeadLetter<D extends EventDefs> = ContractEvent<D> & {
+  lane: string | null
+  /** Durable dispatch claims, including claims abandoned by hard kills. */
+  attemptCount: number
+  /** Caught handler failures since the event was appended or last retried. */
+  failureCount: number
+  /** The last handler failure, stringified. */
+  lastError: string | null
+  failedAt: Date
+}
+
+/** Dead-letter resolution; root sessions only. */
+export type SessionDeadLetters<D extends EventDefs> = {
+  /** The session's dead-lettered events, oldest first. */
+  deadLetters(): Promise<DeadLetter<D>[]>
+  /**
+   * Atomically resolve one dead-lettered event, then wake the session like
+   * an append. Any other event is left alone and reports `redriven: false`.
+   */
+  redrive(
+    index: number,
+    options: SessionRedriveOptions,
+  ): Promise<SessionRedriveResult>
+}
+
 /**
  * The presence members of a session — intersected in via
  * `WithPresence`, so they exist exactly when the contract declares
@@ -293,7 +331,9 @@ export type A2Server<
     (request: Request): Promise<Response>
     (request: Request, options: ServerFetchOptions<D, P>): Promise<Response>
   }
-  session(id: string): Session<D, SessionAppend<D>, P, PublicState<R>>
+  session(
+    id: string,
+  ): Session<D, SessionAppend<D>, P, PublicState<R>> & SessionDeadLetters<D>
   /**
    * Process every currently eligible event. `settled` means nothing
    * actionable remains, including work blocked behind a dead letter.
@@ -603,6 +643,27 @@ const devDefaultStore = retryableLazy(() =>
   import('./store-sqlite.ts').then((m) => m.sqlite()),
 )
 
+/** Waits for a scheduler arm for at most `schedulerArmTimeoutMs`. */
+async function joinSchedulerArm(arm: Promise<void>): Promise<void> {
+  let timeout: ReturnType<typeof setTimeout> | null = null
+  try {
+    await Promise.race([
+      arm,
+      new Promise<never>((_, reject) => {
+        timeout = setTimeout(() => {
+          reject(
+            new Error(
+              `a2 scheduler arm timed out after ${DRAIN_TIMINGS.schedulerArmTimeoutMs}ms`,
+            ),
+          )
+        }, DRAIN_TIMINGS.schedulerArmTimeoutMs)
+      }),
+    ])
+  } finally {
+    if (timeout) clearTimeout(timeout)
+  }
+}
+
 function environment(): 'development' | 'test' | 'production' {
   const env =
     typeof process === 'undefined' ? undefined : process.env?.['NODE_ENV']
@@ -1081,26 +1142,12 @@ export function createServer<
           await dispatchArm
         }
         if (initialArm) {
-          let timeout: ReturnType<typeof setTimeout> | null = null
           try {
-            await Promise.race([
-              initialArm,
-              new Promise<never>((_, reject) => {
-                timeout = setTimeout(() => {
-                  reject(
-                    new Error(
-                      `a2 scheduler arm timed out after ${DRAIN_TIMINGS.schedulerArmTimeoutMs}ms`,
-                    ),
-                  )
-                }, DRAIN_TIMINGS.schedulerArmTimeoutMs)
-              }),
-            ])
+            await joinSchedulerArm(initialArm)
           } catch (err) {
             span.setAttribute('a2.append.armed', false)
             span.recordError(err)
             if (requireDurableInitialArm) throw err
-          } finally {
-            if (timeout) clearTimeout(timeout)
           }
         }
         return appended
@@ -2483,6 +2530,102 @@ export function createServer<
     return { cleared: result.cleared, index: result.index }
   }
 
+  // ── dead letters ─────────────────────────────────────────────────
+
+  const redrivingStore = async (
+    method: 'deadLetters' | 'redrive',
+  ): Promise<A2Store & Required<Pick<A2Store, 'deadLetters' | 'redrive'>>> => {
+    const store = await resolveStore()
+    if (!store.deadLetters || !store.redrive) {
+      throw new TypeError(
+        `session.${method} requires a store that supports redriving dead letters`,
+      )
+    }
+    return store as A2Store & Required<Pick<A2Store, 'deadLetters' | 'redrive'>>
+  }
+
+  const listDeadLetters = async (
+    sessionId: string,
+  ): Promise<DeadLetter<D>[]> => {
+    const store = await redrivingStore('deadLetters')
+    let rows: StoredEvent[]
+    try {
+      rows = await store.deadLetters(nsId(sessionId))
+    } catch (err) {
+      throw asStoreUnavailable(err)
+    }
+    return rows.flatMap((row) =>
+      row.failedAt === null
+        ? []
+        : [
+            {
+              ...toPublic(row),
+              lane: row.lane,
+              attemptCount: row.attemptCount,
+              failureCount: row.failureCount,
+              lastError: row.lastError,
+              failedAt: row.failedAt,
+            },
+          ],
+    )
+  }
+
+  const redriveEvent = async (
+    sessionId: string,
+    index: number,
+    redriveOptions: SessionRedriveOptions,
+  ): Promise<SessionRedriveResult> => {
+    assertRedriveIndex(index)
+    const action = redriveOptions?.action
+    if (action !== 'retry' && action !== 'skip') {
+      throw new TypeError("redrive requires action 'retry' or 'skip'")
+    }
+    return telemetry.span(
+      'a2.redrive',
+      {
+        'a2.contract': name,
+        'a2.session_id': sessionId,
+        'a2.redrive.index': index,
+        'a2.redrive.action': action,
+      },
+      async (span) => {
+        const store = await redrivingStore('redrive')
+        let result: StoreRedriveResult
+        try {
+          result = await store.redrive({
+            sessionId: nsId(sessionId),
+            index,
+            action,
+          })
+        } catch (err) {
+          throw asStoreUnavailable(err)
+        }
+        span.setAttribute('a2.redrive.redriven', result.redriven)
+        // Healing like a top-level append, even when nothing was redriven, so
+        // repeating a redrive whose wakeup was lost resumes the session.
+        if (result.hasPending) {
+          const watchdogAt = scheduler
+            ? schedulerSlot(nextClaimWindow().watchdogAtMs)
+            : undefined
+          const arm =
+            watchdogAt === undefined
+              ? null
+              : startSchedulerArm(sessionId, watchdogAt)
+          scheduleDrain(sessionId, watchdogAt)
+          if (arm) {
+            try {
+              await joinSchedulerArm(arm)
+            } catch (err) {
+              span.setAttribute('a2.redrive.armed', false)
+              span.recordError(err)
+            }
+          }
+        }
+        return { redriven: result.redriven }
+      },
+    )
+  }
+
   const makeSession = <Append>(
     id: string,
     append: Append,
@@ -2584,7 +2727,7 @@ export function createServer<
         generatedIds,
       )
     }
-    return {
+    const ctx: HandlerContext<D> = {
       event: toPublic(trigger),
       attempt,
       // The handler-scoped `setPresence` stamps `seen` with the
@@ -2597,6 +2740,9 @@ export function createServer<
       }),
       signal,
     }
+    if (attempt > 1 && trigger.lastFailedAttempt === attempt - 1)
+      failedPreviousAttempt.add(ctx)
+    return ctx
   }
 
   const abortHubs = new Map<string, AbortHub>()
@@ -3276,7 +3422,9 @@ export function createServer<
     )
   }
 
-  const session = (id: string): Session<D, SessionAppend<D>, P> => {
+  const session = (
+    id: string,
+  ): Session<D, SessionAppend<D>, P> & SessionDeadLetters<D> => {
     assertSessionId(id)
     const appendInline = async (...events: AppendInput<D>[]) =>
       appendExternal(id, 'inline', events)
@@ -3285,7 +3433,11 @@ export function createServer<
     const append = Object.assign(appendInline, {
       dispatch: appendDispatch,
     }) as SessionAppend<D>
-    return makeSession(id, append, null)
+    return Object.assign(makeSession(id, append, null), {
+      deadLetters: () => listDeadLetters(id),
+      redrive: (index: number, redriveOptions: SessionRedriveOptions) =>
+        redriveEvent(id, index, redriveOptions),
+    })
   }
 
   const historyPage = async (
@@ -3393,6 +3545,12 @@ function assertStateIndex(value: number): void {
   }
 }
 
+function assertRedriveIndex(value: number): void {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new TypeError('redrive requires a positive safe-integer index')
+  }
+}
+
 function assertClearIndex(value: number): void {
   if (!Number.isSafeInteger(value) || value < 0) {
     throw new TypeError('ifLastIndex must be a non-negative safe integer')

package/src/store-memory.ts

@@ -259,16 +259,18 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
     return row
   }
 
-  const appendResult = (sessionId: string, events: StoredEvent[]) => {
+  const hasPending = (sessionId: string): boolean => {
     const log = sessions.get(sessionId)
-    return {
-      events,
-      hasPending: (log?.rows ?? []).some(
-        (row) => row.index > log!.clearIndex && row.processedAt === null,
-      ),
-    }
+    return (log?.rows ?? []).some(
+      (row) => row.index > log!.clearIndex && row.processedAt === null,
+    )
   }
 
+  const appendResult = (sessionId: string, events: StoredEvent[]) => ({
+    events,
+    hasPending: hasPending(sessionId),
+  })
+
   /** The attempt-currency fence shared by handler appends and clears. */
   const assertCurrentCause = (sessionId: string, cause: EventCause): void => {
     if (cause.index <= clearIndexOf(sessionId)) {
@@ -719,6 +721,39 @@ export function memory(options: MemoryStoreOptions = {}): A2Store {
       return { outcome: 'failed', failureCount: row.failureCount }
     },
 
+    async deadLetters(sessionId) {
+      const log = sessions.get(sessionId)
+      return (log?.rows ?? [])
+        .filter(
+          (row) =>
+            row.index > log!.clearIndex &&
+            row.processedAt === null &&
+            row.failedAt !== null,
+        )
+        .map(toStored)
+    },
+
+    async redrive({ sessionId, index, action }) {
+      const row =
+        index > clearIndexOf(sessionId) ? rowAt(sessionId, index) : undefined
+      if (!row || row.processedAt !== null || row.failedAt === null) {
+        return { redriven: false, hasPending: hasPending(sessionId) }
+      }
+      if (action === 'skip') {
+        row.processedAt = clock.now()
+        settle(sessionId, row)
+      } else {
+        row.failedAt = null
+        row.failureCount = 0
+        const state = dispatchOf(sessionId)
+        const queue = row.lane === null ? undefined : state.lanes.get(row.lane)
+        if (row.lane === null || queue?.rows[queue.head] === row) {
+          state.ready.set(row.index, row)
+        }
+      }
+      return { redriven: true, hasPending: hasPending(sessionId) }
+    },
+
     async readState(sessionId, reducerName, stateOptions) {
       return readStateOf(
         sessionId,

package/src/store-postgres.ts

@@ -1364,6 +1364,86 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
       )
     },
 
+    async deadLetters(sessionId) {
+      return wrap(async () => {
+        const c = await client()
+        const { rows } = await c.query(
+          `select * from a2_events
+            where session_id = $1
+              and processed_at is null
+              and failed_at is not null
+              and idx > ${clearIndexOf('$1')}
+            order by idx`,
+          [sessionId],
+        )
+        return rows.map(toStored)
+      })
+    },
+
+    async redrive({ sessionId, index, action }) {
+      return wrap(() =>
+        withTx(async (query) => {
+          // Lane promotion must see every append that committed before it,
+          // so a skip takes the session lock first, in append's order. The
+          // row lock serializes both actions with claims and handler appends.
+          if (action === 'skip') {
+            await query('select pg_advisory_xact_lock(hashtext($1))', [
+              sessionId,
+            ])
+          }
+          const { rows } = await query(
+            `select lane from a2_events
+              where session_id = $1 and idx = $2
+                and idx > ${clearIndexOf('$1')}
+                and processed_at is null
+                and failed_at is not null
+              for update`,
+            [sessionId, index],
+          )
+          const row = rows[0]
+          if (row && action === 'retry') {
+            await query(
+              `update a2_events set failed_at = null, failure_count = 0
+                where session_id = $1 and idx = $2`,
+              [sessionId, index],
+            )
+          } else if (row) {
+            await query(
+              `update a2_events set processed_at = $3, lane_ready = false
+                where session_id = $1 and idx = $2`,
+              [sessionId, index, clock.now().getTime()],
+            )
+            const lane = (row['lane'] as string | null) ?? null
+            if (lane !== null) {
+              await query(
+                `update a2_events set lane_ready = true
+                  where session_id = $1
+                    and idx = (
+                      select idx from a2_events
+                       where session_id = $1 and lane = $2 and processed_at is null
+                         and idx > ${clearIndexOf('$1')}
+                       order by idx limit 1
+                    )`,
+                [sessionId, lane],
+              )
+            }
+          }
+          const { rows: status } = await query(
+            `select exists(
+               select 1 from a2_events
+               where session_id = $1 and processed_at is null
+                 and idx > ${clearIndexOf('$1')}
+             ) as has_pending`,
+            [sessionId],
+          )
+          return {
+            redriven: row !== undefined,
+            hasPending: status[0]?.['has_pending'] === true,
+          }
+        }),
+      )
+    },
+
     async readState(sessionId, reducerName, stateOptions) {
       return wrap(
         async () =>