@nuxtagentic/automations 0.0.0-stage → 0.1.1

package/runtime/server/utils/automation-launch.ts

@@ -0,0 +1,205 @@
+import { createError, type H3Event } from 'h3'
+import { useRuntimeConfig } from '#imports'
+import type { InferServerSelect } from '@cloudconfig/core'
+import { CloudConfigClientError } from '@cloudconfig/core/client'
+import { toAgentDefinition } from '@nuxtagentic/agents/runtime/shared/definition'
+import { isLocalAgentRuntimeEnabled } from '@nuxtagentic/agents/runtime/server/utils/local-agent-runtime'
+import { isWorkspaceModuleInstalled } from '@nuxtagentic/workspace/runtime/shared/modules'
+import type { AutomationsConfig } from '../../../schema'
+import type { AutomationDraft } from '../../shared/automation'
+import { executeAutomation, type RunSnapshot } from './automation-engine'
+import type { createJobContext, AutomationJob } from './automation-context'
+
+type Context = Awaited<ReturnType<typeof createJobContext>>
+export type AutomationRecord = InferServerSelect<AutomationsConfig, 'automation'>
+export interface AutomationLaunch {
+  automation: AutomationRecord
+  payload: unknown
+  /** Deduplicates deliveries: a second launch with the same id returns the first run. */
+  eventId: string
+}
+
+/** Queues one run of an automation for an incoming event, webhook call, schedule or test. */
+export async function launchAutomation(
+  event: H3Event | null,
+  context: Context,
+  { automation, payload, eventId }: AutomationLaunch,
+  options: { test?: boolean; forceError?: boolean; managed?: boolean } = {},
+) {
+  if (automation.workspace !== context.workspace) {
+    throw new Error('Choose an automation in this workspace.')
+  }
+  if (!options.test && !automation.active) {
+    throw new Error('Activate this automation before sending events.')
+  }
+  const managed = event
+    ? !!(useRuntimeConfig(event).public.workspaceModules as { managed?: boolean }).managed
+    : options.managed === true
+  if (managed) {
+    const installations = await context.api.workspaceInstallation.list({
+      pageSize: 1,
+      filters: {
+        type: 'group',
+        logic: 'and',
+        children: [
+          { type: 'condition', field: 'workspace', operator: 'eq', value: context.workspace },
+        ],
+      },
+    })
+    const installed = installations.data[0]?.installed as string[] | undefined
+    if (!isWorkspaceModuleInstalled('agents', installed)) {
+      throw new Error('Install Agents from Modules before running this automation.')
+    }
+  }
+  if (!automation.agentId) {
+    throw new Error('Choose an agent before running this automation.')
+  }
+  const agent = await context.api.agent.get(automation.agentId).catch((cause: unknown) => {
+    if (cause instanceof CloudConfigClientError && cause.status === 404) {
+      throw new Error('The selected agent is no longer available. Choose another agent.')
+    }
+    throw cause
+  })
+  const executionAgent = toAgentDefinition(agent)
+  const localEnabled = event ? isLocalAgentRuntimeEnabled(event) : false
+  if (!localEnabled) {
+    executionAgent.runtime = 'api'
+  }
+  const local = executionAgent.runtime === 'codex' || executionAgent.runtime === 'claude'
+  if (import.meta.dev && local) {
+    const { findLocalAgentBinary } =
+      await import('@nuxtagentic/agents/runtime/server/utils/local-agent-catalog')
+    if (!(await findLocalAgentBinary(executionAgent.runtime as 'codex' | 'claude'))) {
+      throw new Error(`Install ${executionAgent.runtime} and sign in before running this agent.`)
+    }
+  }
+  if (!options.test && agent.status !== 'active') {
+    throw new Error('Activate the selected agent before receiving events.')
+  }
+  if (!local && !context.keys[agent.provider] && !options.forceError) {
+    throw new Error(
+      `Configure the ${agent.provider} API key on the server before running this automation.`,
+    )
+  }
+  const uniqueId = `${automation.id}:${eventId}`
+  const existing = await context.api.automationRun.list({
+    pageSize: 1,
+    filters: {
+      type: 'group',
+      logic: 'and',
+      children: [{ type: 'condition', field: 'eventId', operator: 'eq', value: uniqueId }],
+    },
+  })
+  if (existing.data[0]) {
+    return { runId: existing.data[0].id, status: existing.data[0].status, duplicate: true }
+  }
+  // CLI execution belongs on this Node server, never in a proxied Workflow worker.
+  const env = (local ? undefined : (event?.context.cloudflare?.env ?? context.env)) as
+    Record<string, unknown> | undefined
+  const binding = env?.AUTOMATION_RUNNER as
+    { create: (input: { id: string; params: AutomationJob }) => Promise<unknown> } | undefined
+  if (env && !binding) {
+    throw createError({
+      statusCode: 503,
+      statusMessage:
+        'Deploy with the Automations module to provision the AUTOMATION_RUNNER workflow binding.',
+    })
+  }
+  const { webhookToken: _token, ownerId: _owner, ...definition } = automation
+  const snapshot: RunSnapshot = {
+    automation: definition as unknown as AutomationDraft,
+    agent: executionAgent,
+    managed,
+  }
+  let run
+  try {
+    run = await context.api.automationRun.create({
+      automationId: automation.id,
+      workspace: context.workspace,
+      eventId: uniqueId,
+      payload,
+      snapshot,
+      ownerId: context.ownerId,
+    })
+  } catch (cause) {
+    const duplicate = await context.api.automationRun.list({
+      pageSize: 1,
+      filters: {
+        type: 'group',
+        logic: 'and',
+        children: [{ type: 'condition', field: 'eventId', operator: 'eq', value: uniqueId }],
+      },
+    })
+    if (duplicate.data[0]) {
+      return { runId: duplicate.data[0].id, status: duplicate.data[0].status, duplicate: true }
+    }
+    throw cause
+  }
+  const job: AutomationJob = {
+    ownerId: context.ownerId,
+    workspace: context.workspace,
+    runId: run.id,
+    forceError: options.forceError,
+  }
+  try {
+    if (binding) {
+      await binding.create({ id: `automation-${run.id}`, params: job })
+    } else {
+      const task = executeAutomation(job, context).catch(async () => {
+        await context.api.automationRun.update(run.id, {
+          ownerId: context.ownerId,
+          status: 'failed',
+          error: 'Execution stopped unexpectedly. Check the server logs.',
+          finishedAt: new Date().toISOString(),
+        })
+      })
+      if (event) {
+        event.waitUntil(task)
+      } else {
+        await task
+      }
+    }
+  } catch (cause) {
+    await context.api.automationRun.update(run.id, {
+      ownerId: context.ownerId,
+      status: 'failed',
+      error: 'Could not enqueue this automation. Check the workflow binding.',
+      finishedAt: new Date().toISOString(),
+    })
+    throw cause
+  }
+  return { runId: run.id, status: 'queued', duplicate: false }
+}
+export interface AutomationEvent {
+  name: string
+  payload: unknown
+  id: string
+}
+
+/** Launches every active automation in the workspace that listens for this event. */
+export async function emitAutomationEvent(
+  event: H3Event,
+  context: Context,
+  { name, payload, id }: AutomationEvent,
+) {
+  const result = await context.api.automation.list({
+    pageSize: 100,
+    filters: {
+      type: 'group',
+      logic: 'and',
+      children: [
+        { type: 'condition', field: 'workspace', operator: 'eq', value: context.workspace },
+        { type: 'condition', field: 'active', operator: 'eq', value: true },
+      ],
+    },
+  })
+  const matching = result.data.filter((row) => {
+    const trigger = row.trigger as { kind: string; event: string }
+    return trigger.kind === 'event' && trigger.event === name
+  })
+  const runs = []
+  for (const automation of matching) {
+    runs.push(await launchAutomation(event, context, { automation, payload, eventId: id }))
+  }
+  return { runs }
+}

package/runtime/server/utils/automation-schedule.ts

@@ -0,0 +1,31 @@
+import { createJobContext } from './automation-context'
+import { launchAutomation } from './automation-launch'
+
+export async function fireAutomationSchedule(
+  payload: Record<string, unknown>,
+  dueAt: string,
+  env: Record<string, unknown> | undefined,
+) {
+  const ownerId = Number(payload.ownerId)
+  const workspace = String(payload.workspace)
+  const context = await createJobContext({ ownerId, workspace, runId: 0 }, env)
+  const automation = await context.api.automation
+    .get(Number(payload.automationId))
+    .catch(() => null)
+  if (
+    !automation ||
+    !automation.active ||
+    automation.workspace !== workspace ||
+    (automation.trigger as { kind?: string }).kind !== 'schedule'
+  ) {
+    return
+  }
+  await launchAutomation(
+    null,
+    context,
+    { automation, payload: { scheduledAt: dueAt }, eventId: `schedule:${dueAt}` },
+    {
+      managed: true,
+    },
+  )
+}

package/runtime/server/utils/connection-events.ts

@@ -0,0 +1,114 @@
+import type { H3Event } from 'h3'
+import { createServerClient, type DbProvider } from '@cloudconfig/core/server'
+import type { Connect, ConnectionSummary } from '@cloudconfig/core/connect'
+import type { IntegrationsConfig } from '@nuxtagentic/integrations/schema'
+import {
+  backendConfig,
+  type Binding,
+} from '@nuxtagentic/integrations/runtime/server/utils/integration-context'
+import { getWebhookContext } from './automation-context'
+import { emitAutomationEvent } from './automation-launch'
+
+interface ConnectionEventOptions {
+  connection: ConnectionSummary
+  context: { database: DbProvider; env: Record<string, unknown> | undefined }
+  delivery: { name: string; payload: unknown; id: string }
+}
+
+/**
+ * Integration connections that use one Connect connection. A vendor event has
+ * no signed-in user, so the query runs as the connection's own owner — or, for
+ * a shared connection, as its organization — and the normal access rules apply.
+ */
+async function bindingsFor(
+  summary: ConnectionSummary,
+  context: { database: DbProvider; env: Record<string, unknown> | undefined },
+): Promise<Binding[]> {
+  const { organizationId, userId, id } = summary
+  const { database, env } = context
+  const organization = organizationId === null ? null : { id: organizationId, role: 'member' }
+  const api = createServerClient(
+    backendConfig as IntegrationsConfig,
+    { env, request: new Request('https://events.internal') },
+    { db: database, getUser: () => ({ id: userId ?? 0, organization }) },
+  )
+  const result = await api.integrationConnection.list({
+    pageSize: 100,
+    filters: {
+      type: 'group',
+      logic: 'and',
+      children: [
+        { type: 'condition', field: 'connectionId', operator: 'eq', value: id },
+        { type: 'condition', field: 'enabled', operator: 'eq', value: true },
+      ],
+    },
+  })
+  return result.data
+}
+
+/**
+ * Start the automations listening for `name` that may see this connection:
+ * its owner's for a personal connection, every current member's for a shared one.
+ */
+export async function emitConnectionEvent(
+  event: H3Event,
+  connect: Connect,
+  { connection, context, delivery }: ConnectionEventOptions,
+) {
+  const { name, payload, id } = delivery
+  const runs = []
+  for (const binding of await bindingsFor(connection, context)) {
+    const { provider, scope, organizationId, ownerId: connectionOwnerId, workspace } = binding
+    // Events are namespaced by integration: `github.*` only reaches GitHub bindings.
+    if (!name.startsWith(`${provider}.`)) {
+      continue
+    }
+    const owners =
+      scope === 'workspace' && organizationId !== null
+        ? (await connect.members(organizationId)).map((member) => member.userId)
+        : [connectionOwnerId]
+    for (const ownerId of owners) {
+      const job = await getWebhookContext(event, ownerId)
+      job.workspace = workspace
+      const result = await emitAutomationEvent(event, job, {
+        name,
+        payload,
+        id: `${provider}:${id}`,
+      })
+      runs.push(...result.runs)
+    }
+  }
+  return { runs }
+}
+
+/** Maps a vendor's own event name to the catalog trigger it represents, if any. */
+export function catalogTrigger(provider: string, event: string, payload: unknown): string {
+  const record = (payload && typeof payload === 'object' ? payload : {}) as Record<string, unknown>
+  switch (provider) {
+    case 'github':
+      if (event === 'pull_request.opened') {
+        return 'github.a_pull_request_is_opened'
+      }
+      if (event === 'issues.opened') {
+        return 'github.an_issue_is_created'
+      }
+      return event === 'push' ? 'github.a_branch_is_pushed' : ''
+    case 'slack':
+      if (event === 'app_mention') {
+        return 'slack.an_agent_is_mentioned'
+      }
+      return event === 'message' ? 'slack.a_message_is_posted' : ''
+    case 'shopify':
+      return event === 'orders/create' ? 'shopify.an_order_is_placed' : ''
+    case 'hubspot':
+      return event === 'deal.propertyChange' && record.propertyName === 'dealstage'
+        ? 'hubspot.a_deal_changes_stage'
+        : ''
+    case 'intercom':
+      return event === 'conversation.user.created' ? 'intercom.a_conversation_is_opened' : ''
+    case 'notion':
+      return event === 'page.created' ? 'notion.a_database_row_is_added' : ''
+    default:
+      return ''
+  }
+}

package/runtime/server/utils/maintenance.ts

@@ -0,0 +1,255 @@
+import { CronExpressionParser } from 'cron-parser'
+import { createJobContext } from './automation-context'
+import { closeUnfinishedRun } from './automation-engine'
+import { fireAutomationSchedule } from './automation-schedule'
+import { syncAutomationSchedule } from './schedule-sync'
+
+/** The D1 surface the job needs, so tests can supply any SQLite database. */
+export interface MaintenanceDatabase {
+  prepare(sql: string): {
+    bind(...values: unknown[]): {
+      all<T = Record<string, unknown>>(): Promise<{ results: T[] }>
+      run(): Promise<{ meta?: { changes?: number } }>
+    }
+  }
+}
+interface WorkflowBinding {
+  get(id: string): Promise<{ status(): Promise<{ status: string }> }>
+}
+export interface MaintenanceReport {
+  schedulesRepaired: number
+  runsRecovered: number
+  runsClosed: number
+  receiptsDeleted: number
+  runsDeleted: number
+}
+
+const MINUTE = 60_000
+/** A Durable Object alarm fires within seconds; a missing run after this is a lost schedule. */
+const SCHEDULE_GRACE = 10 * MINUTE
+/** Missed occurrences older than this are re-armed but not run late. */
+const CATCH_UP_WINDOW = 60 * MINUTE
+/** Runs untouched for this long are checked against their Workflow instance. */
+const STALE_RUN = 15 * MINUTE
+/** Receipts only matter while a run can still replay. */
+const RECEIPT_RETENTION = 24 * 60 * MINUTE
+const MAX_SCHEDULES = 2_000
+const MAX_STALE_RUNS = 25
+const DELETE_BATCH = 500
+const FINISHED = `'succeeded', 'failed'`
+
+/**
+ * Repairs what durable execution cannot repair from inside one run: schedules
+ * whose alarm was lost, runs whose Workflow instance ended without closing
+ * them, and receipts or history past their retention. Every task is bounded
+ * and idempotent, so the Worker cron may run it as often as it likes.
+ */
+export async function runAutomationMaintenance(
+  env: Record<string, unknown>,
+  database: MaintenanceDatabase,
+  options: { now?: Date; retentionDays?: number } = {},
+): Promise<MaintenanceReport> {
+  const now = options.now ?? new Date()
+  const report: MaintenanceReport = {
+    schedulesRepaired: 0,
+    runsRecovered: 0,
+    runsClosed: 0,
+    receiptsDeleted: 0,
+    runsDeleted: 0,
+  }
+  const tasks = [
+    () => repairSchedules(env, database, now, report),
+    () => closeOrphanedRuns(env, database, now, report),
+    () => pruneHistory(database, now, options.retentionDays ?? 0, report),
+  ]
+  // One failing task never stops the others; the next cron run retries it.
+  for (const task of tasks) {
+    await task().catch((cause: unknown) => {
+      console.error('[automations] A maintenance task failed.', cause)
+    })
+  }
+  return report
+}
+
+interface ScheduledRow {
+  id: number
+  owner_id: number
+  workspace: string
+  trigger: string
+  updated_at: string
+}
+
+async function repairSchedules(
+  env: Record<string, unknown>,
+  database: MaintenanceDatabase,
+  now: Date,
+  report: MaintenanceReport,
+) {
+  const { results } = await database
+    .prepare(
+      `SELECT id, owner_id, workspace, "trigger", updated_at FROM automations
+       WHERE active = 1 AND json_extract("trigger", '$.kind') = 'schedule' ORDER BY id LIMIT ?`,
+    )
+    .bind(MAX_SCHEDULES)
+    .all<ScheduledRow>()
+  const cutoff = new Date(now.getTime() - SCHEDULE_GRACE)
+  const expected = results.flatMap((row) => {
+    const dueAt = lastOccurrence(row.trigger, cutoff)
+    // A schedule saved after this occurrence was never armed for it.
+    if (!dueAt || dueAt.getTime() <= Date.parse(row.updated_at)) {
+      return []
+    }
+    return [{ row, dueAt, eventId: `${row.id}:schedule:${dueAt.toISOString()}` }]
+  })
+  const launched = new Set<string>()
+  for (const ids of chunks(
+    expected.map((entry) => entry.eventId),
+    90,
+  )) {
+    const { results: rows } = await database
+      .prepare(
+        `SELECT event_id FROM automationRuns WHERE event_id IN (${ids.map(() => '?').join(', ')})`,
+      )
+      .bind(...ids)
+      .all<{ event_id: string }>()
+    for (const row of rows) {
+      launched.add(row.event_id)
+    }
+  }
+  for (const { row, dueAt } of expected.filter((entry) => !launched.has(entry.eventId))) {
+    try {
+      const trigger = JSON.parse(row.trigger) as Record<string, unknown>
+      await syncAutomationSchedule(env, row.owner_id, row.id, {
+        active: true,
+        workspace: row.workspace,
+        trigger,
+      })
+      report.schedulesRepaired++
+      if (now.getTime() - dueAt.getTime() <= SCHEDULE_GRACE + CATCH_UP_WINDOW) {
+        // The same entry point the alarm uses; the event ID deduplicates a late alarm.
+        await fireAutomationSchedule(
+          { ownerId: row.owner_id, workspace: row.workspace, automationId: row.id },
+          dueAt.toISOString(),
+          env,
+        )
+        report.runsRecovered++
+      }
+    } catch (cause) {
+      console.error(`[automations] Could not repair the schedule of automation ${row.id}.`, cause)
+    }
+  }
+}
+
+function lastOccurrence(trigger: string, before: Date) {
+  try {
+    const { cron, timeZone } = JSON.parse(trigger) as { cron?: string; timeZone?: string }
+    return CronExpressionParser.parse(String(cron), { tz: timeZone, currentDate: before })
+      .prev()
+      .toDate()
+  } catch {
+    return undefined
+  }
+}
+
+async function closeOrphanedRuns(
+  env: Record<string, unknown>,
+  database: MaintenanceDatabase,
+  now: Date,
+  report: MaintenanceReport,
+) {
+  const workflows = env.AUTOMATION_RUNNER as WorkflowBinding | undefined
+  if (!workflows) {
+    return
+  }
+  // Random sampling keeps long, legitimately waiting runs from starving the rest.
+  const { results } = await database
+    .prepare(
+      `SELECT id, owner_id, workspace FROM automationRuns
+       WHERE status IN ('queued', 'running', 'waiting') AND updated_at < ?
+       ORDER BY RANDOM() LIMIT ?`,
+    )
+    .bind(new Date(now.getTime() - STALE_RUN).toISOString(), MAX_STALE_RUNS)
+    .all<{ id: number; owner_id: number; workspace: string }>()
+  for (const row of results) {
+    try {
+      const state = await workflowState(workflows, `automation-${row.id}`)
+      if (!['errored', 'terminated', 'complete', 'missing'].includes(state)) {
+        continue
+      }
+      const job = { ownerId: row.owner_id, workspace: row.workspace, runId: row.id }
+      const reason =
+        state === 'missing'
+          ? 'Its Workflow instance no longer exists.'
+          : `Its Workflow instance is ${state}.`
+      await closeUnfinishedRun(job, await createJobContext(job, env), new Error(reason))
+      report.runsClosed++
+    } catch (cause) {
+      console.error(`[automations] Could not check automation run ${row.id}.`, cause)
+    }
+  }
+}
+
+async function workflowState(workflows: WorkflowBinding, id: string) {
+  let instance
+  try {
+    instance = await workflows.get(id)
+  } catch (cause) {
+    if (/not.?found|does not exist/i.test(cause instanceof Error ? cause.message : '')) {
+      return 'missing'
+    }
+    throw cause
+  }
+  return (await instance.status()).status
+}
+
+async function pruneHistory(
+  database: MaintenanceDatabase,
+  now: Date,
+  retentionDays: number,
+  report: MaintenanceReport,
+) {
+  const receipts = await database
+    .prepare(
+      `DELETE FROM automationRunReceipts WHERE id IN (
+         SELECT receipt.id FROM automationRunReceipts AS receipt
+         JOIN automationRuns AS run ON run.id = receipt.run_id
+         WHERE run.status IN (${FINISHED}) AND run.finished_at < ? LIMIT ?)`,
+    )
+    .bind(new Date(now.getTime() - RECEIPT_RETENTION).toISOString(), DELETE_BATCH * 2)
+    .run()
+  report.receiptsDeleted += receipts.meta?.changes ?? 0
+  if (!(retentionDays > 0)) {
+    return
+  }
+  const { results } = await database
+    .prepare(
+      // Unary + keeps SQLite on the finished_at index; nearly every row has a finished status.
+      `SELECT id FROM automationRuns WHERE finished_at < ? AND +status IN (${FINISHED})
+       ORDER BY finished_at LIMIT ?`,
+    )
+    .bind(new Date(now.getTime() - retentionDays * 24 * 60 * MINUTE).toISOString(), DELETE_BATCH)
+    .all<{ id: number }>()
+  for (const ids of chunks(
+    results.map((row) => row.id),
+    90,
+  )) {
+    const placeholders = ids.map(() => '?').join(', ')
+    await database
+      .prepare(`DELETE FROM automationRunReceipts WHERE run_id IN (${placeholders})`)
+      .bind(...ids)
+      .run()
+    const deleted = await database
+      .prepare(`DELETE FROM automationRuns WHERE id IN (${placeholders})`)
+      .bind(...ids)
+      .run()
+    report.runsDeleted += deleted.meta?.changes ?? 0
+  }
+}
+
+function chunks<T>(values: T[], size: number) {
+  const groups: T[][] = []
+  for (let index = 0; index < values.length; index += size) {
+    groups.push(values.slice(index, index + size))
+  }
+  return groups
+}

package/runtime/server/utils/schedule-sync.ts

@@ -0,0 +1,32 @@
+import { getAgentPlatform } from '@cloudconfig/core/agents/platform'
+
+interface ScheduledAutomation {
+  active?: unknown
+  workspace?: unknown
+  trigger?: unknown
+}
+
+/** Replace or cancel the automation's durable schedule to match its saved trigger. */
+export async function syncAutomationSchedule(
+  env: Record<string, unknown> | undefined,
+  ownerId: number,
+  automationId: number,
+  row: ScheduledAutomation | undefined,
+) {
+  // Hosts without declared agents, such as the CLI, keep no schedules.
+  const platform = getAgentPlatform(env)
+  if (!platform) {
+    return
+  }
+  const trigger = row?.trigger as { kind?: string; cron?: string; timeZone?: string } | undefined
+  await platform.schedule(
+    `automation:${ownerId}:${automationId}`,
+    row?.active && trigger?.kind === 'schedule'
+      ? {
+          cron: String(trigger.cron),
+          timeZone: String(trigger.timeZone),
+          payload: { ownerId, workspace: row.workspace, automationId },
+        }
+      : null,
+  )
+}

package/runtime/server/utils/transient-retry.ts

@@ -0,0 +1,34 @@
+/** Failures Cloudflare documents as temporary for D1 queries and Worker subrequests. */
+const TRANSIENT_FAILURE =
+  /network connection lost|object to be reset|reset because its code was updated|overloaded|queued for too long|internal error|connection reset|econnreset|socket hang up/i
+const ATTEMPTS = 4
+
+export function isTransientFailure(cause: unknown) {
+  let error = cause
+  for (let depth = 0; error instanceof Error && depth < 5; depth++) {
+    if (TRANSIENT_FAILURE.test(error.message)) {
+      return true
+    }
+    error = error.cause
+  }
+  return false
+}
+
+/**
+ * Retry an operation through brief storage and network interruptions. Only
+ * idempotent work belongs here: journaled tool calls are, because a retry after
+ * dispatch finds their "started" receipt and stops instead of repeating them.
+ */
+export async function retryTransientFailures<T>(operation: () => Promise<T>): Promise<T> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await operation()
+    } catch (cause) {
+      if (attempt >= ATTEMPTS || !isTransientFailure(cause)) {
+        throw cause
+      }
+      const delay = 250 * 4 ** (attempt - 1) * (0.5 + Math.random() / 2)
+      await new Promise((resolve) => setTimeout(resolve, delay))
+    }
+  }
+}