experimental-a2 0.3.0 → 0.5.0

package/src/scheduler-qstash.ts

@@ -0,0 +1,915 @@
+/**
+ * experimental-a2/scheduler-qstash — HTTP scheduler over Upstash QStash.
+ *
+ * QStash publishes one signed HTTP delivery per scheduler task. Drain tasks
+ * use an absolute, whole-second `notBefore` and a fixed-size deduplication id;
+ * append tasks hash their stable task id with the callback URL.
+ * Store-outage successors carry a signed private generation and back off to
+ * one day without deriving their deduplication identity from the local clock.
+ * `handler(...servers)` verifies the signature over the raw request body
+ * before parsing it, then applies the same drain protocol as the Vercel
+ * scheduler adapter.
+ *
+ * `@upstash/qstash` is an optional peer dependency, imported lazily. The
+ * structural `transport` option keeps its types out of A2's public surface and
+ * gives tests a provider-faithful injection seam.
+ */
+
+import {
+  assertSchedulerTargets,
+  classifySchedulerSendFailure,
+  DRAIN_TIMINGS,
+  serverInternals,
+  type DrainResult,
+} from './internal.ts'
+import { A2Error } from './errors.ts'
+import { platformVercelOidcToken } from './platform.ts'
+import { retryableLazy } from './retryable-lazy.ts'
+import { normalizeSchedulerTask, type SchedulerTask } from './scheduler-task.ts'
+import {
+  deliverSchedulerAppend,
+  type A2Scheduler,
+  type DrainableServer,
+} from './server.ts'
+import { SYSTEM_CLOCK, type Clock } from './store.ts'
+
+type QStashStoreOutage = {
+  version: 1
+  /** Zero is the first successor created for a store outage. */
+  generation: number
+  /** The first failing drain task's due time. Append identity lives on the task. */
+  rootDueAt?: number
+}
+
+type QStashMessage = SchedulerTask & {
+  /** Provider-private state authenticated as part of QStash's signed body. */
+  _a2QStash?: QStashStoreOutage
+}
+
+export type QStashPublishRequest = {
+  url: string
+  body: QStashMessage
+  /** Headers the transport must deliver to the callback. */
+  headers?: Readonly<Record<string, string>>
+  /** Header names the transport must redact from provider dashboard/API views. */
+  redact?: {
+    header: readonly string[]
+  }
+  /** Whole-second Unix timestamp. Omitted for an immediate delivery. */
+  notBefore?: number
+  deduplicationId: string
+  /** Delivery retries after the initial attempt. */
+  retries?: number
+}
+
+/** The request shape passed to an injected `@upstash/qstash` client. */
+export type QStashClientPublishRequest = Omit<
+  QStashPublishRequest,
+  'headers' | 'redact'
+> & {
+  headers?: Record<string, string>
+  redact?: {
+    header: string[]
+  }
+}
+
+/** The outbound part of `@upstash/qstash` used by the built-in transport. */
+export type QStashPublishingClient = {
+  publishJSON(request: QStashClientPublishRequest): Promise<unknown>
+}
+
+export type QStashVerifyRequest = {
+  signature: string
+  /** The untouched request body covered by the signature. */
+  body: string
+  /** Exact destination URL, checked against the signature subject. */
+  url: string
+  upstashRegion?: string
+}
+
+/** The small part of `@upstash/qstash` this adapter needs. */
+export type QStashTransport = {
+  publish(request: QStashPublishRequest): Promise<unknown>
+  verify(request: QStashVerifyRequest): Promise<boolean>
+}
+
+export type QStashSchedulerOptions = {
+  /** Exact URL mounted with `schedulerHandler(...)`. Inferred in local dev and on Vercel. */
+  url?: string
+  /** QStash API token. Default: `QSTASH_TOKEN`. */
+  token?: string
+  /** QStash signing key. Default: `QSTASH_CURRENT_SIGNING_KEY`. */
+  currentSigningKey?: string
+  /** Next QStash signing key. Default: `QSTASH_NEXT_SIGNING_KEY`. */
+  nextSigningKey?: string
+  /** Delivery retries after the initial attempt. Default: provider default. */
+  retries?: number
+  /** Maximum callback body and outbound task with outage headroom. Default 1 MiB. */
+  maxBodyBytes?: number
+  /** Static callback headers. QStash receives them; A2 requests dashboard/API redaction. */
+  headers?: Readonly<Record<string, string>>
+  /** Injectable publishing client. The built-in receiver still verifies callbacks. */
+  client?: QStashPublishingClient
+  /** Injectable transport (tests). Default: `@upstash/qstash`. */
+  transport?: QStashTransport
+  /** Injectable clock for due-time calculations (tests). */
+  clock?: Clock
+}
+
+const qstashTransport = async (
+  options: Pick<
+    QStashSchedulerOptions,
+    'token' | 'currentSigningKey' | 'nextSigningKey' | 'client'
+  >,
+  developmentMode: boolean | undefined,
+): Promise<QStashTransport> => {
+  const sdk = await import('@upstash/qstash').catch(() => {
+    throw classifySchedulerSendFailure(
+      new Error(
+        "experimental-a2/scheduler-qstash needs the '@upstash/qstash' package (optional peer dependency) — install it, or inject a transport",
+      ),
+      'terminal',
+    )
+  })
+  const developmentOverride =
+    developmentMode === undefined ? {} : { devMode: developmentMode }
+  const client: QStashPublishingClient =
+    options.client ??
+    new sdk.Client({
+      ...(options.token === undefined ? {} : { token: options.token }),
+      ...developmentOverride,
+    })
+  const receiver = new sdk.Receiver({
+    ...(options.currentSigningKey === undefined
+      ? {}
+      : { currentSigningKey: options.currentSigningKey }),
+    ...(options.nextSigningKey === undefined
+      ? {}
+      : { nextSigningKey: options.nextSigningKey }),
+    ...developmentOverride,
+  })
+  return {
+    publish: (request) => {
+      const clientRequest: QStashClientPublishRequest = {
+        url: request.url,
+        body: request.body,
+        ...(request.headers === undefined
+          ? {}
+          : { headers: { ...request.headers } }),
+        ...(request.redact === undefined
+          ? {}
+          : { redact: { header: [...request.redact.header] } }),
+        deduplicationId: request.deduplicationId,
+        ...(request.retries === undefined ? {} : { retries: request.retries }),
+        ...(request.notBefore === undefined
+          ? {}
+          : { notBefore: request.notBefore }),
+      }
+      return client.publishJSON(clientRequest)
+    },
+    verify: (request) => receiver.verify(request),
+  }
+}
+
+const digestId = async (
+  parts: readonly (string | number)[],
+): Promise<string> => {
+  const digest = await crypto.subtle.digest(
+    'SHA-256',
+    new TextEncoder().encode(JSON.stringify(parts)),
+  )
+  const hex = Array.from(new Uint8Array(digest), (byte) =>
+    byte.toString(16).padStart(2, '0'),
+  ).join('')
+  return `a2_${hex}`
+}
+
+type ParsedQStashMessage = {
+  task: SchedulerTask
+  outage?: QStashStoreOutage
+}
+
+const recordOf = (value: unknown): Record<string, unknown> | null =>
+  value !== null && typeof value === 'object' && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null
+
+const TRUSTED_SOURCE_HEADER = 'x-vercel-trusted-oidc-idp-token'
+const TRUSTED_SOURCE_DELIVERY_MARGIN_MS = 60_000
+const RESERVED_CALLBACK_HEADERS = new Set([
+  'connection',
+  'content-length',
+  'host',
+  'keep-alive',
+  'proxy-authenticate',
+  'proxy-authorization',
+  'te',
+  'trailer',
+  'transfer-encoding',
+  'upgrade',
+  'x-vercel-oidc-token',
+  TRUSTED_SOURCE_HEADER,
+])
+
+const callbackHeaders = (
+  value: Readonly<Record<string, string>> | undefined,
+): Readonly<Record<string, string>> | undefined => {
+  if (value === undefined) return undefined
+
+  let entries: [string, string][]
+  try {
+    entries = Object.entries(value)
+  } catch {
+    throw new TypeError('qstash() headers must be an HTTP header record')
+  }
+
+  const normalized: [string, string][] = []
+  const seen = new Set<string>()
+  for (const [name, headerValue] of entries) {
+    if (typeof headerValue !== 'string') {
+      throw new TypeError('qstash() header values must be strings')
+    }
+    let entry: [string, string] | undefined
+    try {
+      entry = [...new Headers([[name, headerValue]]).entries()][0]
+    } catch {
+      throw new TypeError(
+        'qstash() headers must contain valid HTTP header names and values',
+      )
+    }
+    if (entry === undefined) {
+      throw new TypeError(
+        'qstash() headers must contain valid HTTP header names and values',
+      )
+    }
+    const [normalizedName, normalizedValue] = entry
+    if (seen.has(normalizedName)) {
+      throw new TypeError(
+        `qstash() headers contain duplicate name '${normalizedName}'`,
+      )
+    }
+    if (
+      normalizedName.startsWith('upstash-') ||
+      normalizedName.startsWith('content-type') ||
+      RESERVED_CALLBACK_HEADERS.has(normalizedName)
+    ) {
+      throw new TypeError(
+        `qstash() headers cannot set reserved header '${normalizedName}'`,
+      )
+    }
+    seen.add(normalizedName)
+    normalized.push([normalizedName, normalizedValue])
+  }
+
+  return normalized.length === 0
+    ? undefined
+    : Object.freeze(Object.fromEntries(normalized))
+}
+
+const publishHeaders = (
+  configured: Readonly<Record<string, string>> | undefined,
+  generated: Pick<QStashPublishRequest, 'headers' | 'redact'> | undefined,
+): Pick<QStashPublishRequest, 'headers' | 'redact'> | undefined => {
+  if (configured === undefined && generated === undefined) return undefined
+
+  const headers = new Headers()
+  const redacted = new Set<string>()
+  for (const [name, value] of Object.entries(configured ?? {})) {
+    headers.set(name, value)
+    redacted.add(name)
+  }
+  for (const [name, value] of Object.entries(generated?.headers ?? {})) {
+    headers.set(name, value)
+    redacted.add(name)
+  }
+  for (const name of generated?.redact?.header ?? []) {
+    const normalized = [...new Headers([[name, 'redacted']]).keys()][0]
+    if (normalized === undefined) {
+      throw new TypeError('qstash generated an invalid redaction header')
+    }
+    redacted.add(normalized)
+  }
+
+  const headerEntries = [...headers.entries()]
+  return {
+    ...(headerEntries.length === 0
+      ? {}
+      : { headers: Object.fromEntries(headerEntries) }),
+    ...(redacted.size === 0
+      ? {}
+      : { redact: { header: [...redacted].toSorted() } }),
+  }
+}
+
+const jwtExpirationMs = (token: string): number | null => {
+  const parts = token.split('.')
+  const payloadPart = parts.length === 3 ? parts[1] : undefined
+  if (
+    payloadPart === undefined ||
+    payloadPart.length === 0 ||
+    !/^[A-Za-z0-9_-]+$/.test(payloadPart) ||
+    payloadPart.length % 4 === 1
+  ) {
+    return null
+  }
+
+  try {
+    const decode = globalThis.atob
+    if (typeof decode !== 'function') return null
+    const base64 = payloadPart
+      .replaceAll('-', '+')
+      .replaceAll('_', '/')
+      .padEnd(payloadPart.length + ((4 - (payloadPart.length % 4)) % 4), '=')
+    const binary = decode(base64)
+    const bytes = Uint8Array.from(binary, (character) =>
+      character.charCodeAt(0),
+    )
+    const payload = recordOf(JSON.parse(new TextDecoder().decode(bytes)))
+    const expiration = payload?.['exp']
+    if (
+      typeof expiration !== 'number' ||
+      !Number.isSafeInteger(expiration) ||
+      expiration <= 0 ||
+      expiration > Number.MAX_SAFE_INTEGER / 1_000
+    ) {
+      return null
+    }
+    return expiration * 1_000
+  } catch {
+    return null
+  }
+}
+
+const previewTrustedSource = (
+  firstDeliveryAt: number,
+): Pick<QStashPublishRequest, 'headers' | 'redact'> | undefined => {
+  const token = platformVercelOidcToken()
+  if (token === null) return undefined
+  const expiration = jwtExpirationMs(token)
+  if (
+    expiration === null ||
+    firstDeliveryAt + TRUSTED_SOURCE_DELIVERY_MARGIN_MS > expiration
+  ) {
+    return undefined
+  }
+  return {
+    headers: { [TRUSTED_SOURCE_HEADER]: token },
+    redact: { header: [TRUSTED_SOURCE_HEADER] },
+  }
+}
+
+const RETRYABLE_HTTP_CLIENT_STATUSES = new Set([408, 409, 425, 429])
+
+const httpStatusOf = (error: unknown): number | undefined => {
+  try {
+    const record = recordOf(error)
+    if (!record) return undefined
+    const response = recordOf(record['response'])
+    const statuses = [
+      record['status'],
+      record['statusCode'],
+      response?.['status'],
+    ].filter(
+      (value): value is number =>
+        typeof value === 'number' &&
+        Number.isInteger(value) &&
+        value >= 100 &&
+        value <= 599,
+    )
+    const [status] = statuses
+    if (status === undefined) return undefined
+    return statuses.every((candidate) => candidate === status)
+      ? status
+      : undefined
+  } catch {
+    return undefined
+  }
+}
+
+const classifyQStashPublishFailure = (error: unknown): unknown => {
+  const status = httpStatusOf(error)
+  const terminal =
+    status !== undefined &&
+    status >= 400 &&
+    status < 500 &&
+    !RETRYABLE_HTTP_CLIENT_STATUSES.has(status)
+  return classifySchedulerSendFailure(
+    error,
+    terminal ? 'terminal' : 'retryable',
+  )
+}
+
+const parseTask = (body: string): ParsedQStashMessage | null => {
+  let value: unknown
+  try {
+    value = JSON.parse(body)
+  } catch {
+    return null
+  }
+  const task = normalizeSchedulerTask(value)
+  const record = recordOf(value)
+  if (!task || !record) return null
+
+  const encodedOutage = record['_a2QStash']
+  if (encodedOutage === undefined) return { task }
+  const outage = recordOf(encodedOutage)
+  const generation = outage?.['generation']
+  if (
+    !outage ||
+    outage['version'] !== 1 ||
+    typeof generation !== 'number' ||
+    !Number.isSafeInteger(generation) ||
+    generation < 0 ||
+    generation >= Number.MAX_SAFE_INTEGER
+  ) {
+    return null
+  }
+
+  const rootDueAt = outage['rootDueAt']
+  if (task.kind === 'drain') {
+    if (
+      typeof rootDueAt !== 'number' ||
+      !Number.isInteger(rootDueAt) ||
+      rootDueAt % 1_000 !== 0
+    ) {
+      return null
+    }
+  } else if (rootDueAt !== undefined) {
+    return null
+  }
+
+  return {
+    task,
+    outage: {
+      version: 1,
+      generation,
+      ...(rootDueAt === undefined ? {} : { rootDueAt }),
+    },
+  }
+}
+
+const DEFAULT_MAX_BODY_BYTES = 1024 * 1024
+const STORE_OUTAGE_BASE_DELAY_MS = 30_000
+const STORE_OUTAGE_MAX_DELAY_MS = 24 * 60 * 60 * 1_000
+
+const readBoundedBody = async (
+  request: Request,
+  maxBytes: number,
+): Promise<string | null> => {
+  const declared = request.headers.get('content-length')
+  if (
+    declared !== null &&
+    /^\d+$/.test(declared) &&
+    Number(declared) > maxBytes
+  ) {
+    return null
+  }
+
+  if (request.body === null) return ''
+  const reader = request.body.getReader()
+  const chunks: Uint8Array[] = []
+  let length = 0
+  try {
+    for (;;) {
+      // oxlint-disable-next-line no-await-in-loop
+      const { done, value } = await reader.read()
+      if (done) break
+      length += value.byteLength
+      if (length > maxBytes) {
+        // oxlint-disable-next-line no-await-in-loop
+        await reader.cancel().catch(() => {})
+        return null
+      }
+      chunks.push(value)
+    }
+  } finally {
+    reader.releaseLock()
+  }
+
+  const bytes = new Uint8Array(length)
+  let offset = 0
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset)
+    offset += chunk.byteLength
+  }
+  return new TextDecoder().decode(bytes)
+}
+
+const storeUnavailable = (error: unknown): error is A2Error =>
+  error instanceof A2Error && error.code === 'STORE_UNAVAILABLE'
+
+const response = {
+  ok: (): Response => new Response('ok'),
+  unauthorized: (): Response => new Response('unauthorized', { status: 401 }),
+  malformed: (): Response =>
+    new Response('invalid a2 scheduler message', {
+      status: 489,
+      headers: { 'Upstash-NonRetryable-Error': 'true' },
+    }),
+  tooLarge: (): Response =>
+    new Response('a2 scheduler message is too large', {
+      status: 489,
+      headers: { 'Upstash-NonRetryable-Error': 'true' },
+    }),
+}
+
+const CALLBACK_PATH = '/api/a2/scheduler'
+const DEFAULT_DEVELOPMENT_PORT = 3_000
+
+const developmentCallbackUrl = (portValue: string | undefined): string => {
+  const port =
+    portValue === undefined || portValue === ''
+      ? DEFAULT_DEVELOPMENT_PORT
+      : Number(portValue)
+  if (
+    !Number.isSafeInteger(port) ||
+    port < 1 ||
+    port > 65_535 ||
+    (portValue !== undefined && portValue !== '' && !/^\d+$/.test(portValue))
+  ) {
+    throw new TypeError(
+      'qstash() could not infer url: PORT must be an integer from 1 to 65535 for local development',
+    )
+  }
+  return `http://localhost:${String(port)}${CALLBACK_PATH}`
+}
+
+type CallbackResolution = {
+  callbackUrl: string
+  /** An SDK override. Undefined preserves the SDK's environment behavior. */
+  developmentMode?: boolean
+  inferredVercelPreview?: true
+}
+
+const inferCallbackUrl = (): CallbackResolution => {
+  const env = typeof process === 'undefined' ? undefined : process.env
+  const vercelEnv = env?.['VERCEL_ENV']
+  const variable =
+    env?.['VERCEL'] !== '1'
+      ? null
+      : vercelEnv === 'production'
+        ? 'VERCEL_PROJECT_PRODUCTION_URL'
+        : vercelEnv === 'preview'
+          ? 'VERCEL_BRANCH_URL'
+          : null
+  if (variable !== null) {
+    const host = env?.[variable]
+    if (!host) {
+      throw new TypeError(
+        `qstash() could not infer url: ${variable} is missing`,
+      )
+    }
+    let origin: URL
+    try {
+      origin = new URL(`https://${host}`)
+    } catch {
+      throw new TypeError(
+        `qstash() could not infer url: ${variable} must be a hostname`,
+      )
+    }
+    if (
+      origin.host !== host ||
+      origin.pathname !== '/' ||
+      origin.search !== '' ||
+      origin.hash !== '' ||
+      origin.username !== '' ||
+      origin.password !== '' ||
+      origin.port !== ''
+    ) {
+      throw new TypeError(
+        `qstash() could not infer url: ${variable} must be a hostname`,
+      )
+    }
+    return {
+      callbackUrl: new URL(CALLBACK_PATH, origin).toString(),
+      developmentMode: false,
+      ...(vercelEnv === 'preview' ? { inferredVercelPreview: true } : {}),
+    }
+  }
+
+  if (env === undefined) {
+    throw new TypeError(
+      'qstash() needs url when the runtime cannot identify local development',
+    )
+  }
+
+  if (env['NODE_ENV'] === 'production') {
+    throw new TypeError(
+      'qstash() needs url in production outside a Vercel production or preview deployment',
+    )
+  }
+
+  return {
+    callbackUrl: developmentCallbackUrl(env['PORT']),
+    developmentMode: true,
+  }
+}
+
+export function qstash(options: QStashSchedulerOptions = {}): A2Scheduler {
+  const env = typeof process === 'undefined' ? undefined : process.env
+  const resolution: CallbackResolution =
+    options.url === undefined
+      ? inferCallbackUrl()
+      : {
+          callbackUrl: options.url,
+          ...(env?.['NODE_ENV'] === 'production'
+            ? { developmentMode: false }
+            : {}),
+        }
+  const { callbackUrl, developmentMode, inferredVercelPreview } = resolution
+  let parsedUrl: URL
+  try {
+    parsedUrl = new URL(callbackUrl)
+  } catch {
+    throw new TypeError('qstash() needs an absolute callback url')
+  }
+  if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') {
+    throw new TypeError('qstash() callback url must use http or https')
+  }
+  if (options.client !== undefined && options.transport !== undefined) {
+    throw new TypeError('qstash() cannot combine client and transport')
+  }
+  if (options.client !== undefined && options.token !== undefined) {
+    throw new TypeError('qstash() cannot combine client and token')
+  }
+  const url = parsedUrl.toString()
+  const configuredHeaders = callbackHeaders(options.headers)
+  const { retries } = options
+  if (
+    retries !== undefined &&
+    (!Number.isSafeInteger(retries) || retries < 0)
+  ) {
+    throw new TypeError('qstash() retries must be a non-negative integer')
+  }
+  const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES
+  if (!Number.isSafeInteger(maxBodyBytes) || maxBodyBytes < 1) {
+    throw new TypeError('qstash() maxBodyBytes must be a positive integer')
+  }
+  const clock = options.clock ?? SYSTEM_CLOCK
+
+  // Lazy: construction does no I/O and does not require the optional peer.
+  const transport = retryableLazy(() =>
+    options.transport
+      ? Promise.resolve(options.transport)
+      : qstashTransport(options, developmentMode),
+  ).get
+
+  const storeOutageDeduplicationId = (
+    task: SchedulerTask,
+    outage: QStashStoreOutage,
+  ): Promise<string> => {
+    if (task.kind === 'append') {
+      return digestId([
+        'a2:scheduler-qstash:store-outage:append:v1',
+        url,
+        task.contract,
+        task.sessionId,
+        task.id,
+        outage.generation,
+      ])
+    }
+    const { rootDueAt } = outage
+    if (rootDueAt === undefined) {
+      throw new TypeError('qstash store-outage drain needs a root dueAt')
+    }
+    return digestId([
+      'a2:scheduler-qstash:store-outage:drain:v1',
+      url,
+      task.contract,
+      task.sessionId,
+      rootDueAt,
+      outage.generation,
+    ])
+  }
+
+  const sendTask = async (
+    task: SchedulerTask,
+    outage?: QStashStoreOutage,
+  ): Promise<void> => {
+    const nowMs = clock.now().getTime()
+    const requestedDueAt = task.dueAt
+    if (!Number.isFinite(requestedDueAt)) {
+      throw classifySchedulerSendFailure(
+        new TypeError('qstash scheduler dueAt must be a finite number'),
+        'terminal',
+      )
+    }
+    const dueAt =
+      requestedDueAt <= nowMs
+        ? Math.floor(nowMs / 1_000) * 1_000
+        : Math.ceil(requestedDueAt / 1_000) * 1_000
+    const normalizedTask: SchedulerTask =
+      task.kind === 'append'
+        ? {
+            version: 1,
+            kind: 'append',
+            id: task.id,
+            contract: task.contract,
+            sessionId: task.sessionId,
+            dueAt,
+            events: task.events,
+          }
+        : {
+            version: 1,
+            kind: 'drain',
+            contract: task.contract,
+            sessionId: task.sessionId,
+            dueAt,
+          }
+    const message: QStashMessage = {
+      ...normalizedTask,
+      ...(outage === undefined ? {} : { _a2QStash: outage }),
+    }
+    const sizeCheckedMessage: QStashMessage =
+      outage === undefined
+        ? {
+            ...normalizedTask,
+            _a2QStash:
+              normalizedTask.kind === 'append'
+                ? { version: 1, generation: Number.MAX_SAFE_INTEGER }
+                : {
+                    version: 1,
+                    generation: Number.MAX_SAFE_INTEGER,
+                    rootDueAt: dueAt,
+                  },
+          }
+        : message
+    let bodyBytes: number
+    try {
+      bodyBytes = new TextEncoder().encode(
+        JSON.stringify(sizeCheckedMessage),
+      ).byteLength
+    } catch (error) {
+      throw classifySchedulerSendFailure(error, 'terminal')
+    }
+    if (bodyBytes > maxBodyBytes) {
+      throw classifySchedulerSendFailure(
+        new RangeError('qstash scheduler message exceeds maxBodyBytes'),
+        'terminal',
+      )
+    }
+    const deduplicationId =
+      outage !== undefined
+        ? await storeOutageDeduplicationId(task, outage)
+        : task.kind === 'append'
+          ? await digestId(['a2:scheduler-qstash:append:v1', url, task.id])
+          : await digestId([
+              'a2:scheduler-qstash:v1',
+              url,
+              task.contract,
+              task.sessionId,
+              dueAt,
+            ])
+    const trustedSource =
+      inferredVercelPreview === true
+        ? previewTrustedSource(Math.max(nowMs, dueAt))
+        : undefined
+    const deliveryHeaders = publishHeaders(configuredHeaders, trustedSource)
+    const t = await transport()
+    try {
+      await t.publish({
+        url,
+        body: message,
+        ...deliveryHeaders,
+        ...(dueAt <= nowMs ? {} : { notBefore: dueAt / 1_000 }),
+        deduplicationId,
+        ...(retries === undefined ? {} : { retries }),
+      })
+    } catch (error) {
+      throw classifyQStashPublishFailure(error)
+    }
+  }
+
+  const sendStoreOutageSuccessor = async (
+    task: SchedulerTask,
+    outage?: QStashStoreOutage,
+  ): Promise<void> => {
+    const generation = (outage?.generation ?? -1) + 1
+    const exponent = Math.min(generation, 6)
+    const delayMs = Math.min(
+      STORE_OUTAGE_MAX_DELAY_MS,
+      STORE_OUTAGE_BASE_DELAY_MS * 4 ** exponent,
+    )
+    const nowMs = clock.now().getTime()
+    const dueAt = Math.ceil((nowMs + delayMs) / 1_000) * 1_000
+    const nextOutage: QStashStoreOutage =
+      task.kind === 'append'
+        ? { version: 1, generation }
+        : { version: 1, generation, rootDueAt: outage?.rootDueAt ?? task.dueAt }
+    await sendTask({ ...task, dueAt }, nextOutage)
+  }
+
+  const successorDueAt = (message: { dueAt: number }): number => {
+    const minimumDueAt =
+      clock.now().getTime() +
+      DRAIN_TIMINGS.claimTtlMs +
+      DRAIN_TIMINGS.watchdogGraceMs
+    const beats = Math.max(
+      1,
+      Math.ceil(
+        (minimumDueAt - message.dueAt) / DRAIN_TIMINGS.claimHeartbeatMs,
+      ),
+    )
+    return message.dueAt + beats * DRAIN_TIMINGS.claimHeartbeatMs
+  }
+
+  return {
+    async schedule(task) {
+      await sendTask(task)
+    },
+
+    handler(...servers: DrainableServer[]) {
+      assertSchedulerTargets(servers, { allowStructural: true })
+      const byContract = new Map<string, DrainableServer>()
+      for (const server of servers) {
+        byContract.set(server.contract.name, server)
+      }
+      return async (req: Request): Promise<Response> => {
+        const signature = req.headers.get('upstash-signature')
+        if (!signature) return response.unauthorized()
+
+        const body = await readBoundedBody(req, maxBodyBytes)
+        if (body === null) return response.tooLarge()
+        const upstashRegion = req.headers.get('upstash-region')
+        const t = await transport()
+        try {
+          const valid = await t.verify({
+            signature,
+            body,
+            url,
+            ...(upstashRegion === null ? {} : { upstashRegion }),
+          })
+          if (!valid) return response.unauthorized()
+        } catch {
+          return response.unauthorized()
+        }
+
+        const parsed = parseTask(body)
+        if (!parsed) return response.malformed()
+        const { task, outage } = parsed
+
+        const server = byContract.get(task.contract)
+        if (!server) {
+          throw new Error(
+            `a2 scheduler: no server for contract '${task.contract}' — pass it to schedulerHandler(...)`,
+          )
+        }
+        const internals = serverInternals.get(server)
+        if (task.kind === 'append') {
+          try {
+            await deliverSchedulerAppend(server, task)
+          } catch (error) {
+            if (!storeUnavailable(error)) throw error
+            await sendStoreOutageSuccessor(task, outage)
+          }
+          return response.ok()
+        }
+
+        const nextDueAt = successorDueAt(task)
+        let result: DrainResult | { settled: boolean; outcome: 'unknown' }
+        try {
+          result = internals
+            ? await internals.schedulerDrain(task.sessionId, {
+                watchdogDueAt: nextDueAt,
+              })
+            : {
+                ...(await server.drain(task.sessionId)),
+                outcome: 'unknown' as const,
+              }
+        } catch (error) {
+          if (!storeUnavailable(error)) throw error
+          await sendStoreOutageSuccessor(task, outage)
+          return response.ok()
+        }
+        if (result.settled) return response.ok()
+
+        if (result.outcome === 'busy') {
+          if (result.schedulerArm) await result.schedulerArm
+          else {
+            await sendTask({
+              version: 1,
+              kind: 'drain',
+              contract: task.contract,
+              sessionId: task.sessionId,
+              dueAt: nextDueAt,
+            })
+          }
+          return response.ok()
+        }
+
+        // QStash's per-message retry default can be lower than A2's failure
+        // budget. Hand stalled work to a fresh durable message so recovery
+        // continues until the store reports success or dead-letter. If this
+        // publish fails, the throw keeps the current delivery retryable.
+        await sendTask({
+          version: 1,
+          kind: 'drain',
+          contract: task.contract,
+          sessionId: task.sessionId,
+          dueAt: nextDueAt,
+        })
+        return response.ok()
+      }
+    },
+  }
+}