@tanstack/ai-cloudflare 0.0.0 → 0.1.0

package/src/utils/fetch.ts

@@ -0,0 +1,131 @@
+import type { Ai } from '@cloudflare/workers-types'
+import type { CloudflareGatewayOptions, FetchLike } from './config'
+
+/**
+ * Workers AI streams end with a usage-only trailer (`{"response":"","usage":
+ * {...}}`) that has no `choices` field. The OpenAI Chat Completions stream
+ * reader indexes `chunk.choices[0]` on every event, so give such events an
+ * empty `choices` array (OpenAI's own usage-only trailer shape) and keep the
+ * usage totals they carry.
+ */
+export function normalizeSseResponse(response: Response): Response {
+  const contentType = response.headers.get('content-type') ?? ''
+  if (!response.body || !contentType.includes('text/event-stream')) {
+    return response
+  }
+  let buffer = ''
+  const fixLine = (line: string): string => {
+    if (!line.startsWith('data: ') || line === 'data: [DONE]') return line
+    try {
+      const event = JSON.parse(line.slice(6)) as Record<string, unknown>
+      if (event && typeof event === 'object' && !('choices' in event)) {
+        return `data: ${JSON.stringify({ ...event, choices: [] })}`
+      }
+    } catch {
+      // Not JSON: forward untouched.
+    }
+    return line
+  }
+  const body = response.body
+    .pipeThrough(new TextDecoderStream())
+    .pipeThrough(
+      new TransformStream<string, string>({
+        transform(chunk, controller) {
+          buffer += chunk
+          const lines = buffer.split('\n')
+          buffer = lines.pop() ?? ''
+          for (const line of lines) controller.enqueue(`${fixLine(line)}\n`)
+        },
+        flush(controller) {
+          if (buffer) controller.enqueue(fixLine(buffer))
+        },
+      }),
+    )
+    .pipeThrough(new TextEncoderStream())
+  return new Response(body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers: response.headers,
+  })
+}
+
+/**
+ * Cloudflare error bodies look like `{ name, message, internalCode }` or
+ * `{ errors: [{ code, message }] }`. The OpenAI SDK only reads
+ * `body.error.message`, so rewrap them or every failure reads as
+ * "status code (no body)".
+ */
+export async function normalizeErrorResponse(
+  response: Response,
+): Promise<Response> {
+  const text = await response.text()
+  let body = text
+  try {
+    const json = JSON.parse(text) as {
+      error?: unknown
+      errors?: Array<{ code?: number; message?: string }>
+      message?: string
+      name?: string
+      internalCode?: number
+    }
+    if (json && typeof json === 'object' && !('error' in json)) {
+      const first = json.errors?.[0]
+      body = JSON.stringify({
+        error: {
+          message: json.message ?? first?.message ?? text,
+          type: json.name ?? 'cloudflare_error',
+          code: json.internalCode ?? first?.code ?? null,
+        },
+      })
+    }
+  } catch {
+    // Not JSON: forward the text as-is.
+  }
+  return new Response(body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers: response.headers,
+  })
+}
+
+/** Applies the error and SSE normalizations a raw Cloudflare response needs. */
+export async function normalizeResponse(response: Response): Promise<Response> {
+  return response.ok
+    ? normalizeSseResponse(response)
+    : await normalizeErrorResponse(response)
+}
+
+/**
+ * Makes `env.AI` look like an OpenAI-compatible HTTP endpoint to the OpenAI
+ * SDK: the JSON request body becomes `binding.run(model, inputs)` and the
+ * raw inference `Response` (OpenAI-format JSON or SSE) is handed back.
+ */
+export function createBindingFetch(
+  binding: Ai,
+  gateway?: CloudflareGatewayOptions,
+): FetchLike {
+  // `Ai` is typed against the bundled model catalog; the adapter accepts any
+  // model id, so widen the binding to the open catalog shape for this call.
+  const run = binding.run.bind(binding) as (
+    model: string,
+    inputs: Record<string, unknown>,
+    options: Record<string, unknown>,
+  ) => Promise<unknown>
+  return async (_input, init) => {
+    const { model, ...inputs } = JSON.parse(
+      typeof init?.body === 'string' ? init.body : '{}',
+    ) as { model: string } & Record<string, unknown>
+    const response = (await run(model, inputs, {
+      returnRawResponse: true,
+      ...(gateway && { gateway }),
+    })) as Response
+    return await normalizeResponse(response)
+  }
+}
+
+/** Wraps a REST fetch so responses get the same error and trailer fixes. */
+export function createRestFetch(baseFetch: FetchLike | undefined): FetchLike {
+  const fetchImpl = baseFetch ?? fetch
+  return async (input, init) =>
+    await normalizeResponse(await fetchImpl(input, init))
+}

package/src/utils/models.ts

@@ -0,0 +1,41 @@
+import type {
+  AiModels,
+  BaseAiAutomaticSpeechRecognition,
+  BaseAiTextEmbeddings,
+  BaseAiTextGeneration,
+  BaseAiTextToImage,
+  BaseAiTextToSpeech,
+} from '@cloudflare/workers-types'
+
+/** Model ids from the Workers AI catalog whose task shape matches `TTask`. */
+type ModelsFor<TTask> = {
+  [K in keyof AiModels]: AiModels[K] extends TTask ? K : never
+}[keyof AiModels]
+
+/**
+ * Chat model id. Catalog ids get autocomplete; any other id works too,
+ * including third-party `provider/model` ids routed through AI Gateway
+ * (for example `openai/gpt-5.5`).
+ */
+export type CloudflareTextModel =
+  | ModelsFor<BaseAiTextGeneration>
+  | (string & {})
+
+export type CloudflareEmbeddingModel =
+  | ModelsFor<BaseAiTextEmbeddings>
+  | (string & {})
+
+export type CloudflareImageModel = ModelsFor<BaseAiTextToImage> | (string & {})
+
+export type CloudflareTTSModel =
+  | ModelsFor<BaseAiTextToSpeech>
+  | '@cf/deepgram/aura-1'
+  | '@cf/deepgram/aura-2-en'
+  | '@cf/deepgram/aura-2-es'
+  | (string & {})
+
+export type CloudflareTranscriptionModel =
+  | ModelsFor<BaseAiAutomaticSpeechRecognition>
+  | '@cf/openai/whisper-large-v3-turbo'
+  | '@cf/deepgram/nova-3'
+  | (string & {})

package/src/utils/run.ts

@@ -0,0 +1,116 @@
+import { arrayBufferToBase64 } from '@tanstack/ai-utils'
+import { CLOUDFLARE_API_BASE, gatewayHeaders, isBindingConfig } from './config'
+import type { CloudflareConfig } from './config'
+
+export type RunInputs = Record<string, unknown>
+
+export interface RunBinary {
+  /** Input field that carries the bytes on the binding path. */
+  field: string
+  body: Uint8Array | ArrayBuffer | Blob
+  contentType: string
+}
+
+/**
+ * Runs a Workers AI model with its native task inputs (embeddings, image,
+ * speech, transcription) through the binding or the REST `/ai/run` endpoint.
+ *
+ * Returns the model's decoded output: an object for JSON tasks, or bytes
+ * (`Uint8Array` / `ReadableStream`) for binary media outputs.
+ */
+export async function runModel(
+  config: CloudflareConfig,
+  model: string,
+  inputs: RunInputs,
+  options?: { signal?: AbortSignal; binary?: RunBinary },
+): Promise<unknown> {
+  if (isBindingConfig(config)) {
+    const run = config.binding.run.bind(config.binding) as (
+      model: string,
+      inputs: RunInputs,
+      options?: Record<string, unknown>,
+    ) => Promise<unknown>
+    // The binding serializes inputs as JSON; binary bodies must travel as a
+    // ReadableStream, which it forwards as the raw request body.
+    const bindingInputs = options?.binary
+      ? {
+          ...inputs,
+          [options.binary.field]: {
+            body: new Response(options.binary.body as BodyInit).body,
+            contentType: options.binary.contentType,
+          },
+        }
+      : inputs
+    return await run(
+      model,
+      bindingInputs,
+      config.gateway ? { gateway: config.gateway } : undefined,
+    )
+  }
+
+  const url = new URL(
+    `${CLOUDFLARE_API_BASE}/accounts/${config.accountId}/ai/run/${model}`,
+  )
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${config.apiKey}`,
+    ...gatewayHeaders(config.gateway),
+  }
+  let body: BodyInit
+  if (options?.binary) {
+    // Binary tasks take the bytes as the body and the other inputs as query.
+    for (const [key, value] of Object.entries(inputs)) {
+      if (value !== undefined) url.searchParams.set(key, String(value))
+    }
+    headers['Content-Type'] = options.binary.contentType
+    body = options.binary.body as BodyInit
+  } else {
+    headers['Content-Type'] = 'application/json'
+    body = JSON.stringify(inputs)
+  }
+  const fetchImpl = config.fetch ?? fetch
+  const response = await fetchImpl(url, {
+    method: 'POST',
+    headers,
+    body,
+    signal: options?.signal,
+  })
+  if (!response.ok) {
+    throw new Error(
+      `Workers AI request for ${model} failed (${response.status}): ${await response.text()}`,
+    )
+  }
+  if (response.headers.get('content-type')?.includes('application/json')) {
+    const json = (await response.json()) as {
+      success?: boolean
+      result?: unknown
+      errors?: Array<{ message?: string }>
+    }
+    if (json.success === false) {
+      throw new Error(
+        `Workers AI request for ${model} failed: ${json.errors?.map((e) => e.message).join('; ')}`,
+      )
+    }
+    return 'result' in json ? json.result : json
+  }
+  return new Uint8Array(await response.arrayBuffer())
+}
+
+/** Base64-encodes a binary model output, whatever shape it arrived in. */
+export async function outputToBase64(output: unknown): Promise<string> {
+  if (typeof output === 'string') return output
+  let bytes: Uint8Array
+  if (output instanceof Uint8Array) {
+    bytes = output
+  } else if (output instanceof ArrayBuffer) {
+    bytes = new Uint8Array(output)
+  } else if (output instanceof ReadableStream) {
+    bytes = new Uint8Array(await new Response(output).arrayBuffer())
+  } else {
+    throw new Error(
+      `Unexpected Workers AI output type: ${Object.prototype.toString.call(output)}`,
+    )
+  }
+  const copy = new Uint8Array(bytes.byteLength)
+  copy.set(bytes)
+  return arrayBufferToBase64(copy.buffer)
+}