@tanstack/ai-fal 0.6.17 → 0.7.0

package/src/adapters/transcription.ts

@@ -0,0 +1,169 @@
+import { fal } from '@fal-ai/client'
+import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters'
+import {
+  configureFalClient,
+  dataUrlToBlob,
+  generateId as utilGenerateId,
+} from '../utils'
+import type { OutputType, Result } from '@fal-ai/client'
+import type {
+  TranscriptionOptions,
+  TranscriptionResult,
+  TranscriptionSegment,
+} from '@tanstack/ai'
+import type { FalClientConfig } from '../utils'
+import type { FalModel, FalModelInput } from '../model-meta'
+
+/**
+ * Provider options for transcription, excluding fields TanStack AI handles.
+ */
+export type FalTranscriptionProviderOptions<TModel extends string> = Omit<
+  FalModelInput<TModel>,
+  'audio_url'
+>
+
+interface FalChunk {
+  text: string
+  timestamp?: [number, number] | null
+  speaker?: string
+}
+
+/**
+ * fal.ai transcription (speech-to-text) adapter.
+ *
+ * Supports fal.ai STT models like whisper, wizper, etc.
+ *
+ * @example
+ * ```typescript
+ * const adapter = falTranscription('fal-ai/whisper')
+ * const result = await generateTranscription({
+ *   adapter,
+ *   audio: 'https://example.com/audio.mp3',
+ *   language: 'en',
+ * })
+ * ```
+ */
+export class FalTranscriptionAdapter<
+  TModel extends FalModel,
+> extends BaseTranscriptionAdapter<
+  TModel,
+  FalTranscriptionProviderOptions<TModel>
+> {
+  readonly name = 'fal' as const
+
+  constructor(model: TModel, config?: FalClientConfig) {
+    super(model, {})
+    configureFalClient(config)
+  }
+
+  async transcribe(
+    options: TranscriptionOptions<FalTranscriptionProviderOptions<TModel>>,
+  ): Promise<TranscriptionResult> {
+    const { logger } = options
+    logger.request(
+      `activity=generateTranscription provider=fal model=${this.model}`,
+      {
+        provider: 'fal',
+        model: this.model,
+      },
+    )
+    try {
+      const input = this.buildInput(options)
+      const result = await fal.subscribe(this.model, { input })
+      return this.transformResponse(result)
+    } catch (error) {
+      logger.errors('fal.generateTranscription fatal', {
+        error,
+        source: 'fal.generateTranscription',
+      })
+      throw error
+    }
+  }
+
+  private buildInput(
+    options: TranscriptionOptions<FalTranscriptionProviderOptions<TModel>>,
+  ): FalModelInput<TModel> {
+    // fal-client auto-uploads Blob/File inputs via fal.storage.upload, but
+    // passes strings through unchanged — so a `data:` URL would reach fal's
+    // API and get rejected with a 422 "Unsupported data URL". Decode data
+    // URLs to a Blob up front so the auto-upload path handles them.
+    let audioInput: string | Blob | File
+    if (options.audio instanceof ArrayBuffer) {
+      audioInput = new Blob([options.audio])
+    } else if (typeof options.audio === 'string') {
+      audioInput = dataUrlToBlob(options.audio) ?? options.audio
+    } else {
+      audioInput = options.audio
+    }
+
+    const input = {
+      ...options.modelOptions,
+      audio_url: audioInput,
+      ...(options.language ? { language: options.language } : {}),
+      ...(options.prompt ? { prompt: options.prompt } : {}),
+    } as FalModelInput<TModel>
+    return input
+  }
+
+  protected override generateId(): string {
+    return utilGenerateId(this.name)
+  }
+
+  private transformResponse(
+    response: Result<OutputType<TModel>>,
+  ): TranscriptionResult {
+    const data = response.data as Record<string, unknown>
+
+    const text = (data.text as string) || ''
+
+    // Map fal chunks to TanStack segments. fal whisper can return
+    // `timestamp: null` on some chunks (e.g. when word-level timing is
+    // disabled), and the runtime payload is not actually constrained to a
+    // 2-tuple — treat it as unknown and validate before indexing.
+    let segments: Array<TranscriptionSegment> | undefined
+    const chunks = data.chunks as Array<FalChunk> | undefined
+    if (chunks && Array.isArray(chunks)) {
+      segments = chunks.flatMap((chunk, index) => {
+        const ts = chunk.timestamp as unknown
+        if (
+          !Array.isArray(ts) ||
+          ts.length < 2 ||
+          typeof ts[0] !== 'number' ||
+          typeof ts[1] !== 'number'
+        ) {
+          return []
+        }
+        return [
+          {
+            id: index,
+            start: ts[0],
+            end: ts[1],
+            text: chunk.text,
+            ...(chunk.speaker ? { speaker: chunk.speaker } : {}),
+          },
+        ]
+      })
+    }
+
+    // Extract language from response
+    const language =
+      (data.language as string) ||
+      (data.inferred_languages as Array<string> | undefined)?.[0] ||
+      (data.languages as Array<string> | undefined)?.[0]
+
+    return {
+      id: response.requestId || this.generateId(),
+      model: this.model,
+      text,
+      language,
+      segments,
+    }
+  }
+}
+
+export function falTranscription<TModel extends FalModel>(
+  model: TModel,
+  config?: FalClientConfig,
+): FalTranscriptionAdapter<TModel> {
+  return new FalTranscriptionAdapter(model, config)
+}

package/src/image/image-provider-options.ts

@@ -8,13 +8,17 @@ export function mapSizeToFalFormat<TModel extends string>(
   // "16:9_4K" → { aspect_ratio, resolution }
   // "16:9"    → { aspect_ratio }
   // no match  → { image_size }
-  const match = size.match(/^(\d+:\d+)(?:_(.+))?$/)
+  if (typeof size === 'string') {
+    const match = size.match(/^(\d+:\d+)(?:_(.+))?$/)
+    if (match) {
+      return {
+        aspect_ratio: match[1],
+        ...(match[2] ? { resolution: match[2] } : {}),
+      } as FalModelImageSizeInput<TModel>
+    }
+  }
 
   return {
     image_size: size,
-    aspect_ratio: match?.[1],
-    ...(match?.[2] && {
-      resolution: match?.[2],
-    }),
-  }
+  } as FalModelImageSizeInput<TModel>
 }

package/src/index.ts

@@ -10,6 +10,27 @@ export { FalImageAdapter, falImage } from './adapters/image'
 
 export { FalVideoAdapter, falVideo } from './adapters/video'
 
+// ============================================================================
+// Speech Adapter (TTS)
+// ============================================================================
+
+export { FalSpeechAdapter, falSpeech } from './adapters/speech'
+
+// ============================================================================
+// Transcription Adapter (STT)
+// ============================================================================
+
+export {
+  FalTranscriptionAdapter,
+  falTranscription,
+} from './adapters/transcription'
+
+// ============================================================================
+// Audio Adapter (Music/Sound Generation)
+// ============================================================================
+
+export { FalAudioAdapter, falAudio } from './adapters/audio'
+
 // ============================================================================
 // Model Types (from fal.ai's type system)
 // ============================================================================
@@ -17,6 +38,9 @@ export { FalVideoAdapter, falVideo } from './adapters/video'
 export {
   type FalImageProviderOptions,
   type FalVideoProviderOptions,
+  type FalSpeechProviderOptions,
+  type FalTranscriptionProviderOptions,
+  type FalAudioProviderOptions,
   type FalModel,
   type FalModelInput,
   type FalModelOutput,

package/src/model-meta.ts

@@ -119,3 +119,30 @@ export type FalVideoProviderOptions<TModel extends string> =
   TModel extends keyof EndpointTypeMap
     ? Omit<FalModelInput<TModel>, 'prompt'>
     : Record<string, any>
+
+/**
+ * Provider options for TTS, excluding fields TanStack AI handles.
+ * Use this for the `modelOptions` parameter in speech generation.
+ */
+export type FalSpeechProviderOptions<TModel extends string> = Omit<
+  FalModelInput<TModel>,
+  'prompt' | 'text'
+>
+
+/**
+ * Provider options for transcription, excluding fields TanStack AI handles.
+ * Use this for the `modelOptions` parameter in transcription.
+ */
+export type FalTranscriptionProviderOptions<TModel extends string> = Omit<
+  FalModelInput<TModel>,
+  'audio_url'
+>
+
+/**
+ * Provider options for audio generation, excluding fields TanStack AI handles.
+ * Use this for the `modelOptions` parameter in audio generation.
+ */
+export type FalAudioProviderOptions<TModel extends string> = Omit<
+  FalModelInput<TModel>,
+  'prompt'
+>

package/src/utils/client.ts

@@ -40,21 +40,129 @@ export function getFalApiKeyFromEnv(): string {
 }
 
 export function configureFalClient(config?: FalClientConfig): void {
-  if (config?.proxyUrl) {
-    fal.config({
-      proxyUrl: config.proxyUrl,
-    })
-  } else {
-    const apiKey = config?.apiKey ?? getFalApiKeyFromEnv()
-    if (!apiKey) {
-      throw new Error('API key is required')
+  const apiKey = config?.apiKey ?? getFalApiKeyFromEnv()
+  fal.config({
+    credentials: apiKey,
+    ...(config?.proxyUrl ? { proxyUrl: config.proxyUrl } : {}),
+  })
+}
+
+export function generateId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).substring(2)}`
+}
+
+/**
+ * Extract a safe file extension from a URL. Strips query strings, URL
+ * fragments, and any trailing slashes, and only returns the extension when
+ * it looks like a real one (2-5 alphanumeric chars). Returns undefined
+ * otherwise so callers can fall back to a default.
+ */
+export function extractUrlExtension(url: string): string | undefined {
+  // Parse via URL when possible so we only look at the pathname and never
+  // mistake a TLD (e.g. the `.com` in `https://x.com/`) for a file extension.
+  let pathname: string
+  try {
+    const parsed = new URL(url)
+    pathname = parsed.pathname
+  } catch {
+    // Fall back to treating the input as a raw path when URL parsing fails
+    // (e.g. the caller passed a bare path). Still strip ?query and #fragment.
+    pathname = url.split('?')[0]!.split('#')[0]!
+  }
+  // Drop trailing slashes so `/path/audio.mp3/` still yields `mp3`.
+  const normalized = pathname.replace(/\/+$/, '')
+  // Require at least one `/` — otherwise we're looking at an empty pathname
+  // (bare-host URLs like `https://x.com/` land here after stripping the
+  // trailing slash).
+  if (!normalized.includes('/')) return undefined
+  const lastSegment = normalized.split('/').pop()
+  if (!lastSegment) return undefined
+  const extension = lastSegment.split('.').pop()
+  if (!extension || extension === lastSegment) return undefined
+  return /^[a-z0-9]{2,5}$/i.test(extension) ? extension : undefined
+}
+
+/**
+ * Derive a reasonable audio content-type. Prefers the explicit MIME (stripped
+ * of parameters), then an extension-based lookup for common audio formats,
+ * otherwise falls back to audio/mpeg — fal URLs virtually always serve mp3.
+ */
+export function deriveAudioContentType(
+  explicitContentType: string | undefined,
+  url: string,
+): string {
+  const stripped = explicitContentType?.split(';')[0]?.trim()
+  if (stripped) return stripped
+
+  const ext = extractUrlExtension(url)?.toLowerCase()
+  switch (ext) {
+    case 'mp3':
+      return 'audio/mpeg'
+    case 'wav':
+      return 'audio/wav'
+    case 'ogg':
+    case 'oga':
+      return 'audio/ogg'
+    case 'flac':
+      return 'audio/flac'
+    case 'aac':
+      return 'audio/aac'
+    case 'm4a':
+    case 'mp4':
+      return 'audio/mp4'
+    case 'webm':
+      return 'audio/webm'
+    default:
+      return 'audio/mpeg'
+  }
+}
+
+/**
+ * Decode a `data:` URL into a Blob so fal-client can auto-upload it via
+ * `fal.storage.upload`. fal's inference API rejects data URLs with a 422
+ * "Unsupported data URL", so we convert them before handing them off.
+ *
+ * Supports both base64 and URL-encoded data URLs. Returns `undefined` for
+ * anything that isn't a data URL, so callers can fall through to other
+ * handling (http URLs are passed to fal as-is).
+ */
+export function dataUrlToBlob(value: string): Blob | undefined {
+  if (!value.startsWith('data:')) return undefined
+  const commaIndex = value.indexOf(',')
+  if (commaIndex === -1) return undefined
+
+  const header = value.slice(5, commaIndex)
+  const payload = value.slice(commaIndex + 1)
+  const isBase64 = /;base64$/i.test(header)
+  const mimeType = header.split(';')[0] || 'application/octet-stream'
+
+  if (isBase64) {
+    const binary = atob(payload)
+    const bytes = new Uint8Array(binary.length)
+    for (let i = 0; i < binary.length; i += 1) {
+      bytes[i] = binary.charCodeAt(i)
     }
-    fal.config({
-      credentials: apiKey,
-    })
+    return new Blob([bytes], { type: mimeType })
   }
+
+  return new Blob([decodeURIComponent(payload)], { type: mimeType })
 }
 
-export function generateId(prefix: string): string {
-  return `${prefix}-${Date.now()}-${Math.random().toString(36).substring(7)}`
+/**
+ * Convert an ArrayBuffer to base64 in a cross-runtime way.
+ *
+ * The naive `btoa(String.fromCharCode(...bytes))` form blows up V8's argument
+ * limit (~65k) on realistic audio payloads, so we either use `Buffer` (Node /
+ * Bun) or walk the byte array in a single loop (browser).
+ */
+export function arrayBufferToBase64(bytes: ArrayBuffer): string {
+  if (typeof Buffer !== 'undefined' && typeof Buffer.from === 'function') {
+    return Buffer.from(bytes).toString('base64')
+  }
+  const view = new Uint8Array(bytes)
+  let binary = ''
+  for (let i = 0; i < view.byteLength; i += 1) {
+    binary += String.fromCharCode(view[i]!)
+  }
+  return btoa(binary)
 }

package/src/utils/index.ts

@@ -2,5 +2,9 @@ export {
   getFalApiKeyFromEnv,
   configureFalClient,
   generateId,
+  arrayBufferToBase64,
+  dataUrlToBlob,
+  extractUrlExtension,
+  deriveAudioContentType,
   type FalClientConfig,
 } from './client'