@tanstack/ai-lovable 0.0.0

package/src/adapters/transcription.ts

@@ -0,0 +1,226 @@
+import OpenAI from 'openai'
+import { BaseTranscriptionAdapter } from '@tanstack/ai/adapters'
+import { toRunErrorPayload } from '@tanstack/ai/adapter-internals'
+import { base64ToArrayBuffer, generateId } from '@tanstack/ai-utils'
+import {
+  getLovableApiKeyFromEnv,
+  openaiRequestOptions,
+  withLovableDefaults,
+} from '../utils/client'
+import type {
+  TokenUsage,
+  TranscriptionOptions,
+  TranscriptionResponseFormat,
+  TranscriptionResult,
+} from '@tanstack/ai'
+import type OpenAI_SDK from 'openai'
+import type { LovableTranscriptionModel } from '../model-meta'
+import type { LovableTranscriptionProviderOptions } from '../audio/transcription-provider-options'
+import type { LovableClientConfig } from '../utils/client'
+
+function isPlainFormat(format: string): format is 'json' | 'text' {
+  return format === 'json' || format === 'text'
+}
+
+export interface LovableTranscriptionConfig extends LovableClientConfig {}
+
+function buildTranscriptionUsage(
+  response?: OpenAI_SDK.Audio.TranscriptionCreateResponse,
+): TokenUsage | undefined {
+  const usage = response?.usage
+  if (!usage || usage.type === 'duration') {
+    return undefined
+  }
+
+  const result: TokenUsage = {
+    promptTokens: usage.input_tokens || 0,
+    completionTokens: usage.output_tokens || 0,
+    totalTokens: usage.total_tokens || 0,
+  }
+
+  const inputDetails = usage.input_token_details
+  const promptTokensDetails = {
+    ...(inputDetails?.audio_tokens
+      ? { audioTokens: inputDetails.audio_tokens }
+      : {}),
+    ...(inputDetails?.text_tokens
+      ? { textTokens: inputDetails.text_tokens }
+      : {}),
+  }
+  if (Object.keys(promptTokensDetails).length > 0) {
+    result.promptTokensDetails = promptTokensDetails
+  }
+  if (usage.output_tokens) {
+    result.completionTokensDetails = { textTokens: usage.output_tokens }
+  }
+
+  return result
+}
+
+export class LovableTranscriptionAdapter<
+  TModel extends LovableTranscriptionModel,
+> extends BaseTranscriptionAdapter<
+  TModel,
+  LovableTranscriptionProviderOptions
+> {
+  readonly name = 'lovable' as const
+
+  protected client: OpenAI
+
+  constructor(config: LovableTranscriptionConfig, model: TModel) {
+    super(model, {})
+    this.client = new OpenAI(withLovableDefaults(config))
+  }
+
+  async transcribe(
+    options: TranscriptionOptions<LovableTranscriptionProviderOptions>,
+  ): Promise<TranscriptionResult> {
+    const { model, language } = options
+
+    try {
+      const request = this.buildTranscriptionRequest(options)
+
+      options.logger.request(
+        `activity=transcription provider=${this.name} model=${model}`,
+        { provider: this.name, model },
+      )
+
+      const response = await this.client.audio.transcriptions.create(
+        request,
+        openaiRequestOptions(options.abortSignal),
+      )
+      const usage =
+        typeof response === 'string'
+          ? undefined
+          : buildTranscriptionUsage(response)
+
+      return {
+        id: generateId(this.name),
+        model,
+        text: typeof response === 'string' ? response : response.text,
+        ...(language !== undefined && { language }),
+        ...(usage !== undefined && { usage }),
+      }
+    } catch (error: unknown) {
+      options.logger.errors(`${this.name}.transcribe fatal`, {
+        error: toRunErrorPayload(error, `${this.name}.transcribe failed`),
+        source: `${this.name}.transcribe`,
+      })
+      throw error
+    }
+  }
+
+  private buildTranscriptionRequest(
+    options: TranscriptionOptions<LovableTranscriptionProviderOptions>,
+  ): OpenAI_SDK.Audio.TranscriptionCreateParamsNonStreaming {
+    const { model, audio, language, prompt, responseFormat, modelOptions } =
+      options
+    const file = this.prepareAudioFile(audio)
+    const topLevelResponseFormat = responseFormat
+    const effectiveResponseFormat =
+      topLevelResponseFormat ?? modelOptions?.response_format
+
+    if (
+      topLevelResponseFormat !== undefined &&
+      modelOptions?.response_format !== undefined &&
+      topLevelResponseFormat !== modelOptions.response_format
+    ) {
+      throw new Error(
+        `Conflicting response formats: responseFormat="${topLevelResponseFormat}" and modelOptions.response_format="${modelOptions.response_format}". Provide only one.`,
+      )
+    }
+
+    if (
+      effectiveResponseFormat !== undefined &&
+      !isPlainFormat(effectiveResponseFormat)
+    ) {
+      throw new Error(
+        `lovable: model "${model}" only supports json and text response formats; received "${effectiveResponseFormat}".`,
+      )
+    }
+
+    const request: OpenAI_SDK.Audio.TranscriptionCreateParamsNonStreaming = {
+      ...modelOptions,
+      model,
+      file,
+    }
+    delete request.stream
+    if (language !== undefined) {
+      request.language = language
+    }
+    if (prompt !== undefined) {
+      request.prompt = prompt
+    }
+    request.response_format = mapResponseFormat(effectiveResponseFormat)
+
+    return request
+  }
+
+  protected prepareAudioFile(audio: string | File | Blob | ArrayBuffer): File {
+    if (typeof File !== 'undefined' && audio instanceof File) {
+      return audio
+    }
+    if (typeof Blob !== 'undefined' && audio instanceof Blob) {
+      this.ensureFileSupport()
+      return new File([audio], 'audio.mp3', {
+        type: audio.type || 'audio/mpeg',
+      })
+    }
+    if (typeof ArrayBuffer !== 'undefined' && audio instanceof ArrayBuffer) {
+      this.ensureFileSupport()
+      return new File([audio], 'audio.mp3', { type: 'audio/mpeg' })
+    }
+    if (typeof audio === 'string') {
+      this.ensureFileSupport()
+
+      if (audio.startsWith('data:')) {
+        const parts = audio.split(',')
+        const header = parts[0]
+        const base64Data = parts[1] || ''
+        const mimeMatch = header?.match(/data:([^;]+)/)
+        const mimeType = mimeMatch?.[1] || 'audio/mpeg'
+        const bytes = base64ToArrayBuffer(base64Data)
+        const extension = mimeType.split('/')[1] || 'mp3'
+        return new File([bytes], `audio.${extension}`, { type: mimeType })
+      }
+
+      const bytes = base64ToArrayBuffer(audio)
+      return new File([bytes], 'audio.mp3', { type: 'audio/mpeg' })
+    }
+
+    throw new Error('Invalid audio input type')
+  }
+
+  private ensureFileSupport(): void {
+    if (typeof File === 'undefined') {
+      throw new Error(
+        '`File` is not available in this environment. ' +
+          'Use Node.js 20 or newer, or pass a File object directly.',
+      )
+    }
+  }
+}
+
+function mapResponseFormat(
+  format?: TranscriptionResponseFormat,
+): 'json' | 'text' {
+  if (format === 'text') return 'text'
+  return 'json'
+}
+
+export function createLovableTranscription<
+  TModel extends LovableTranscriptionModel,
+>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<LovableTranscriptionConfig, 'apiKey'>,
+): LovableTranscriptionAdapter<TModel> {
+  return new LovableTranscriptionAdapter({ apiKey, ...config }, model)
+}
+
+export function lovableTranscription<TModel extends LovableTranscriptionModel>(
+  model: TModel,
+  config?: Omit<LovableTranscriptionConfig, 'apiKey'>,
+): LovableTranscriptionAdapter<TModel> {
+  return createLovableTranscription(model, getLovableApiKeyFromEnv(), config)
+}

package/src/adapters/tts.ts

@@ -0,0 +1,97 @@
+import OpenAI from 'openai'
+import { BaseTTSAdapter } from '@tanstack/ai/adapters'
+import { toRunErrorPayload } from '@tanstack/ai/adapter-internals'
+import { arrayBufferToBase64, generateId } from '@tanstack/ai-utils'
+import {
+  getLovableApiKeyFromEnv,
+  openaiRequestOptions,
+  withLovableDefaults,
+} from '../utils/client'
+import type { TTSOptions, TTSResult } from '@tanstack/ai'
+import type OpenAI_SDK from 'openai'
+import type { LovableTTSModel } from '../model-meta'
+import type { LovableTTSProviderOptions } from '../audio/tts-provider-options'
+import type { LovableClientConfig } from '../utils/client'
+
+export interface LovableTTSConfig extends LovableClientConfig {}
+
+const CONTENT_TYPES: Record<string, string> = {
+  mp3: 'audio/mpeg',
+  opus: 'audio/opus',
+  aac: 'audio/aac',
+  flac: 'audio/flac',
+  wav: 'audio/wav',
+  pcm: 'audio/pcm',
+}
+
+export class LovableTTSAdapter<
+  TModel extends LovableTTSModel,
+> extends BaseTTSAdapter<TModel, LovableTTSProviderOptions> {
+  readonly name = 'lovable' as const
+
+  protected client: OpenAI
+
+  constructor(config: LovableTTSConfig, model: TModel) {
+    super(model, {})
+    this.client = new OpenAI(withLovableDefaults(config))
+  }
+
+  async generateSpeech(
+    options: TTSOptions<LovableTTSProviderOptions>,
+  ): Promise<TTSResult> {
+    const { model, text, voice, format, speed, modelOptions } = options
+
+    const request: OpenAI_SDK.Audio.SpeechCreateParams = {
+      model,
+      input: text,
+      voice: voice || 'alloy',
+      response_format: format,
+      ...(speed !== undefined && { speed }),
+      ...(modelOptions ?? {}),
+    }
+
+    try {
+      options.logger.request(
+        `activity=tts provider=${this.name} model=${model} format=${request.response_format ?? 'default'} voice=${request.voice}`,
+        { provider: this.name, model },
+      )
+      const response = await this.client.audio.speech.create(
+        request,
+        openaiRequestOptions(options.abortSignal),
+      )
+      const arrayBuffer = await response.arrayBuffer()
+      const base64 = arrayBufferToBase64(arrayBuffer)
+      const outputFormat = (request.response_format as string) || 'mp3'
+      const contentType = CONTENT_TYPES[outputFormat] || 'audio/mpeg'
+
+      return {
+        id: generateId(this.name),
+        model,
+        audio: base64,
+        format: outputFormat,
+        contentType,
+      }
+    } catch (error: unknown) {
+      options.logger.errors(`${this.name}.generateSpeech fatal`, {
+        error: toRunErrorPayload(error, `${this.name}.generateSpeech failed`),
+        source: `${this.name}.generateSpeech`,
+      })
+      throw error
+    }
+  }
+}
+
+export function createLovableSpeech<TModel extends LovableTTSModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<LovableTTSConfig, 'apiKey'>,
+): LovableTTSAdapter<TModel> {
+  return new LovableTTSAdapter({ apiKey, ...config }, model)
+}
+
+export function lovableSpeech<TModel extends LovableTTSModel>(
+  model: TModel,
+  config?: Omit<LovableTTSConfig, 'apiKey'>,
+): LovableTTSAdapter<TModel> {
+  return createLovableSpeech(model, getLovableApiKeyFromEnv(), config)
+}

package/src/adapters/video.ts

@@ -0,0 +1,303 @@
+import OpenAI from 'openai'
+import { resolveMediaPrompt } from '@tanstack/ai'
+import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters'
+import { toRunErrorPayload } from '@tanstack/ai/adapter-internals'
+import { arrayBufferToBase64 } from '@tanstack/ai-utils'
+import {
+  getLovableApiKeyFromEnv,
+  openaiRequestOptions,
+  withLovableDefaults,
+} from '../utils/client'
+import { imagePartToFile } from '../image/image-input-to-file'
+import {
+  toApiSeconds,
+  validateHighResDuration,
+  validateVideoSeconds,
+  validateVideoSize,
+} from '../video/video-provider-options'
+import type { DurationOptions } from '@tanstack/ai/adapters'
+import type {
+  VideoGenerationOptions,
+  VideoJobResult,
+  VideoStatusResult,
+  VideoUrlResult,
+} from '@tanstack/ai'
+import type OpenAI_SDK from 'openai'
+import type {
+  LovableVideoModel,
+  LovableVideoModelDurationByName,
+  LovableVideoModelInputModalitiesByName,
+  LovableVideoModelProviderOptionsByName,
+  LovableVideoModelSizeByName,
+} from '../model-meta'
+import type {
+  LovableVideoDuration,
+  LovableVideoProviderOptions,
+} from '../video/video-provider-options'
+import type { LovableClientConfig } from '../utils/client'
+
+const LARGE_MEDIA_BUFFER_BYTES = 10 * 1024 * 1024
+const VIDEO_DURATIONS = [
+  4, 6, 8,
+] as const satisfies ReadonlyArray<LovableVideoDuration>
+
+function warnIfLargeMediaBuffer(byteLength: number, source: string): void {
+  if (byteLength <= LARGE_MEDIA_BUFFER_BYTES) return
+  console.warn(
+    `[lovable.${source}] downloaded ${(byteLength / 1024 / 1024).toFixed(1)} MiB into memory before base64 encoding. ` +
+      `Workers/serverless runtimes commonly run out of memory above ~10 MiB. ` +
+      `Consider streaming the video through a CDN or your own storage layer instead.`,
+  )
+}
+
+/**
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export interface LovableVideoConfig extends LovableClientConfig {
+  /**
+   * Opt into fetching HTTP(S) image URL inputs for `input_reference`.
+   * The endpoint requires uploaded file bytes, so an HTTP(S) URL has to be
+   * downloaded and buffered in memory. When `false` (the default), HTTP(S)
+   * URL image inputs throw.
+   */
+  allowUrlFetch?: boolean
+}
+
+/**
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export class LovableVideoAdapter<
+  TModel extends LovableVideoModel,
+> extends BaseVideoAdapter<
+  TModel,
+  LovableVideoProviderOptions,
+  LovableVideoModelProviderOptionsByName,
+  LovableVideoModelSizeByName,
+  LovableVideoModelInputModalitiesByName,
+  LovableVideoModelDurationByName
+> {
+  readonly name = 'lovable' as const
+
+  protected client: OpenAI
+  protected clientConfig: LovableVideoConfig
+
+  constructor(config: LovableVideoConfig, model: TModel) {
+    super({}, model)
+    this.clientConfig = config
+    const { allowUrlFetch: _allowUrlFetch, ...clientOptions } = config
+    this.client = new OpenAI(withLovableDefaults(clientOptions))
+  }
+
+  async createVideoJob(
+    options: VideoGenerationOptions<
+      LovableVideoProviderOptions,
+      LovableVideoModelSizeByName[TModel],
+      LovableVideoModelDurationByName[TModel]
+    >,
+  ): Promise<VideoJobResult> {
+    const { model, size, duration, modelOptions } = options
+
+    const resolvedSize = size ?? modelOptions?.size
+    validateVideoSize(model, resolvedSize)
+    const seconds = duration ?? modelOptions?.seconds
+    validateVideoSeconds(model, seconds)
+    validateHighResDuration(model, resolvedSize, seconds)
+
+    const resolved = resolveMediaPrompt(options.prompt)
+
+    if (resolved.videos.length > 0) {
+      throw new Error(
+        `${this.name}.createVideoJob does not support video prompt parts (model: ${model}).`,
+      )
+    }
+    if (resolved.audios.length > 0) {
+      throw new Error(
+        `${this.name}.createVideoJob does not support audio prompt parts (model: ${model}).`,
+      )
+    }
+    if (resolved.images.length > 1) {
+      throw new Error(
+        `${this.name}: video models accept at most one input_reference image; received ${resolved.images.length}.`,
+      )
+    }
+
+    const request: OpenAI_SDK.Videos.VideoCreateParams = {
+      model,
+      prompt: resolved.text,
+    }
+    const [inputReference] = resolved.images
+    if (inputReference) {
+      request.input_reference = await imagePartToFile(
+        inputReference,
+        'input-reference',
+        this.clientConfig.allowUrlFetch ?? false,
+        options.abortSignal,
+      )
+    }
+    if (resolvedSize) {
+      // Gateway Veo sizes include 1080p and 4K, which are not in the OpenAI SDK union.
+      request.size = resolvedSize as OpenAI_SDK.Videos.VideoSize
+    }
+    if (seconds !== undefined) {
+      const apiSeconds = toApiSeconds(seconds)
+      if (apiSeconds !== undefined) {
+        // Gateway Veo clips can be 6 seconds. The OpenAI SDK union is 4/8/12.
+        request.seconds = apiSeconds as OpenAI_SDK.Videos.VideoSeconds
+      }
+    }
+
+    try {
+      options.logger.request(
+        `activity=video.create provider=${this.name} model=${model} size=${request.size ?? 'default'} seconds=${request.seconds ?? 'default'}`,
+        { provider: this.name, model },
+      )
+      const response = await this.client.videos.create(
+        request,
+        openaiRequestOptions(options.abortSignal),
+      )
+      return { jobId: response.id, model }
+    } catch (error: unknown) {
+      options.logger.errors(`${this.name}.createVideoJob fatal`, {
+        error: toRunErrorPayload(error, `${this.name}.createVideoJob failed`),
+        source: `${this.name}.createVideoJob`,
+      })
+      throw error
+    }
+  }
+
+  async getVideoStatus(jobId: string): Promise<VideoStatusResult> {
+    try {
+      const response = await this.client.videos.retrieve(jobId)
+      return {
+        jobId,
+        status: this.mapStatus(response.status),
+        progress: response.progress,
+        ...(response.error?.message !== undefined && {
+          error: response.error.message,
+        }),
+      }
+    } catch (error: unknown) {
+      if (isHttpStatus(error, 404)) {
+        return { jobId, status: 'failed', error: 'Job not found' }
+      }
+      throw error
+    }
+  }
+
+  async getVideoUrl(jobId: string): Promise<VideoUrlResult> {
+    try {
+      const videoInfo = await this.client.videos.retrieve(jobId)
+      const directUrl = videoResourceUrl(videoInfo)
+      if (directUrl) {
+        return {
+          jobId,
+          url: directUrl,
+          ...(videoInfo.expires_at != null && {
+            expiresAt: new Date(videoInfo.expires_at * 1000),
+          }),
+        }
+      }
+
+      const contentResponse = await this.client.videos.downloadContent(jobId)
+      return dataUrlFromResponse(
+        jobId,
+        contentResponse,
+        'video.downloadContent',
+      )
+    } catch (error: unknown) {
+      if (isHttpStatus(error, 404)) {
+        throw new Error(`Video job not found: ${jobId}`)
+      }
+      if (isHttpStatus(error, 400)) {
+        throw new Error(
+          `Video is not ready for download. Check status first. Job ID: ${jobId}`,
+        )
+      }
+      throw error
+    }
+  }
+
+  override availableDurations(): DurationOptions<LovableVideoDuration> {
+    return { kind: 'discrete', values: VIDEO_DURATIONS }
+  }
+
+  override snapDuration(seconds: number): LovableVideoDuration | undefined {
+    return snapToDurationOption(seconds, this.availableDurations())
+  }
+
+  protected mapStatus(
+    apiStatus: string,
+  ): 'pending' | 'processing' | 'completed' | 'failed' {
+    switch (apiStatus) {
+      case 'queued':
+      case 'pending':
+        return 'pending'
+      case 'processing':
+      case 'in_progress':
+        return 'processing'
+      case 'completed':
+      case 'succeeded':
+        return 'completed'
+      case 'failed':
+      case 'error':
+      case 'cancelled':
+        return 'failed'
+      default:
+        return 'processing'
+    }
+  }
+}
+
+async function dataUrlFromResponse(
+  jobId: string,
+  contentResponse: Response,
+  source: string,
+): Promise<VideoUrlResult> {
+  const videoBlob = await contentResponse.blob()
+  const buffer = await videoBlob.arrayBuffer()
+  warnIfLargeMediaBuffer(buffer.byteLength, source)
+  const base64 = arrayBufferToBase64(buffer)
+  const mimeType = contentResponse.headers.get('content-type') || 'video/mp4'
+  return {
+    jobId,
+    url: `data:${mimeType};base64,${base64}`,
+  }
+}
+
+function videoResourceUrl(video: OpenAI_SDK.Videos.Video): string | undefined {
+  const extra = video as OpenAI_SDK.Videos.Video & { url?: string }
+  if (extra.url && extra.url.length > 0) {
+    return extra.url
+  }
+  return undefined
+}
+
+function isHttpStatus(error: unknown, status: number): boolean {
+  return (
+    typeof error === 'object' &&
+    error !== null &&
+    'status' in error &&
+    (error as { status: unknown }).status === status
+  )
+}
+
+/**
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export function createLovableVideo<TModel extends LovableVideoModel>(
+  model: TModel,
+  apiKey: string,
+  config?: Omit<LovableVideoConfig, 'apiKey'>,
+): LovableVideoAdapter<TModel> {
+  return new LovableVideoAdapter({ apiKey, ...config }, model)
+}
+
+/**
+ * @experimental Video generation is an experimental feature and may change.
+ */
+export function lovableVideo<TModel extends LovableVideoModel>(
+  model: TModel,
+  config?: Omit<LovableVideoConfig, 'apiKey'>,
+): LovableVideoAdapter<TModel> {
+  return createLovableVideo(model, getLovableApiKeyFromEnv(), config)
+}

package/src/audio/transcription-provider-options.ts

@@ -0,0 +1,7 @@
+export type LovableTranscriptionResponseFormat = 'json' | 'text'
+
+export interface LovableTranscriptionProviderOptions {
+  temperature?: number
+  response_format?: LovableTranscriptionResponseFormat
+  prompt?: string
+}

package/src/audio/tts-provider-options.ts

@@ -0,0 +1,21 @@
+export type LovableTTSVoice =
+  | 'alloy'
+  | 'ash'
+  | 'ballad'
+  | 'coral'
+  | 'echo'
+  | 'fable'
+  | 'onyx'
+  | 'nova'
+  | 'sage'
+  | 'shimmer'
+  | 'verse'
+
+export type LovableTTSFormat = 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm'
+
+export interface LovableTTSProviderOptions {
+  /**
+   * Extra voice direction in plain language, for example "speak slowly and warmly".
+   */
+  instructions?: string
+}

package/src/byok.ts

@@ -0,0 +1,7 @@
+import { defineByokProvider } from '@tanstack/ai/byok'
+
+export const lovableByok = defineByokProvider({
+  id: 'lovable',
+  label: 'Lovable AI Gateway',
+  env: 'LOVABLE_API_KEY',
+})

package/src/embedding/embedding-provider-options.ts

@@ -0,0 +1,9 @@
+/**
+ * Provider options for Lovable embedding models.
+ *
+ * `dimensions` is a top-level option on `embed()`. `encoding_format` is
+ * pinned to `float` so vectors are always `number[]`.
+ */
+export interface LovableEmbeddingProviderOptions {
+  user?: string
+}