@tanstack/ai-remix 0.0.0

package/src/create-transcription.ts

@@ -0,0 +1,105 @@
+import { createGeneration } from './create-generation.ts'
+import { reconstructTranscriptionResult } from '@tanstack/ai-client'
+import type { Handle } from 'remix/ui'
+import type { TranscriptionResult } from '@tanstack/ai'
+import type {
+  GenerationPersistenceOptions,
+  TranscriptionGenerateInput,
+} from '@tanstack/ai-client'
+import type {
+  CreateGenerationOptions,
+  CreateGenerationReturn,
+} from './create-generation.ts'
+
+/**
+ * Options for the createTranscription helper.
+ *
+ * Handle is the first argument of the helper. It is not part of this type.
+ *
+ * @template TOutput - The output type after optional transform (defaults to TranscriptionResult)
+ */
+export type CreateTranscriptionOptions<TOutput = TranscriptionResult> = Omit<
+  CreateGenerationOptions<
+    TranscriptionGenerateInput,
+    TranscriptionResult,
+    TOutput
+  >,
+  'onResult' | 'reconstructResult'
+> & {
+  onResult?: (result: TranscriptionResult) => TOutput | null | void
+}
+
+/**
+ * Return type for the createTranscription helper.
+ *
+ * @template TOutput - The output type (after optional transform)
+ */
+export type CreateTranscriptionReturn<TOutput = TranscriptionResult> =
+  CreateGenerationReturn<TOutput, TranscriptionGenerateInput>
+
+/**
+ * Creates an audio transcription helper for Remix setup.
+ *
+ * Call this in a Remix component setup function. Pass the component Handle as
+ * the first argument.
+ *
+ * @example
+ * ```tsx
+ * import { createTranscription } from '@tanstack/ai-remix'
+ * import { fetchServerSentEvents } from '@tanstack/ai-client'
+ * import type { Handle } from 'remix/ui'
+ *
+ * function Transcriber(handle: Handle) {
+ *   const transcription = createTranscription(handle, {
+ *     connection: fetchServerSentEvents('/api/transcribe'),
+ *   })
+ *
+ *   return () => (
+ *     <div>
+ *       <input
+ *         type="file"
+ *         accept="audio/*"
+ *         onChange={(event) => {
+ *           const file = event.currentTarget.files?.[0]
+ *           if (!file) return
+ *           const reader = new FileReader()
+ *           reader.onload = () => {
+ *             const audio = reader.result
+ *             if (typeof audio === 'string') {
+ *               transcription.generate({ audio, language: 'en' })
+ *             }
+ *           }
+ *           reader.readAsDataURL(file)
+ *         }}
+ *       />
+ *       {transcription.isLoading ? <p>Transcribing...</p> : null}
+ *       {transcription.result ? <p>{transcription.result.text}</p> : null}
+ *     </div>
+ *   )
+ * }
+ * ```
+ */
+export function createTranscription<TTransformed = void>(
+  handle: Pick<Handle, 'id' | 'update' | 'signal'>,
+  options: Omit<
+    CreateTranscriptionOptions,
+    'onResult' | 'persistence' | 'threadId'
+  > & {
+    onResult?: (result: TranscriptionResult) => TTransformed
+  } & GenerationPersistenceOptions,
+) {
+  const devtools = {
+    ...options.devtools,
+    hookName: 'createTranscription',
+    outputKind: 'text' as const,
+  }
+  return createGeneration<
+    TranscriptionGenerateInput,
+    TranscriptionResult,
+    TTransformed
+  >(handle, {
+    ...options,
+    devtools,
+    reconstructResult: reconstructTranscriptionResult,
+  })
+}

package/src/index.ts

@@ -0,0 +1,92 @@
+export { createChat } from './create-chat.ts'
+export { createByok } from './create-byok.ts'
+export { createRealtimeChat } from './create-realtime-chat.ts'
+export { createMcpAppBridge } from './create-mcp-app-bridge.ts'
+export type { CreateMcpAppBridgeOptions } from './create-mcp-app-bridge.ts'
+export type {
+  DeepPartial,
+  CreateChatOptions,
+  CreateChatReturn,
+  UIMessage,
+  ChatRequestBody,
+  QueuedMessage,
+  SendMessageOptions,
+  WhenBusy,
+  QueueConfig,
+  QueueStrategy,
+  QueueOption,
+} from './types.ts'
+export type {
+  CreateRealtimeChatOptions,
+  CreateRealtimeChatReturn,
+} from './realtime-types.ts'
+
+export { createGeneration } from './create-generation.ts'
+export type {
+  CreateGenerationOptions,
+  CreateGenerationReturn,
+} from './create-generation.ts'
+export { createGenerateImage } from './create-generate-image.ts'
+export type {
+  CreateGenerateImageOptions,
+  CreateGenerateImageReturn,
+} from './create-generate-image.ts'
+export { createGenerateAudio } from './create-generate-audio.ts'
+export type {
+  CreateGenerateAudioOptions,
+  CreateGenerateAudioReturn,
+} from './create-generate-audio.ts'
+export { createGenerateSpeech } from './create-generate-speech.ts'
+export type {
+  CreateGenerateSpeechOptions,
+  CreateGenerateSpeechReturn,
+} from './create-generate-speech.ts'
+export { createTranscription } from './create-transcription.ts'
+export type {
+  CreateTranscriptionOptions,
+  CreateTranscriptionReturn,
+} from './create-transcription.ts'
+export { createSummarize } from './create-summarize.ts'
+export type {
+  CreateSummarizeOptions,
+  CreateSummarizeReturn,
+} from './create-summarize.ts'
+export { createGenerateVideo } from './create-generate-video.ts'
+export type {
+  CreateGenerateVideoOptions,
+  CreateGenerateVideoReturn,
+} from './create-generate-video.ts'
+export { createAudioRecorder } from './create-audio-recorder.ts'
+export type { CreateAudioRecorderOptions } from './create-audio-recorder.ts'
+
+// Re-export from ai-client for convenience (mirror octane index.ts).
+// createMcpAppBridge / CreateMcpAppBridgeOptions come from ./create-mcp-app-bridge.
+export {
+  fetchServerSentEvents,
+  fetchHttpStream,
+  xhrServerSentEvents,
+  xhrHttpStream,
+  stream,
+  rpcStream,
+  createChatClientOptions,
+  type McpAppBridge,
+  type ChatFetcher,
+  type ChatFetcherInput,
+  type ChatFetcherOptions,
+  type ConnectionAdapter,
+  type ConnectConnectionAdapter,
+  type SubscribeConnectionAdapter,
+  type RunAgentInputContext,
+  type FetchConnectionOptions,
+  type XhrConnectionOptions,
+  type InferChatMessages,
+  type GenerationClientState,
+  type ImageGenerateInput,
+  type AudioGenerateInput,
+  type SpeechGenerateInput,
+  type TranscriptionGenerateInput,
+  type SummarizeGenerateInput,
+  type VideoGenerateInput,
+  type VideoGenerateResult,
+  type VideoStatusInfo,
+} from '@tanstack/ai-client'

package/src/realtime-types.ts

@@ -0,0 +1,150 @@
+import type {
+  AnyClientTool,
+  RealtimeMessage,
+  RealtimeMode,
+  RealtimeSessionConfig,
+  RealtimeStatus,
+  RealtimeToken,
+  UsageInfo,
+} from '@tanstack/ai'
+import type { RealtimeAdapter } from '@tanstack/ai-client'
+
+/**
+ * Options for the createRealtimeChat helper.
+ *
+ * Called in Remix setup with Handle. Handle is not part of this type.
+ */
+export interface CreateRealtimeChatOptions {
+  /**
+   * Function to fetch a realtime token from the server.
+   * Called on connect and when token needs refresh.
+   */
+  getToken: () => Promise<RealtimeToken>
+
+  /**
+   * The realtime adapter to use (e.g., openaiRealtime())
+   */
+  adapter: RealtimeAdapter
+
+  /**
+   * Client-side tools with execution logic
+   */
+  tools?: ReadonlyArray<AnyClientTool>
+
+  /**
+   * Auto-play assistant audio (default: true)
+   */
+  autoPlayback?: boolean
+
+  /**
+   * Request microphone access on connect (default: true)
+   */
+  autoCapture?: boolean
+
+  /**
+   * System instructions for the assistant
+   */
+  instructions?: string
+
+  /**
+   * Voice to use for audio output
+   */
+  voice?: string
+
+  /**
+   * Voice activity detection mode (default: 'server')
+   */
+  vadMode?: 'server' | 'semantic' | 'manual'
+
+  /**
+   * Output modalities for responses (e.g., ['audio', 'text'])
+   */
+  outputModalities?: Array<'audio' | 'text'>
+
+  /**
+   * Temperature for generation (provider-specific range)
+   */
+  temperature?: number
+
+  /**
+   * Maximum number of tokens in a response
+   */
+  maxOutputTokens?: number | 'inf'
+
+  /**
+   * Eagerness level for semantic VAD ('low', 'medium', 'high')
+   */
+  semanticEagerness?: 'low' | 'medium' | 'high'
+
+  // Callbacks
+  onConnect?: () => void
+  onDisconnect?: () => void
+  onError?: (error: Error) => void
+  onMessage?: (message: RealtimeMessage) => void
+  onModeChange?: (mode: RealtimeMode) => void
+  onInterrupted?: () => void
+  onUsage?: (usage: UsageInfo) => void
+  onGoAway?: (timeLeft?: string) => void
+  onStatusChange?: (status: RealtimeStatus) => void
+}
+
+/**
+ * Return type for the createRealtimeChat helper.
+ *
+ * Called in Remix setup with Handle. Fields are plain values.
+ */
+export interface CreateRealtimeChatReturn {
+  // Connection state
+  /** Current connection status */
+  status: RealtimeStatus
+  /** Current error, if any */
+  error: Error | null
+  /** Connect to the realtime session */
+  connect: () => Promise<void>
+  /** Disconnect from the realtime session */
+  disconnect: () => Promise<void>
+
+  // Conversation state
+  /** Current mode (idle, listening, thinking, speaking) */
+  mode: RealtimeMode
+  /** Conversation messages */
+  messages: Array<RealtimeMessage>
+  /** User transcript while speaking (before finalized) */
+  pendingUserTranscript: string | null
+  /** Assistant transcript while speaking (before finalized) */
+  pendingAssistantTranscript: string | null
+
+  // Voice control
+  /** Start listening for voice input (manual VAD mode) */
+  startListening: () => void
+  /** Stop listening for voice input (manual VAD mode) */
+  stopListening: () => void
+  /** Interrupt the current assistant response */
+  interrupt: () => void
+
+  // Text input
+  /** Send a text message instead of voice */
+  sendText: (text: string) => void
+
+  // Image input
+  /** Send an image to the conversation */
+  sendImage: (imageData: string, mimeType: string) => void
+
+  // Audio visualization (0-1 normalized)
+  /** Current input (microphone) volume level */
+  inputLevel: number
+  /** Current output (speaker) volume level */
+  outputLevel: number
+  /** Get frequency data for input audio visualization */
+  getInputFrequencyData: () => Uint8Array
+  /** Get frequency data for output audio visualization */
+  getOutputFrequencyData: () => Uint8Array
+  /** Get time domain data for input waveform */
+  getInputTimeDomainData: () => Uint8Array
+  /** Get time domain data for output waveform */
+  getOutputTimeDomainData: () => Uint8Array
+
+  // Session control
+  /** Update the active session and persist the configuration for reconnects. */
+  updateSession: (config: RealtimeSessionConfig) => void
+}

package/src/types.ts

@@ -0,0 +1,313 @@
+import type {
+  AnyClientTool,
+  InterruptDefinition,
+  InferSchemaType,
+  ModelMessage,
+  RunAgentResumeItem,
+  SchemaInput,
+} from '@tanstack/ai/client'
+import type {
+  AIDevtoolsDisplayOptions,
+  BoundInterrupts,
+  ChatClientOptions,
+  ChatClientState,
+  ResolvableChatInterrupt,
+  ChatInterruptState,
+  ChatRequestBody,
+  ChatResumeState,
+  ClientContextOptionFromTools,
+  ConnectionStatus,
+  DistributedOmit,
+  InferredClientContext,
+  MultimodalContent,
+  QueueConfig,
+  QueueOption,
+  QueueStrategy,
+  QueuedMessage,
+  SendMessageOptions,
+  UIMessage,
+  WhenBusy,
+} from '@tanstack/ai-client'
+
+// Re-export types from ai-client
+export type {
+  ChatRequestBody,
+  MultimodalContent,
+  QueueConfig,
+  QueuedMessage,
+  QueueOption,
+  QueueStrategy,
+  SendMessageOptions,
+  UIMessage,
+  WhenBusy,
+}
+
+/**
+ * Recursive partial. Every property and every nested array element is optional.
+ * Used to type the in-flight `partial` value the helper exposes while a
+ * structured output stream is still arriving (the JSON has shape but is
+ * incomplete).
+ */
+export type DeepPartial<T> =
+  T extends ReadonlyArray<infer U>
+    ? Array<DeepPartial<U>>
+    : T extends object
+      ? { [K in keyof T]?: DeepPartial<T[K]> }
+      : T
+
+/**
+ * Options for the createChat helper.
+ *
+ * Call `createChat(handle, options)` in Remix setup with Handle from
+ * `remix/ui`. Handle is the first argument of the helper. It is not part of
+ * this type. The default id is `options.threadId ?? handle.id`.
+ *
+ * Pass either `connection` or `fetcher`. The XOR is enforced at the type
+ * level via `ChatTransport`.
+ *
+ * This extends ChatClientOptions but omits the state change callbacks that
+ * createChat manages internally:
+ * - `onMessagesChange` - Managed internally (exposed as `messages`)
+ * - `onLoadingChange` - Managed internally (exposed as `isLoading`)
+ * - `onErrorChange` - Managed internally (exposed as `error`)
+ * - `onStatusChange` - Managed internally (exposed as `status`)
+ *
+ * All other callbacks (onResponse, onChunk, onFinish, onError) are
+ * passed through to the underlying ChatClient and can be used for side effects.
+ *
+ * When `outputSchema` is supplied, the helper returns a typed `partial` (live
+ * progressive object, updated from `TEXT_MESSAGE_CONTENT` deltas via
+ * `parsePartialJSON`) and `final` (validated terminal payload from the
+ * `structured-output.complete` event). The schema is used purely for type
+ * inference on the client. Server-side validation still runs against the
+ * schema you pass to `chat({ outputSchema })` on the server route.
+ *
+ * Changing `connection` or `fetcher` updates the active ChatClient in place,
+ * preserving its state. Changing `threadId` creates a fresh client.
+ */
+export type CreateChatOptions<
+  TTools extends ReadonlyArray<AnyClientTool> = any,
+  TSchema extends SchemaInput | undefined = undefined,
+  TContext = InferredClientContext<TTools>,
+  TInterrupts extends ReadonlyArray<InterruptDefinition<any, any, any, any>> =
+    readonly [],
+> = DistributedOmit<
+  ChatClientOptions<TTools, TContext, TInterrupts>,
+  | 'onMessagesChange'
+  | 'onLoadingChange'
+  | 'onErrorChange'
+  | 'onStatusChange'
+  | 'onSubscriptionChange'
+  | 'onConnectionStatusChange'
+  | 'onSessionGeneratingChange'
+  | 'onQueueChange'
+  | 'onResumeStateChange'
+  | 'onRunIdChange'
+  | 'context'
+  | 'devtools'
+> & {
+  /** Display options for TanStack AI Devtools. */
+  devtools?: AIDevtoolsDisplayOptions
+  /**
+   * Opt into live subscription behavior when the helper is called in Remix
+   * setup with Handle. When enabled, the helper subscribes on setup and
+   * unsubscribes on dispose.
+   */
+  live?: boolean
+  /**
+   * Standard-schema-compatible schema (Zod, Valibot, ArkType, or a plain JSON
+   * Schema). Used to infer the shape of `partial` and `final` in the return.
+   * The schema is **not** sent to the server. Server-side validation runs
+   * against the schema passed to `chat({ outputSchema })` on the server route.
+   */
+  outputSchema?: TSchema
+} & ClientContextOptionFromTools<TTools, TContext>
+
+/**
+ * Discriminated return shape from the createChat helper. When `outputSchema`
+ * is supplied, the helper adds typed `partial` / `final` fields. When it is
+ * omitted (default), the return is unchanged. Fields are plain values.
+ */
+export type CreateChatReturn<
+  TTools extends ReadonlyArray<AnyClientTool> = any,
+  TSchema extends SchemaInput | undefined = undefined,
+  TInterrupts extends ReadonlyArray<InterruptDefinition<any, any, any, any>> =
+    readonly [],
+> = BaseCreateChatReturn<
+  TTools,
+  TSchema extends SchemaInput ? InferSchemaType<TSchema> : unknown,
+  TInterrupts
+> &
+  (TSchema extends SchemaInput
+    ? {
+        /**
+         * Live, progressively-parsed structured output. Updated from
+         * `TEXT_MESSAGE_CONTENT` deltas via `parsePartialJSON` while the stream
+         * is still arriving, and snapped to the validated payload when
+         * `structured-output.complete` fires. Resets on every new run
+         * (`sendMessage` / `reload`).
+         */
+        partial: DeepPartial<InferSchemaType<TSchema>>
+        /**
+         * Final, schema-validated structured output. `null` until the terminal
+         * `structured-output.complete` event arrives. Resets on every new run.
+         */
+        final: InferSchemaType<TSchema> | null
+      }
+    : Record<never, never>)
+
+interface BaseCreateChatReturn<
+  TTools extends ReadonlyArray<AnyClientTool> = any,
+  TData = unknown,
+  TInterrupts extends ReadonlyArray<InterruptDefinition<any, any, any, any>> =
+    readonly [],
+> {
+  /**
+   * Current messages in the conversation. When `outputSchema` is supplied,
+   * `messages[i].parts.find(p => p.type === 'structured-output')` is typed
+   * with the schema's inferred shape: `data: T`, `partial: DeepPartial<T>`.
+   */
+  messages: Array<UIMessage<TTools, TData>>
+
+  /**
+   * Send a message and get a response.
+   * Can be a simple string or multimodal content with images, audio, etc.
+   * By default, sends while busy are queued until the run settles successfully
+   * (`queue: 'drop'` restores the old drop-while-busy behavior).
+   * Pass `{ whenBusy }` to override the policy for a single send, or
+   * `{ body }` to merge per-call JSON into this request's `forwardedProps`.
+   */
+  sendMessage: (
+    content: string | MultimodalContent,
+    options?: SendMessageOptions,
+  ) => Promise<void>
+
+  /**
+   * Pending messages queued while the client is busy (streaming, claiming a
+   * send, or draining). Separate from `messages` until they drain.
+   */
+  queue: Array<QueuedMessage>
+
+  /**
+   * Cancel a queued message before it drains. No-op if already sent.
+   */
+  cancelQueued: (id: string) => void
+
+  /**
+   * Append a message to the conversation
+   */
+  append: (message: ModelMessage | UIMessage<TTools, TData>) => Promise<void>
+
+  /**
+   * Add the result of a client-side tool execution
+   */
+  addToolResult: (result: {
+    toolCallId: string
+    tool: string
+    output: any
+    state?: 'output-available' | 'output-error'
+    errorText?: string
+  }) => Promise<void>
+
+  /**
+   * @deprecated Use a bound `tool-approval` interrupt and
+   * `interrupt.resolveInterrupt`.
+   */
+  addToolApprovalResponse: (response: {
+    id: string // approval.id, not toolCallId
+    approved: boolean
+  }) => Promise<void>
+
+  /**
+   * The id of the run this client has in flight (one it started or rejoined),
+   * or `null` when there is none (including while a run sits paused on an
+   * interrupt, waiting on approval).
+   *
+   * A run is one turn of the conversation, so this changes from turn to turn. A
+   * whole tool loop stays inside one run, while resuming after an interrupt
+   * continues the turn under a new id — so one user message can produce several
+   * run ids. Use it to talk to your own server about that run (cancel it, poll
+   * it, correlate a log line).
+   */
+  runId: string | null
+  interrupts: BoundInterrupts<TTools, TInterrupts>
+  /** @deprecated Use `interrupts`. */
+  pendingInterrupts: BoundInterrupts<TTools, TInterrupts>
+  interruptErrors: ChatInterruptState<TTools, TInterrupts>['interruptErrors']
+  resuming: boolean
+  resolveInterrupts: {
+    (approved: boolean): void
+    (
+      resolver: (
+        interrupt: ResolvableChatInterrupt<TTools, TInterrupts>,
+      ) => undefined,
+    ): void
+  }
+  cancelInterrupts: () => void
+  retryInterrupts: () => void
+  resumeInterruptsUnsafe: (
+    resume: Array<RunAgentResumeItem>,
+    state?: ChatResumeState,
+  ) => Promise<boolean>
+  /** @deprecated Use bound interrupt methods or `resumeInterruptsUnsafe`. */
+  resumeInterrupts: (
+    resume: Array<RunAgentResumeItem>,
+    state?: ChatResumeState,
+  ) => Promise<boolean>
+
+  /**
+   * Reload the last assistant message
+   */
+  reload: () => Promise<void>
+
+  /**
+   * Stop the current response generation
+   */
+  stop: () => void
+
+  /**
+   * Whether a response is currently being generated
+   */
+  isLoading: boolean
+
+  /**
+   * Current error, if any
+   */
+  error: Error | undefined
+
+  /**
+   * Current status of the chat client
+   */
+  status: ChatClientState
+
+  /**
+   * Whether the subscription loop is currently active
+   */
+  isSubscribed: boolean
+
+  /**
+   * Current connection lifecycle status
+   */
+  connectionStatus: ConnectionStatus
+
+  /**
+   * Whether the shared session is actively generating.
+   * Derived from stream run events (RUN_STARTED / RUN_FINISHED / RUN_ERROR).
+   * Unlike `isLoading` (request-local), this reflects shared generation
+   * activity visible to all subscribers (e.g. across tabs/devices).
+   */
+  sessionGenerating: boolean
+
+  /**
+   * Set messages manually
+   */
+  setMessages: (messages: Array<UIMessage<TTools, TData>>) => void
+
+  /**
+   * Clear all messages
+   */
+  clear: () => void
+}
+
+// createChatClientOptions and InferChatMessages live in @tanstack/ai-client
+// and are re-exported from there.

package/src/ui.ts

@@ -0,0 +1,41 @@
+// Barrel entry for the `@tanstack/ai-remix/ui` subpath.
+export {
+  createChatUI,
+  type ChatUIComponents,
+  type ChatUIFactoryConfig,
+  type ChatUIHost,
+  type ChatUIQueueItem,
+  type InputProps,
+  type InterruptProps,
+  type LayoutProps,
+  type MessageProps,
+  type PartProps,
+  type QueueProps,
+  type ToolProps,
+} from './chat-ui/create-ui.tsx'
+export { createChatHook } from './chat-ui/create-chat-hook.ts'
+export { Chat, useChatContext, type ChatProps } from './chat-ui/chat.tsx'
+export {
+  ChatMessages,
+  type ChatMessagesProps,
+} from './chat-ui/chat-messages.tsx'
+export {
+  ChatMessage,
+  type ChatMessageProps,
+  type ToolCallRenderProps,
+} from './chat-ui/chat-message.tsx'
+export {
+  ChatInput,
+  type ChatInputProps,
+  type ChatInputRenderProps,
+} from './chat-ui/chat-input.tsx'
+export {
+  ToolApproval,
+  type ToolApprovalProps,
+  type ToolApprovalRenderProps,
+} from './chat-ui/tool-approval.tsx'
+export { TextPart, type TextPartProps } from './chat-ui/text-part.tsx'
+export {
+  ThinkingPart,
+  type ThinkingPartProps,
+} from './chat-ui/thinking-part.tsx'