@tanstack/ai-client 0.18.5 → 0.19.0

package/src/audio-recorder.ts

@@ -0,0 +1,322 @@
+import { arrayBufferToBase64 } from '@tanstack/ai-utils'
+import type { AudioPart } from '@tanstack/ai/client'
+
+/** Lifecycle state of an {@link AudioRecorder}. */
+export type AudioRecorderState = 'idle' | 'recording' | 'stopping'
+
+export interface AudioRecorderOptions {
+  /** Constraints forwarded to `getUserMedia({ audio })`. Defaults to `true`. */
+  audio?: MediaTrackConstraints | boolean
+  /**
+   * Preferred recorder mime type. Used only when
+   * `MediaRecorder.isTypeSupported` reports it; otherwise the browser default
+   * is used.
+   */
+  mimeType?: string
+  /** Fired on `getUserMedia` rejection (permission denied) or recorder error. */
+  onError?: (error: Error) => void
+}
+
+/**
+ * Resolves the value `stop()` produces from a recorder transform callback.
+ *
+ * - If the callback returns a value — sync or async — that value's awaited type
+ *   is used (a returned `null` is a real value and is preserved).
+ * - If the callback returns nothing (`void`/`undefined`), or is absent, falls
+ *   back to {@link AudioRecording}.
+ *
+ * @template TFn - The transform callback type (or undefined if not provided)
+ */
+export type InferAudioRecordingOutput<TFn> = TFn extends (
+  recording: AudioRecording,
+) => infer R
+  ? [Exclude<Awaited<R>, void | undefined>] extends [never]
+    ? AudioRecording
+    : Exclude<Awaited<R>, void | undefined>
+  : AudioRecording
+
+export interface AudioRecording {
+  /** The raw recorded media blob. */
+  blob: Blob
+  /** Base64 of the recorded bytes (no `data:` prefix). */
+  base64: string
+  /** The recorder's native mime type, e.g. `audio/webm;codecs=opus`. */
+  mimeType: string
+  /** Recording length in milliseconds. */
+  durationMs: number
+  /**
+   * Ready-to-use audio content part for `sendMessage`/generation prompts:
+   * `{ type: 'audio', source: { type: 'data', value: base64, mimeType } }`.
+   */
+  part: AudioPart
+}
+
+/**
+ * Framework-agnostic browser audio recorder. Wraps `getUserMedia` +
+ * `MediaRecorder`, returns the recorder's native output (no transcode), and
+ * builds a plug-and-play {@link AudioRecording.part}.
+ */
+export class AudioRecorder {
+  private readonly options: AudioRecorderOptions
+  private recorder: MediaRecorder | null = null
+  private stream: MediaStream | null = null
+  private chunks: Array<Blob> = []
+  private startedAt = 0
+  private _state: AudioRecorderState = 'idle'
+  // True while start() is awaiting getUserMedia (state is still 'idle' then).
+  private starting = false
+  // Set by cancel()/teardown during that window so start() releases the
+  // freshly acquired stream instead of beginning a leaked recording.
+  private pendingCancel = false
+  private readonly listeners = new Set<(state: AudioRecorderState) => void>()
+  private stopResolve: ((recording: AudioRecording) => void) | null = null
+  private stopReject: ((error: Error) => void) | null = null
+
+  constructor(options: AudioRecorderOptions = {}) {
+    this.options = options
+  }
+
+  /** Feature-detect the browser media APIs. SSR/Worker-safe. */
+  static isSupported(): boolean {
+    return (
+      typeof navigator !== 'undefined' &&
+      typeof navigator.mediaDevices !== 'undefined' &&
+      typeof navigator.mediaDevices.getUserMedia === 'function' &&
+      typeof MediaRecorder !== 'undefined'
+    )
+  }
+
+  get state(): AudioRecorderState {
+    return this._state
+  }
+
+  subscribe(cb: (state: AudioRecorderState) => void): () => void {
+    this.listeners.add(cb)
+    return () => {
+      this.listeners.delete(cb)
+    }
+  }
+
+  private setState(state: AudioRecorderState): void {
+    this._state = state
+    for (const cb of this.listeners) {
+      cb(state)
+    }
+  }
+
+  async start(): Promise<void> {
+    if (this._state !== 'idle' || this.starting) {
+      return
+    }
+    this.starting = true
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: this.options.audio ?? true,
+      })
+      // cancel()/teardown ran while we were awaiting the mic: release the
+      // freshly acquired stream and bail rather than starting a recording the
+      // caller can no longer stop (a leaked live microphone).
+      if (this.pendingCancel) {
+        stream.getTracks().forEach((t) => t.stop())
+        return
+      }
+      this.stream = stream
+      const wanted = this.options.mimeType
+      const useMimeType =
+        wanted &&
+        typeof MediaRecorder.isTypeSupported === 'function' &&
+        MediaRecorder.isTypeSupported(wanted)
+          ? wanted
+          : undefined
+      const recorder = useMimeType
+        ? new MediaRecorder(stream, { mimeType: useMimeType })
+        : new MediaRecorder(stream)
+      this.chunks = []
+      recorder.ondataavailable = (e) => {
+        if (e.data.size > 0) {
+          this.chunks.push(e.data)
+        }
+      }
+      recorder.onstop = () => {
+        void this.finalize()
+      }
+      recorder.onerror = (event) => {
+        const detail =
+          'error' in event && event.error instanceof Error
+            ? event.error
+            : new Error('Audio recording failed')
+        this.handleError(detail)
+      }
+      this.recorder = recorder
+      this.startedAt = Date.now()
+      recorder.start()
+      this.setState('recording')
+    } catch (err) {
+      this.releaseStream()
+      this.recorder = null
+      const error =
+        err instanceof Error ? err : new Error('Failed to start recording')
+      this.setState('idle')
+      this.notifyError(error)
+      throw error
+    } finally {
+      this.starting = false
+      // Reset here (not before the await) so a cancel() that arrives mid-start
+      // is observed above; clearing it afterward keeps the next start() clean.
+      this.pendingCancel = false
+    }
+  }
+
+  stop(): Promise<AudioRecording> {
+    if (this._state !== 'recording' || !this.recorder) {
+      return Promise.reject(
+        new Error('AudioRecorder.stop() called while not recording'),
+      )
+    }
+    this.setState('stopping')
+    const recorder = this.recorder
+    return new Promise<AudioRecording>((resolve, reject) => {
+      // Some browsers/codecs never fire onstop; this watchdog unwedges the
+      // recorder instead of leaking this promise forever.
+      const watchdog = setTimeout(() => {
+        if (this._state !== 'stopping') {
+          return
+        }
+        // Detach handlers so a late onstop/onerror from the stalled recorder
+        // can't reach back in and fire finalize()/onError a second time.
+        this.detachRecorder()
+        if (this.chunks.length > 0) {
+          // onstop never fired, but ondataavailable already delivered the
+          // audio — finalize from the buffered chunks rather than discarding a
+          // recording the user successfully captured.
+          void this.finalize()
+        } else {
+          this.handleError(
+            new Error('Recording stop timed out after 10s with no audio'),
+          )
+        }
+      }, 10_000)
+      this.stopResolve = (rec) => {
+        clearTimeout(watchdog)
+        resolve(rec)
+      }
+      this.stopReject = (err) => {
+        clearTimeout(watchdog)
+        reject(err)
+      }
+      recorder.stop()
+    })
+  }
+
+  cancel(): void {
+    if (this.starting) {
+      // A start() is awaiting getUserMedia; flag it so the resolved stream is
+      // released instead of beginning a recording with no handle to stop it.
+      this.pendingCancel = true
+      return
+    }
+    if (this._state === 'idle') {
+      return
+    }
+    const recorder = this.recorder
+    if (recorder) {
+      // Detach handlers so finalize()/onError never run for a discarded
+      // recording.
+      this.detachRecorder()
+      try {
+        recorder.stop()
+      } catch (err) {
+        // Stopping an already-inactive recorder throws InvalidStateError —
+        // that's expected here. Anything else is unexpected; surface it rather
+        // than swallowing it silently.
+        if (
+          !(err instanceof DOMException && err.name === 'InvalidStateError')
+        ) {
+          this.notifyError(
+            err instanceof Error ? err : new Error('Failed to stop recorder'),
+          )
+        }
+      }
+    }
+    this.releaseStream()
+    this.recorder = null
+    this.chunks = []
+    const reject = this.stopReject
+    this.stopResolve = null
+    this.stopReject = null
+    this.setState('idle')
+    reject?.(new Error('Recording cancelled'))
+  }
+
+  private async finalize(): Promise<void> {
+    const mimeType = this.recorder?.mimeType || 'audio/webm'
+    const durationMs = Date.now() - this.startedAt
+    try {
+      const blob = new Blob(this.chunks, { type: mimeType })
+      const base64 = arrayBufferToBase64(await blob.arrayBuffer())
+      const recording: AudioRecording = {
+        blob,
+        base64,
+        mimeType,
+        durationMs,
+        part: {
+          type: 'audio',
+          source: { type: 'data', value: base64, mimeType },
+        },
+      }
+      this.releaseStream()
+      this.recorder = null
+      this.chunks = []
+      const resolve = this.stopResolve
+      this.stopResolve = null
+      this.stopReject = null
+      this.setState('idle')
+      resolve?.(recording)
+    } catch (err) {
+      this.handleError(
+        err instanceof Error ? err : new Error('Failed to finalize recording'),
+      )
+    }
+  }
+
+  private handleError(error: Error): void {
+    this.releaseStream()
+    this.recorder = null
+    this.chunks = []
+    const reject = this.stopReject
+    this.stopResolve = null
+    this.stopReject = null
+    this.setState('idle')
+    // Settle the pending stop() promise before invoking the user callback so a
+    // throwing onError can't strand the awaiter (the two error channels are
+    // independent — see start()/stop() docs).
+    reject?.(error)
+    this.notifyError(error)
+  }
+
+  /** Invoke the user onError callback, isolating a throw so it can't disrupt
+   * internal state teardown or strand a pending promise. */
+  private notifyError(error: Error): void {
+    try {
+      this.options.onError?.(error)
+    } catch {
+      // A user onError that throws must not propagate into recorder internals.
+    }
+  }
+
+  private releaseStream(): void {
+    this.stream?.getTracks().forEach((t) => t.stop())
+    this.stream = null
+  }
+
+  /** Detach all event handlers so a stale recorder can't mutate our state. */
+  private detachRecorder(): void {
+    const recorder = this.recorder
+    if (!recorder) {
+      return
+    }
+    recorder.onstop = null
+    recorder.onerror = null
+    recorder.ondataavailable = null
+  }
+}

package/src/generation-types.ts

@@ -11,7 +11,29 @@ import type {
 // ===========================
 
 /**
- * Infers the output type from an `onResult` callback's return type.
+ * Maps an `onResult` transform's raw return type to the stored output type.
+ *
+ * - A concrete return (excluding null/void/undefined) becomes the output type.
+ * - A return of only null/void/undefined falls back to TResult (the transform
+ *   reacted to the result or chose to keep it, rather than replacing it).
+ *
+ * Hooks infer `TReturn` directly from the `onResult` return position — a
+ * covariant inference site that works even for an optional nested property —
+ * which both contextually types the callback parameter as `TResult` and
+ * narrows `result`. See issue #848.
+ *
+ * @template TResult - The raw result type from the generation
+ * @template TReturn - The transform's return type (defaults to `void` when no
+ *   transform is provided)
+ */
+export type InferGenerationOutputFromReturn<TResult, TReturn> = [
+  Exclude<TReturn, null | void | undefined>,
+] extends [never]
+  ? TResult
+  : Exclude<TReturn, null | void | undefined>
+
+/**
+ * Infers the output type from an `onResult` callback's type.
  *
  * - If the callback returns a concrete type (excluding null/void/undefined), uses that type.
  * - If the callback only returns null/void/undefined, or is not provided, falls back to TResult.
@@ -22,9 +44,7 @@ import type {
 export type InferGenerationOutput<TResult, TFn> = TFn extends (
   result: any,
 ) => infer R
-  ? [Exclude<R, null | void | undefined>] extends [never]
-    ? TResult
-    : Exclude<R, null | void | undefined>
+  ? InferGenerationOutputFromReturn<TResult, R>
   : TResult
 
 // ===========================

package/src/index.ts

@@ -1,4 +1,13 @@
+export { AudioRecorder } from './audio-recorder'
+export type {
+  AudioRecorderOptions,
+  AudioRecorderState,
+  AudioRecording,
+  InferAudioRecordingOutput,
+} from './audio-recorder'
 export { ChatClient } from './chat-client'
+export { createMcpAppBridge } from './mcp-app-bridge'
+export type { McpAppBridge, CreateMcpAppBridgeOptions } from './mcp-app-bridge'
 export { RealtimeClient } from './realtime-client'
 export { GenerationClient } from './generation-client'
 export { VideoGenerationClient } from './video-generation-client'
@@ -30,6 +39,7 @@ export type {
 // Generation client types
 export type {
   InferGenerationOutput,
+  InferGenerationOutputFromReturn,
   GenerationClientState,
   GenerationClientOptions,
   GenerationFetcher,

package/src/mcp-app-bridge.ts

@@ -0,0 +1,120 @@
+export interface CreateMcpAppBridgeOptions {
+  threadId: string
+  callEndpoint: string
+  chat: {
+    sendMessage: (
+      content: string,
+      body?: Record<string, unknown>,
+    ) => Promise<void>
+  }
+  fetchImpl?: typeof fetch
+  onLink?: (url: string) => void
+}
+
+export interface McpAppBridge {
+  callTool: (input: {
+    serverId?: string
+    toolName: string
+    args?: Record<string, unknown>
+    /**
+     * Reserved — forwarded to the call handler for correlation purposes but
+     * not consumed by the handler. Accepted on the wire; the handler does not
+     * read it (mirrors the `meta` convention on `UIResourcePart`).
+     */
+    messageId?: string
+  }) => Promise<unknown>
+  sendPrompt: (text: string) => Promise<void>
+  openLink: (url: string) => { isError: boolean }
+}
+
+interface ToolCallResponse {
+  ok: boolean
+  result?: unknown
+  error?: string
+}
+
+function isToolCallResponse(value: unknown): value is ToolCallResponse {
+  return (
+    value !== null &&
+    typeof value === 'object' &&
+    'ok' in value &&
+    typeof value.ok === 'boolean'
+  )
+}
+
+// Links arrive from an untrusted sandboxed widget. Only hand http(s)/mailto
+// URLs to the host's onLink; reject javascript:/data:/file:/etc. so a widget
+// can't smuggle a script-executing or local-resource URL through the bridge.
+const SAFE_LINK_SCHEMES = new Set(['http:', 'https:', 'mailto:'])
+function isSafeLink(url: string): boolean {
+  try {
+    return SAFE_LINK_SCHEMES.has(new URL(url).protocol)
+  } catch {
+    return false
+  }
+}
+
+export function createMcpAppBridge(
+  options: CreateMcpAppBridgeOptions,
+): McpAppBridge {
+  const { threadId, callEndpoint, chat, fetchImpl, onLink } = options
+  const doFetch = fetchImpl ?? fetch
+
+  return {
+    async callTool(input) {
+      const response = await doFetch(callEndpoint, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+          threadId,
+          serverId: input.serverId,
+          toolName: input.toolName,
+          args: input.args,
+          messageId: input.messageId,
+        }),
+      })
+
+      if (!response.ok) {
+        throw new Error(`MCP app tool call failed: HTTP ${response.status}`)
+      }
+
+      const raw: unknown = await response.json()
+      if (!isToolCallResponse(raw)) {
+        throw new Error('MCP app tool call failed')
+      }
+
+      if (!raw.ok) {
+        throw new Error(raw.error ?? 'MCP app tool call failed')
+      }
+
+      return raw.result
+    },
+
+    async sendPrompt(text) {
+      await chat.sendMessage(text)
+    },
+
+    openLink(url) {
+      if (!isSafeLink(url)) {
+        console.warn(
+          '[mcp-app-bridge] openLink rejected: unsupported URL scheme',
+          url,
+        )
+        return { isError: true }
+      }
+      if (onLink) {
+        try {
+          onLink(url)
+          return { isError: false }
+        } catch (err) {
+          console.warn('[mcp-app-bridge] openLink: onLink handler threw', err)
+          return { isError: true }
+        }
+      }
+      console.warn(
+        '[mcp-app-bridge] openLink ignored: no onLink handler configured',
+      )
+      return { isError: true }
+    },
+  }
+}

package/src/types.ts

@@ -10,6 +10,7 @@ import type {
   ModelMessage,
   StreamChunk,
   StructuredOutputPart,
+  UIResourcePart,
   VideoPart,
 } from '@tanstack/ai/client'
 import type { ConnectionAdapter } from './connection-adapters'
@@ -244,6 +245,7 @@ export type MessagePart<
   | ToolResultPart
   | ThinkingPart
   | StructuredOutputPart<TData>
+  | UIResourcePart
 
 /**
  * UIMessage - Domain-specific message format optimized for building chat UIs