agents 0.22.0 → 0.23.0

package/dist/types-_Faxb570.d.ts

@@ -0,0 +1,439 @@
+//#region src/voice/types.d.ts
+/**
+ * Shared types for the voice pipeline.
+ *
+ * Used by both the server (index.ts) and client (client.ts)
+ * to ensure protocol consistency.
+ */
+/**
+ * Current voice protocol version.
+ * Bump this when making backwards-incompatible wire protocol changes.
+ * The server sends this in the initial `welcome` message so clients
+ * can detect version mismatches.
+ */
+declare const VOICE_PROTOCOL_VERSION = 1;
+type VoiceStatus = "idle" | "listening" | "thinking" | "speaking";
+/** Server-side diagnostic configuration shared by both voice mixins. */
+interface VoiceDiagnosticsOptions {
+  /** Forward safe server diagnostics and enable browser console logging. */
+  browserConsole?: boolean;
+}
+/**
+ * A bounded diagnostic record forwarded by the voice protocol.
+ * Event names and metadata are intentionally open and are not stable API.
+ */
+interface VoiceDiagnosticEvent {
+  event: string;
+  timestamp: number;
+  data?: Record<string, unknown>;
+}
+type VoiceTurnSource = "speech" | "text";
+type VoiceTurnOutcome =
+  | "completed"
+  | "no_output"
+  | "output_limit"
+  | "content_filtered"
+  | "model_error"
+  | "tts_error"
+  | "aborted"
+  | "skipped"
+  | "error";
+/**
+ * Stable, content-free summary of one allocated voice turn.
+ *
+ * `turnId`, `source`, and `outcome` are dimensions used to correlate and
+ * interpret the measurements; every other field is a duration. Optional
+ * timings are omitted when their lifecycle landmark was not reached. Timings
+ * use one server clock, overlap, and are not additive. Browser playback timing
+ * is excluded because browser and Worker clocks are independent.
+ */
+interface VoiceTurnMetrics {
+  /** SDK-assigned correlation ID for this turn. */
+  turnId: string;
+  /** Whether the turn originated from finalized speech or a text message. */
+  source: VoiceTurnSource;
+  /** Terminal result, emitted exactly once for the allocated turn. */
+  outcome: VoiceTurnOutcome;
+  /** Turn allocation to terminal summary, in milliseconds. */
+  turnTotalMs: number;
+  /** Provider speech start to the first interim transcript. */
+  speechStartToFirstInterimMs?: number;
+  /** Provider speech start to the finalized transcript. */
+  speechStartToFinalMs?: number;
+  /** Time spent in the server's `afterTranscribe` hook. */
+  afterTranscribeMs?: number;
+  /** Model invocation to the first non-whitespace text delta. */
+  modelToFirstTextMs?: number;
+  /** Cumulative duration of reasoning blocks exposed by the model stream. */
+  exposedReasoningMs?: number;
+  /** Model invocation through normalized stream consumption. */
+  modelStreamConsumptionMs?: number;
+  /** Finalized input through the first server audio send. */
+  finalInputToFirstAudioMs?: number;
+  /** First TTS provider invocation through the first server audio send. */
+  ttsToFirstAudioMs?: number;
+  /** First TTS provider invocation through completion of all sentence work. */
+  ttsWallMs?: number;
+  /** Cumulative overlapping TTS sentence hook and provider work. */
+  ttsWorkMs?: number;
+}
+/** Audio format the server uses for binary audio payloads. */
+type VoiceAudioFormat = "mp3" | "pcm16" | "wav" | "opus";
+type VoiceRole = "user" | "assistant";
+/** Stable machine-readable error codes emitted by the voice protocol. */
+type VoiceErrorCode = "stt_startup_failed" | "stt_connection_lost";
+/** Stable pipeline stage associated with a structured voice error. */
+type VoiceErrorStage = "stt";
+/** Client-safe error detail. `message` remains for string-event compatibility. */
+interface VoiceError {
+  message: string;
+  code?: VoiceErrorCode;
+  stage?: VoiceErrorStage;
+  retryable?: boolean;
+}
+/** Stable machine-readable outcomes for non-ordinary LLM completions. */
+type VoiceCompletionOutcomeCode =
+  | "no_output"
+  | "output_limit"
+  | "content_filtered"
+  | "model_error";
+/** Normalized finish reasons emitted by the supported AI SDK stream shape. */
+type VoiceModelFinishReason =
+  | "stop"
+  | "length"
+  | "content-filter"
+  | "tool-calls"
+  | "error"
+  | "other";
+/** Bounded completion metadata exposed by the voice protocol. */
+interface VoiceCompletionOutcome {
+  code: VoiceCompletionOutcomeCode;
+  stage: "llm";
+  finishReason?: VoiceModelFinishReason;
+  partialOutput: boolean;
+}
+type VoiceClientMessage =
+  | {
+      type: "hello";
+      protocol_version?: number;
+    }
+  | {
+      type: "start_call";
+      preferred_format?: VoiceAudioFormat;
+    }
+  | {
+      type: "end_call";
+    }
+  | {
+      type: "start_of_speech";
+    }
+  | {
+      type: "end_of_speech";
+    }
+  | {
+      type: "interrupt";
+    }
+  | {
+      type: "text_message";
+      text: string;
+    };
+type VoiceServerMessage =
+  | {
+      type: "welcome";
+      protocol_version: number;
+      diagnostics?: {
+        browser_console: true;
+      };
+    }
+  | ({
+      type: "diagnostic";
+    } & VoiceDiagnosticEvent)
+  | {
+      type: "status";
+      status: VoiceStatus;
+    }
+  | {
+      type: "audio_config";
+      format: VoiceAudioFormat;
+      sampleRate?: number;
+    }
+  | {
+      type: "transcript";
+      role: VoiceRole;
+      text: string;
+    }
+  | {
+      type: "transcript_start";
+      role: VoiceRole;
+    }
+  | {
+      type: "transcript_delta";
+      text: string;
+    }
+  | {
+      type: "transcript_end";
+      text: string;
+    }
+  | {
+      type: "transcript_interim";
+      text: string;
+    }
+  | {
+      type: "playback_interrupt";
+    }
+  | {
+      type: "metrics";
+      llm_ms: number;
+      tts_ms: number;
+      first_audio_ms: number;
+      total_ms: number;
+    }
+  | ({
+      type: "turn_metrics";
+    } & VoiceTurnMetrics)
+  | ({
+      type: "completion_outcome";
+    } & VoiceCompletionOutcome)
+  | ({
+      type: "error";
+    } & VoiceError);
+/**
+ * Compact compatibility summary for successful, non-empty speech turns.
+ * These overlapping latency landmarks and work totals are not additive.
+ * Use `VoiceTurnMetrics` for stable detailed timing and terminal summaries of
+ * unsuccessful, aborted, skipped, or text turns.
+ */
+interface VoicePipelineMetrics {
+  /**
+   * Time from immediately before `onTurn()` until normalized model-stream
+   * consumption completes. This can include tool work and consumer waits while
+   * consuming the stream. It is not time to first text.
+   */
+  llm_ms: number;
+  /**
+   * Cumulative per-sentence work from immediately before `beforeSynthesize`
+   * until that sentence's synthesis and hook work settles. Sentence work can
+   * overlap both other sentences and model consumption, so this value can
+   * exceed wall time and overlaps the other metrics.
+   */
+  tts_ms: number;
+  /**
+   * Time from turn-pipeline start, before `afterTranscribe`, to the first server
+   * audio send. Includes post-STT hooks, model work, and TTS, but excludes STT
+   * and browser playback. `0` means the server sent no audio.
+   */
+  first_audio_ms: number;
+  /**
+   * Time from the same turn-pipeline start until model consumption and TTS
+   * draining complete. Measured before final context, persistence, and status
+   * work. Excludes STT and browser playback.
+   */
+  total_ms: number;
+}
+interface TranscriptMessage {
+  role: VoiceRole;
+  text: string;
+  timestamp: number;
+}
+interface TTSProvider {
+  synthesize(text: string, signal?: AbortSignal): Promise<ArrayBuffer | null>;
+}
+interface StreamingTTSProvider {
+  synthesizeStream(
+    text: string,
+    signal?: AbortSignal
+  ): AsyncGenerator<ArrayBuffer>;
+}
+/**
+ * Continuous speech-to-text provider.
+ *
+ * Creates a per-call session that receives audio continuously from
+ * `start_call` to `end_call`. The model handles turn detection
+ * internally — there is no client-side speech boundary signaling
+ * required for STT.
+ *
+ * The session fires `onUtterance` when the model detects a complete
+ * utterance (e.g. Flux `EndOfTurn`, Nova 3 `speech_final` +
+ * endpointing). The voice pipeline maps this to `onTurn` (withVoice)
+ * or `onTranscript` (withVoiceInput).
+ */
+interface Transcriber {
+  /** Create a new transcription session for one call. */
+  createSession(options?: TranscriberSessionOptions): TranscriberSession;
+}
+interface TranscriberSessionOptions {
+  /** Language code (e.g. "en"). */
+  language?: string;
+  /**
+   * Called when the provider produces an interim (unstable) transcript.
+   * This text may change as more audio arrives.
+   */
+  onInterim?: (text: string) => void;
+  /**
+   * Called when the model detects the start of user speech.
+   *
+   * Providers can use this for low-latency barge-in before a final
+   * utterance is available. The transcript may be omitted or unstable.
+   */
+  onSpeechStart?: (text?: string) => void;
+  /**
+   * Called when the model detects a complete utterance.
+   * The transcript is the stable text for this turn.
+   *
+   * For Flux: fires on `EndOfTurn`.
+   * For Nova 3: fires on `Results` with `speech_final: true`.
+   */
+  onUtterance?: (transcript: string) => void;
+  /**
+   * Called when the session can no longer transcribe because its provider
+   * connection failed or closed unexpectedly. Providers must not call this
+   * for teardown initiated by {@link TranscriberSession.close}.
+   */
+  onFatalError?: (error: Error) => void;
+}
+/**
+ * A per-call transcription session. Lives for the entire call duration.
+ *
+ * Unlike per-utterance sessions, this session is never finished or
+ * aborted mid-call. It receives all audio continuously and the model
+ * handles speech boundary detection. On interrupt, the LLM+TTS
+ * pipeline is aborted but the transcriber session stays alive.
+ */
+interface TranscriberSession {
+  /**
+   * Feed raw PCM audio (16kHz mono 16-bit LE).
+   * Fire-and-forget — the session buffers internally as needed.
+   */
+  feed(chunk: ArrayBuffer): void;
+  /**
+   * Resolves when the session is ready to accept audio and emit transcripts.
+   * Optional so existing custom transcribers can start synchronously.
+   */
+  waitUntilReady?(): Promise<void>;
+  /**
+   * Optional. Provide the agent's most recent spoken reply (the text sent to
+   * TTS) as conversational context for the next user turn.
+   *
+   * The pipeline calls this after the agent finishes speaking each reply and
+   * greeting. Providers that support context carryover (e.g. AssemblyAI's
+   * `agent_context`) use it to better recognize short or contextual answers
+   * ("yes", "7pm", an email spelled aloud). Providers that don't support it
+   * simply omit this method — it is a no-op for them.
+   */
+  updateAgentContext?(text: string): void;
+  /**
+   * Close the session and release resources.
+   * Called at end_call or disconnect — not on interrupt.
+   */
+  close(): void;
+}
+/**
+ * Pluggable audio input source for VoiceClient.
+ *
+ * When provided via `VoiceClientOptions.audioInput`, VoiceClient delegates
+ * mic capture to this object instead of using its built-in AudioWorklet.
+ * The audio input is responsible for capturing audio and routing it to the
+ * server (however it chooses — WebRTC, SFU, direct binary, etc.).
+ *
+ * It must call `onAudioLevel` with RMS values so VoiceClient can run
+ * silence detection, interrupt detection, and update the audio level UI.
+ *
+ * @example
+ * ```typescript
+ * class SFUAudioInput implements VoiceAudioInput {
+ *   onAudioLevel: ((rms: number) => void) | null = null;
+ *   async start() {
+ *     // Set up WebRTC peer connection, SFU session, etc.
+ *     // In a monitoring loop, call this.onAudioLevel?.(rms)
+ *   }
+ *   stop() {
+ *     // Tear down WebRTC
+ *   }
+ * }
+ * ```
+ */
+interface VoiceAudioInput {
+  /** Start capturing audio. Called by VoiceClient on startCall(). */
+  start(): Promise<void>;
+  /** Stop capturing audio. Called by VoiceClient on endCall() or disconnect(). */
+  stop(): void;
+  /**
+   * Set by VoiceClient before start(). The audio input must call this
+   * with RMS audio level values on each frame so VoiceClient can run
+   * silence detection, interrupt detection, and update the UI.
+   */
+  onAudioLevel: ((rms: number) => void) | null;
+  /**
+   * Set by VoiceClient before start(). If the audio input provides
+   * raw PCM audio (16kHz mono 16-bit LE), call this callback and
+   * VoiceClient will forward the data to the server via its transport.
+   *
+   * This is needed when audio reaches the server through the same
+   * WebSocket as protocol messages (e.g. SFU in local dev where the
+   * SFU adapter can't connect back to localhost).
+   *
+   * If the audio input routes audio to the server through an external
+   * path (e.g. SFU WebSocket adapter in production), this can be left
+   * unused — the audio will arrive on a separate connection.
+   */
+  onAudioData?: ((pcm: ArrayBuffer) => void) | null;
+}
+/** Details a transport can provide when its connection closes. */
+interface VoiceTransportCloseInfo {
+  code?: number;
+  reason?: string;
+  wasClean?: boolean;
+}
+/**
+ * Abstraction over the data channel between client and server.
+ * The default implementation wraps PartySocket (WebSocket).
+ * Implement this interface to use WebRTC, SFU, or other transports.
+ */
+interface VoiceTransport {
+  /** Send a JSON-serializable message to the server. */
+  sendJSON(data: Record<string, unknown>): void;
+  /** Send raw binary audio to the server. */
+  sendBinary(data: ArrayBuffer): void;
+  /** Open the connection. */
+  connect(): void;
+  /** Close the connection and release resources. */
+  disconnect(): void;
+  /** Whether the transport is currently connected and ready to send. */
+  readonly connected: boolean;
+  onopen: (() => void) | null;
+  onclose: ((info?: VoiceTransportCloseInfo) => void) | null;
+  onerror: ((error?: unknown) => void) | null;
+  /** Called when a JSON string message arrives from the server. */
+  onmessage: ((data: string | ArrayBuffer | Blob) => void) | null;
+}
+//#endregion
+export {
+  VoiceTransport as C,
+  VoiceTurnSource as D,
+  VoiceTurnOutcome as E,
+  VoiceStatus as S,
+  VoiceTurnMetrics as T,
+  VoiceErrorStage as _,
+  TranscriberSessionOptions as a,
+  VoiceRole as b,
+  VoiceAudioFormat as c,
+  VoiceCompletionOutcome as d,
+  VoiceCompletionOutcomeCode as f,
+  VoiceErrorCode as g,
+  VoiceError as h,
+  TranscriberSession as i,
+  VoiceAudioInput as l,
+  VoiceDiagnosticsOptions as m,
+  TTSProvider as n,
+  TranscriptMessage as o,
+  VoiceDiagnosticEvent as p,
+  Transcriber as r,
+  VOICE_PROTOCOL_VERSION as s,
+  StreamingTTSProvider as t,
+  VoiceClientMessage as u,
+  VoiceModelFinishReason as v,
+  VoiceTransportCloseInfo as w,
+  VoiceServerMessage as x,
+  VoicePipelineMetrics as y
+};
+//# sourceMappingURL=types-_Faxb570.d.ts.map

package/dist/voice/client.d.ts

@@ -0,0 +1,226 @@
+import {
+  C as VoiceTransport,
+  D as VoiceTurnSource,
+  E as VoiceTurnOutcome,
+  S as VoiceStatus,
+  T as VoiceTurnMetrics,
+  _ as VoiceErrorStage,
+  b as VoiceRole,
+  c as VoiceAudioFormat,
+  d as VoiceCompletionOutcome,
+  f as VoiceCompletionOutcomeCode,
+  g as VoiceErrorCode,
+  h as VoiceError,
+  l as VoiceAudioInput,
+  m as VoiceDiagnosticsOptions,
+  o as TranscriptMessage,
+  p as VoiceDiagnosticEvent,
+  s as VOICE_PROTOCOL_VERSION,
+  v as VoiceModelFinishReason,
+  w as VoiceTransportCloseInfo,
+  y as VoicePipelineMetrics
+} from "../types-_Faxb570.js";
+
+//#region src/voice/client.d.ts
+interface VoiceClientOptions {
+  /** Agent name (matches the server-side Durable Object class). */
+  agent: string;
+  /** Instance name for the agent. @default "default" */
+  name?: string;
+  /** Host to connect to. @default window.location.host */
+  host?: string;
+  /** Query parameters appended to the WebSocket URL. */
+  query?: Record<string, string | null | undefined>;
+  /**
+   * Custom transport for sending/receiving data.
+   * Defaults to a WebSocket transport via PartySocket.
+   * Provide a custom implementation for WebRTC, SFU, or other transports.
+   */
+  transport?: VoiceTransport;
+  /**
+   * Custom audio input source. When provided, VoiceClient does NOT
+   * use its built-in AudioWorklet mic capture. The audio input is
+   * responsible for capturing and routing audio to the server.
+   * It must report audio levels via `onAudioLevel` for silence and
+   * interrupt detection to work.
+   */
+  audioInput?: VoiceAudioInput;
+  /**
+   * Preferred audio format for server responses. Sent in `start_call`
+   * as a hint — the server may ignore it if it cannot produce that format.
+   * The actual format is declared in the server's `audio_config` message.
+   */
+  preferredFormat?: VoiceAudioFormat;
+  /** RMS threshold below which audio is considered silence. @default 0.04 */
+  silenceThreshold?: number;
+  /** How long silence must last before sending end_of_speech (ms). @default 500 */
+  silenceDurationMs?: number;
+  /** RMS threshold for detecting user speech during agent playback. @default 0.05 */
+  interruptThreshold?: number;
+  /** Consecutive high-RMS chunks needed to trigger an interrupt. @default 2 */
+  interruptChunks?: number;
+  /** Maximum transcript messages to keep in memory. @default 200 */
+  maxTranscriptMessages?: number;
+  /**
+   * Preferred audio output device for assistant playback.
+   * Pass a MediaDeviceInfo.deviceId from an audiooutput device.
+   * Unsupported browsers continue playing through the default output.
+   * @default "default"
+   */
+  outputDeviceId?: string;
+}
+/**
+ * Low-level browser transport information for diagnosing connection failures.
+ * Error causes stay as the raw `unknown` value because browsers do not
+ * guarantee useful fields on WebSocket error events. The SDK does not
+ * serialize or log this local event; consumers must narrow it before use.
+ */
+type VoiceConnectionDiagnostic =
+  | {
+      type: "error";
+      cause: unknown;
+    }
+  | ({
+      type: "close";
+    } & VoiceTransportCloseInfo);
+/** Maps each event name to the data type passed to its listeners. */
+interface VoiceClientEventMap {
+  statuschange: VoiceStatus;
+  transcriptchange: TranscriptMessage[];
+  interimtranscript: string | null;
+  metricschange: VoicePipelineMetrics | null;
+  turnmetrics: VoiceTurnMetrics;
+  audiolevelchange: number;
+  connectionchange: boolean;
+  connectiondiagnostic: VoiceConnectionDiagnostic;
+  voiceerror: VoiceError;
+  completionoutcome: VoiceCompletionOutcome;
+  error: string | null;
+  outputdeviceerror: string | null;
+  mutechange: boolean;
+  custommessage: unknown;
+}
+type VoiceClientEvent = keyof VoiceClientEventMap;
+/**
+ * Default VoiceTransport backed by PartySocket (reconnecting WebSocket).
+ * Created automatically when no custom transport is provided.
+ */
+declare class WebSocketVoiceTransport implements VoiceTransport {
+  #private;
+  onopen: (() => void) | null;
+  onclose: ((info?: VoiceTransportCloseInfo) => void) | null;
+  onerror: ((error?: unknown) => void) | null;
+  onmessage: ((data: string | ArrayBuffer | Blob) => void) | null;
+  constructor(options: {
+    agent: string;
+    name?: string;
+    host?: string;
+    query?: Record<string, string | null | undefined>;
+  });
+  get connected(): boolean;
+  sendJSON(data: Record<string, unknown>): void;
+  sendBinary(data: ArrayBuffer): void;
+  connect(): void;
+  disconnect(): void;
+}
+declare class VoiceClient {
+  #private;
+  constructor(options: VoiceClientOptions);
+  get status(): VoiceStatus;
+  get transcript(): TranscriptMessage[];
+  get metrics(): VoicePipelineMetrics | null;
+  /** Last stable terminal summary received for any speech or text turn. */
+  get turnMetrics(): VoiceTurnMetrics | null;
+  get audioLevel(): number;
+  get isMuted(): boolean;
+  get connected(): boolean;
+  get error(): string | null;
+  get outputDeviceError(): string | null;
+  /**
+   * The current interim (partial) transcript from streaming STT.
+   * Updates in real time as the user speaks. Cleared when the final
+   * transcript is produced or the call reaches a terminal reset.
+   * null when no interim text is available.
+   */
+  get interimTranscript(): string | null;
+  /**
+   * The protocol version reported by the server.
+   * null until the server sends its welcome message.
+   */
+  get serverProtocolVersion(): number | null;
+  addEventListener<K extends VoiceClientEvent>(
+    event: K,
+    listener: (data: VoiceClientEventMap[K]) => void
+  ): void;
+  removeEventListener<K extends VoiceClientEvent>(
+    event: K,
+    listener: (data: VoiceClientEventMap[K]) => void
+  ): void;
+  connect(): void;
+  disconnect(): void;
+  startCall(): Promise<void>;
+  endCall(): void;
+  toggleMute(): void;
+  /**
+   * Send a text message to the agent. The agent processes it through
+   * `onTurn()` (bypassing STT) and responds with text transcript and
+   * TTS audio (if in a call) or text-only (if not).
+   */
+  sendText(text: string): void;
+  /**
+   * Send arbitrary JSON to the agent. Use this for app-level messages
+   * that are not part of the voice protocol (e.g. `{ type: "kick_speaker" }`).
+   * The server receives these in the consumer's `onMessage()` handler.
+   */
+  sendJSON(data: Record<string, unknown>): void;
+  /**
+   * Set the preferred audio output device for assistant playback.
+   * Unsupported browsers continue playing through the default output.
+   */
+  setOutputDevice(outputDeviceId?: string): Promise<void>;
+  /**
+   * The last custom (non-voice-protocol) message received from the server.
+   * Listen for the `"custommessage"` event to be notified when this changes.
+   */
+  get lastCustomMessage(): unknown;
+  /**
+   * The audio format the server declared for binary payloads.
+   * Set when the server sends `audio_config` at call start.
+   */
+  get audioFormat(): VoiceAudioFormat | null;
+  /**
+   * The sample rate (Hz) the server declared for raw pcm16 payloads.
+   * Set when the server sends `audio_config` at call start. Defaults to 16000.
+   */
+  get sampleRate(): number;
+}
+//#endregion
+export {
+  type TranscriptMessage,
+  VOICE_PROTOCOL_VERSION,
+  type VoiceAudioFormat,
+  type VoiceAudioInput,
+  VoiceClient,
+  VoiceClientEvent,
+  VoiceClientEventMap,
+  VoiceClientOptions,
+  type VoiceCompletionOutcome,
+  type VoiceCompletionOutcomeCode,
+  VoiceConnectionDiagnostic,
+  type VoiceDiagnosticEvent,
+  type VoiceDiagnosticsOptions,
+  type VoiceError,
+  type VoiceErrorCode,
+  type VoiceErrorStage,
+  type VoiceModelFinishReason,
+  type VoicePipelineMetrics,
+  type VoiceRole,
+  type VoiceStatus,
+  type VoiceTransport,
+  type VoiceTransportCloseInfo,
+  type VoiceTurnMetrics,
+  type VoiceTurnOutcome,
+  type VoiceTurnSource,
+  WebSocketVoiceTransport
+};
+//# sourceMappingURL=client.d.ts.map