@vellumai/plugin-api 0.10.10-dev.202607211337.6d2617c → 0.10.10-dev.202607211519.4a4e758

package/index.d.ts

@@ -2393,6 +2393,11 @@ declare interface CopyBlockSurfaceData {
     language?: string;
 }
 
+/** Create a live voice connection bound to the caller's `send` transport. */
+export declare function createLiveVoiceConnection(options: {
+    send: LiveVoiceFrameSender;
+}): LiveVoiceConnection;
+
 /**
  * Plugin-facing credential resolution.
  *
@@ -3717,6 +3722,203 @@ declare interface ListSurfaceData {
     selectionMode: "single" | "multiple" | "none";
 }
 
+declare const _LIVE_VOICE_SERVER_FRAME_TYPES: readonly ["ready", "busy", "speech_started", "utterance_end", "utterance_discarded", "stt_partial", "stt_final", "thinking", "assistant_text_delta", "tts_audio", "tts_done", "turn_cancelled", "metrics", "archived", "error"];
+
+declare const LIVE_VOICE_TURN_DETECTION_MODES: readonly ["manual", "server_vad"];
+
+declare interface LiveVoiceArchivedServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "archived";
+    readonly conversationId: string;
+    readonly sessionId: string;
+    readonly turnId?: string;
+    readonly role?: "user" | "assistant";
+    readonly attachmentId?: string;
+    readonly attachmentIds?: string[];
+    readonly warning?: {
+        readonly code: string;
+        readonly message: string;
+    };
+}
+
+declare interface LiveVoiceAssistantTextDeltaServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "assistant_text_delta";
+    readonly text: string;
+}
+
+declare interface LiveVoiceBusyServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "busy";
+    readonly activeSessionId: string;
+}
+
+export declare interface LiveVoiceConnection {
+    /**
+     * The active session id once a `start` frame has been accepted, else
+     * `undefined`. Read-only — the connection assigns it on `start` and clears
+     * it on `end`, teardown, or a session that the manager no longer knows.
+     */
+    readonly sessionId: string | undefined;
+    /**
+     * Handle one inbound transport message — a JSON text frame or a binary
+     * audio chunk. Never rejects: protocol and handler failures are reported
+     * back to the client as `error` frames.
+     */
+    handleMessage(message: string | ArrayBuffer | ArrayBufferView): Promise<void>;
+    /**
+     * Release the session bound to this connection when the transport closes.
+     * Idempotent — a no-op when no session is active.
+     */
+    release(reason?: LiveVoiceSessionCloseReason): void;
+}
+
+declare interface LiveVoiceErrorServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "error";
+    readonly code: LiveVoiceProtocolErrorCode;
+    readonly message: string;
+    /**
+     * True when the session continues past the error (e.g. a transient
+     * transcriber blip or one failed TTS segment). Absent (including on frames
+     * from older daemons) means the error is terminal for the session.
+     */
+    readonly recoverable?: boolean;
+}
+
+/**
+ * Sends one fully-sequenced server frame to the client over the caller's
+ * transport. The frame is a plain object; the caller is responsible for wire
+ * encoding (e.g. `JSON.stringify` for a text WebSocket).
+ */
+export declare type LiveVoiceFrameSender = (frame: LiveVoiceServerFrame) => void;
+
+declare interface LiveVoiceMetricsServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "metrics";
+    readonly event?: string;
+    readonly sessionId?: string;
+    readonly conversationId?: string;
+    readonly turnId: string;
+    readonly metrics?: unknown;
+    readonly sttMs: number | null;
+    readonly llmFirstDeltaMs: number | null;
+    readonly ttsFirstAudioMs: number | null;
+    /** End-of-speech (utterance_end, or ptt_release in manual mode) to first TTS audio. */
+    readonly roundTripMs: number | null;
+    readonly totalMs: number | null;
+    /**
+     * Semantic-endpointing "hold" decisions taken during the turn. Present only
+     * when the endpoint decider was consulted (otherwise the field is absent,
+     * keeping frames unchanged).
+     */
+    readonly endpointHoldCount?: number;
+    /** Worst endpoint-decision latency observed during the turn. */
+    readonly endpointDecisionMaxLatencyMs?: number;
+    /** Which floor-holding ack actually spoke during the turn, if any. */
+    readonly ackSpoken?: "first_delta" | "tool_use";
+}
+
+declare const LiveVoiceProtocolErrorCode: {
+    readonly InvalidJson: "invalid_json";
+    readonly InvalidFrame: "invalid_frame";
+    readonly UnknownType: "unknown_type";
+    readonly MissingRequiredField: "missing_required_field";
+    readonly InvalidField: "invalid_field";
+    readonly InvalidAudioPayload: "invalid_audio_payload";
+    /**
+     * Session startup was rejected because the daemon cannot run both audio
+     * legs (STT/TTS providers or credentials are unresolved). The error
+     * `message` names the offending provider(s) and missing credential(s).
+     */
+    readonly CredentialsUnavailable: "credentials_unavailable";
+};
+
+declare type LiveVoiceProtocolErrorCode = (typeof LiveVoiceProtocolErrorCode)[keyof typeof LiveVoiceProtocolErrorCode];
+
+declare interface LiveVoiceReadyServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "ready";
+    readonly sessionId: string;
+    readonly conversationId: string;
+    /**
+     * Echoes the turn-detection mode the session is actually running, so
+     * clients can detect a daemon that ignored a requested mode. Absent
+     * (older daemons) means "manual".
+     */
+    readonly turnDetection?: LiveVoiceTurnDetectionMode;
+}
+
+export declare type LiveVoiceServerFrame = LiveVoiceReadyServerFrame | LiveVoiceBusyServerFrame | LiveVoiceSpeechStartedServerFrame | LiveVoiceUtteranceEndServerFrame | LiveVoiceUtteranceDiscardedServerFrame | LiveVoiceSttPartialServerFrame | LiveVoiceSttFinalServerFrame | LiveVoiceThinkingServerFrame | LiveVoiceAssistantTextDeltaServerFrame | LiveVoiceTtsAudioServerFrame | LiveVoiceTtsDoneServerFrame | LiveVoiceTurnCancelledServerFrame | LiveVoiceMetricsServerFrame | LiveVoiceArchivedServerFrame | LiveVoiceErrorServerFrame;
+
+declare interface LiveVoiceServerFrameBase {
+    readonly type: LiveVoiceServerFrameType;
+    readonly seq: number;
+}
+
+declare type LiveVoiceServerFrameType = (typeof _LIVE_VOICE_SERVER_FRAME_TYPES)[number];
+
+export declare type LiveVoiceSessionCloseReason = "client_end" | "error" | "websocket_close" | "transport_closed" | "manager_shutdown";
+
+/**
+ * Emitted when the server VAD detects user speech. The client MUST
+ * immediately stop local TTS playback — this doubles as the flush-tail-audio
+ * signal (the in-app analog of the phone stack's buffered-audio clear).
+ */
+declare interface LiveVoiceSpeechStartedServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "speech_started";
+}
+
+declare interface LiveVoiceSttFinalServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "stt_final";
+    readonly text: string;
+}
+
+declare interface LiveVoiceSttPartialServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "stt_partial";
+    readonly text: string;
+}
+
+declare interface LiveVoiceThinkingServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "thinking";
+    readonly turnId: string;
+}
+
+declare interface LiveVoiceTtsAudioServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "tts_audio";
+    readonly mimeType: string;
+    readonly sampleRate: number;
+    readonly dataBase64: string;
+}
+
+declare interface LiveVoiceTtsDoneServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "tts_done";
+    readonly turnId: string;
+}
+
+/**
+ * Emitted when an in-flight assistant turn is aborted by barge-in. The client
+ * must drop any buffered tts_audio for that turn; no tts_done will follow.
+ */
+declare interface LiveVoiceTurnCancelledServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "turn_cancelled";
+    readonly turnId: string;
+}
+
+declare type LiveVoiceTurnDetectionMode = (typeof LIVE_VOICE_TURN_DETECTION_MODES)[number];
+
+/**
+ * Emitted only in server_vad mode when the closed utterance produced no
+ * usable speech: it is dropped without an assistant turn and the client
+ * should return to listening.
+ */
+declare interface LiveVoiceUtteranceDiscardedServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "utterance_discarded";
+}
+
+/**
+ * Emitted when the server VAD closes the utterance and the turn's
+ * transcription begins (plays the role ptt_release plays in manual mode).
+ */
+declare interface LiveVoiceUtteranceEndServerFrame extends LiveVoiceServerFrameBase {
+    readonly type: "utterance_end";
+    readonly reason: "silence" | "max-duration";
+}
+
 export declare type LLMCallSite = z.infer<typeof LLMCallSiteEnum>;
 
 /**

package/index.js

@@ -10,6 +10,7 @@ export const TtsSynthesisError = api.TtsSynthesisError;
 export const addMessage = api.addMessage;
 export const assistantEventHub = api.assistantEventHub;
 export const buildMessageExcerpt = api.buildMessageExcerpt;
+export const createLiveVoiceConnection = api.createLiveVoiceConnection;
 export const deleteConversation = api.deleteConversation;
 export const doesSupportVision = api.doesSupportVision;
 export const embedAndUpsert = api.embedAndUpsert;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/plugin-api",
-  "version": "0.10.10-dev.202607211337.6d2617c",
+  "version": "0.10.10-dev.202607211519.4a4e758",
   "description": "Public TypeScript authoring contract for Vellum assistant plugins.",
   "license": "MIT",
   "type": "module",