@llmrtc/llmrtc-core 1.2.0 → 1.3.0

package/CHANGELOG.md

@@ -1,5 +1,63 @@
 # @llmrtc/llmrtc-core
 
+## 1.3.0
+
+### Minor Changes
+
+- 6a74624: Experimental realtime speech-to-speech relay mode (RFC 0001, M1).
+  - New opt-in realtimeSpeech server mode: sessions connect to a native
+    speech-to-speech model over the provider's WebSocket instead of the
+    STT-LLM-TTS pipeline, for ~300-500ms voice-to-voice latency.
+  - OpenAIRealtimeSpeechProvider (gpt-realtime-2.1 family): bidirectional
+    24kHz PCM, user/assistant transcripts, provider-side turn detection,
+    barge-in with response cancellation and history truncation, per-
+    response usage, session-expiry warning ahead of the 60-minute cap.
+  - Relay orchestrator with a decoupled control loop, epoch-tagged
+    playback queue, and independent pacer, so interruption reaction is
+    bounded regardless of response length; final transcripts are mirrored
+    into session history.
+  - Protocol (additive): assistant-transcript and usage messages,
+    ready.mode, REALTIME_ERROR/BUDGET_EXCEEDED error codes.
+  - M1 scope: tool bridging, budgets, session renewal, playbooks, client
+    reconnect grace, and the Gemini adapter land in subsequent milestones.
+
+- 2cc97a0: Realtime relay milestone 2 (RFC 0001): tools, budgets, session renewal.
+  - Tool bridging: the same ToolRegistry definitions drive provider-native
+    function calling; calls execute through ToolExecutor (timeouts, abort
+    support) without blocking the relay control loop, with results
+    returned to the live session and tool-call-start/end relayed to
+    clients. Provider-side cancellations abort in-flight executions.
+  - Session budgets: maxSessionMs (default 120 minutes) and maxTokens
+    guardrails with warn or end-session behavior; ending emits
+    BUDGET_EXCEEDED and closes the provider session.
+  - OpenAI 60-minute session renewal: near expiry (session-expiring
+    events now carry a renewable flag in core) the orchestrator opens a
+    fresh provider session seeded from the mirrored transcript history
+    and swaps it in at a quiet moment; in-flight work is aborted cleanly
+    if quiescence cannot be reached. The onSessionRenewed hook is
+    deferred to a later milestone.
+  - Budget end-session reports exactly one BUDGET_EXCEEDED error to the
+    client; tool bridge failures (unserializable results) are contained
+    per call instead of crashing the process.
+
+- 3db8818: Realtime relay milestone 3 (RFC 0001): playbooks, client events,
+  reconnect grace.
+  - Playbooks work in relay mode (llm_decision transitions;
+    clearHistory and per-stage llmConfig are not applied in relay mode): playbook_transition tool calls
+    reconfigure the live session's instructions and tools via the shared
+    PlaybookEngine, emit stage-change to clients, and nudge the model to
+    speak the new stage.
+  - Web client: new assistantTranscript and usage events, plus a
+    reserved modeChanged event (mid-session pipeline fallback ships in a
+    later milestone);
+    new mode-changed protocol message; session interface gains optional
+    requestResponse (OpenAI adapter implements it).
+  - Client reconnect grace: a dropped client has clientReconnectGraceMs
+    (default 30s) to reconnect and adopt its still-live provider session
+    (honest historyRecovered semantics); playback re-targets the new
+    peer.
+  - New docs page: Realtime Speech-to-Speech (experimental).
+
 ## 1.2.0
 
 ### Minor Changes

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export * from './types.js';
 export * from './async-event-queue.js';
+export * from './realtime-speech.js';
 export * from './tools.js';
 export * from './tool-executor.js';
 export * from './playbook.js';

package/dist/index.js

@@ -1,5 +1,6 @@
 export * from './types.js';
 export * from './async-event-queue.js';
+export * from './realtime-speech.js';
 export * from './tools.js';
 export * from './tool-executor.js';
 export * from './playbook.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,wBAAwB,CAAC;AACvC,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,sBAAsB,CAAC;AACrC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC"}

package/dist/protocol.d.ts

@@ -81,6 +81,12 @@ export interface ReadyMessage extends BaseMessage {
     protocolVersion: number;
     /** ICE servers for WebRTC (STUN/TURN) - provided by server for client use */
     iceServers?: RTCIceServer[];
+    /**
+     * Session mode: 'pipeline' (STT-LLM-TTS, the default) or 'realtime'
+     * (native speech-to-speech relay). Absent from older servers, which
+     * are always pipeline.
+     */
+    mode?: 'pipeline' | 'realtime';
 }
 /**
  * Heartbeat pong response
@@ -122,6 +128,35 @@ export interface TranscriptMessage extends BaseMessage {
     /** Whether this is the final transcription */
     isFinal: boolean;
 }
+/**
+ * Assistant-side transcript (realtime relay mode): what the assistant
+ * is saying, streamed as it speaks.
+ */
+export interface AssistantTranscriptMessage extends BaseMessage {
+    type: 'assistant-transcript';
+    /** Transcript text */
+    text: string;
+    /** Whether this is the final transcript for the response */
+    isFinal: boolean;
+}
+/**
+ * Per-response token usage (realtime relay mode).
+ */
+export interface UsageMessage extends BaseMessage {
+    type: 'usage';
+    inputTokens: number;
+    outputTokens: number;
+    audioInputTokens?: number;
+    audioOutputTokens?: number;
+    cachedTokens?: number;
+}
+/**
+ * Mid-session mode change (realtime relay fell back to pipeline).
+ */
+export interface ModeChangedMessage extends BaseMessage {
+    type: 'mode-changed';
+    mode: 'pipeline' | 'realtime';
+}
 /**
  * LLM response chunk (streaming)
  */
@@ -249,7 +284,7 @@ export interface StageChangeMessage extends BaseMessage {
  * - Processing: AUDIO_*, VAD_*, INVALID_*
  * - Generic: INTERNAL_*, RATE_LIMITED
  */
-export type ErrorCode = 'WEBRTC_UNAVAILABLE' | 'CONNECTION_FAILED' | 'SESSION_NOT_FOUND' | 'SESSION_EXPIRED' | 'STT_ERROR' | 'STT_TIMEOUT' | 'LLM_ERROR' | 'LLM_TIMEOUT' | 'TTS_ERROR' | 'TTS_TIMEOUT' | 'AUDIO_PROCESSING_ERROR' | 'VAD_ERROR' | 'INVALID_MESSAGE' | 'INVALID_AUDIO_FORMAT' | 'TOOL_ERROR' | 'PLAYBOOK_ERROR' | 'INTERNAL_ERROR' | 'RATE_LIMITED';
+export type ErrorCode = 'WEBRTC_UNAVAILABLE' | 'CONNECTION_FAILED' | 'SESSION_NOT_FOUND' | 'SESSION_EXPIRED' | 'STT_ERROR' | 'STT_TIMEOUT' | 'LLM_ERROR' | 'LLM_TIMEOUT' | 'TTS_ERROR' | 'TTS_TIMEOUT' | 'AUDIO_PROCESSING_ERROR' | 'VAD_ERROR' | 'INVALID_MESSAGE' | 'INVALID_AUDIO_FORMAT' | 'TOOL_ERROR' | 'PLAYBOOK_ERROR' | 'REALTIME_ERROR' | 'BUDGET_EXCEEDED' | 'INTERNAL_ERROR' | 'RATE_LIMITED';
 /**
  * Error message from server
  */
@@ -263,7 +298,7 @@ export interface ErrorMessage extends BaseMessage {
 /**
  * Union of all server-to-client message types
  */
-export type ServerMessage = ReadyMessage | PongMessage | SignalMessage | ReconnectAckMessage | TranscriptMessage | LLMChunkMessage | LLMMessage | TTSStartMessage | TTSChunkMessage | TTSMessage | TTSCompleteMessage | TTSCancelledMessage | SpeechStartMessage | SpeechEndMessage | ToolCallStartMessage | ToolCallEndMessage | StageChangeMessage | ErrorMessage;
+export type ServerMessage = ReadyMessage | PongMessage | SignalMessage | ReconnectAckMessage | TranscriptMessage | AssistantTranscriptMessage | UsageMessage | ModeChangedMessage | LLMChunkMessage | LLMMessage | TTSStartMessage | TTSChunkMessage | TTSMessage | TTSCompleteMessage | TTSCancelledMessage | SpeechStartMessage | SpeechEndMessage | ToolCallStartMessage | ToolCallEndMessage | StageChangeMessage | ErrorMessage;
 /**
  * Union of all protocol messages
  */
@@ -288,7 +323,7 @@ export declare function parseMessage(json: string): ProtocolMessage | null;
 /**
  * Create a ready message
  */
-export declare function createReadyMessage(id: string, iceServers?: RTCIceServer[]): ReadyMessage;
+export declare function createReadyMessage(id: string, iceServers?: RTCIceServer[], mode?: 'pipeline' | 'realtime'): ReadyMessage;
 /**
  * Create an error message
  */

package/dist/protocol.js

@@ -32,6 +32,9 @@ const SERVER_MESSAGE_TYPES = new Set([
     'signal',
     'reconnect-ack',
     'transcript',
+    'assistant-transcript',
+    'usage',
+    'mode-changed',
     'llm-chunk',
     'llm',
     'tts-start',
@@ -89,8 +92,8 @@ export function parseMessage(json) {
 /**
  * Create a ready message
  */
-export function createReadyMessage(id, iceServers) {
-    return { type: 'ready', id, protocolVersion: PROTOCOL_VERSION, iceServers };
+export function createReadyMessage(id, iceServers, mode) {
+    return { type: 'ready', id, protocolVersion: PROTOCOL_VERSION, iceServers, ...(mode && { mode }) };
 }
 /**
  * Create an error message

package/dist/protocol.js.map

@@ -1 +1 @@
-{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAoWlC,gFAAgF;AAChF,cAAc;AACd,gFAAgF;AAEhF,mCAAmC;AACnC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,MAAM;IACN,OAAO;IACP,WAAW;IACX,OAAO;IACP,aAAa;CACd,CAAC,CAAC;AAEH,mCAAmC;AACnC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,OAAO;IACP,MAAM;IACN,QAAQ;IACR,eAAe;IACf,YAAY;IACZ,WAAW;IACX,KAAK;IACL,WAAW;IACX,WAAW;IACX,KAAK;IACL,cAAc;IACd,eAAe;IACf,cAAc;IACd,YAAY;IACZ,iBAAiB;IACjB,eAAe;IACf,cAAc;IACd,OAAO;CACR,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAoB;IAClD,OAAO,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAoB;IAClD,OAAO,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,GAAoB;IACjD,OAAO,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,GAAsB,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,wDAAwD;AACxD,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAAU,EAAE,UAA2B;IACxE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,eAAe,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC;AAC9E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAe,EAAE,OAAe;IACjE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,OAAgB;IACpE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAe,EAAE,IAAa;IAClE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY,EACZ,MAAc,EACd,UAAkB;IAElB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AACzD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAc;IAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CACxC,IAAY,EACZ,MAAc,EACd,IAA6B;IAE7B,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AACpE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACtC,MAAc,EACd,UAAkB,EAClB,MAAgB,EAChB,KAAc;IAEd,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;AACtE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAY,EACZ,EAAU,EACV,MAAc;IAEd,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;AACpD,CAAC"}
+{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAiZlC,gFAAgF;AAChF,cAAc;AACd,gFAAgF;AAEhF,mCAAmC;AACnC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,MAAM;IACN,OAAO;IACP,WAAW;IACX,OAAO;IACP,aAAa;CACd,CAAC,CAAC;AAEH,mCAAmC;AACnC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,OAAO;IACP,MAAM;IACN,QAAQ;IACR,eAAe;IACf,YAAY;IACZ,sBAAsB;IACtB,OAAO;IACP,cAAc;IACd,WAAW;IACX,KAAK;IACL,WAAW;IACX,WAAW;IACX,KAAK;IACL,cAAc;IACd,eAAe;IACf,cAAc;IACd,YAAY;IACZ,iBAAiB;IACjB,eAAe;IACf,cAAc;IACd,OAAO;CACR,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAoB;IAClD,OAAO,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAoB;IAClD,OAAO,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,GAAoB;IACjD,OAAO,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,GAAsB,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,wDAAwD;AACxD,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,EAAU,EACV,UAA2B,EAC3B,IAA8B;IAE9B,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,eAAe,EAAE,gBAAgB,EAAE,UAAU,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AACrG,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAe,EAAE,OAAe;IACjE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY,EAAE,OAAgB;IACpE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAe,EAAE,IAAa;IAClE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAY,EACZ,MAAc,EACd,UAAkB;IAElB,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AACzD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,MAAc;IAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CACxC,IAAY,EACZ,MAAc,EACd,IAA6B;IAE7B,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AACpE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACtC,MAAc,EACd,UAAkB,EAClB,MAAgB,EAChB,KAAc;IAEd,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;AACtE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAY,EACZ,EAAU,EACV,MAAc;IAEd,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;AACpD,CAAC"}

package/dist/realtime-speech.d.ts

@@ -0,0 +1,159 @@
+/**
+ * Realtime speech-to-speech provider abstraction (RFC 0001).
+ *
+ * A RealtimeSpeechProvider connects a voice session to a provider's
+ * native speech-to-speech model over a bidirectional stream: mic PCM
+ * goes up, assistant PCM comes down, with transcripts, tool calls, and
+ * turn events normalized into RealtimeSpeechEvent. Turn detection,
+ * comprehension, and voice synthesis are provider-side; the orchestrator
+ * relays audio and reacts to events.
+ *
+ * Experimental: shipped behind the opt-in `realtimeSpeech` server mode.
+ */
+import type { ToolDefinition } from './tools.js';
+export interface RealtimeSpeechConfig {
+    /** System prompt for the realtime model. */
+    instructions?: string;
+    /** Provider voice id (pass-through, e.g. OpenAI 'marin'). */
+    voice?: string;
+    /** Tool definitions - same shape as pipeline mode. */
+    tools?: ToolDefinition[];
+    /**
+     * Emit user-transcript events (default true). Input transcription is
+     * a separate transcription-model pass billed on top of realtime audio
+     * tokens.
+     */
+    inputTranscription?: boolean;
+    /**
+     * Transcription model for user transcripts
+     * (default: 'gpt-4o-mini-transcribe' on OpenAI; Gemini transcribes
+     * natively).
+     */
+    transcriptionModel?: string;
+    /** Provider-native VAD tuning. */
+    turnDetection?: {
+        type: 'server_vad';
+        silenceDurationMs?: number;
+        thresholdOverride?: number;
+    } | {
+        type: 'semantic';
+        eagerness?: 'low' | 'medium' | 'high' | 'auto';
+    };
+    /** Per-response output token cap (the primary runaway-cost bound). */
+    maxOutputTokens?: number;
+    /** Context-size cost lever, provider-mapped. */
+    contextManagement?: {
+        strategy: 'truncate' | 'compress';
+        /** OpenAI truncation retention ratio. */
+        retentionRatio?: number;
+        /** Gemini compression trigger tokens. */
+        triggerTokens?: number;
+    };
+}
+export interface RealtimeUsage {
+    inputTokens: number;
+    outputTokens: number;
+    audioInputTokens?: number;
+    audioOutputTokens?: number;
+    cachedTokens?: number;
+}
+/** Events pushed by the provider session, normalized across providers. */
+export type RealtimeSpeechEvent = 
+/** Assistant audio delta, tagged so stale deltas of a cancelled response can be dropped. */
+{
+    type: 'audio';
+    pcm: Buffer;
+    sampleRate: number;
+    responseId: string;
+    itemId?: string;
+} | {
+    type: 'user-transcript';
+    text: string;
+    isFinal: boolean;
+} | {
+    type: 'assistant-transcript';
+    text: string;
+    isFinal: boolean;
+}
+/** Barge-in signal (OpenAI server VAD; not emitted by Gemini). */
+ | {
+    type: 'user-speech-started';
+} | {
+    type: 'user-speech-stopped';
+} | {
+    type: 'response-started';
+    responseId: string;
+} | {
+    type: 'response-done';
+    responseId: string;
+    usage?: RealtimeUsage;
+} | {
+    type: 'tool-call';
+    callId: string;
+    name: string;
+    arguments: Record<string, unknown>;
+}
+/** Provider cancelled in-flight tool calls (Gemini interruption). */
+ | {
+    type: 'tool-call-cancelled';
+    callIds: string[];
+}
+/** Provider-driven barge-in (Gemini). */
+ | {
+    type: 'interrupted';
+}
+/**
+ * Session nearing its lifetime cap. renewable: true means the adapter
+ * has no internal recovery (OpenAI expires_at) and the orchestrator
+ * should renew by reseeding; adapters that recover internally (Gemini
+ * resumption) emit it as informational only.
+ */
+ | {
+    type: 'session-expiring';
+    inMs?: number;
+    renewable?: boolean;
+} | {
+    type: 'error';
+    error: Error;
+    recoverable: boolean;
+};
+export interface RealtimeSpeechSession {
+    /** PCM rate the session expects from sendAudio (16k Gemini, 24k OpenAI). */
+    readonly inputSampleRate: number;
+    /** PCM rate of emitted 'audio' events (24k for both current targets). */
+    readonly outputSampleRate: number;
+    /**
+     * Ordered control stream. Consumers must never block this stream on
+     * realtime-paced playback: move 'audio' events into a playback queue
+     * synchronously and handle everything else immediately.
+     */
+    events(): AsyncIterable<RealtimeSpeechEvent>;
+    /**
+     * Fire-and-forget mic PCM (16-bit signed LE mono at inputSampleRate).
+     * During an adapter-internal reconnect, frames are buffered (bounded)
+     * and replayed once the session is re-established; frames beyond the
+     * bound are dropped.
+     */
+    sendAudio(frame: Buffer): void;
+    /**
+     * Cancel the in-progress response, if any; a safe no-op when nothing
+     * is active. playedMs (from the playback pacer) trims provider-side
+     * history to the audio the user actually heard, where the provider
+     * supports truncation.
+     */
+    cancelResponse(playedMs?: number): void;
+    sendToolResult(callId: string, output: unknown): void;
+    /**
+     * Ask the model to produce a response now (e.g. a stage onEnter
+     * announcement after a playbook transition). Optional: providers
+     * whose responses are purely VAD-driven may omit it.
+     */
+    requestResponse?(): void;
+    /** Live re-configuration (playbook stage changes swap instructions/tools). */
+    update(config: Partial<RealtimeSpeechConfig>): Promise<void>;
+    close(): Promise<void>;
+}
+export interface RealtimeSpeechProvider {
+    name: string;
+    connect(config: RealtimeSpeechConfig): Promise<RealtimeSpeechSession>;
+}

package/dist/realtime-speech.js

@@ -0,0 +1,14 @@
+/**
+ * Realtime speech-to-speech provider abstraction (RFC 0001).
+ *
+ * A RealtimeSpeechProvider connects a voice session to a provider's
+ * native speech-to-speech model over a bidirectional stream: mic PCM
+ * goes up, assistant PCM comes down, with transcripts, tool calls, and
+ * turn events normalized into RealtimeSpeechEvent. Turn detection,
+ * comprehension, and voice synthesis are provider-side; the orchestrator
+ * relays audio and reacts to events.
+ *
+ * Experimental: shipped behind the opt-in `realtimeSpeech` server mode.
+ */
+export {};
+//# sourceMappingURL=realtime-speech.js.map

package/dist/realtime-speech.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-speech.js","sourceRoot":"","sources":["../src/realtime-speech.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llmrtc/llmrtc-core",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Core contracts and orchestrators for @llmrtc/LLMRTC",
   "homepage": "https://www.llmrtc.org",
   "license": "Apache-2.0",