@kuralle-syrinx/realtime 4.4.1 → 4.5.0

package/dist/realtime-bridge.js

@@ -0,0 +1,659 @@
+// SPDX-License-Identifier: MIT
+import { Route, categorizeLlmError, } from "@kuralle-syrinx/core";
+import { pcm16BytesToSamples } from "@kuralle-syrinx/core/audio";
+const ENGINE_SAMPLE_RATE_HZ = 16_000;
+const FRAME_SAMPLES_20MS = 320;
+const FRAME_BYTES_20MS = FRAME_SAMPLES_20MS * 2;
+export class RealtimeBridge {
+    adapter;
+    reasoner;
+    delegateToolName;
+    opts;
+    bus = null;
+    contextId = "";
+    turnUserText = "";
+    /** Authoritative full assistant transcript(s) — providers that emit a final transcript (OpenAI). */
+    turnAssistantText = "";
+    /** Concatenated streamed transcript fragments — providers that emit deltas only, no final (Gemini Live). */
+    turnAssistantDeltas = "";
+    pendingSpeechEndedAtMs = null;
+    sessionAbort = null;
+    inflight;
+    delegateTask = null;
+    playedMs = 0;
+    audioRemainder = new Uint8Array(0);
+    disposers = [];
+    constructor(adapter, reasoner, delegateToolName = "consult_knowledge", opts = {}) {
+        this.adapter = adapter;
+        this.reasoner = reasoner;
+        this.delegateToolName = delegateToolName;
+        this.opts = opts;
+    }
+    injectContext(text) {
+        // Provider history differs here: OpenAI retains system items, while Gemini drops
+        // system/developer history, so the adapter selects the provider-safe representation.
+        if (this.adapter.injectContext) {
+            this.adapter.injectContext(text);
+            return;
+        }
+        console.warn("RealtimeBridge: context injection requested but the adapter does not support it");
+    }
+    async initialize(bus, _cfg) {
+        this.bus = bus;
+        this.sessionAbort = new AbortController();
+        this.disposers.push(bus.on("user.audio_received", (pkt) => {
+            const resampled = resamplePcm16Bytes(pkt.audio, ENGINE_SAMPLE_RATE_HZ, this.adapter.caps.inputSampleRateHz);
+            this.adapter.sendAudio(resampled);
+        }), bus.on("user.text_received", (pkt) => {
+            if (pkt.text.trim().length > 0)
+                this.adapter.sendText?.(pkt.text);
+        }), bus.on("tts.playout_progress", (pkt) => {
+            if (pkt.contextId === this.contextId)
+                this.playedMs = pkt.playedOutMs;
+        }), bus.on("interrupt.tts", () => {
+            this.adapter.cancelResponse(this.playedMs);
+            this.inflight?.abort();
+        }));
+        if (this.opts.syrinxTurns) {
+            this.disposers.push(bus.on("eos.turn_complete", () => {
+                this.adapter.requestResponse?.();
+            }));
+        }
+        await this.adapter.open(this.sessionAbort.signal);
+        void this.pump();
+    }
+    async close() {
+        this.sessionAbort?.abort();
+        this.inflight?.abort();
+        for (const dispose of this.disposers.splice(0))
+            dispose();
+        await this.adapter.close();
+        this.bus = null;
+        this.sessionAbort = null;
+    }
+    async pump() {
+        const bus = this.bus;
+        if (!bus)
+            return;
+        try {
+            for await (const ev of this.adapter.events) {
+                if (!this.bus)
+                    return;
+                await this.handleEvent(bus, ev);
+            }
+        }
+        catch (err) {
+            if (!this.bus || isAbortError(err))
+                return;
+            this.onError(this.bus, err instanceof Error ? err : new Error(String(err)), false);
+        }
+    }
+    async handleEvent(bus, ev) {
+        switch (ev.type) {
+            case "response_started":
+                this.onResponseStarted(bus);
+                break;
+            case "audio":
+                // textOnly: provider audio must not be used (REQ-4) — TTS plugin owns playout.
+                if (!this.opts.textOnly)
+                    this.onAudio(bus, ev.pcm16, ev.sampleRateHz);
+                break;
+            case "transcript":
+                if (ev.role === "user") {
+                    if (ev.final)
+                        this.onFinalTranscript(bus, ev.text);
+                }
+                else if (this.opts.textOnly) {
+                    // Drive cascade TTS path as text arrives; do not buffer for display-only re-emit.
+                    this.onTextOnlyAssistantTranscript(bus, ev.text, ev.final);
+                }
+                else if (ev.final && ev.text.trim()) {
+                    this.turnAssistantText = this.turnAssistantText
+                        ? `${this.turnAssistantText} ${ev.text.trim()}`
+                        : ev.text.trim();
+                }
+                else if (!ev.final && ev.text) {
+                    // Streamed fragments already carry their own leading spaces — concatenate verbatim.
+                    this.turnAssistantDeltas += ev.text;
+                    bus.push(Route.Main, {
+                        kind: "llm.delta",
+                        contextId: this.contextId,
+                        timestampMs: Date.now(),
+                        text: ev.text,
+                    });
+                }
+                break;
+            case "tool_call":
+                await this.handleToolCall(bus, ev);
+                break;
+            case "response_done":
+                this.onResponseDone(bus);
+                // Meter the native front — previously native turns produced no usage at all.
+                if (ev.usage && this.contextId) {
+                    bus.push(Route.Background, {
+                        kind: "usage.recorded",
+                        contextId: this.contextId,
+                        timestampMs: Date.now(),
+                        stage: "llm",
+                        ...ev.usage,
+                    });
+                }
+                break;
+            case "speech_started":
+                this.onSpeechStarted(bus);
+                break;
+            case "speech_stopped":
+                this.onSpeechStopped(bus);
+                break;
+            case "resumption_handle": {
+                // G4: persist-worthy native-resume handle (Gemini). Background route —
+                // a durable host stores the latest and passes it back on reconnect.
+                const packet = {
+                    kind: "realtime.resumption_handle",
+                    contextId: this.contextId,
+                    timestampMs: Date.now(),
+                    handle: ev.handle,
+                };
+                bus.push(Route.Background, packet);
+                break;
+            }
+            case "error":
+                this.onError(bus, ev.cause, ev.recoverable);
+                break;
+            default:
+                break;
+        }
+    }
+    async handleToolCall(bus, ev) {
+        if (ev.toolName === this.delegateToolName) {
+            if (this.reasoner)
+                this.dispatchDelegate(bus, ev);
+            return;
+        }
+        if (this.opts.onFrontToolCall) {
+            await this.handleFrontTool(bus, ev);
+            return;
+        }
+        // No front-tool handler: with a reasoner, an unexpected tool is a recoverable error
+        // (without one, tool calls are ignored, as before).
+        if (this.reasoner) {
+            const cause = new Error(`Unexpected tool call "${ev.toolName}" (expected "${this.delegateToolName}")`);
+            if (this.opts.debug)
+                console.error("[realtime-bridge]", cause.message);
+            this.onError(bus, cause, true);
+        }
+    }
+    async handleFrontTool(bus, ev) {
+        if (!this.opts.onFrontToolCall)
+            return;
+        let result;
+        try {
+            result = (await this.opts.onFrontToolCall({ toolId: ev.toolId, toolName: ev.toolName, args: ev.args })) ?? "";
+        }
+        catch (err) {
+            this.onError(bus, err instanceof Error ? err : new Error(String(err)), true);
+            return;
+        }
+        this.adapter.injectToolResult(ev.toolId, result);
+    }
+    /**
+     * Start the reasoner delegate OFF the serial event pump. The pump must keep draining adapter
+     * events during the multi-second reasoner "thinking gap" — above all `speech_started`, so a
+     * barge-in mid-delegate still fires `interrupt.detected` and (via `interrupt.tts`) aborts the
+     * now-unwanted delegate before its answer is voiced over the user. Awaiting the delegate inside
+     * the pump loop (the old shape) froze barge-in for the entire gap. `runDelegate` sets
+     * `this.inflight` synchronously before its first await, so a subsequently-pumped barge-in sees
+     * a live controller to abort.
+     */
+    dispatchDelegate(bus, ev) {
+        this.delegateTask = this.runDelegate(bus, ev).catch((err) => {
+            if (!this.bus || isAbortError(err))
+                return;
+            this.onError(bus, err instanceof Error ? err : new Error(String(err)), false);
+        });
+    }
+    async runDelegate(bus, ev) {
+        // Pin the turn's contextId now: the pump keeps advancing while the delegate streams, so
+        // this.contextId may move to a later turn before we finish. The tool_call/tool_result pair
+        // must stay bound to the turn that issued the call (R1: never merge parts across turns).
+        const contextId = this.contextId;
+        if (!contextId)
+            return;
+        const toolCall = {
+            kind: "llm.tool_call",
+            contextId,
+            timestampMs: Date.now(),
+            toolId: ev.toolId,
+            toolName: ev.toolName,
+            toolArgs: ev.args,
+        };
+        bus.push(Route.Main, toolCall);
+        const queryArg = this.opts.delegateQueryArg ?? "query";
+        const rawQuery = ev.args[queryArg];
+        const userText = typeof rawQuery === "string" ? rawQuery : "";
+        if (userText.trim().length === 0) {
+            this.onError(bus, new Error(`delegate tool "${ev.toolName}" called without a string "${queryArg}" argument`), true);
+            return;
+        }
+        const controller = new AbortController();
+        this.inflight = controller;
+        let answer = "";
+        let grounded = false;
+        let blockedMessage;
+        let blockedPayload;
+        let passStartedMs = Date.now();
+        let passTtftRecorded = false;
+        const pushConversationMetric = (name, value, timestampMs) => {
+            bus.push(Route.Main, {
+                kind: "metric.conversation",
+                contextId,
+                timestampMs,
+                name,
+                value,
+            });
+        };
+        const recordPassTtft = (timestampMs) => {
+            if (passTtftRecorded)
+                return;
+            passTtftRecorded = true;
+            pushConversationMetric("llm.pass_ttft_ms", String(timestampMs - passStartedMs), timestampMs);
+        };
+        pushConversationMetric("llm.call_started", "1", passStartedMs);
+        // G2 observability: the query is on its way to the reasoner (Background route, R4).
+        const queryStartedMs = Date.now();
+        const delegateQuery = {
+            kind: "delegate.query",
+            contextId,
+            timestampMs: queryStartedMs,
+            query: userText,
+            toolId: ev.toolId,
+            toolName: ev.toolName,
+        };
+        bus.push(Route.Background, delegateQuery);
+        try {
+            const messages = this.opts.contextProvider?.() ?? [];
+            for await (const part of this.reasoner.stream({
+                userText,
+                toolArgs: ev.args,
+                messages,
+                signal: controller.signal,
+            })) {
+                switch (part.type) {
+                    case "text-delta":
+                        recordPassTtft(Date.now());
+                        answer += part.text;
+                        break;
+                    case "tool-call":
+                        recordPassTtft(Date.now());
+                        break;
+                    case "tool-result":
+                        grounded = true;
+                        passStartedMs = Date.now();
+                        passTtftRecorded = false;
+                        pushConversationMetric("llm.call_started", "1", passStartedMs);
+                        break;
+                    case "control": {
+                        const controlMs = Date.now();
+                        bus.push(Route.Background, {
+                            kind: "delegate.result",
+                            contextId,
+                            timestampMs: controlMs,
+                            query: userText,
+                            answer,
+                            durationMs: controlMs - queryStartedMs,
+                            grounded,
+                            toolId: ev.toolId,
+                            toolName: ev.toolName,
+                            control: { name: part.name, payload: part.payload },
+                        });
+                        break;
+                    }
+                    case "blocked":
+                        blockedMessage = part.userFacingMessage;
+                        blockedPayload = part.payload;
+                        break;
+                    case "finish":
+                        if (!answer && part.text)
+                            answer = part.text;
+                        break;
+                    case "suspended": {
+                        const cause = new Error(part.prompt ?? "delegate suspended — cannot voice inline without resume");
+                        if (this.opts.debug)
+                            console.error("[realtime-bridge]", cause.message, part.payload);
+                        this.onError(bus, cause, false);
+                        return;
+                    }
+                    case "error":
+                        if (!part.recoverable)
+                            throw part.cause;
+                        this.onError(bus, part.cause, true);
+                        return;
+                }
+                if (blockedMessage !== undefined)
+                    break;
+            }
+        }
+        catch (err) {
+            if (isAbortError(err))
+                return;
+            this.onError(bus, err instanceof Error ? err : new Error(String(err)), false);
+            return;
+        }
+        finally {
+            // Only clear if still ours — a newer delegate may have replaced this.inflight.
+            if (this.inflight === controller)
+                this.inflight = undefined;
+        }
+        if (blockedMessage !== undefined) {
+            const blockedMs = Date.now();
+            bus.push(Route.Background, {
+                kind: "delegate.result",
+                contextId,
+                timestampMs: blockedMs,
+                query: userText,
+                answer: blockedMessage,
+                durationMs: blockedMs - queryStartedMs,
+                grounded,
+                toolId: ev.toolId,
+                toolName: ev.toolName,
+                blocked: { userFacingMessage: blockedMessage, payload: blockedPayload },
+            });
+            this.adapter.injectToolResult(ev.toolId, this.formatToolResult(blockedMessage));
+            return;
+        }
+        if (answer.length === 0) {
+            this.onError(bus, new Error("delegate produced no output"), false);
+            return;
+        }
+        const answeredMs = Date.now();
+        const toolResult = {
+            kind: "llm.tool_result",
+            contextId,
+            timestampMs: answeredMs,
+            toolId: ev.toolId,
+            toolName: ev.toolName,
+            result: answer,
+        };
+        bus.push(Route.Main, toolResult);
+        // G2 observability: the reasoner produced the final answer (Background route, R4).
+        const delegateResult = {
+            kind: "delegate.result",
+            contextId,
+            timestampMs: answeredMs,
+            query: userText,
+            answer,
+            durationMs: answeredMs - queryStartedMs,
+            grounded,
+            toolId: ev.toolId,
+            toolName: ev.toolName,
+        };
+        bus.push(Route.Background, delegateResult);
+        this.adapter.injectToolResult(ev.toolId, this.formatToolResult(answer));
+    }
+    /** G1: wrap the buffered delegate answer for the front model (synchronous — R2). */
+    formatToolResult(answer) {
+        if (this.opts.toolResultFormat === "string")
+            return answer;
+        const envelope = {
+            response_text: answer,
+            require_repeat_verbatim: true,
+            ...(this.opts.renderDirective ? { render: this.opts.renderDirective } : {}),
+        };
+        return JSON.stringify(envelope);
+    }
+    onSpeechStarted(bus) {
+        if (!this.adapter.caps.emitsServerSpeechStarted || !this.contextId)
+            return;
+        const packet = {
+            kind: "interrupt.detected",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            source: "vad",
+        };
+        bus.push(Route.Critical, packet);
+    }
+    onSpeechStopped(bus) {
+        const timestampMs = Date.now();
+        if (!this.contextId) {
+            // Server VAD reports speech end before response.created mints the response context.
+            this.pendingSpeechEndedAtMs = timestampMs;
+            return;
+        }
+        bus.push(Route.Main, {
+            kind: "vad.speech_ended",
+            contextId: this.contextId,
+            timestampMs,
+        });
+    }
+    onResponseStarted(bus) {
+        const previousContextId = this.contextId;
+        this.contextId = crypto.randomUUID();
+        this.turnUserText = "";
+        this.turnAssistantText = "";
+        this.turnAssistantDeltas = "";
+        this.playedMs = 0;
+        this.audioRemainder = new Uint8Array(0);
+        if (this.pendingSpeechEndedAtMs !== null) {
+            bus.push(Route.Main, {
+                kind: "vad.speech_ended",
+                contextId: this.contextId,
+                timestampMs: this.pendingSpeechEndedAtMs,
+            });
+            this.pendingSpeechEndedAtMs = null;
+        }
+        const packet = {
+            kind: "turn.change",
+            contextId: this.contextId,
+            previousContextId,
+            reason: "realtime_response_started",
+            timestampMs: Date.now(),
+        };
+        bus.push(Route.Main, packet);
+    }
+    onAudio(bus, pcm16, sampleRateHz) {
+        if (!this.contextId)
+            return;
+        const resampled = resamplePcm16Bytes(pcm16, sampleRateHz, ENGINE_SAMPLE_RATE_HZ);
+        this.audioRemainder = concatBytes(this.audioRemainder, resampled);
+        this.emitCoalescedAudio(bus, false);
+    }
+    onFinalTranscript(bus, text) {
+        if (!this.contextId || text.trim().length === 0)
+            return;
+        this.turnUserText = text;
+        const packet = {
+            kind: "stt.result",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            text,
+            confidence: 0,
+        };
+        bus.push(Route.Main, packet);
+    }
+    /**
+     * textOnly: stream assistant transcript into the cascade LLM→segmenter→TTS path.
+     * Non-final deltas drive `llm.delta` immediately (no buffer). Final emits `llm.done` only —
+     * the C0 adapter's final carries the full accumulated text, so re-emitting as delta would
+     * duplicate every streamed fragment.
+     */
+    onTextOnlyAssistantTranscript(bus, text, final) {
+        if (!this.contextId)
+            return;
+        const timestampMs = Date.now();
+        if (!final && text) {
+            const delta = {
+                kind: "llm.delta",
+                contextId: this.contextId,
+                timestampMs,
+                text,
+            };
+            bus.push(Route.Main, delta);
+            return;
+        }
+        if (final && text.trim()) {
+            const done = {
+                kind: "llm.done",
+                contextId: this.contextId,
+                timestampMs,
+                text,
+            };
+            bus.push(Route.Main, done);
+        }
+    }
+    onResponseDone(bus) {
+        if (!this.contextId)
+            return;
+        if (!this.opts.textOnly) {
+            this.emitCoalescedAudio(bus, true);
+            this.audioRemainder = new Uint8Array(0);
+        }
+        const timestampMs = Date.now();
+        const transcripts = this.turnUserText
+            ? [{
+                    kind: "stt.result",
+                    contextId: this.contextId,
+                    timestampMs,
+                    text: this.turnUserText,
+                    confidence: 0,
+                }]
+            : [];
+        const turnComplete = {
+            kind: "eos.turn_complete",
+            contextId: this.contextId,
+            timestampMs,
+            text: this.turnUserText,
+            transcripts,
+            endpointingOwner: "timer",
+            endpointingReason: "end_of_speech",
+        };
+        if (this.opts.textOnly) {
+            // Already streamed llm.delta/llm.done above. TTS plugin owns tts.end (REQ-5) — forcing
+            // it here would cut playout / break barge-in heard-prefix timing.
+            // syrinxTurns: endpointing already emitted the user-turn eos.turn_complete that drove
+            // requestResponse; re-emitting here would re-subscribe-fire requestResponse → loop.
+            if (!this.opts.syrinxTurns)
+                bus.push(Route.Main, turnComplete);
+            return;
+        }
+        const ttsEnd = {
+            kind: "tts.end",
+            contextId: this.contextId,
+            timestampMs,
+        };
+        // Surface the assistant's spoken transcript as agent text so UIs (e.g. the studio) render it.
+        // The realtime adapter already produced the audio directly; these packets are display-only.
+        // Prefer the authoritative final transcript (OpenAI); fall back to the streamed fragments for
+        // providers that only emit non-final deltas (Gemini Live).
+        const assistantText = this.turnAssistantText.trim() || this.turnAssistantDeltas.trim();
+        if (assistantText) {
+            const done = {
+                kind: "llm.done",
+                contextId: this.contextId,
+                timestampMs,
+                text: assistantText,
+            };
+            if (this.turnAssistantDeltas.length === 0) {
+                bus.push(Route.Main, {
+                    kind: "llm.delta",
+                    contextId: this.contextId,
+                    timestampMs,
+                    text: assistantText,
+                });
+            }
+            bus.push(Route.Main, done);
+        }
+        bus.push(Route.Main, turnComplete, ttsEnd);
+    }
+    emitCoalescedAudio(bus, flush) {
+        if (!this.contextId)
+            return;
+        let buf = this.audioRemainder;
+        if (buf.byteLength === 0)
+            return;
+        let processLen = buf.byteLength;
+        if (processLen % 2 !== 0) {
+            processLen -= 1;
+        }
+        if (processLen < 2)
+            return;
+        let offset = 0;
+        while (offset + FRAME_BYTES_20MS <= processLen) {
+            const frame = buf.subarray(offset, offset + FRAME_BYTES_20MS);
+            this.pushAudioFrame(bus, frame);
+            offset += FRAME_BYTES_20MS;
+        }
+        const trailingEven = processLen - offset;
+        if (flush && trailingEven >= 2) {
+            this.pushAudioFrame(bus, buf.subarray(offset, offset + trailingEven));
+            offset += trailingEven;
+        }
+        const leftoverStart = offset;
+        const leftoverEnd = buf.byteLength;
+        this.audioRemainder =
+            leftoverStart < leftoverEnd
+                ? new Uint8Array(buf.subarray(leftoverStart, leftoverEnd))
+                : new Uint8Array(0);
+    }
+    pushAudioFrame(bus, frame) {
+        if (frame.byteLength % 2 !== 0 || frame.byteLength === 0)
+            return;
+        const packet = {
+            kind: "tts.audio",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            audio: frame,
+            sampleRateHz: ENGINE_SAMPLE_RATE_HZ,
+        };
+        bus.push(Route.Main, packet);
+    }
+    onError(bus, cause, isRecoverable) {
+        const packet = {
+            kind: "llm.error",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            component: "bridge",
+            category: categorizeLlmError(cause),
+            cause,
+            isRecoverable,
+        };
+        bus.push(Route.Critical, packet);
+    }
+}
+function concatBytes(a, b) {
+    const out = new Uint8Array(a.byteLength + b.byteLength);
+    out.set(a, 0);
+    out.set(b, a.byteLength);
+    return out;
+}
+function resamplePcm16Bytes(pcm16, fromHz, toHz) {
+    if (fromHz === toHz)
+        return new Uint8Array(pcm16);
+    // pcm16 may be a view at an ODD byteOffset (decoded envelope payload) — `new Int16Array(buffer, offset)`
+    // throws "start offset … multiple of 2". pcm16BytesToSamples copies via DataView, which is offset-safe.
+    // It requires an even byteLength, so drop a trailing odd byte first (matches the prior truncating view).
+    const even = pcm16.byteLength % 2 === 0 ? pcm16 : pcm16.subarray(0, pcm16.byteLength - 1);
+    const samples = pcm16BytesToSamples(even);
+    const out = resamplePcm16(samples, fromHz, toHz);
+    const bytes = new Uint8Array(out.byteLength);
+    bytes.set(new Uint8Array(out.buffer, out.byteOffset, out.byteLength));
+    return bytes;
+}
+function resamplePcm16(samples, fromHz, toHz) {
+    if (fromHz === toHz)
+        return samples;
+    const outLength = Math.max(1, Math.round((samples.length * toHz) / fromHz));
+    const out = new Int16Array(outLength);
+    const ratio = fromHz / toHz;
+    for (let i = 0; i < out.length; i += 1) {
+        const src = i * ratio;
+        const lo = Math.floor(src);
+        const hi = Math.min(samples.length - 1, lo + 1);
+        const frac = src - lo;
+        out[i] = Math.round(samples[lo] * (1 - frac) + samples[hi] * frac);
+    }
+    return out;
+}
+function isAbortError(err) {
+    return err instanceof Error && err.name === "AbortError";
+}
+//# sourceMappingURL=realtime-bridge.js.map