@kuralle-syrinx/realtime 4.4.0 → 4.5.0

package/dist/realtime-adapter.d.ts

@@ -0,0 +1,100 @@
+export interface RealtimeAdapter {
+    readonly caps: {
+        readonly inputSampleRateHz: number;
+        readonly outputSampleRateHz: number;
+        readonly supportsConcurrentToolAudio: boolean;
+        readonly supportsTruncate: boolean;
+        readonly emitsServerSpeechStarted: boolean;
+        /**
+         * G4: the provider resumes a prior session server-side from a handle (Gemini Live
+         * `sessionResumption`). Absent/false → resuming requires the host to replay the
+         * transcript (e.g. OpenAI `resumeHistory`). A native-resume provider must NOT also
+         * be replayed — that would double-apply history (RFC bimodel-delegate-seam R6).
+         */
+        readonly supportsNativeResume?: boolean;
+        /** The front model owns full-duplex interaction decisions (turn-taking, barge-in). When true,
+         *  Syrinx's InteractionPolicy runs observe-only and does not drive its own turn/interrupt decisions
+         *  (RFC InteractionPolicy REQ-4). Absent/false → Syrinx drives. No current adapter sets this. */
+        readonly supportsFullDuplex?: boolean;
+        /** The front emits its own backchannels ("mhmm"). When true, Syrinx suppresses its own backchannel
+         *  cues (RFC InteractionPolicy REQ-4). Absent/false → Syrinx may emit. */
+        readonly emitsBackchannel?: boolean;
+        /** Half-cascade: provider can run text-only modality (`modalities:["text"]`) so Syrinx TTS
+         *  drives speech from assistant transcript events. Absent/false → no text-only half-cascade. */
+        readonly supportsTextOnlyModality?: boolean;
+    };
+    open(signal: AbortSignal): Promise<void>;
+    sendAudio(pcm16: Uint8Array): void;
+    /**
+     * Send a typed user turn to the front model and request a response. Optional: adapters whose
+     * provider cannot accept text input omit it, and the bridge silently ignores typed turns for them.
+     */
+    sendText?(text: string): void;
+    /** Inject transient context without requesting a provider response. */
+    injectContext?(text: string): void;
+    /**
+     * Commit any buffered user input and request a response. For Syrinx-OWNED turn detection
+     * (provider server VAD disabled via turnDetection:null): the host calls this when its own
+     * endpointing (InteractionPolicy / VAD) signals end-of-turn. Optional — adapters without
+     * manual turn control omit it.
+     */
+    requestResponse?(): void;
+    cancelResponse(audioEndMs: number): void;
+    injectToolResult(toolId: string, text: string): void;
+    /** Close the provider socket and end the event stream. */
+    close(): Promise<void>;
+    readonly events: AsyncIterable<RealtimeEvent>;
+}
+/**
+ * A function tool advertised to the front model so it can decide when to delegate. Domain-neutral:
+ * the caller (example/app) supplies these — the provider adapter never hardcodes any tool.
+ */
+export interface RealtimeToolDef {
+    readonly name: string;
+    readonly description: string;
+    /** JSON Schema for the tool arguments. */
+    readonly parameters: Record<string, unknown>;
+}
+/** A prior-conversation message replayed into a front model on resume (G4). */
+export interface RealtimeResumeMessage {
+    readonly role: "user" | "assistant";
+    readonly content: string;
+}
+/** Token usage a realtime provider reports on response completion (e.g. OpenAI response.done). */
+export interface RealtimeUsage {
+    readonly inputTokens?: number;
+    readonly outputTokens?: number;
+    readonly totalTokens?: number;
+}
+export type RealtimeEvent = {
+    type: "audio";
+    pcm16: Uint8Array;
+    sampleRateHz: number;
+} | {
+    type: "speech_started";
+} | {
+    type: "speech_stopped";
+} | {
+    type: "transcript";
+    role: "user" | "assistant";
+    text: string;
+    final: boolean;
+} | {
+    type: "tool_call";
+    toolId: string;
+    toolName: string;
+    args: Record<string, unknown>;
+} | {
+    type: "response_started";
+} | {
+    type: "response_done";
+    usage?: RealtimeUsage;
+} | {
+    type: "resumption_handle";
+    handle: string;
+} | {
+    type: "error";
+    cause: Error;
+    recoverable: boolean;
+};
+//# sourceMappingURL=realtime-adapter.d.ts.map

package/dist/realtime-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-adapter.d.ts","sourceRoot":"","sources":["../src/realtime-adapter.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;QACnC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;QACpC,QAAQ,CAAC,2BAA2B,EAAE,OAAO,CAAC;QAC9C,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;QACnC,QAAQ,CAAC,wBAAwB,EAAE,OAAO,CAAC;QAC3C;;;;;WAKG;QACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,CAAC;QACxC;;yGAEiG;QACjG,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QACtC;kFAC0E;QAC1E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;QACpC;wGACgG;QAChG,QAAQ,CAAC,wBAAwB,CAAC,EAAE,OAAO,CAAC;KAC7C,CAAC;IAEF,IAAI,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACnC;;;OAGG;IACH,QAAQ,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uEAAuE;IACvE,aAAa,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC;;;;;OAKG;IACH,eAAe,CAAC,IAAI,IAAI,CAAC;IACzB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACrD,0DAA0D;IAC1D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;CAC/C;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,0CAA0C;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC9C;AAED,+EAA+E;AAC/E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,kGAAkG;AAClG,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC1D;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAChF;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GACtF;IAAE,IAAI,EAAE,kBAAkB,CAAA;CAAE,GAC5B;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,KAAK,CAAC,EAAE,aAAa,CAAA;CAAE,GAGhD;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,WAAW,EAAE,OAAO,CAAA;CAAE,CAAC"}

package/dist/realtime-adapter.js

@@ -0,0 +1,3 @@
+// SPDX-License-Identifier: MIT
+export {};
+//# sourceMappingURL=realtime-adapter.js.map

package/dist/realtime-adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-adapter.js","sourceRoot":"","sources":["../src/realtime-adapter.ts"],"names":[],"mappings":"AAAA,+BAA+B"}

package/dist/realtime-bridge.d.ts

@@ -0,0 +1,130 @@
+import { type PipelineBus, type PluginConfig, type Reasoner, type ReasonerMessage, type VoicePlugin } from "@kuralle-syrinx/core";
+import type { RealtimeAdapter } from "./realtime-adapter.js";
+/**
+ * G1 — the structured delegate-result envelope (RFC bimodel-delegate-seam §4).
+ * The bridge wraps the accumulated reasoner answer in this JSON shape before
+ * `injectToolResult`, so the front model treats it as authoritative tool data
+ * and repeats it faithfully instead of paraphrasing or inventing facts. Field
+ * names mirror OpenAI's Realtime Prompting Guide "Tool Output Formatting"
+ * (`response_text` + `require_repeat_verbatim`) — the only field-validated
+ * envelope shape, and gpt-realtime is the primary front (OQ1).
+ */
+export interface DelegateResultEnvelope {
+    /** The authoritative answer — the front model's ONLY source of facts for the turn. */
+    readonly response_text: string;
+    readonly require_repeat_verbatim: true;
+    /** Optional app directive the front prompt keys on, e.g. "translate_faithfully". */
+    readonly render?: string;
+}
+export interface RealtimeBridgeOptions {
+    readonly debug?: boolean;
+    /**
+     * How the delegate answer reaches the front model at `injectToolResult` time.
+     * `"envelope"` (default) wraps it as a {@link DelegateResultEnvelope}; `"string"`
+     * injects the raw answer (pre-envelope behavior). The envelope is applied to the
+     * already-buffered delegate answer — a synchronous JSON.stringify, no added latency
+     * (R2) — and never touches the cascade text-delta path. Bus packets
+     * (`llm.tool_result`, `delegate.result`) always carry the raw answer.
+     */
+    readonly toolResultFormat?: "envelope" | "string";
+    /**
+     * Optional `render` directive included in the envelope (e.g. `"translate_faithfully"`),
+     * for front prompts that key a rendering rule on it. Ignored with `toolResultFormat: "string"`.
+     */
+    readonly renderDirective?: string;
+    /**
+     * Supplies prior conversation context for delegate Reasoner turns. When omitted the delegate is
+     * stateless — each tool call receives only the query extracted from the front-model tool args.
+     */
+    readonly contextProvider?: () => readonly ReasonerMessage[];
+    /**
+     * Name of the front-model tool argument that carries the user's query for the delegate Reasoner.
+     * Must match the argument name in the registered delegate tool's schema. Defaults to "query".
+     */
+    readonly delegateQueryArg?: string;
+    /**
+     * Handle a front-model tool call whose name is NOT the delegate tool — front-level tools like
+     * `wait_for_user`, `escalate_to_human`, or `finish_session` that must not hit the reasoner.
+     * Return the string to inject back to the front model as the tool result (default `""`). When
+     * omitted, a non-delegate tool call remains a recoverable error (the prior behavior).
+     */
+    readonly onFrontToolCall?: (call: {
+        readonly toolId: string;
+        readonly toolName: string;
+        readonly args: Record<string, unknown>;
+    }) => string | undefined | Promise<string | undefined>;
+    /**
+     * Half-cascade text-only mode: stream assistant transcript into the cascade
+     * `llm.delta → segmenter → tts.text` path and ignore provider audio.
+     * REQUIRES a TTS plugin registered on the session bus (the bridge does not own the registry).
+     */
+    readonly textOnly?: boolean;
+    /**
+     * Syrinx-owned turn detection (REQ-6): on `eos.turn_complete`, call
+     * `adapter.requestResponse` so Syrinx endpointing/VAD/InteractionPolicy drives
+     * the provider response instead of server VAD.
+     *
+     * MUST be paired with `turnDetection: null` on the adapter — otherwise the
+     * provider's server VAD and Syrinx would BOTH trigger responses (double-turn).
+     * Native mode (`syrinxTurns` absent/false) is unchanged.
+     */
+    readonly syrinxTurns?: boolean;
+}
+export declare class RealtimeBridge implements VoicePlugin {
+    private readonly adapter;
+    private readonly reasoner?;
+    private readonly delegateToolName;
+    private readonly opts;
+    private bus;
+    private contextId;
+    private turnUserText;
+    /** Authoritative full assistant transcript(s) — providers that emit a final transcript (OpenAI). */
+    private turnAssistantText;
+    /** Concatenated streamed transcript fragments — providers that emit deltas only, no final (Gemini Live). */
+    private turnAssistantDeltas;
+    private pendingSpeechEndedAtMs;
+    private sessionAbort;
+    private inflight;
+    private delegateTask;
+    private playedMs;
+    private audioRemainder;
+    private readonly disposers;
+    constructor(adapter: RealtimeAdapter, reasoner?: Reasoner | undefined, delegateToolName?: string, opts?: RealtimeBridgeOptions);
+    injectContext(text: string): void;
+    initialize(bus: PipelineBus, _cfg: PluginConfig): Promise<void>;
+    close(): Promise<void>;
+    private pump;
+    private handleEvent;
+    private handleToolCall;
+    private handleFrontTool;
+    /**
+     * Start the reasoner delegate OFF the serial event pump. The pump must keep draining adapter
+     * events during the multi-second reasoner "thinking gap" — above all `speech_started`, so a
+     * barge-in mid-delegate still fires `interrupt.detected` and (via `interrupt.tts`) aborts the
+     * now-unwanted delegate before its answer is voiced over the user. Awaiting the delegate inside
+     * the pump loop (the old shape) froze barge-in for the entire gap. `runDelegate` sets
+     * `this.inflight` synchronously before its first await, so a subsequently-pumped barge-in sees
+     * a live controller to abort.
+     */
+    private dispatchDelegate;
+    private runDelegate;
+    /** G1: wrap the buffered delegate answer for the front model (synchronous — R2). */
+    private formatToolResult;
+    private onSpeechStarted;
+    private onSpeechStopped;
+    private onResponseStarted;
+    private onAudio;
+    private onFinalTranscript;
+    /**
+     * textOnly: stream assistant transcript into the cascade LLM→segmenter→TTS path.
+     * Non-final deltas drive `llm.delta` immediately (no buffer). Final emits `llm.done` only —
+     * the C0 adapter's final carries the full accumulated text, so re-emitting as delta would
+     * duplicate every streamed fragment.
+     */
+    private onTextOnlyAssistantTranscript;
+    private onResponseDone;
+    private emitCoalescedAudio;
+    private pushAudioFrame;
+    private onError;
+}
+//# sourceMappingURL=realtime-bridge.d.ts.map

package/dist/realtime-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-bridge.d.ts","sourceRoot":"","sources":["../src/realtime-bridge.ts"],"names":[],"mappings":"AAEA,OAAO,EAaL,KAAK,WAAW,EAChB,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,eAAe,EAQpB,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,uBAAuB,CAAC;AAM5E;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACrC,sFAAsF;IACtF,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,uBAAuB,EAAE,IAAI,CAAC;IACvC,oFAAoF;IACpF,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB;;;;;;;OAOG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,UAAU,GAAG,QAAQ,CAAC;IAClD;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,SAAS,eAAe,EAAE,CAAC;IAC5D;;;OAGG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC;;;;;OAKG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE;QAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACxC,KAAK,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACvD;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;;;;;;;;OAQG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,qBAAa,cAAe,YAAW,WAAW;IAiB9C,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAC1B,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAnBvB,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,SAAS,CAAM;IACvB,OAAO,CAAC,YAAY,CAAM;IAC1B,oGAAoG;IACpG,OAAO,CAAC,iBAAiB,CAAM;IAC/B,4GAA4G;IAC5G,OAAO,CAAC,mBAAmB,CAAM;IACjC,OAAO,CAAC,sBAAsB,CAAuB;IACrD,OAAO,CAAC,YAAY,CAAgC;IACpD,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,cAAc,CAAkD;IACxE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;gBAGhC,OAAO,EAAE,eAAe,EACxB,QAAQ,CAAC,EAAE,QAAQ,YAAA,EACnB,gBAAgB,SAAsB,EACtC,IAAI,GAAE,qBAA0B;IAGnD,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU3B,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAqC/D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YASd,IAAI;YAmBJ,WAAW;YAwEX,cAAc;YAqBd,eAAe;IAe7B;;;;;;;;OAQG;IACH,OAAO,CAAC,gBAAgB;YAUV,WAAW;IA0LzB,oFAAoF;IACpF,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,eAAe;IAcvB,OAAO,CAAC,iBAAiB;IA0BzB,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,iBAAiB;IAazB;;;;;OAKG;IACH,OAAO,CAAC,6BAA6B;IAwBrC,OAAO,CAAC,cAAc;IAkEtB,OAAO,CAAC,kBAAkB;IAiC1B,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,OAAO;CAYhB"}