@urun-sh/openai 0.3.0 → 0.3.2

package/dist/translator-CcDBEfvm.d.cts

@@ -0,0 +1,227 @@
+import { k as VideoOutRecord } from './video-out-D20UuJ8G.cjs';
+
+/**
+ * Gemini Live (BidiGenerateContent) ⇄ uRun Responses translation — PURE
+ * functions only, no I/O and no session state. The wire shapes mirror the
+ * official WebSocket reference (https://ai.google.dev/api/live): client
+ * messages are a union of exactly one of `setup` / `clientContent` /
+ * `realtimeInput` / `toolResponse`; server messages are a union of
+ * `setupComplete` / `serverContent` / `toolCall` / … with `usageMetadata` as
+ * a SIBLING of the union field on `BidiGenerateContentServerMessage`.
+ *
+ * Scope (this slice): setup + clientContent + realtimeInput TEXT + AUDIO +
+ * VIDEO + serverContent streaming + toolCall/toolResponse + sessionResumption
+ * (handles over the proxy's pooled-session identity seam) + goAway + MANUAL
+ * VAD/activity (setup.realtimeInputConfig.automaticActivityDetection.disabled
+ * + realtimeInput.activityStart/activityEnd + serverContent.interrupted
+ * barge-in). AUTOMATIC (server-side) activity detection is the DECLARED
+ * follow-up slice — requesting it fails loud naming the follow-up, never
+ * silently ignored (repo mandate: one canonical path, no silent fallbacks).
+ *
+ * AUTOMATIC activity detection is deliberately NOT emulated here: server-side
+ * VAD is a PLATFORM primitive (urun-python `urun.serve.turn_detection`
+ * `detect_turns` — `turn_start`/`turn_end` events on the session's §5 stream
+ * lane), never adapter-local DSP in the proxy. Until the Live lane's session
+ * shapes emit those events, requesting automatic detection fails loud.
+ */
+
+/** The Live WS endpoint path (v1beta), verbatim from the Google reference. */
+declare const GEMINI_LIVE_PATH = "/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
+/** WS close code for protocol violations (invalid/unsupported payloads). */
+declare const CLOSE_PROTOCOL = 1007;
+/** WS close code for policy rejections (unknown model — sendUnknownModel's WS mirror). */
+declare const CLOSE_UNKNOWN_MODEL = 1008;
+/** WS close code for upstream failures (backhaul errored or ended without completing). */
+declare const CLOSE_UPSTREAM = 1011;
+/**
+ * WS close code when a `sessionResumption.handle` names a session that no
+ * longer exists (proxy restarted, handle expired/evicted, or never issued).
+ * Policy-shaped like CLOSE_UNKNOWN_MODEL: the client must reconnect WITHOUT
+ * the handle to start a new session — resume is never silently faked.
+ */
+declare const CLOSE_SESSION_GONE = 1008;
+/** A loud, translatable failure: `closeCode` is the WS close code to die with. */
+declare class GeminiLiveError extends Error {
+    readonly closeCode: number;
+    constructor(message: string, closeCode?: number);
+}
+type Rec = Record<string, unknown>;
+/** The parsed client message union — exactly one of the four Bidi kinds. */
+type ClientMessage = {
+    kind: 'setup';
+    setup: Rec;
+} | {
+    kind: 'clientContent';
+    clientContent: Rec;
+} | {
+    kind: 'realtimeInput';
+    realtimeInput: Rec;
+} | {
+    kind: 'toolResponse';
+    toolResponse: Rec;
+};
+/** Parse one WS frame into the client-message union; anything else is loud. */
+declare function parseClientMessage(raw: string): ClientMessage;
+/** RealtimeInputConfig.ActivityHandling — the Live API's barge-in policy. */
+type ActivityHandling = 'START_OF_ACTIVITY_INTERRUPTS' | 'NO_INTERRUPTION';
+/** The VAD/activity posture distilled from setup.realtimeInputConfig. */
+interface ActivityConfig {
+    /**
+     * True when `automaticActivityDetection.disabled === true`: the CLIENT owns
+     * activity boundaries and marks them with realtimeInput.activityStart/
+     * activityEnd (the Live API's manual-VAD mode). False (default) means no
+     * activity signaling at all in this slice — AUTOMATIC detection is the
+     * declared platform follow-up (serve turn_detection events, see header).
+     */
+    manualActivity: boolean;
+    /** Whether activityStart barge-ins interrupt an in-flight generation (default) or not. */
+    activityHandling: ActivityHandling;
+}
+/** The per-connection config distilled from BidiGenerateContentSetup. */
+interface LiveSessionConfig {
+    /** setup.model with the `models/` resource prefix stripped — fed to model routing. */
+    model: string;
+    /** systemInstruction as Responses input items (seeded once into the transcript). */
+    systemItems: Rec[];
+    /** functionDeclarations flattened to Responses `{type:'function', …}` tools. */
+    tools: Rec[] | undefined;
+    temperature: number | undefined;
+    maxOutputTokens: number | undefined;
+    /**
+     * `generationConfig.responseModalities` includes AUDIO — the handler opens
+     * the native audio lane (ProxyClients.openAudio → AudioBridge over
+     * rt-audio-in/rt-audio-out) EAGERLY at setup instead of on first audio chunk.
+     */
+    audioModality: boolean;
+    /** VAD/activity posture from setup.realtimeInputConfig (manual mode only). */
+    activity: ActivityConfig;
+    /**
+     * setup.sessionResumption (SessionResumptionConfig): `enabled` when the
+     * field is present at all (the server then emits sessionResumptionUpdate
+     * messages); `handle` when the client is RESUMING a previous Live session.
+     */
+    sessionResumption: {
+        enabled: boolean;
+        handle: string | null;
+    };
+    /**
+     * The client opted into the uRun video OUTPUT extension
+     * (`setup["urun.videoOut"]: {}` — a vendor-prefixed field the real Google
+     * schema can never collide with; Gemini Live itself has NO video OUT
+     * modality). The handler answers in setupComplete with the negotiated named
+     * §5 `rt-video-out` WT/QUIC endpoint, or omits the field (= unsupported).
+     */
+    videoOut: boolean;
+}
+/**
+ * The vendor-prefixed extension key, used verbatim in BOTH directions:
+ * `setup["urun.videoOut"]` opts in; `setupComplete["urun.videoOut"]` carries
+ * the granted {@link import('../../transport/video-out').VideoOutEndpoint}.
+ * Vanilla clients never send it and ignore it in setupComplete — parity holds.
+ */
+declare const VIDEO_OUT_SETUP_KEY = "urun.videoOut";
+/**
+ * The videoOut capability ack (spec v2, WS-inline): the WS IS the transport,
+ * so the grant carries ONLY what the client needs to interpret the inline
+ * chunks — the framing id and the capability-level default codec. No urls,
+ * tokens, or cert hashes ever appear here.
+ */
+declare function videoOutAck(codec: string): Rec;
+/**
+ * One native video-OUT record as the extension SERVER MESSAGE riding the SAME
+ * Bidi WS as the rest of the surface (spec v2): base64 AU inside JSON, the
+ * exact idiom the surface uses for audio bytes (serverContent inlineData) —
+ * vendor-keyed so vanilla clients ignore it.
+ */
+declare function videoOutChunkMsg(record: VideoOutRecord): Rec;
+/** BidiGenerateContentSetup → session config. Everything unsupported is loud. */
+declare function translateSetup(setup: Rec): LiveSessionConfig;
+/**
+ * SessionResumptionUpdate per the Live WS reference: `newHandle` names a
+ * resumable state; sent ONLY when setup.sessionResumption was set.
+ */
+declare function sessionResumptionUpdateMsg(newHandle: string): Rec;
+/**
+ * GoAway per the Live WS reference — `timeLeft` is a protobuf Duration in its
+ * JSON form ("3.5s"). null timeLeft (no native deadline known) renders "0s":
+ * the connection is being terminated NOW (see the missing-primitive note in
+ * handler.ts — core exposes no pre-expiry notice event yet).
+ */
+declare function goAwayMsg(timeLeftMs: number | null): Rec;
+/** BidiGenerateContentClientContent → Responses input items + the turn trigger. */
+declare function translateClientContent(cc: Rec): {
+    items: Rec[];
+    turnComplete: boolean;
+};
+/** Live spec input format: 16-bit little-endian PCM, 16 kHz, mono. */
+declare const GEMINI_AUDIO_IN_RATE = 16000;
+/** Mirror of the transport's native rate (media.ts NATIVE_AUDIO_SAMPLE_RATE). */
+declare const GEMINI_AUDIO_OUT_RATE = 24000;
+/** Live spec output format: 24 kHz 16-bit PCM — the native rate, so out is 1:1. */
+declare const GEMINI_AUDIO_OUT_MIME = "audio/pcm;rate=24000";
+/**
+ * One native Opus frame (20 ms @ 24 kHz = 480 samples) of silence, base64.
+ * `audioStreamEnd` appends this through the SAME `appendInputAudio` lane: the
+ * bridge's outbound framer accumulates 480-sample frames, so a sub-frame
+ * residue of real audio would otherwise sit in the accumulator forever — one
+ * full silence frame provably pushes every real sample out (the flush).
+ */
+declare const OUTBOUND_AUDIO_FLUSH_B64: string;
+/**
+ * Linear-interpolation PCM16 resample (mono). Adapter-edge only: Gemini's
+ * 16 kHz input → the transport's native 24 kHz. Output length is
+ * floor(n·to/from); positions past the last input sample clamp to it.
+ */
+declare function resamplePcm16(samples: Int16Array, from: number, to: number): Int16Array;
+/** The realtimeInput action union — exactly one lane per message. */
+type RealtimeInputAction = {
+    kind: 'text';
+    text: string;
+} | {
+    kind: 'audio';
+    base64Pcm24k: string;
+} | {
+    kind: 'audioStreamEnd';
+} | {
+    kind: 'activityStart';
+} | {
+    kind: 'activityEnd';
+} | {
+    kind: 'video';
+    jpegFrame: Uint8Array;
+};
+/**
+ * BidiGenerateContentRealtimeInput → the text lane, the audio lane
+ * (adapter-edge resampled onto the native AudioBridge path), the
+ * audioStreamEnd flush, the video frame lane (adapter-edge base64-decoded
+ * onto the native rt-video-in named-DATA path), or a manual activity marker.
+ * activityStart/activityEnd (ActivityStart/ActivityEnd — EMPTY marker
+ * messages per the reference) are only translated here; the handler enforces
+ * that they arrive in manual activity mode with a coherent open/close
+ * ordering. NOTHING is silently droppable.
+ */
+declare function translateRealtimeInput(ri: Rec): RealtimeInputAction;
+/** BidiGenerateContentToolResponse → next-turn `function_call_output` items. */
+declare function translateToolResponse(tr: Rec): Rec[];
+/** One streamed text delta → a serverContent.modelTurn message (else null). */
+declare function serverContentOfDelta(ev: {
+    type?: unknown;
+    delta?: unknown;
+}): Rec | null;
+/**
+ * One native `rt-audio-out` chunk (base64 PCM16 @24 kHz, exactly as the
+ * AudioBridge emits it — no conversion: the Live spec output rate IS the
+ * native rate) → a serverContent.modelTurn inlineData message.
+ */
+declare function serverContentOfAudio(base64Pcm24k: string): Rec;
+/** What one completed upstream response contributes: wire messages + transcript items. */
+interface TurnEnd {
+    /** Server messages to send, in order: toolCall? → generationComplete → turnComplete. */
+    messages: Rec[];
+    /** Items to append to the connection transcript (assistant text + function_call items). */
+    assistantItems: Rec[];
+}
+/** `response.completed` → toolCall / generationComplete / turnComplete (+usage). */
+declare function turnEndOfCompleted(completed: unknown): TurnEnd;
+
+export { type ActivityConfig as A, CLOSE_PROTOCOL as C, GEMINI_AUDIO_IN_RATE as G, type LiveSessionConfig as L, OUTBOUND_AUDIO_FLUSH_B64 as O, type RealtimeInputAction as R, type TurnEnd as T, VIDEO_OUT_SETUP_KEY as V, type ActivityHandling as a, CLOSE_SESSION_GONE as b, CLOSE_UNKNOWN_MODEL as c, CLOSE_UPSTREAM as d, type ClientMessage as e, GEMINI_AUDIO_OUT_MIME as f, GEMINI_AUDIO_OUT_RATE as g, GEMINI_LIVE_PATH as h, GeminiLiveError as i, goAwayMsg as j, serverContentOfDelta as k, sessionResumptionUpdateMsg as l, translateRealtimeInput as m, translateSetup as n, translateToolResponse as o, parseClientMessage as p, turnEndOfCompleted as q, resamplePcm16 as r, serverContentOfAudio as s, translateClientContent as t, videoOutChunkMsg as u, videoOutAck as v };

package/dist/{ResponsesClient-y8g6OfNN.d.cts → types-lsVTbNcH.d.cts}

@@ -54,6 +54,21 @@ interface UrunSessionLike {
      * lease stayed alive. Core's surface is `end()` / `disconnect()`.
      */
     end?(): Promise<unknown>;
+    /**
+     * Negotiate this session's §5-over-QUIC data-lane endpoint (urun-infra#1608:
+     * the `TransportOfferResponse.webtransport` surface — url + sha-256 cert
+     * hashes (hex) + the per-leg gateway token — offered with `wtData: true`).
+     * The video OUT lane (`enableSessionVideoOut`) hands this endpoint to a
+     * consumer that dials the relay's WT gateway directly and reads the named
+     * uni streams. Optional structurally; the video OUT wiring fails LOUD when
+     * a session lacks it — a missing platform primitive is never a silent skip.
+     */
+    wtDataEndpoint?(): Promise<{
+        url: string;
+        certHashes: string[];
+        token: string;
+        sessionId?: string;
+    }>;
 }
 /**
  * One OpenAI chat-completions-chunk-style tool_call delta fragment as the
@@ -96,27 +111,4 @@ type LaneMessage = {
     code?: string;
 };
 
-interface ResponsesStream extends AsyncIterable<unknown> {
-    readonly requestId: string;
-}
-declare class UrunResponses {
-    private readonly transport;
-    constructor(session: UrunSessionLike);
-    readonly responses: {
-        create: (params: {
-            model?: string;
-            input: unknown;
-            stream?: boolean;
-            tools?: unknown;
-            tool_choice?: unknown;
-            temperature?: number;
-            max_output_tokens?: number;
-            top_p?: number;
-            stop?: string[];
-            reasoning_effort?: string;
-            chat_template_kwargs?: Record<string, unknown>;
-        }) => Promise<ResponsesStream>;
-    };
-}
-
-export { type LaneMessage as L, UrunResponses as U, type UrunSessionLike as a };
+export type { LaneMessage as L, UrunDocLike as U, LaneToolCallDelta as a, UrunSessionLike as b, UrunStreamLike as c };

package/dist/{ResponsesClient-y8g6OfNN.d.ts → types-lsVTbNcH.d.ts}

@@ -25,6 +25,13 @@ interface UrunSessionLike {
     stream(name: string): UrunStreamLike;
     
     end?(): Promise<unknown>;
+    
+    wtDataEndpoint?(): Promise<{
+        url: string;
+        certHashes: string[];
+        token: string;
+        sessionId?: string;
+    }>;
 }
 
 interface LaneToolCallDelta {
@@ -54,27 +61,4 @@ type LaneMessage = {
     code?: string;
 };
 
-interface ResponsesStream extends AsyncIterable<unknown> {
-    readonly requestId: string;
-}
-declare class UrunResponses {
-    private readonly transport;
-    constructor(session: UrunSessionLike);
-    readonly responses: {
-        create: (params: {
-            model?: string;
-            input: unknown;
-            stream?: boolean;
-            tools?: unknown;
-            tool_choice?: unknown;
-            temperature?: number;
-            max_output_tokens?: number;
-            top_p?: number;
-            stop?: string[];
-            reasoning_effort?: string;
-            chat_template_kwargs?: Record<string, unknown>;
-        }) => Promise<ResponsesStream>;
-    };
-}
-
-export { type LaneMessage as L, UrunResponses as U, type UrunSessionLike as a };
+export type { LaneMessage as L, UrunDocLike as U, LaneToolCallDelta as a, UrunSessionLike as b, UrunStreamLike as c };

package/dist/video-out-CWesbk12.d.ts

@@ -0,0 +1,155 @@
+import { b as UrunSessionLike } from './types-lsVTbNcH.js';
+
+declare const NATIVE_AUDIO_SAMPLE_RATE = 24000;
+
+interface AudioSource {
+    track: unknown;
+    onData(frame: Int16Array): void;
+}
+interface AudioSink {
+    onframe: ((frame: {
+        samples: Int16Array;
+    }) => void) | null;
+}
+interface AudioBackend {
+    createSource(): AudioSource;
+    
+    createSink(remoteTrack: unknown): AudioSink;
+}
+declare class AudioBridge {
+    private readonly backend;
+    private readonly opts;
+    private source;
+    private sink;
+    private outHandlers;
+    constructor(backend: AudioBackend, opts: {
+        sampleRate: number;
+    });
+    startOutbound(): Promise<unknown>;
+    
+    appendInputAudio(base64Pcm16: string): void;
+    
+    startInbound(remoteTrack: unknown): Promise<void>;
+    
+    onOutputAudio(handler: (base64Pcm16: string) => void): () => void;
+}
+
+declare function enableSessionAudio(session: UrunSessionLike, backend: AudioBackend, timeoutMs?: number): Promise<AudioBridge>;
+
+declare const RT_VIDEO_IN_STREAM = "rt-video-in";
+
+declare const VIDEO_FRAME_MESSAGE_BUDGET = 256000;
+
+interface VideoFrameLane {
+    sendInputFrame(jpegFrame: Uint8Array): Promise<void>;
+}
+
+declare function enableSessionVideo(session: UrunSessionLike): VideoFrameLane;
+
+declare function weriftAudioBackend(): Promise<AudioBackend>;
+
+declare function nodeAudioBackend(): Promise<AudioBackend>;
+
+declare const RT_VIDEO_OUT_STREAM = "rt-video-out";
+
+declare const VIDEO_OUT_FRAMING = "urun.videoOut.v1";
+
+declare const VIDEO_OUT_DEFAULT_CODEC = "h264";
+
+interface VideoOutRecordHeader {
+    v: 1;
+    seq: number;
+    ptsUs: number;
+    codec: string;
+    key: boolean;
+}
+
+interface VideoOutRecord {
+    header: VideoOutRecordHeader;
+    au: Uint8Array;
+}
+
+declare function encodeVideoOutRecord(header: Omit<VideoOutRecordHeader, 'v'>, au: Uint8Array): Uint8Array;
+
+declare function decodeVideoOutRecord(payload: Uint8Array): VideoOutRecord;
+
+interface VideoOutEndpoint {
+    transport: 'webtransport';
+    url: string;
+    
+    certHashes: string[];
+    
+    token: string;
+    
+    sessionId: string;
+    stream: typeof RT_VIDEO_OUT_STREAM;
+    codec: string;
+    framing: typeof VIDEO_OUT_FRAMING;
+}
+
+declare function negotiateSessionVideoOutQuic(session: UrunSessionLike): Promise<VideoOutEndpoint>;
+
+declare const VIDEO_OUT_WS_FRAMING = "urun.videoOut.v2-ws";
+
+interface VideoOutLane {
+    
+    codec: string;
+    onOutputFrame(handler: (record: VideoOutRecord) => void): () => void;
+    
+    onLaneError(handler: (err: Error) => void): () => void;
+    
+    onEnd(handler: () => void): () => void;
+}
+
+declare function openSessionVideoOutLane(session: UrunSessionLike): VideoOutLane;
+
+interface VideoOutSink {
+    onFrame(record: VideoOutRecord): void;
+    
+    onEnd?(): void;
+    
+    onLaneError(err: Error): void;
+}
+interface WtByteReader {
+    read(): Promise<{
+        value?: Uint8Array;
+        done: boolean;
+    }>;
+    cancel?(reason?: unknown): Promise<void>;
+}
+
+interface VideoOutWtSession {
+    ready: Promise<unknown>;
+    incomingUnidirectionalStreams: {
+        getReader(): {
+            read(): Promise<{
+                value?: {
+                    getReader(): WtByteReader;
+                };
+                done: boolean;
+            }>;
+        };
+    };
+    createBidirectionalStream(): Promise<{
+        writable: {
+            getWriter(): {
+                write(chunk: Uint8Array): Promise<void>;
+            };
+        };
+        readable: {
+            getReader(): WtByteReader;
+        };
+    }>;
+    close?(info?: unknown): void;
+}
+
+type VideoOutWtConnect = (url: string, options: {
+    serverCertificateHashes: Array<{
+        algorithm: 'sha-256';
+        value: Uint8Array;
+    }>;
+}) => VideoOutWtSession;
+
+declare function consumeVideoOut(endpoint: VideoOutEndpoint, sink: VideoOutSink, connect?: VideoOutWtConnect): Promise<void>;
+
+export { type AudioBackend as A, NATIVE_AUDIO_SAMPLE_RATE as N, RT_VIDEO_IN_STREAM as R, VIDEO_FRAME_MESSAGE_BUDGET as V, AudioBridge as a, type AudioSink as b, type AudioSource as c, RT_VIDEO_OUT_STREAM as d, VIDEO_OUT_DEFAULT_CODEC as e, VIDEO_OUT_FRAMING as f, VIDEO_OUT_WS_FRAMING as g, type VideoFrameLane as h, type VideoOutEndpoint as i, type VideoOutLane as j, type VideoOutRecord as k, type VideoOutRecordHeader as l, type VideoOutSink as m, type VideoOutWtConnect as n, type VideoOutWtSession as o, consumeVideoOut as p, decodeVideoOutRecord as q, enableSessionAudio as r, enableSessionVideo as s, encodeVideoOutRecord as t, negotiateSessionVideoOutQuic as u, nodeAudioBackend as v, openSessionVideoOutLane as w, weriftAudioBackend as x };