@urun-sh/sense 0.0.0-stage → 0.1.0

package/dist/session.js

@@ -0,0 +1,472 @@
+/** Session: inputs, the §1–§7 wire loop over `wire`, and wave assembly. */
+import { connectWire, DialRefused } from "./transport.js";
+import { Question, instructionsEnvelope } from "./question.js";
+import { Wave } from "./wave.js";
+import { Context } from "./context.js";
+import { ProtocolError } from "./errors.js";
+const MAX_FRAME_BYTES = 256_000;
+const TURN_TOKEN_BUDGET = 250_000;
+const MAX_BACKOFF_MS = 30_000;
+const MAX_DIALS = 600;
+/** Parse a server event object into the SDK's session-event shape. */
+export function sessionEvent(raw) {
+    const obj = (typeof raw === "object" && raw !== null ? raw : {});
+    const err = (obj.error ?? null);
+    return {
+        type: typeof obj.type === "string" ? obj.type : "unknown",
+        code: typeof err?.code === "string" ? err.code : typeof obj.code === "string" ? obj.code : undefined,
+        message: typeof err?.message === "string" ? err.message : undefined,
+        raw,
+    };
+}
+function wireDefaults(opts) {
+    return { transport: opts.transport ?? "auto", wireFactory: opts.wireFactory ?? connectWire };
+}
+export class Session {
+    static DEFAULT_BASE_URL = "wss://inference.urun.sh";
+    static DEFAULT_MODEL = "sense-latest:bf16";
+    apiKey;
+    baseUrl;
+    model;
+    hz;
+    transport;
+    context = new Context(this);
+    audio;
+    video;
+    text;
+    stats = { waves: 0, turns: 0, errors: 0, reconnects: 0, decisions: 0, protocolErrors: 0 };
+    isClosed = false;
+    wire = null;
+    wireFactory;
+    waveSeq = 0;
+    sourceQueues = new Map();
+    rrCursor = 0;
+    oneShotText = [];
+    transcript = "";
+    waveQueue = [];
+    waveWaiters = [];
+    accumulator = new Map();
+    accTMs = 0;
+    turnOpen = false;
+    audioPending = false;
+    inputSinceTurn = 0;
+    registry = [];
+    eventListeners = [];
+    constructor(opts = {}) {
+        this.apiKey = opts.apiKey ?? process.env.SENSE_API_KEY ?? "";
+        this.baseUrl = opts.baseUrl ?? process.env.SENSE_BASE_URL ?? Session.DEFAULT_BASE_URL;
+        this.model = opts.model ?? Session.DEFAULT_MODEL;
+        this.hz = opts.hz ?? 25;
+        ({ transport: this.transport, wireFactory: this.wireFactory } = wireDefaults(opts));
+        this.audio = new AudioInput(this);
+        this.video = new VideoInput(this);
+        this.text = new TextInput(this);
+        for (const q of opts.questions ?? [])
+            this.addQuestionRaw(q);
+    }
+    get questions() {
+        return this.registry.slice();
+    }
+    get closed() {
+        return this.isClosed;
+    }
+    /** Register (or replace) a question; takes effect at the next turn open. */
+    addQuestion(id, kind, opts = {}) {
+        return this.addQuestionRaw(new Question({ id, kind, ...opts }));
+    }
+    addQuestionRaw(q) {
+        const i = this.registry.findIndex((x) => x.id === q.id);
+        if (i >= 0)
+            this.registry[i] = q;
+        else
+            this.registry.push(q);
+        return q;
+    }
+    realtimeUrl() {
+        return `${this.baseUrl.replace(/\/+$/, "")}/v1/realtime?model=${encodeURIComponent(this.model)}`;
+    }
+    // ---- inputs -------------------------------------------------------------
+    /** @internal */
+    emitAudio(b64, source) {
+        if (source !== undefined && source !== "default") {
+            let queue = this.sourceQueues.get(source);
+            if (!queue) {
+                queue = [];
+                this.sourceQueues.set(source, queue);
+            }
+            queue.push(b64);
+            return;
+        }
+        this.inputSinceTurn++;
+        this.audioPending = true;
+        this.sendJson({ type: "input_audio_buffer.append", audio: b64 });
+        this.drainSourceQueues();
+    }
+    /** ponytail: round-robin one queued non-default chunk per default push (PR-2
+     * degradation); replace with labeled lanes when PR-2 lands. */
+    drainSourceQueues() {
+        const sources = [...this.sourceQueues.keys()];
+        const source = sources[this.rrCursor++ % Math.max(sources.length, 1)];
+        if (source === undefined)
+            return;
+        const b64 = this.sourceQueues.get(source)?.shift();
+        if (b64 !== undefined)
+            this.sendJson({ type: "input_audio_buffer.append", audio: b64, source });
+    }
+    /** @internal */
+    emitVideo(jpeg) {
+        if (jpeg.byteLength > MAX_FRAME_BYTES) {
+            throw new ProtocolError("frame_too_large", `frame ${jpeg.byteLength}B > ${MAX_FRAME_BYTES}B`);
+        }
+        this.inputSinceTurn++;
+        this.sendJson({
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "user",
+                content: [{ type: "input_image", image_url: `data:image/jpeg;base64,${toB64(jpeg)}` }],
+            },
+        });
+    }
+    /** @internal */
+    emitText(text, mode) {
+        this.inputSinceTurn++;
+        if (mode === "event") {
+            this.oneShotText.push(text);
+            return;
+        }
+        if (mode === "replace")
+            this.transcript = text;
+        else
+            this.transcript += (this.transcript ? "\n" : "") + text;
+    }
+    // ---- wire ---------------------------------------------------------------
+    sendJson(obj) {
+        this.wire?.sendJson(obj);
+    }
+    sendSessionUpdate() {
+        this.sendJson({
+            type: "session.update",
+            session: {
+                model: this.model,
+                modalities: ["text"],
+                instructions: JSON.stringify(instructionsEnvelope(this.registry, this.hz)),
+            },
+        });
+    }
+    openTurn() {
+        const text = [this.transcript, ...this.oneShotText].filter(Boolean).join("\n");
+        this.oneShotText = [];
+        if (text) {
+            this.sendJson({
+                type: "conversation.item.create",
+                item: { type: "message", role: "user", content: [{ type: "input_text", text }] },
+            });
+        }
+        // qvs semantics (live-verified 2026-11-03): commit must follow BUFFERED
+        // audio — an empty commit is refused (empty_buffer) and the turn then
+        // opens with no audio bound, so later appends never reach the model.
+        // Appends sent before the commit are held server-side (GA semantics).
+        if (this.audioPending) {
+            this.sendJson({ type: "input_audio_buffer.commit" });
+            this.audioPending = false;
+        }
+        this.sendJson({
+            type: "response.create",
+            response: {
+                instructions: JSON.stringify(instructionsEnvelope(this.registry, this.hz)),
+                max_output_tokens: TURN_TOKEN_BUDGET,
+            },
+        });
+        this.accumulator.clear();
+        this.turnOpen = true;
+        this.inputSinceTurn = 0;
+        this.stats.turns++;
+    }
+    /** Barge-in: stop the active turn's cadence. */
+    bargeIn() {
+        this.sendJson({ type: "response.cancel" });
+        this.turnOpen = false;
+    }
+    /** Connect with the §7 dial policy (503/429 → backoff and redial). The
+     * transport selection (auto: quic → webrtc → ws) happens in the wire
+     * factory; a refused dial restarts the whole order. */
+    async connect() {
+        if (this.isClosed)
+            throw new ProtocolError("protocol", "session is closed");
+        let delay = 1_000;
+        for (let attempt = 1;; attempt++) {
+            try {
+                const wire = await this.wireFactory({
+                    transport: this.transport,
+                    apiBaseUrl: this.baseUrl.replace(/^wss:/, "https:").replace(/^ws:/, "http:").replace(/\/+$/, ""),
+                    apiKey: this.apiKey,
+                    model: this.model,
+                    wsUrl: this.realtimeUrl(),
+                });
+                this.wire = wire;
+                wire.onMessage((raw) => this.onMessage(raw));
+                wire.onClose((code) => this.onClose(code ?? 0));
+                this.sendSessionUpdate();
+                return;
+            }
+            catch (err) {
+                if (attempt >= MAX_DIALS || !(err instanceof DialRefused))
+                    throw err;
+                await new Promise((r) => setTimeout(r, delay));
+                delay = Math.min(delay * 2, MAX_BACKOFF_MS);
+            }
+        }
+    }
+    onClose(code) {
+        this.wire = null;
+        this.turnOpen = false;
+        if (code === 1011) {
+            // session_gone: discard local state, reconnect fresh (§7).
+            this.registry = [];
+            this.transcript = "";
+            this.stats.reconnects++;
+            void this.connect();
+            return;
+        }
+        // any other wire death ends the streams instead of hanging the run loop
+        for (const w of this.waveWaiters.splice(0))
+            w(null);
+    }
+    onMessage(raw) {
+        let msg;
+        try {
+            msg = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        const type = typeof msg.type === "string" ? msg.type : "";
+        if (type === "response.output_text.delta" || type === "response.text.delta") {
+            this.onDeltaMessage(msg);
+            return;
+        }
+        if (type === "response.completed" || type === "response.done" || type === "response.failed") {
+            this.onTurnCompleted();
+            return;
+        }
+        if (type === "error") {
+            // A handled error (benign code, or commit sent) stops here; a mid-turn
+            // failure falls through so listeners still see the event.
+            if (this.onErrorMessage(msg))
+                return;
+        }
+        this.eventListeners.forEach((h) => h(sessionEvent(msg)));
+    }
+    onDeltaMessage(msg) {
+        for (const line of String(msg.delta ?? "").split("\n"))
+            this.onDecisionLine(line);
+    }
+    onTurnCompleted() {
+        this.flushAccumulator();
+        this.turnOpen = false;
+    }
+    /** Returns true when the error was handled locally and needs no listener dispatch. */
+    onErrorMessage(msg) {
+        this.stats.errors++;
+        const code = msg.error?.code;
+        if (code === "empty_buffer" || code === "audio_input_transcript_unavailable")
+            return true; // benign (§5.1, §7)
+        if (code === "buffer_full") {
+            this.sendJson({ type: "input_audio_buffer.commit" });
+            return true;
+        }
+        // mid-turn failure: end the turn; the run loop re-opens.
+        console.error(`[sense] lane error: ${JSON.stringify(msg.error)}`);
+        this.turnOpen = false;
+        return false;
+    }
+    onEvent(handler) {
+        this.eventListeners.push(handler);
+        return {
+            cancel: () => {
+                this.eventListeners = this.eventListeners.filter((h) => h !== handler);
+            },
+        };
+    }
+    /** §6.3 client-side wave assembly: one decision line per question per wave;
+     * a repeat qid closes the previous wave. */
+    onDecisionLine(line) {
+        const trimmed = line.trim();
+        if (!trimmed.startsWith("{") || !trimmed.includes('"sense.decision"'))
+            return;
+        let ev;
+        try {
+            ev = JSON.parse(trimmed);
+        }
+        catch {
+            return;
+        }
+        const qid = String(ev.id ?? ev.question ?? "");
+        if (!qid)
+            return;
+        if (this.accumulator.has(qid))
+            this.flushAccumulator();
+        this.accumulator.set(qid, ev);
+        if (typeof ev.t_ms === "number")
+            this.accTMs = ev.t_ms;
+        this.stats.decisions++;
+    }
+    flushAccumulator() {
+        if (this.accumulator.size === 0)
+            return;
+        const payload = {
+            wave_id: this.waveSeq,
+            t_ms: this.accTMs,
+            results: Object.fromEntries(this.accumulator),
+        };
+        this.accumulator.clear();
+        try {
+            const wave = Wave.decode(this.registry, payload);
+            this.waveSeq++;
+            this.stats.waves++;
+            this.deliver(wave);
+        }
+        catch (err) {
+            if (err instanceof ProtocolError) {
+                // Owner ruling: a wave that does not answer every registered
+                // question is a PROTOCOL ERROR — surface it loudly, count it, and
+                // drop the partial group; never treat it as false/hold, never let
+                // one cut wave kill the session.
+                this.stats.protocolErrors++;
+                console.error(`[sense] PROTOCOL ERROR: ${err.message}`);
+                return;
+            }
+            throw err;
+        }
+    }
+    deliver(wave) {
+        this.context.ingest(wave);
+        this.waveQueue.push(wave);
+        while (this.waveWaiters.length && this.waveQueue.length) {
+            this.waveWaiters.shift()(this.waveQueue.shift());
+        }
+    }
+    nextWave() {
+        const queued = this.waveQueue.shift();
+        if (queued)
+            return Promise.resolve(queued);
+        if (this.isClosed)
+            return Promise.resolve(null);
+        const { promise, resolve } = Promise.withResolvers();
+        this.waveWaiters.push(resolve);
+        return promise;
+    }
+    // ---- public streaming surface -------------------------------------------
+    /** Async-iterator of waves: push any given inputs, then run the turn loop. */
+    async *run(inputs) {
+        if (!this.wire)
+            await this.connect();
+        if (inputs) {
+            for await (const frame of inputs)
+                this.applyFrame(frame);
+        }
+        for (;;) {
+            // The turn opens on FIRST input (qvs semantics): an empty commit is
+            // refused and leaves the turn audio-less forever.
+            if (!this.turnOpen && !this.isClosed && this.inputSinceTurn > 0)
+                this.openTurn();
+            const wave = await this.nextWave();
+            if (!wave)
+                return;
+            yield wave;
+        }
+    }
+    applyFrame(frame) {
+        if ("audio" in frame)
+            this.emitAudio(toB64(frame.audio), frame.source);
+        else if ("video" in frame)
+            this.emitVideo(frame.video instanceof Uint8Array ? frame.video : new Uint8Array(frame.video));
+        else if ("text" in frame)
+            this.emitText(frame.text, frame.mode ?? "append");
+    }
+    /** Latest result for a question, or null before the first wave. */
+    latest(question) {
+        return this.context.get(question);
+    }
+    /** Iterate waves live; coalesce drops earlier waves in runs of identical
+     * argmax values (latest wins). */
+    async *waves(opts = {}) {
+        let pending = null;
+        for (;;) {
+            const wave = await this.nextWave();
+            if (!wave)
+                break;
+            if (!opts.coalesce) {
+                yield wave;
+                continue;
+            }
+            if (pending && coalesceSignature(pending) === coalesceSignature(wave))
+                pending = wave;
+            else {
+                if (pending)
+                    yield pending;
+                pending = wave;
+            }
+        }
+        if (pending)
+            yield pending;
+    }
+    when(pred, handler) {
+        return this.context.when(pred, handler);
+    }
+    /** Observe waves live; `coalesce` delivers the FIRST wave of a run of
+     * consecutive same-argmax waves — later waves update `latest()` without
+     * another callback. */
+    onWave(handler, opts = {}) {
+        return this.context.onWave(handler, opts);
+    }
+    async close() {
+        this.isClosed = true;
+        for (const w of this.waveWaiters.splice(0))
+            w(null);
+        const wire = this.wire;
+        this.wire = null;
+        if (wire)
+            await wire.close();
+    }
+}
+function toB64(data) {
+    const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
+    let bin = "";
+    for (let i = 0; i < bytes.length; i += 0x8000) {
+        bin += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+    }
+    return btoa(bin);
+}
+function coalesceSignature(w) {
+    return w.questionIds().map((qid) => `${qid}=${w.result(qid).top}`).join(";");
+}
+export class AudioInput {
+    session;
+    constructor(session) {
+        this.session = session;
+    }
+    /** Push PCM16 24 kHz mono (encoded base64 on the wire). */
+    push(pcm, source) {
+        this.session.emitAudio(toB64(pcm), source);
+    }
+}
+export class VideoInput {
+    session;
+    constructor(session) {
+        this.session = session;
+    }
+    push(jpeg) {
+        this.session.emitVideo(jpeg instanceof Uint8Array ? jpeg : new Uint8Array(jpeg));
+    }
+}
+export class TextInput {
+    session;
+    constructor(session) {
+        this.session = session;
+    }
+    push(text, mode = "append") {
+        this.session.emitText(text, mode);
+    }
+}

package/dist/transport.d.ts

@@ -0,0 +1,106 @@
+export type Transport = 'auto' | 'quic' | 'webrtc' | 'ws';
+export declare const TRANSPORT_DIAL_TIMEOUT_MS: Record<'quic' | 'webrtc', number>;
+/** A non-WS transport could not be established (fallback signal in auto). */
+export declare class TransportDialError extends Error {
+}
+/** The WS dial was refused with 503/429 — worth a backoff redial (§7). */
+export declare class DialRefused extends Error {
+    readonly status: number;
+    constructor(status: number);
+}
+/** Node `ws` emitter surface (plus the browser-WebSocket addEventListener). */
+interface WsLike {
+    send(data: string): void;
+    close(): void;
+    addEventListener?(type: string, cb: () => void, opts?: {
+        once?: boolean;
+    }): void;
+    on?(event: string, cb: (...args: never[]) => void): unknown;
+    once?(event: string, cb: (...args: never[]) => void): unknown;
+    off?(event: string, cb: (...args: never[]) => void): unknown;
+    removeAllListeners?(): unknown;
+}
+export interface Wire {
+    readonly kind: Transport;
+    /** Fire-and-forget send of one JSON event. Throws once the wire is closed. */
+    sendJson(obj: unknown): void;
+    onMessage(cb: (raw: string) => void): void;
+    onClose(cb: (code?: number) => void): void;
+    close(): Promise<void>;
+}
+export interface WireOptions {
+    transport: Transport;
+    /** https:// base of the inference proxy (POST /v1/realtime/calls). */
+    apiBaseUrl: string;
+    apiKey: string;
+    model: string;
+    /** wss://…/v1/realtime?model=… (the WS dial URL). */
+    wsUrl: string;
+}
+export type WireDialer = (opts: WireOptions) => Promise<Wire>;
+export declare function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T>;
+/** certHashes entries: the gateway serves base64 sha-256; tolerate lowercase hex. */
+export declare function hashToBytes(h: string): Uint8Array;
+/** Incremental NDJSON splitter: feed bytes, get complete lines. */
+export declare class NdjsonSplitter {
+    private buf;
+    push(text: string): string[];
+}
+export declare class WsWire implements Wire {
+    private ws;
+    readonly kind: Transport;
+    private messageCb;
+    private closeCb;
+    constructor(ws: WsLike);
+    sendJson(obj: unknown): void;
+    onMessage(cb: (raw: string) => void): void;
+    onClose(cb: (code?: number) => void): void;
+    close(): Promise<void>;
+}
+/** Dial the WS wire. Node uses the `ws` package (Authorization header
+ * support); a browser global WebSocket cannot set headers, so browser WS
+ * fallback is only viable where the proxy accepts URL-carried auth.
+ *
+ * Dynamic import: `ws` is a Node-only package — a static import would break
+ * every browser bundle of this SDK (the platform-specific-module exception). */
+export declare const dialWs: WireDialer;
+export declare class QuicWire implements Wire {
+    private opts;
+    readonly kind: Transport;
+    private messageCb;
+    private closeCb;
+    private writer;
+    private splitter;
+    private wt;
+    private closedSent;
+    static available(): boolean;
+    constructor(opts: WireOptions);
+    connect(): Promise<void>;
+    private signalClosed;
+    sendJson(obj: unknown): void;
+    onMessage(cb: (raw: string) => void): void;
+    onClose(cb: (code?: number) => void): void;
+    close(): Promise<void>;
+}
+export declare class RtcWire implements Wire {
+    private opts;
+    readonly kind: Transport;
+    private messageCb;
+    private closeCb;
+    private pc;
+    private dc;
+    static available(): boolean;
+    constructor(opts: WireOptions);
+    connect(): Promise<void>;
+    private waitDcOpen;
+    sendJson(obj: unknown): void;
+    onMessage(cb: (raw: string) => void): void;
+    onClose(cb: (code?: number) => void): void;
+    close(): Promise<void>;
+}
+/** Dial order. Auto = quic → webrtc → ws, skipping platforms without the
+ * client stack. Explicit overrides dial exactly one. */
+export declare function transportOrder(transport: Transport): Array<'quic' | 'webrtc' | 'ws'>;
+export declare const defaultDialers: Record<'quic' | 'webrtc' | 'ws', WireDialer>;
+export declare function connectWire(opts: WireOptions, dialers?: Partial<Record<'quic' | 'webrtc' | 'ws', WireDialer>>): Promise<Wire>;
+export {};