@lumavox/client 0.0.0-stage → 0.1.1

package/dist/conversation.js

@@ -0,0 +1,763 @@
+import { wireBytes } from "./pcm.js";
+import { KEEPALIVE_S, MAX_AUDIO_BYTES, MAX_MESSAGE_CHARS, RECORDING_RATES, } from "./protocol.gen.js";
+import { dial, isFinalClose, reconnectAfter, socketUrl, } from "./socket.js";
+/** No `ready` this long after the dial — a proxy that took the upgrade and says nothing — is a dropped connection. */
+export const READY_DEADLINE_MS = 15000;
+const FRESH = {
+    id: null,
+    text: "",
+    chunks: [],
+    mood: null,
+    tool: null,
+    knowledge: [],
+    heard: null,
+    done: false,
+    interrupted: false,
+};
+const HANDLER = {
+    ready: "onReady",
+    "turn.start": "onTurnStart",
+    heard: "onHeard",
+    "heard.partial": "onHeardPartial",
+    knowledge: "onKnowledge",
+    mood: "onMood",
+    "thought.delta": "onThought",
+    "text.delta": "onText",
+    "text.chunk": "onChunk",
+    "tool.start": "onToolStart",
+    "tool.end": "onToolEnd",
+    usage: "onUsage",
+    muted: "onMuted",
+    "audio.start": "onAudioStart",
+    "audio.end": "onAudioEnd",
+    transcript: "onTranscript",
+    "turn.done": "onDone",
+    interrupted: "onInterrupted",
+    error: "onError",
+};
+/**
+ * Call an app's handler. One that throws is rethrown on its own turn of
+ * the event loop, where the app's error reporting sees it — not here,
+ * where it would leave the conversation half-changed.
+ */
+function call(fn, value) {
+    if (!fn)
+        return;
+    try {
+        fn(value);
+    }
+    catch (error) {
+        setTimeout(() => {
+            throw error;
+        });
+    }
+}
+const dropped = (reason, detail) => detail
+    ? { status: "dropped", reason, detail }
+    : { status: "dropped", reason };
+/** Characters as the edge counts them: code points. */
+function characters(text) {
+    let n = 0;
+    for (const _ of text)
+        n++;
+    return n;
+}
+export class Conversation {
+    constructor(setup) {
+        this.snap = {
+            state: "connecting",
+            ready: null,
+            turn: null,
+            caption: null,
+            relation: null,
+            queued: 0,
+        };
+        this.listeners = new Set();
+        this.socket = null;
+        /** Moves with every dial and every end: a callback from an older one is stale. */
+        this.generation = 0;
+        this.key = null;
+        this.attempts = 0;
+        /** A new key was taken for a refusal, and no turn has finished on it yet. */
+        this.replaced = false;
+        this.closed = false;
+        /** Closed by giving up, not by a final code: `reconnect()` may dial again. */
+        this.revivable = false;
+        this.dormant = false;
+        this.everReady = false;
+        /** The socket now open reached `ready`. */
+        this.socketReady = false;
+        /** A turn this conversation did not send is running: the greeting. */
+        this.running = false;
+        this.inFlight = null;
+        this.queue = [];
+        this.open = null;
+        /** `transcribeOnly` recordings waiting for their words, oldest first. */
+        this.hearing = [];
+        /** Recordings waiting for `ready`. */
+        this.held = [];
+        this.recordings = 0;
+        /** The turn whose terminal frame was taken: a second for it says nothing new. */
+        this.finished = null;
+        this.relationSeen = "";
+        this.retry = null;
+        this.deadline = null;
+        this.watchdog = null;
+        this.heardAt = 0;
+        this.saidAt = 0;
+        this.sessionId = setup.sessionId;
+        this.setup = setup;
+        this.speech =
+            setup.options.speech ?? (setup.handlers.onAudio ? undefined : false);
+        void this.dial();
+    }
+    /** What is known now. A new object on every change: compare by identity. */
+    get snapshot() {
+        return this.snap;
+    }
+    get state() {
+        return this.snap.state;
+    }
+    /** `fn` is called after every change of `snapshot`. Answers the unsubscribe. */
+    subscribe(fn) {
+        this.listeners.add(fn);
+        return () => {
+            this.listeners.delete(fn);
+        };
+    }
+    /**
+     * Take a turn by typing. Queued behind the greeting, a running turn, a
+     * reconnect or a quiet spell; answers how it ended, and never rejects.
+     * 1 to 32,000 characters, or it is `dropped: refused` here.
+     */
+    send(text, options = {}) {
+        const length = characters(text);
+        if (length < 1 || length > MAX_MESSAGE_CHARS) {
+            return Promise.resolve(dropped("refused", `A message is 1 to ${MAX_MESSAGE_CHARS} characters; this is ${length}.`));
+        }
+        return this.enqueue({ type: "user_message", text }, options);
+    }
+    /**
+     * Open a recording at `rate`, which must be one of `RECORDING_RATES`.
+     * Answers its id. A recording already open is abandoned, as the edge
+     * abandons it.
+     */
+    startAudio(options) {
+        const { rate, language, model } = options;
+        if (!RECORDING_RATES.includes(rate)) {
+            throw new RangeError(`${rate} Hz is not a rate the socket accepts: ${RECORDING_RATES.join(", ")}.`);
+        }
+        if (this.open)
+            this.abandon("cancelled");
+        const id = `r${++this.recordings}`;
+        const recording = {
+            id,
+            bytes: 0,
+            held: null,
+            socket: -1,
+            over: false,
+        };
+        this.open = recording;
+        this.emit({ caption: null });
+        if (this.closed) {
+            recording.over = true;
+            recording.dropped = "closed";
+            return id;
+        }
+        const frame = this.turnFrame({ type: "user_audio.start", id, rate }, options);
+        if (language)
+            frame.language = language;
+        if (model)
+            frame.model = model;
+        if (this.socket?.ready) {
+            this.socket.send(frame);
+            this.said(recording);
+        }
+        else {
+            recording.held = [frame];
+            this.held.push(recording);
+            if (this.dormant)
+                void this.dial();
+        }
+        return id;
+    }
+    /**
+     * PCM for the open recording: 16-bit samples, or their little-endian
+     * bytes. Answers whether it was taken — not past the 10 MB a recording
+     * may carry (`onWarning`), nor with no recording open.
+     */
+    sendAudio(pcm) {
+        const recording = this.open;
+        if (!recording || recording.over)
+            return false;
+        const bytes = recording.bytes + pcm.byteLength;
+        if (bytes > MAX_AUDIO_BYTES) {
+            call(this.setup.handlers.onWarning, { type: "audio_limit", bytes });
+            return false;
+        }
+        recording.bytes = bytes;
+        const data = wireBytes(pcm);
+        // Held, it is copied: the recorder may reuse its buffer before `ready`.
+        if (recording.held)
+            recording.held.push(data.slice(0));
+        else if (recording.socket === this.generation && this.socket) {
+            this.socket.sendBinary(data);
+            this.saidAt = Date.now();
+        }
+        return true;
+    }
+    endAudio(options = {}) {
+        const recording = this.open;
+        this.open = null;
+        if (!recording) {
+            return Promise.resolve(dropped("refused", "No recording is open: call startAudio first."));
+        }
+        recording.over = true;
+        const failed = recording.failed;
+        if (failed) {
+            return Promise.resolve(options.transcribeOnly
+                ? { status: "failed", frame: failed }
+                : dropped("refused", failed.detail));
+        }
+        if (recording.dropped || this.closed) {
+            return Promise.resolve(dropped(recording.dropped ?? "closed"));
+        }
+        if (!options.transcribeOnly) {
+            return this.enqueue({ type: "user_audio.end" }, {}, recording);
+        }
+        return new Promise((resolve) => {
+            recording.settle = resolve;
+            this.hearing.push(recording);
+            const frame = { type: "user_audio.end", transcribe_only: true };
+            if (recording.held)
+                recording.held.push(frame);
+            else if (recording.socket === this.generation)
+                this.socket?.send(frame);
+        });
+    }
+    /** Stop the turn in flight, and drop what is queued and the open recording. What was said is still billed and stored. */
+    cancel() {
+        for (const pending of this.queue.splice(0)) {
+            pending.resolve(dropped("cancelled"));
+        }
+        if (this.open)
+            this.abandon("cancelled");
+        this.socket?.send({ type: "cancel" });
+        this.emit({ queued: 0, caption: null });
+    }
+    /** Stop speaking, and let the turn finish: the rest of the reply is never spoken, nor billed. */
+    mute() {
+        this.socket?.send({ type: "mute" });
+    }
+    /** Liveness only: answered with `pong`, and never counted as activity. Not needed — the SDK's watchdog listens for the edge's own. */
+    ping() {
+        this.socket?.send({ type: "ping" });
+    }
+    /**
+     * Dial now: the app came to the foreground, the network came back. A
+     * dormant conversation, one waiting to retry, and one that gave up (not
+     * a final close) dial at once; a live one is left alone.
+     */
+    reconnect() {
+        if (this.closed) {
+            if (!this.revivable)
+                return;
+            this.closed = false;
+            this.revivable = false;
+            this.attempts = 0;
+        }
+        else if (!this.retry && !this.dormant)
+            return;
+        void this.dial();
+    }
+    /** End it, for good. Every pending promise answers `dropped: closed`. */
+    close() {
+        if (this.closed)
+            return;
+        this.end();
+        this.sync();
+    }
+    // ---------------------------------------------------------------- dial
+    async dial(refused) {
+        this.stopTimers();
+        const generation = ++this.generation;
+        this.dormant = false;
+        this.sync();
+        let key;
+        try {
+            key =
+                refused !== undefined && this.key !== null
+                    ? await this.setup.tokens.replace(this.key, { code: refused })
+                    : await this.setup.tokens.current();
+        }
+        catch (error) {
+            // swallowed: getToken's own throw, the app's way to stop — handed
+            // back to it on `onClose` (`finish`).
+            if (generation === this.generation && !this.closed) {
+                this.finish(refused ?? 0, "getToken threw.", true, error);
+            }
+            return;
+        }
+        if (generation !== this.generation || this.closed)
+            return;
+        this.key = key;
+        try {
+            this.socket = dial({
+                url: socketUrl(this.setup.socketUrl, this.sessionId),
+                key,
+                sessionId: this.sessionId,
+                WebSocket: this.setup.WebSocket,
+                onFrame: (frame) => this.frame(frame),
+                onAudio: (frame) => {
+                    this.heardAt = Date.now();
+                    call(this.setup.handlers.onAudio, frame);
+                },
+                onClose: (code, reason) => this.lost(generation, code, reason),
+                onWarning: (warning) => call(this.setup.handlers.onWarning, warning),
+            });
+        }
+        catch (error) {
+            // swallowed: a URL the runtime's `WebSocket` refuses, which no retry
+            // cures — handed to the app on `onClose` (`finish`).
+            this.finish(0, "The socket could not be opened.", true, error);
+            return;
+        }
+        this.heardAt = this.saidAt = Date.now();
+        this.socketReady = false;
+        this.deadline = setTimeout(() => this.drop(generation, "The connection never became ready."), READY_DEADLINE_MS);
+        this.watchdog = setInterval(() => {
+            const quiet = Date.now() - Math.max(this.heardAt, this.saidAt);
+            if (quiet > 2 * KEEPALIVE_S * 1000) {
+                this.drop(generation, "Nothing came from the edge for a minute.");
+            }
+        }, KEEPALIVE_S * 500);
+    }
+    /** A socket given up on from this side: lost as a dropped network is (1006). */
+    drop(generation, reason) {
+        const socket = this.socket;
+        if (generation !== this.generation || !socket)
+            return;
+        socket.detach();
+        socket.close();
+        this.lost(generation, 1006, reason);
+    }
+    /** The socket closed: settle what it carried, then reconnect, rest or end. */
+    lost(generation, code, reason) {
+        if (generation !== this.generation || this.closed)
+            return;
+        call(this.setup.handlers.onDisconnect, {
+            code,
+            reason,
+            ready: this.socketReady,
+        });
+        // The handler may have closed it: then nothing follows.
+        if (generation !== this.generation || this.closed)
+            return;
+        this.stopTimers();
+        this.socket = null;
+        for (const recording of [this.open, ...this.hearing]) {
+            if (recording && recording.socket === generation) {
+                recording.over = true;
+                recording.dropped = "closed";
+                recording.settle?.(dropped("closed"));
+            }
+        }
+        this.hearing = this.hearing.filter((r) => r.socket !== generation);
+        const cut = this.inFlight;
+        this.inFlight = null;
+        if (cut?.started)
+            cut.resolve({ status: "interrupted", frame: null });
+        else if (cut && cut.sent < 2 && !cut.recording)
+            this.queue.unshift(cut);
+        else
+            cut?.resolve(dropped("closed"));
+        const turn = this.snap.turn;
+        this.running = false;
+        this.emit({
+            caption: null,
+            queued: this.queue.length,
+            turn: turn && !turn.done ? { ...turn, done: true, interrupted: true } : turn,
+        });
+        const next = reconnectAfter(code, this.attempts);
+        if (next.kind === "new-key") {
+            // A second refusal on a fresh key is final: the console's old 4005
+            // loop minted, re-sent and was refused every ~600 ms.
+            if (this.replaced)
+                this.finish(code, reason, true);
+            else {
+                this.replaced = true;
+                this.attempts += 1;
+                void this.dial(code);
+            }
+        }
+        else if (next.kind === "dormant") {
+            this.dormant = true;
+            this.sync();
+        }
+        else if (next.kind === "retry") {
+            this.attempts += 1;
+            this.retry = setTimeout(() => {
+                this.retry = null;
+                void this.dial();
+            }, next.delayMs);
+            this.sync();
+        }
+        else
+            this.finish(code, reason, isFinalClose(code));
+    }
+    /** It will not reconnect: say why, once. */
+    finish(code, reason, final, error) {
+        this.end();
+        this.revivable = !final;
+        this.sync();
+        const close = { code, reason, final };
+        if (error !== undefined)
+            close.error = error;
+        call(this.setup.handlers.onClose, close);
+    }
+    /** Closed: no socket, no timer, and every promise answered. */
+    end() {
+        this.closed = true;
+        this.generation += 1;
+        this.stopTimers();
+        const socket = this.socket;
+        this.socket = null;
+        socket?.detach();
+        socket?.close();
+        const pending = [...this.queue.splice(0)];
+        if (this.inFlight)
+            pending.push(this.inFlight);
+        this.inFlight = null;
+        for (const p of pending)
+            p.resolve(dropped("closed"));
+        for (const recording of [this.open, ...this.hearing, ...this.held]) {
+            if (recording) {
+                recording.over = true;
+                recording.dropped = "closed";
+                recording.settle?.(dropped("closed"));
+            }
+        }
+        this.hearing = [];
+        this.held = [];
+        this.running = false;
+        this.emit({ queued: 0, caption: null });
+    }
+    stopTimers() {
+        if (this.retry)
+            clearTimeout(this.retry);
+        if (this.deadline)
+            clearTimeout(this.deadline);
+        if (this.watchdog)
+            clearInterval(this.watchdog);
+        this.retry = this.deadline = this.watchdog = null;
+    }
+    // -------------------------------------------------------------- frames
+    frame(frame) {
+        this.heardAt = Date.now();
+        let quiet = false;
+        switch (frame.type) {
+            case "ready":
+                this.ready(frame);
+                break;
+            case "turn.start": {
+                const pending = this.inFlight;
+                if (pending && !pending.started)
+                    pending.started = true;
+                else
+                    this.running = true;
+                this.finished = null;
+                this.emit({ turn: { ...FRESH, id: frame.turn } });
+                this.sync();
+                break;
+            }
+            case "heard":
+                this.update({ heard: frame.text }, { caption: null });
+                break;
+            case "heard.partial":
+                if (this.listening())
+                    this.emit({ caption: frame.text || null });
+                break;
+            case "transcript":
+                this.transcript(frame);
+                break;
+            case "mood":
+                this.update({ mood: { mood: frame.mood, intensity: frame.intensity } });
+                break;
+            case "text.delta":
+                this.update({ text: (this.snap.turn?.text ?? "") + frame.text });
+                break;
+            case "text.chunk":
+                this.update({
+                    chunks: [...(this.snap.turn?.chunks ?? []), frame.text],
+                });
+                break;
+            case "tool.start":
+                this.update({ tool: frame.name || "tool" });
+                break;
+            case "tool.end":
+                this.update({ tool: null });
+                break;
+            case "knowledge":
+                this.update({
+                    knowledge: [...(this.snap.turn?.knowledge ?? []), frame],
+                });
+                break;
+            case "relation": {
+                const { type: _t, seq: _s, turn: _turn, ...relation } = frame;
+                this.relate(relation);
+                break;
+            }
+            case "turn.done":
+            case "interrupted":
+                quiet = this.terminal(frame);
+                break;
+            case "error":
+                this.error(frame);
+                break;
+        }
+        const name = HANDLER[frame.type];
+        if (name && !quiet) {
+            call(this.setup.handlers[name], frame);
+        }
+        call(this.setup.handlers.onEvent, frame);
+        if (frame.type === "ready" ||
+            frame.type === "turn.done" ||
+            frame.type === "interrupted" ||
+            frame.type === "error") {
+            this.pump();
+        }
+    }
+    ready(frame) {
+        this.socketReady = true;
+        if (this.deadline)
+            clearTimeout(this.deadline);
+        this.deadline = null;
+        this.attempts = 0;
+        this.everReady = true;
+        this.running = frame.greeting;
+        this.emit(frame.greeting ? { ready: frame, turn: FRESH } : { ready: frame });
+        this.relate(frame.relation);
+        // What was started while the socket was down goes first, in order.
+        for (const recording of this.held.splice(0)) {
+            if (recording.dropped)
+                continue;
+            for (const item of recording.held ?? []) {
+                if (item instanceof ArrayBuffer)
+                    this.socket?.sendBinary(item);
+                else
+                    this.socket?.send(item);
+            }
+            recording.held = null;
+            this.said(recording);
+        }
+        this.sync();
+    }
+    /** The end of a turn: answers whether it was a second for the same turn, which says nothing new. */
+    terminal(frame) {
+        if (frame.turn === this.finished)
+            return true;
+        this.finished = frame.turn;
+        const turn = this.snap.turn ?? FRESH;
+        this.emit({
+            turn: {
+                ...turn,
+                id: frame.turn,
+                text: frame.message || turn.text,
+                mood: frame.mood
+                    ? { mood: frame.mood, intensity: frame.mood_intensity }
+                    : turn.mood,
+                tool: null,
+                done: true,
+                interrupted: frame.type === "interrupted",
+            },
+        });
+        const pending = this.inFlight;
+        if (pending?.started) {
+            this.inFlight = null;
+            pending.resolve(frame.type === "turn.done"
+                ? { status: "done", frame }
+                : { status: "interrupted", frame });
+        }
+        this.running = false;
+        // The new key took a turn: a later refusal may take another.
+        if (frame.type === "turn.done")
+            this.replaced = false;
+        this.sync();
+        return false;
+    }
+    error(frame) {
+        if (frame.id) {
+            this.refuse(frame);
+            return;
+        }
+        if (frame.turn)
+            return;
+        // Outside a turn: the last word a recording or a message will get.
+        this.emit({ caption: null });
+        const pending = this.inFlight;
+        if (pending && !pending.started) {
+            this.inFlight = null;
+            pending.resolve(dropped("refused", frame.detail));
+            this.update({ done: true });
+            this.sync();
+        }
+    }
+    /** An error naming a recording: it was refused, or failed. */
+    refuse(frame) {
+        const { id } = frame;
+        const open = this.open;
+        if (open && open.id === id) {
+            open.failed = frame;
+            open.over = true;
+        }
+        const hearing = this.hearing.find((r) => r.id === id);
+        if (hearing) {
+            this.hearing = this.hearing.filter((r) => r !== hearing);
+            hearing.settle?.({ status: "failed", frame });
+        }
+        const pending = this.inFlight;
+        if (pending && pending.recording?.id === id && !pending.started) {
+            this.inFlight = null;
+            pending.resolve(dropped("refused", frame.detail));
+            this.update({ done: true });
+        }
+        this.emit({ caption: null });
+        this.sync();
+    }
+    transcript(frame) {
+        // By its id; the oldest, from an edge that echoes none.
+        const recording = frame.id
+            ? this.hearing.find((r) => r.id === frame.id)
+            : this.hearing[0];
+        if (recording) {
+            this.hearing = this.hearing.filter((r) => r !== recording);
+            recording.settle?.({
+                status: "done",
+                text: frame.text,
+                seconds: frame.seconds,
+                frame,
+            });
+        }
+        this.emit({ caption: null });
+    }
+    /** Each change of the relationship once: `ready` repeats it on every reconnect. */
+    relate(relation) {
+        if (!relation)
+            return;
+        const seen = JSON.stringify(relation, Object.keys(relation).sort());
+        if (seen === this.relationSeen)
+            return;
+        this.relationSeen = seen;
+        this.emit({ relation });
+        call(this.setup.handlers.onRelation, relation);
+    }
+    // ------------------------------------------------------------- sending
+    enqueue(frame, options, recording) {
+        if (this.closed)
+            return Promise.resolve(dropped("closed"));
+        return new Promise((resolve) => {
+            this.queue.push({
+                frame: this.turnFrame(frame, options),
+                resolve,
+                recording,
+                sent: 0,
+                started: false,
+            });
+            if (this.dormant)
+                void this.dial();
+            else
+                this.pump();
+            this.emit({ queued: this.queue.length });
+        });
+    }
+    /** The next queued turn goes, when nothing is running and the socket is ready. */
+    pump() {
+        while (this.queue.length &&
+            !this.inFlight &&
+            !this.running &&
+            this.socket?.ready) {
+            const next = this.queue.shift();
+            const recording = next.recording;
+            if (recording &&
+                (recording.dropped || recording.socket !== this.generation)) {
+                // Its recording was on a socket that is gone.
+                next.resolve(dropped(recording.dropped ?? "closed"));
+                continue;
+            }
+            // A message abandons an open recording, at the edge and here.
+            if (next.frame.type === "user_message" && this.open) {
+                this.abandon("cancelled");
+            }
+            this.inFlight = next;
+            next.sent += 1;
+            this.socket.send(next.frame);
+            this.saidAt = Date.now();
+            this.emit({ turn: FRESH });
+        }
+        this.emit({ queued: this.queue.length });
+        this.sync();
+    }
+    /** A turn's frame with its `speech` and `thoughts`. */
+    turnFrame(frame, options) {
+        const speech = options.speech ?? this.speech;
+        if (speech !== undefined && frame.type !== "user_audio.end") {
+            frame.speech = speech;
+        }
+        if (options.thoughts ?? this.setup.options.thoughts)
+            frame.thoughts = true;
+        return frame;
+    }
+    /** The open recording is over before its end: the edge discards the rest. */
+    abandon(reason) {
+        const recording = this.open;
+        if (!recording)
+            return;
+        recording.over = true;
+        recording.dropped = reason;
+    }
+    said(recording) {
+        recording.socket = this.generation;
+        this.saidAt = Date.now();
+    }
+    /** A recording's words are still to come: a partial is a caption. */
+    listening() {
+        return Boolean((this.open && !this.open.over) ||
+            this.hearing.length ||
+            this.inFlight?.recording);
+    }
+    // ------------------------------------------------------------ snapshot
+    update(turn, rest = {}) {
+        this.emit({ ...rest, turn: { ...(this.snap.turn ?? FRESH), ...turn } });
+    }
+    emit(change) {
+        let changed = false;
+        for (const key in change) {
+            if (change[key] !== this.snap[key]) {
+                changed = true;
+            }
+        }
+        if (!changed)
+            return;
+        this.snap = { ...this.snap, ...change };
+        for (const listener of this.listeners)
+            call(listener, undefined);
+    }
+    /** The state, from what is true now; `onState` when it moved. */
+    sync() {
+        const state = this.closed
+            ? "closed"
+            : this.dormant
+                ? "dormant"
+                : !this.socket?.ready
+                    ? this.everReady || this.attempts
+                        ? "reconnecting"
+                        : "connecting"
+                    : this.inFlight || this.running
+                        ? "busy"
+                        : "ready";
+        if (state === this.snap.state)
+            return;
+        this.emit({ state });
+        call(this.setup.handlers.onState, state);
+    }
+}