@oxvo/ai-live-assist 7.3.0

package/cjs/control.d.ts

@@ -0,0 +1,31 @@
+import type { ControlServerEnvelope, LaunchResult } from './types.js';
+type Handler = (message: ControlServerEnvelope) => void | Promise<void>;
+export declare class ControlChannel {
+    private readonly result;
+    private readonly tabId;
+    private readonly clientVersion;
+    private readonly onDisconnect;
+    private readonly onClientSequence;
+    private socket;
+    private clientSequence;
+    private serverSequence;
+    private fencingToken;
+    private heartbeat;
+    private handlers;
+    private intentionallyClosed;
+    private pendingAcks;
+    private recentAcks;
+    constructor(result: LaunchResult, tabId: string, clientVersion: string, onDisconnect: () => void, initialClientSequence?: number, onClientSequence?: (sequence: number) => void);
+    get leaseToken(): number;
+    get isOpen(): boolean;
+    setFencingToken(value: number): void;
+    subscribe(handler: Handler): () => void;
+    connect(signal: AbortSignal, resume: boolean): Promise<void>;
+    send(type: string, payload: Record<string, unknown>): string;
+    waitForAck(messageId: string, timeoutMs?: number): Promise<boolean>;
+    startHeartbeat(seconds: number): void;
+    close(code?: number, reason?: string): void;
+    private handleMessage;
+    private stopHeartbeat;
+}
+export {};

package/cjs/control.js

@@ -0,0 +1,190 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ControlChannel = void 0;
+const INVALID_MESSAGE_CLOSE_CODE = 4003;
+const PROTOCOL_VIOLATION_CLOSE_CODE = 4008;
+class ControlChannel {
+    constructor(result, tabId, clientVersion, onDisconnect, initialClientSequence = 0, onClientSequence = () => undefined) {
+        this.result = result;
+        this.tabId = tabId;
+        this.clientVersion = clientVersion;
+        this.onDisconnect = onDisconnect;
+        this.onClientSequence = onClientSequence;
+        this.socket = null;
+        this.serverSequence = 0;
+        this.fencingToken = 0;
+        this.heartbeat = null;
+        this.handlers = new Set();
+        this.intentionallyClosed = false;
+        this.pendingAcks = new Map();
+        this.recentAcks = new Set();
+        this.clientSequence =
+            Number.isSafeInteger(initialClientSequence) && initialClientSequence >= 0
+                ? initialClientSequence
+                : 0;
+    }
+    get leaseToken() {
+        return this.fencingToken;
+    }
+    get isOpen() {
+        return this.socket?.readyState === WebSocket.OPEN;
+    }
+    setFencingToken(value) {
+        if (!Number.isSafeInteger(value) || value < 1) {
+            throw new Error('Invalid AI Live Assist fencing token.');
+        }
+        this.fencingToken = value;
+    }
+    subscribe(handler) {
+        this.handlers.add(handler);
+        return () => this.handlers.delete(handler);
+    }
+    connect(signal, resume) {
+        return new Promise((resolve, reject) => {
+            const url = new URL(this.result.controlUrl);
+            url.searchParams.set('session', this.result.sessionId);
+            const socket = new WebSocket(url, [
+                `oxvo-ai-live-assist-v${this.result.protocolVersion}`,
+                `oxvo-auth.${this.result.controlCredential}`,
+            ]);
+            this.socket = socket;
+            this.intentionallyClosed = false;
+            const abort = () => {
+                socket.close(1000, 'cancelled');
+                reject(new DOMException('The operation was aborted.', 'AbortError'));
+            };
+            signal.addEventListener('abort', abort, { once: true });
+            socket.addEventListener('open', () => {
+                signal.removeEventListener('abort', abort);
+                this.send('hello', {
+                    protocolMin: 1,
+                    protocolMax: this.result.protocolVersion,
+                    clientVersion: this.clientVersion,
+                    resume,
+                });
+                resolve();
+            }, { once: true });
+            socket.addEventListener('message', (event) => {
+                void this.handleMessage(event.data);
+            });
+            socket.addEventListener('error', () => {
+                signal.removeEventListener('abort', abort);
+                if (socket.readyState !== WebSocket.OPEN) {
+                    reject(new Error('The AI Live Assist control channel could not connect.'));
+                }
+            }, { once: true });
+            socket.addEventListener('close', () => {
+                this.stopHeartbeat();
+                if (!this.intentionallyClosed && !signal.aborted)
+                    this.onDisconnect();
+            });
+        });
+    }
+    send(type, payload) {
+        if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
+            throw new Error('The AI Live Assist control channel is not connected.');
+        }
+        const id = `msg_${crypto.randomUUID()}`;
+        this.clientSequence += 1;
+        this.socket.send(JSON.stringify({
+            v: this.result.protocolVersion,
+            id,
+            type,
+            sessionId: this.result.sessionId,
+            tabId: this.tabId,
+            sequence: this.clientSequence,
+            fencingToken: this.fencingToken,
+            sentAt: new Date().toISOString(),
+            payload,
+        }));
+        this.onClientSequence(this.clientSequence);
+        return id;
+    }
+    waitForAck(messageId, timeoutMs = 2500) {
+        if (this.recentAcks.delete(messageId))
+            return Promise.resolve(true);
+        return new Promise((resolve) => {
+            const timeout = setTimeout(() => {
+                this.pendingAcks.delete(messageId);
+                resolve(false);
+            }, Math.max(250, Math.min(10000, timeoutMs)));
+            this.pendingAcks.set(messageId, (acknowledged) => {
+                clearTimeout(timeout);
+                this.pendingAcks.delete(messageId);
+                resolve(acknowledged);
+            });
+        });
+    }
+    startHeartbeat(seconds) {
+        this.stopHeartbeat();
+        const interval = Math.max(5, Math.min(30, seconds)) * 1000;
+        this.heartbeat = setInterval(() => {
+            if (!this.isOpen || this.fencingToken < 1)
+                return;
+            try {
+                this.send('heartbeat', { lastServerSequence: this.serverSequence });
+            }
+            catch {
+                this.onDisconnect();
+            }
+        }, interval);
+    }
+    close(code = 1000, reason = 'closed') {
+        this.intentionallyClosed = true;
+        this.stopHeartbeat();
+        this.socket?.close(code, reason.slice(0, 120));
+        this.socket = null;
+        this.handlers.clear();
+        for (const resolve of this.pendingAcks.values())
+            resolve(false);
+        this.pendingAcks.clear();
+        this.recentAcks.clear();
+    }
+    async handleMessage(raw) {
+        if (typeof raw !== 'string' || raw.length > 131072)
+            return;
+        let message;
+        try {
+            message = JSON.parse(raw);
+        }
+        catch {
+            this.close(INVALID_MESSAGE_CLOSE_CODE, 'invalid message');
+            return;
+        }
+        const directProtocolError = message.type === 'error' && message.sequence === 0;
+        if (message.v !== this.result.protocolVersion ||
+            message.sessionId !== this.result.sessionId ||
+            !Number.isSafeInteger(message.sequence) ||
+            (!directProtocolError && message.sequence <= this.serverSequence) ||
+            typeof message.type !== 'string' ||
+            !message.payload ||
+            typeof message.payload !== 'object') {
+            this.close(PROTOCOL_VIOLATION_CLOSE_CODE, 'protocol violation');
+            return;
+        }
+        if (!directProtocolError)
+            this.serverSequence = message.sequence;
+        if (message.type === 'ack' && typeof message.payload.messageId === 'string') {
+            const messageId = message.payload.messageId;
+            const pending = this.pendingAcks.get(messageId);
+            if (pending)
+                pending(true);
+            else {
+                this.recentAcks.add(messageId);
+                if (this.recentAcks.size > 100) {
+                    const oldest = this.recentAcks.values().next().value;
+                    if (typeof oldest === 'string')
+                        this.recentAcks.delete(oldest);
+                }
+            }
+        }
+        for (const handler of this.handlers)
+            await handler(message);
+    }
+    stopHeartbeat() {
+        if (this.heartbeat)
+            clearInterval(this.heartbeat);
+        this.heartbeat = null;
+    }
+}
+exports.ControlChannel = ControlChannel;

package/cjs/experienceState.d.ts

@@ -0,0 +1,18 @@
+export type VoicePresenceVisualState = "unavailable" | "launcher" | "activation_modal" | "preflight" | "requesting_microphone" | "connecting" | "active.ready" | "active.listening" | "active.thinking" | "active.speaking" | "active.acting" | "active.awaiting_confirmation" | "active.reconnecting" | "ending" | "ended" | "recoverable_error" | "terminal_error";
+export type VoicePresenceStateContext = {
+    sessionId: string | null;
+    tabId: string;
+    pageId: string | null;
+    serverSequence: number;
+    fencingToken: number;
+};
+export declare class VoicePresenceStateMachine {
+    private readonly onChange;
+    private current;
+    private context;
+    constructor(tabId: string, onChange?: (state: VoicePresenceVisualState, context: VoicePresenceStateContext) => void);
+    get state(): VoicePresenceVisualState;
+    get snapshot(): VoicePresenceStateContext;
+    transition(next: VoicePresenceVisualState, update?: Partial<VoicePresenceStateContext>): boolean;
+    reset(): void;
+}

package/cjs/experienceState.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VoicePresenceStateMachine = void 0;
+const ACTIVE_STATES = [
+    "active.ready",
+    "active.listening",
+    "active.thinking",
+    "active.speaking",
+    "active.acting",
+    "active.awaiting_confirmation",
+    "active.reconnecting",
+];
+const transitions = new Map([
+    ["unavailable", new Set(["launcher"])],
+    ["launcher", new Set(["activation_modal", "active.reconnecting", "unavailable"])],
+    ["activation_modal", new Set(["launcher", "preflight", "recoverable_error", "terminal_error"])],
+    ["preflight", new Set(["requesting_microphone", "recoverable_error", "terminal_error", "launcher"])],
+    ["requesting_microphone", new Set(["connecting", "recoverable_error", "terminal_error", "launcher"])],
+    ["connecting", new Set(["active.ready", "active.reconnecting", "recoverable_error", "terminal_error", "ending"])],
+    ["active.reconnecting", new Set(["connecting", "active.ready", "recoverable_error", "terminal_error", "ending"])],
+    ["recoverable_error", new Set(["launcher", "preflight", "requesting_microphone", "connecting", "active.reconnecting", "ending", "terminal_error"])],
+    ["terminal_error", new Set(["launcher", "ended"])],
+    ["ending", new Set(["ended", "terminal_error"])],
+    ["ended", new Set(["launcher"])],
+]);
+for (const state of ACTIVE_STATES) {
+    const allowed = transitions.get(state) ?? new Set();
+    for (const next of ACTIVE_STATES)
+        allowed.add(next);
+    allowed.add("ending");
+    allowed.add("recoverable_error");
+    allowed.add("terminal_error");
+    transitions.set(state, allowed);
+}
+class VoicePresenceStateMachine {
+    constructor(tabId, onChange = () => undefined) {
+        this.onChange = onChange;
+        this.current = "unavailable";
+        this.context = {
+            sessionId: null,
+            tabId,
+            pageId: null,
+            serverSequence: 0,
+            fencingToken: 0,
+        };
+    }
+    get state() {
+        return this.current;
+    }
+    get snapshot() {
+        return { ...this.context };
+    }
+    transition(next, update = {}) {
+        if (update.serverSequence !== undefined &&
+            update.serverSequence <= this.context.serverSequence) {
+            return false;
+        }
+        if (update.sessionId &&
+            this.context.sessionId &&
+            update.sessionId !== this.context.sessionId) {
+            return false;
+        }
+        if (next !== this.current && !transitions.get(this.current)?.has(next)) {
+            return false;
+        }
+        this.context = { ...this.context, ...update };
+        this.current = next;
+        this.onChange(next, this.snapshot);
+        return true;
+    }
+    reset() {
+        this.current = "unavailable";
+        this.context = {
+            ...this.context,
+            sessionId: null,
+            pageId: null,
+            serverSequence: 0,
+            fencingToken: 0,
+        };
+    }
+}
+exports.VoicePresenceStateMachine = VoicePresenceStateMachine;

package/cjs/index.d.ts

@@ -0,0 +1,13 @@
+import type { App, Options as BrowserOptions } from '@oxvo/browser';
+import AiLiveAssist from './AiLiveAssist.js';
+import type { PluginOptions } from './types.js';
+import { VERSION } from './version.js';
+export type { PluginOptions } from './types.js';
+export { AiLiveAssist, VERSION };
+export default function aiLiveAssist(options?: PluginOptions): (app: App | null) => AiLiveAssist | undefined;
+export declare function installAiLiveAssist(input: {
+    tracker: {
+        use<T>(plugin: (app: App | null, options?: Partial<BrowserOptions>) => T): T;
+    };
+    options?: PluginOptions;
+}): AiLiveAssist | undefined;

package/cjs/index.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VERSION = exports.AiLiveAssist = void 0;
+exports.default = aiLiveAssist;
+exports.installAiLiveAssist = installAiLiveAssist;
+const AiLiveAssist_js_1 = require("./AiLiveAssist.js");
+exports.AiLiveAssist = AiLiveAssist_js_1.default;
+const version_js_1 = require("./version.js");
+Object.defineProperty(exports, "VERSION", { enumerable: true, get: function () { return version_js_1.VERSION; } });
+const installed = new WeakMap();
+function aiLiveAssist(options = {}) {
+    return function install(app) {
+        if (!app || options.enabled === false || app.insideIframe)
+            return undefined;
+        const existing = installed.get(app);
+        if (existing)
+            return existing;
+        if (document.querySelector('[data-oxvo-ai-live-assist-root]')) {
+            return undefined;
+        }
+        if (!app.checkRequiredVersion?.('7.3.0')) {
+            console.warn('OXVO AI Live Assist requires @oxvo/browser version 7.3.0 or newer.');
+            return undefined;
+        }
+        const instance = new AiLiveAssist_js_1.default(app, options);
+        installed.set(app, instance);
+        return instance;
+    };
+}
+function installAiLiveAssist(input) {
+    return input.tracker.use(aiLiveAssist(input.options));
+}

package/cjs/media.d.ts

@@ -0,0 +1,34 @@
+import type { AssistMode, LaunchResult } from './types.js';
+import type { RuntimeClient } from './client.js';
+type MediaCallbacks = {
+    onLocalStream: (stream: MediaStream | null) => void;
+    onRemoteStream: (stream: MediaStream) => void;
+    onConnectionState: (state: RTCPeerConnectionState) => void;
+    onMicrophoneUnavailable: () => void;
+};
+export declare class RealtimeMedia {
+    private readonly client;
+    private readonly callbacks;
+    private connection;
+    private localStream;
+    private dataChannel;
+    private manualTurn;
+    private pushActive;
+    private muted;
+    private replacingDevice;
+    constructor(client: RuntimeClient, callbacks: MediaCallbacks);
+    prepare(mode: AssistMode): Promise<void>;
+    connect(result: LaunchResult, mode: AssistMode, signal: AbortSignal): Promise<number>;
+    setMuted(muted: boolean): void;
+    setManualTurn(manual: boolean): void;
+    beginPushToTalk(): void;
+    endPushToTalk(): void;
+    switchToTextOnly(): void;
+    close(): void;
+    private applyTrackState;
+    private bindTrackEnd;
+    private onDeviceChange;
+    private replaceUnavailableDevice;
+    private waitForIce;
+}
+export {};

package/cjs/media.js

@@ -0,0 +1,207 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RealtimeMedia = void 0;
+class RealtimeMedia {
+    constructor(client, callbacks) {
+        this.client = client;
+        this.callbacks = callbacks;
+        this.connection = null;
+        this.localStream = null;
+        this.dataChannel = null;
+        this.manualTurn = false;
+        this.pushActive = false;
+        this.muted = false;
+        this.replacingDevice = false;
+        this.onDeviceChange = () => {
+            void this.replaceUnavailableDevice();
+        };
+    }
+    async prepare(mode) {
+        if (mode !== 'voice')
+            return;
+        if (!navigator.mediaDevices?.getUserMedia) {
+            throw new Error('Microphone access is not available in this browser.');
+        }
+        this.localStream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+                autoGainControl: true,
+                echoCancellation: true,
+                noiseSuppression: true,
+            },
+            video: false,
+        });
+        navigator.mediaDevices.addEventListener?.('devicechange', this.onDeviceChange);
+        this.bindTrackEnd();
+        this.applyTrackState();
+        this.callbacks.onLocalStream(this.localStream);
+    }
+    async connect(result, mode, signal) {
+        const connection = new RTCPeerConnection({ bundlePolicy: 'max-bundle' });
+        this.connection = connection;
+        this.dataChannel = connection.createDataChannel('oai-events', {
+            ordered: true,
+        });
+        if (mode === 'voice') {
+            const stream = this.localStream;
+            if (!stream)
+                throw new Error('Microphone access was not prepared.');
+            for (const track of stream.getAudioTracks())
+                connection.addTrack(track, stream);
+        }
+        else {
+            // OpenAI's WebRTC endpoint requires an audio media section even when the
+            // configured response modality is text-only.
+            connection.addTransceiver('audio', { direction: 'recvonly' });
+        }
+        connection.addEventListener('track', (event) => {
+            const stream = event.streams[0];
+            if (stream)
+                this.callbacks.onRemoteStream(stream);
+        });
+        connection.addEventListener('connectionstatechange', () => {
+            this.callbacks.onConnectionState(connection.connectionState);
+        });
+        const offer = await connection.createOffer();
+        await connection.setLocalDescription(offer);
+        await this.waitForIce(connection, signal);
+        const localSdp = connection.localDescription?.sdp;
+        if (!localSdp)
+            throw new Error('The browser could not create a realtime offer.');
+        const response = await this.client.offer(result, localSdp, signal);
+        await connection.setRemoteDescription({ type: 'answer', sdp: response.answerSdp });
+        return response.fencingToken;
+    }
+    setMuted(muted) {
+        this.muted = muted;
+        this.applyTrackState();
+    }
+    setManualTurn(manual) {
+        this.manualTurn = manual;
+        this.pushActive = false;
+        this.applyTrackState();
+    }
+    beginPushToTalk() {
+        if (!this.manualTurn || this.muted)
+            return;
+        this.pushActive = true;
+        this.applyTrackState();
+    }
+    endPushToTalk() {
+        this.pushActive = false;
+        this.applyTrackState();
+    }
+    switchToTextOnly() {
+        for (const sender of this.connection?.getSenders() ?? []) {
+            if (sender.track?.kind === 'audio') {
+                sender.track.stop();
+                void sender.replaceTrack(null);
+            }
+        }
+        for (const track of this.localStream?.getTracks() ?? [])
+            track.stop();
+        this.localStream = null;
+        this.callbacks.onLocalStream(null);
+        navigator.mediaDevices?.removeEventListener?.('devicechange', this.onDeviceChange);
+    }
+    close() {
+        this.dataChannel?.close();
+        this.dataChannel = null;
+        for (const track of this.localStream?.getTracks() ?? [])
+            track.stop();
+        this.localStream = null;
+        this.callbacks.onLocalStream(null);
+        navigator.mediaDevices?.removeEventListener?.('devicechange', this.onDeviceChange);
+        this.connection?.close();
+        this.connection = null;
+    }
+    applyTrackState() {
+        const enabled = !this.muted && (!this.manualTurn || this.pushActive);
+        for (const track of this.localStream?.getAudioTracks() ?? []) {
+            track.enabled = enabled;
+        }
+    }
+    bindTrackEnd() {
+        for (const track of this.localStream?.getAudioTracks() ?? []) {
+            track.addEventListener('ended', this.onDeviceChange, { once: true });
+        }
+    }
+    async replaceUnavailableDevice() {
+        if (this.replacingDevice || !this.connection || !this.localStream)
+            return;
+        const current = this.localStream.getAudioTracks()[0];
+        if (!current) {
+            this.callbacks.onMicrophoneUnavailable();
+            return;
+        }
+        if (current.readyState === 'live') {
+            try {
+                const devices = await navigator.mediaDevices.enumerateDevices();
+                const currentId = current.getSettings().deviceId;
+                if (!currentId ||
+                    devices.some((device) => device.kind === 'audioinput' && device.deviceId === currentId)) {
+                    return;
+                }
+            }
+            catch {
+                return;
+            }
+        }
+        this.replacingDevice = true;
+        try {
+            const replacement = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    autoGainControl: true,
+                    echoCancellation: true,
+                    noiseSuppression: true,
+                },
+                video: false,
+            });
+            const track = replacement.getAudioTracks()[0];
+            const sender = this.connection
+                .getSenders()
+                .find((candidate) => candidate.track?.kind === 'audio');
+            if (!track || !sender)
+                throw new Error('No replacement microphone track.');
+            await sender.replaceTrack(track);
+            for (const prior of this.localStream.getTracks())
+                prior.stop();
+            this.localStream = replacement;
+            this.bindTrackEnd();
+            this.applyTrackState();
+            this.callbacks.onLocalStream(replacement);
+        }
+        catch {
+            this.callbacks.onMicrophoneUnavailable();
+        }
+        finally {
+            this.replacingDevice = false;
+        }
+    }
+    waitForIce(connection, signal) {
+        if (connection.iceGatheringState === 'complete')
+            return Promise.resolve();
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(finish, 2500);
+            const onState = () => {
+                if (connection.iceGatheringState === 'complete')
+                    finish();
+            };
+            const onAbort = () => {
+                cleanup();
+                reject(new DOMException('The operation was aborted.', 'AbortError'));
+            };
+            const cleanup = () => {
+                clearTimeout(timeout);
+                connection.removeEventListener('icegatheringstatechange', onState);
+                signal.removeEventListener('abort', onAbort);
+            };
+            function finish() {
+                cleanup();
+                resolve();
+            }
+            connection.addEventListener('icegatheringstatechange', onState);
+            signal.addEventListener('abort', onAbort, { once: true });
+        });
+    }
+}
+exports.RealtimeMedia = RealtimeMedia;

package/cjs/messages.d.ts

@@ -0,0 +1,2 @@
+import type { WidgetMessageKey } from "./types.js";
+export declare const defaultMessages: Record<WidgetMessageKey, string>;