@theoremjs/web 0.0.0-stage → 0.3.4-next.375.ga2d1878

package/client/live/live-page-tool.d.ts

@@ -0,0 +1,39 @@
+/** The page's result for a call: the tool's output schema checks it, then the model reads it. */
+export type PageToolAnswer = {
+    output: unknown;
+};
+/** One page tool: what the page does for a call the model made, and what it tells the model. */
+export type PageTool = (args: Record<string, unknown>, call: {
+    callId: string;
+    operationId: string;
+}) => PageToolAnswer | Promise<PageToolAnswer>;
+/**
+ * The page's tools, by name: each answers a tool the profile declares with
+ * `answeredBy: 'page'`. Any other tool runs on the relay, gates and all.
+ */
+export type PageTools = Record<string, PageTool>;
+/** What a host gives a live call. */
+export type LiveCallOptions = {
+    /** The value chosen for each of the profile's `inputs.slots`, read as the call starts. */
+    slots?: Record<string, string>;
+    /**
+     * What the page tells the agent: any JSON. It goes with the call's opening, and
+     * again whenever it changes, as background the agent reads without replying.
+     */
+    context?: unknown;
+    pageTools?: PageTools;
+};
+/**
+ * Where the page's tools and the profile's disagree, for the builder: a page
+ * tool with no handler here leaves the model unanswered, and a handler for a
+ * tool the profile does not give the page never runs.
+ */
+export declare function pageToolMismatch(declared: readonly string[], handled: readonly string[]): {
+    unanswered: string[];
+    unused: string[];
+};
+/**
+ * Says in the console where the page's tools and the profile's disagree. A wrong name is the
+ * builder's to fix, so it is never said to the visitor.
+ */
+export declare function warnPageToolMismatch(declared: readonly string[], handled: readonly string[]): void;

package/client/live/live-page-tool.js

@@ -0,0 +1,26 @@
+/**
+ * Where the page's tools and the profile's disagree, for the builder: a page
+ * tool with no handler here leaves the model unanswered, and a handler for a
+ * tool the profile does not give the page never runs.
+ */
+export function pageToolMismatch(declared, handled) {
+    return {
+        unanswered: declared.filter((name) => !handled.includes(name)),
+        unused: handled.filter((name) => !declared.includes(name)),
+    };
+}
+/**
+ * Says in the console where the page's tools and the profile's disagree. A wrong name is the
+ * builder's to fix, so it is never said to the visitor.
+ */
+export function warnPageToolMismatch(declared, handled) {
+    const { unanswered, unused } = pageToolMismatch(declared, handled);
+    for (const name of unanswered) {
+        // labels-exempt: builder diagnostic
+        console.warn(`Theorem: page tool '${name}' has no handler in pageTools; the agent gets no answer.`);
+    }
+    for (const name of unused) {
+        // labels-exempt: builder diagnostic
+        console.warn(`Theorem: pageTools has '${name}', which the profile does not declare answeredBy: 'page'.`);
+    }
+}

package/client/live/live-state.d.ts

@@ -0,0 +1,13 @@
+export type LiveSessionStatus = 'disconnected' | 'connecting'
+/** The call dropped and the client is taking it up again. */
+ | 'reconnecting' | 'ready' | 'listening' | 'speaking' | 'working' | 'error';
+export type LiveConnectPhase = 'socket' | 'microphone';
+/** What a live call is doing, for its status line; the UI words it. */
+export type LiveState = 'calling_tool' | 'connecting' | 'reconnecting' | 'requesting_mic' | 'speaking' | 'connected' | 'muted' | 'listening' | 'error' | 'ended';
+export declare function liveState(args: {
+    status: LiveSessionStatus;
+    connectPhase: LiveConnectPhase | null;
+    toolName: string | null;
+    isMuted: boolean;
+    voiceEnabled?: boolean;
+}): LiveState;

package/client/live/live-state.js

@@ -0,0 +1,22 @@
+export function liveState(args) {
+    if (args.status === 'reconnecting')
+        return 'reconnecting';
+    if (args.toolName)
+        return 'calling_tool';
+    if (args.connectPhase === 'socket')
+        return 'connecting';
+    if (args.connectPhase === 'microphone')
+        return 'requesting_mic';
+    if (args.status === 'speaking')
+        return 'speaking';
+    if (args.status === 'listening' || args.status === 'ready') {
+        if (args.voiceEnabled === false)
+            return 'connected';
+        return args.isMuted ? 'muted' : 'listening';
+    }
+    if (args.status === 'connecting')
+        return 'connecting';
+    if (args.status === 'error')
+        return 'error';
+    return 'ended';
+}

package/client/live/live-tool.d.ts

@@ -0,0 +1,11 @@
+import type { ToolGate } from '@theoremjs/agents/kernel';
+import type { ToolGateResolution } from '../tool-resume.ts';
+/** A gate the user is asked to answer, for the model's call `callId`. */
+export type LiveToolGatePrompt = {
+    callId: string;
+    toolName: string;
+    input: Record<string, unknown>;
+    gate: ToolGate;
+};
+/** The user's answer to a live gate, or `withdrawn`: the model cancelled the call, so nothing is asked. */
+export type LiveGateAnswer = ToolGateResolution | 'withdrawn';

package/client/live/live-tool.js

@@ -0,0 +1 @@
+export {};

package/client/live-client.d.ts

@@ -0,0 +1,189 @@
+/**
+ * Pure client-side SDK for THEOREM Gemini 3.1 Flash Live sessions over WebSocket relay.
+ *
+ * Handles:
+ * - Bidirectional WebSocket connection to `/api/live/relay`.
+ * - 16-bit 16kHz PCM microphone audio recording & streaming.
+ * - Gapless 24kHz PCM / WAV model voice playback scheduling.
+ * - Barge-in interruption cancellation (instant audio queue flush).
+ * - Quiet-mic gate while model audio plays (reduces false barge-in from speaker bleed).
+ * - UI tool execution and response routing.
+ *
+ * @module
+ */
+import { type SessionEventOf, type TraceRecord } from '@theoremjs/agents';
+import type { LiveConnectPhase, LiveSessionStatus } from './live/live-state.ts';
+import { type ExecuteToolOnRelay, type LiveToolStep } from './live-messages.ts';
+import { type ClientTurnEvent } from './transport.ts';
+export type { LiveConnection, LiveSocket } from './live-messages.ts';
+import type { LiveSocket } from './live-messages.ts';
+export interface LiveClientOptions {
+    profile?: string;
+    /** The host's own JSON for its relay, sent as the open message's `host`. */
+    openMessage?: Record<string, unknown>;
+    /** The value chosen for each of the profile's `inputs.slots`, fixed for the call. */
+    slots?: Record<string, string>;
+    /** What the page tells the agent as the call opens; `setContext` replaces it during the call. */
+    context?: unknown;
+    relayUrl?: string;
+    /** In-process connections reuse the same microphone, playback, and tool client. */
+    createSocket?: () => LiveSocket;
+    /** When false, skip microphone capture; session still receives model audio. */
+    voiceIngress?: boolean;
+    onStatusChange?: (status: LiveSessionStatus) => void;
+    onConnectPhase?: (phase: LiveConnectPhase | null) => void;
+    onTranscript?: (text: string, isUser: boolean, meta?: {
+        interim?: boolean;
+    }) => void;
+    /**
+     * Every event the session sends, `unsupported` for a kind this client does
+     * not know, and `malformed` for one that failed its check (left out; the call goes on).
+     */
+    onTurnEvent?: (event: ClientTurnEvent) => void;
+    /** A trace record the session wrote, when the relay delivers them. */
+    onTrace?: (record: TraceRecord) => void;
+    /** A failure, typed by kind; word it with `clientFailure` and the interface's `labels`. */
+    onError?: (error: Error) => void;
+    /** Provider signalled the upstream session is draining (e.g. goAway). */
+    onSessionClosing?: (timeLeftMs?: number) => void;
+    /**
+     * The provider ended the session after warning it would: not a failure.
+     * `session.message` is the user's line; `session.ended` the close, for the builder.
+     */
+    onSessionEnded?: (session: SessionEventOf<'ended'>) => void;
+    /**
+     * The model called a tool. The host runs it with `executeToolOnRelay` and
+     * answers its gates; the session answers the model. A throw reaches `onError`.
+     */
+    onToolCall: (name: string, args: Record<string, unknown>, meta: {
+        callId: string;
+        operationId: string;
+    }) => Promise<void>;
+    /** The relay opened the call: the id it gave the call, and the profile it runs. */
+    onSessionReady?: (info: {
+        sessionId?: string;
+        profile?: string;
+    }) => void;
+    onVolumeLevel?: (level: number, isUser: boolean) => void;
+}
+export declare class LiveSessionClient {
+    private ws;
+    private audioContext;
+    private micStream;
+    private micSource;
+    private micWorklet;
+    private micWorkletModuleLoaded;
+    private playbackNodes;
+    private playbackBus;
+    private playbackAnalyser;
+    private playbackMeterFrame;
+    private playbackMeterBuffer;
+    private nextPlaybackTime;
+    private status;
+    /** Server reported background work (reasoning / async tools) still in flight. */
+    private serverWorking;
+    private micActivating;
+    private connectTimeout;
+    private isMuted;
+    private options;
+    /** Serialize control-plane handling (tools / status) without waiting on audio decode. */
+    private inboundChain;
+    /** Ordered model-audio playback queue (separate so tools are not stuck behind decode). */
+    private audioChain;
+    /**
+     * Tool calls, one at a time in the order the model made them. Apart from
+     * `inboundChain`: a call waits on the relay's reply, which arrives inbound.
+     */
+    private toolChain;
+    /** Bumped on barge-in / cancel so stale audioChain work is skipped. */
+    private audioEpoch;
+    private pendingExecuteResults;
+    /** The page's context package, and the JSON of the one the relay last got. */
+    private context;
+    private sentContext;
+    /** The provider's latest handle for taking this call up again after a drop. */
+    private history;
+    private providerState;
+    /** The session said it ended, or failed: a close after that is the end, not a drop. */
+    private sessionOver;
+    /** When the call dropped, for the time away a resume reports. */
+    private droppedAt;
+    /** When the call that could not reconnect dropped: the next call takes it up again. */
+    private lostAt;
+    private reconnectTries;
+    private reconnectTimer;
+    constructor(options: LiveClientOptions);
+    private setStatus;
+    /** Still opening: the first connect, or a try at taking a dropped call up again. */
+    private get opening();
+    private setConnectPhase;
+    private clearConnectTimeout;
+    private detachWebSocket;
+    private clearReconnect;
+    private teardownConnection;
+    /** End the session on `error`: a failed connect or a lost socket. */
+    private failSession;
+    private createSocket;
+    /**
+     * Start a call. After one that dropped and could not reconnect, it takes that call up again,
+     * so the agent neither greets twice nor forgets; after a hang-up it is a new call.
+     */
+    connect(): Promise<void>;
+    /** A lost call that cannot be taken up again, its handle refused or expired, starts as a new one. */
+    private startOver;
+    /** The first message: the call's values, and the handle and time away when it takes a dropped call up again. */
+    private openMessage;
+    /** The socket closed while the call was open: take it up again if the provider gave a handle, else end. */
+    private handleDrop;
+    private scheduleReconnect;
+    /** A try at opening failed: a reconnect waits and tries again, a first connect fails the call. */
+    private failOpen;
+    private open;
+    private ensureAudioContext;
+    private activateMicrophone;
+    private sendMicFrame;
+    private reportMicVolume;
+    private forwardMicBuffer;
+    private setupMicrophonePipeline;
+    private ensurePlaybackBus;
+    private stopPlaybackMeter;
+    private startPlaybackMeter;
+    private processServerEnvelope;
+    private enqueueServerMessage;
+    private handleReadyEnvelope;
+    /** The call is open, first or again: the page's context catches up with what changed meanwhile. */
+    private becomeReady;
+    private handleExecuteToolResultEnvelope;
+    private tryHandleControlEnvelope;
+    private collectInboundTurn;
+    private processTurnEvent;
+    private handleEvidenceTurnEvent;
+    private handleSessionTurnEvent;
+    /** Hand the session's closing and end to the host; `true` when it was one of them. */
+    private notifySessionLifecycle;
+    /** Status to settle into once model speech stops. */
+    private restingStatus;
+    private collectMediaTurnEvent;
+    private handleDoneTurnEvent;
+    private scheduleMediaChunks;
+    /** The relay runs `LiveSession.executeTool`; see `ExecuteToolOnRelay`. */
+    executeToolOnRelay(args: Parameters<ExecuteToolOnRelay>[0]): Promise<LiveToolStep>;
+    private enqueueAudioChunk;
+    cancelPlayback(): void;
+    toggleMute(): boolean;
+    sendText(text: string): void;
+    /**
+     * The call's slot values. An open call is taken up again with them when the provider gave a
+     * handle; with none, the next call has them.
+     */
+    setSlots(slots: Record<string, string> | undefined): void;
+    /**
+     * Replace the page's context package. In a call it reaches the agent as
+     * background, with no reply; before one, it goes with the call's opening.
+     */
+    setContext(context: unknown): void;
+    private flushContext;
+    sendVideo(data: string, mimeType?: string): void;
+    disconnect(): void;
+    private cleanupAudio;
+}