@theoremjs/react 0.3.0 → 0.3.1

package/client/live/live-state.d.ts

@@ -1,7 +1,9 @@
-export type LiveSessionStatus = 'disconnected' | 'connecting' | 'ready' | 'listening' | 'speaking' | 'working' | 'error';
+export type LiveSessionStatus = 'disconnected' | 'connecting'
+/** The call dropped and the client is taking it up again. */
+ | 'reconnecting' | 'ready' | 'listening' | 'speaking' | 'working' | 'error';
 export type LiveConnectPhase = 'socket' | 'microphone';
 /** What a live call is doing, for its status line; the UI words it. */
-export type LiveState = 'calling_tool' | 'connecting' | 'requesting_mic' | 'speaking' | 'connected' | 'muted' | 'listening' | 'error' | 'ended';
+export type LiveState = 'calling_tool' | 'connecting' | 'reconnecting' | 'requesting_mic' | 'speaking' | 'connected' | 'muted' | 'listening' | 'error' | 'ended';
 export declare function liveState(args: {
     status: LiveSessionStatus;
     connectPhase: LiveConnectPhase | null;

package/client/live/live-state.js

@@ -1,4 +1,6 @@
 export function liveState(args) {
+    if (args.status === 'reconnecting')
+        return 'reconnecting';
     if (args.toolName)
         return 'calling_tool';
     if (args.connectPhase === 'socket')

package/client/live/live-video.js

@@ -20,7 +20,7 @@ export function startLiveVideoCapture(onFrame, options) {
         if (!context) {
             for (const track of stream.getTracks())
                 track.stop();
-            throw new TheoremError('unsupported', 'no 2d canvas for live video'); // lexicon-exempt: internal diagnostic
+            throw new TheoremError('unsupported', 'no 2d canvas for live video'); // labels-exempt: internal diagnostic
         }
         let timer = null;
         let stopped = false;

package/client/live/run-live-tool-call.d.ts

@@ -1,16 +1,21 @@
 import type { ExecuteToolOnRelay } from '../live-messages.ts';
+import type { PageTools } from './live-page-tool.ts';
 import type { LiveGateAnswer, LiveToolGatePrompt } from './live-tool.ts';
 /**
- * Live provider tool-call handler: run the model's call on the relay; at a
- * gate, ask the user and send their decision. The session answers the model
+ * Live provider tool-call handler: the page answers the call if it is the
+ * page's; else run the model's call on the relay; at a gate, ask the user and
+ * send their decision. The session answers the model
  * however the call ends; its tool events tell the user. A call the model
  * cancels while its gate is open is gone: nothing is sent.
  */
 export declare function runLiveToolCall(args: {
     executeToolOnRelay: ExecuteToolOnRelay;
+    /** The host's page tools, if it has any. */
+    pageTools?: PageTools;
     name: string;
     toolArgs: Record<string, unknown>;
     callId: string;
+    operationId: string;
     sessionPermissions: string[];
     setSessionPermissions: (next: string[]) => void;
     waitForGateDecision: (prompt: LiveToolGatePrompt) => Promise<LiveGateAnswer>;

package/client/live/run-live-tool-call.js

@@ -1,31 +1,69 @@
 import { continueGatedToolInvocation } from "../tool-resume.js";
 /**
- * Live provider tool-call handler: run the model's call on the relay; at a
- * gate, ask the user and send their decision. The session answers the model
+ * Live provider tool-call handler: the page answers the call if it is the
+ * page's; else run the model's call on the relay; at a gate, ask the user and
+ * send their decision. The session answers the model
  * however the call ends; its tool events tell the user. A call the model
  * cancels while its gate is open is gone: nothing is sent.
  */
 export async function runLiveToolCall(args) {
-    const { executeToolOnRelay, name, toolArgs, callId } = args;
+    const { executeToolOnRelay, name, toolArgs, callId, operationId } = args;
     let sessionPermissions = args.sessionPermissions;
-    let step = await executeToolOnRelay({ callId });
-    while (step.status === 'gated') {
+    // why: A name from the model is looked up among the host's own keys only.
+    const pageTool = args.pageTools && Object.hasOwn(args.pageTools, name) ? args.pageTools[name] : undefined;
+    let step = await executeToolOnRelay({ callId, operationId });
+    const dispatched = new Set();
+    while (step.status !== 'settled') {
+        if (step.status === 'dispatched') {
+            if (dispatched.has(step.grantId))
+                return;
+            dispatched.add(step.grantId);
+            const grantId = step.grantId;
+            const input = step.arguments;
+            const answer = pageTool
+                ? await Promise.resolve()
+                    .then(() => pageTool(input, { callId, operationId }))
+                    .catch(() => undefined)
+                : undefined;
+            step = await executeToolOnRelay({
+                callId,
+                operationId,
+                grantId,
+                ...(answer ? { output: answer.output } : { unanswered: true }),
+            });
+            continue;
+        }
         const { gate } = step;
-        const resolution = await args.waitForGateDecision({ callId, toolName: name, input: toolArgs, gate });
+        if (gate.kind === 'page') {
+            await executeToolOnRelay({ callId, operationId, decision: 'deny' });
+            return;
+        }
+        const resolution = await args.waitForGateDecision({
+            callId,
+            toolName: name,
+            input: gate.preparedArguments ?? toolArgs,
+            gate,
+        });
         if (resolution === 'withdrawn')
             return;
-        const reply = continueGatedToolInvocation({ toolName: name, gate, sessionPermissions, resolution });
+        const reply = continueGatedToolInvocation({
+            toolName: name,
+            gate,
+            sessionPermissions,
+            resolution,
+        });
         if (reply.decision === 'deny') {
-            // The session settles the refusal; its tool event tells the user.
-            await executeToolOnRelay({ callId, decision: 'deny' });
+            // why: The session settles the refusal; its tool event tells the user.
+            await executeToolOnRelay({ callId, operationId, decision: 'deny' });
             return;
         }
         sessionPermissions = reply.sessionPermissions;
         args.setSessionPermissions(reply.sessionPermissions);
-        // Signed in: a typed key goes once, with the approval; after an OAuth callback there is none.
+        // why: Signed in: a typed key goes once, with the approval; after an OAuth callback there is none.
         step = await executeToolOnRelay({
             callId,
             decision: 'approve',
+            operationId,
             ...(reply.secret !== undefined ? { secret: reply.secret } : {}),
         });
     }

package/client/live/use-live-caption-log.d.ts

@@ -0,0 +1,17 @@
+import type { TurnEvent } from '@theoremjs/agents';
+import { type LiveCaptionState, type LiveCaptionTurn } from './live-captions.ts';
+/**
+ * The captions a live call shows: this call's lines, and earlier calls kept
+ * when it is started again. Speech arrives through `applyTranscript`, and the
+ * call's turn events (thoughts, tool calls, the reply's end) through
+ * `applyTurnEvent`. A typed send is `noteSentText`, which always starts its own line.
+ */
+export declare function useLiveCaptionLog(): {
+    captions: LiveCaptionState;
+    pastCalls: LiveCaptionTurn[][];
+    applyTranscript: (text: string, isUser: boolean, interim?: boolean) => void;
+    applyTurnEvent: (event: TurnEvent) => void;
+    noteSentText: (text: string) => void;
+    beginNextCall: () => void;
+    clear: () => void;
+};

package/client/live/use-live-caption-log.js

@@ -0,0 +1,51 @@
+import { useCallback, useRef, useState } from 'react';
+import { applyLiveTranscript, applyLiveTurnEvent, emptyLiveCaptionState, stashLiveCaptionCall, } from "./live-captions.js";
+/**
+ * The captions a live call shows: this call's lines, and earlier calls kept
+ * when it is started again. Speech arrives through `applyTranscript`, and the
+ * call's turn events (thoughts, tool calls, the reply's end) through
+ * `applyTurnEvent`. A typed send is `noteSentText`, which always starts its own line.
+ */
+export function useLiveCaptionLog() {
+    const [captions, setCaptions] = useState(emptyLiveCaptionState);
+    const [pastCalls, setPastCalls] = useState([]);
+    const captionsRef = useRef(captions);
+    captionsRef.current = captions;
+    const applyTranscript = useCallback((text, isUser, interim) => {
+        const next = applyLiveTranscript(captionsRef.current, text, isUser, interim);
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const applyTurnEvent = useCallback((event) => {
+        const next = applyLiveTurnEvent(captionsRef.current, event);
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const noteSentText = useCallback((text) => {
+        const next = applyLiveTranscript(captionsRef.current, text, true, false, { forceNew: true });
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const beginNextCall = useCallback(() => {
+        const previous = captionsRef.current.turns;
+        setPastCalls((calls) => stashLiveCaptionCall(calls, previous));
+        const next = emptyLiveCaptionState();
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const clear = useCallback(() => {
+        setPastCalls([]);
+        const next = emptyLiveCaptionState();
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    return {
+        captions,
+        pastCalls,
+        applyTranscript,
+        applyTurnEvent,
+        noteSentText,
+        beginNextCall,
+        clear,
+    };
+}

package/client/live-client.d.ts

@@ -19,10 +19,14 @@ export type { LiveConnection, LiveSocket } from './live-messages.ts';
 import type { LiveSocket } from './live-messages.ts';
 export interface LiveClientOptions {
     profile?: string;
-    /** JSON sent before anything else, as the socket opens. */
+    /** The host's own JSON for its relay, sent as the open message's `host`. */
     openMessage?: Record<string, unknown>;
+    /** The value chosen for each of the profile's `inputs.slots`, fixed for the call. */
+    slots?: Record<string, string>;
+    /** What the page tells the agent as the call opens; `setContext` replaces it during the call. */
+    context?: unknown;
     relayUrl?: string;
-    /** In-process connections reuse the same microphone, playback and tool client. */
+    /** In-process connections reuse the same microphone, playback, and tool client. */
     createSocket?: () => LiveSocket;
     /** When false, skip microphone capture; session still receives model audio. */
     voiceIngress?: boolean;
@@ -38,7 +42,7 @@ export interface LiveClientOptions {
     onTurnEvent?: (event: ClientTurnEvent) => void;
     /** A trace record the session wrote, when the relay delivers them. */
     onTrace?: (record: TraceRecord) => void;
-    /** A failure, typed by kind; word it with `clientFailure` and the interface's `lexicon`. */
+    /** A failure, typed by kind; word it with `clientFailure` and the interface's `labels`. */
     onError?: (error: Error) => void;
     /** Provider signalled the upstream session is draining (e.g. goAway). */
     onSessionClosing?: (timeLeftMs?: number) => void;
@@ -53,6 +57,7 @@ export interface LiveClientOptions {
      */
     onToolCall: (name: string, args: Record<string, unknown>, meta: {
         callId: string;
+        operationId: string;
     }) => Promise<void>;
     /** The relay opened the call: the id it gave the call, and the profile it runs. */
     onSessionReady?: (info: {
@@ -93,16 +98,47 @@ export declare class LiveSessionClient {
     /** Bumped on barge-in / cancel so stale audioChain work is skipped. */
     private audioEpoch;
     private pendingExecuteResults;
+    /** The page's context package, and the JSON of the one the relay last got. */
+    private context;
+    private sentContext;
+    /** The provider's latest handle for taking this call up again after a drop. */
+    private history;
+    private providerState;
+    /** The session said it ended, or failed: a close after that is the end, not a drop. */
+    private sessionOver;
+    /** When the call dropped, for the time away a resume reports. */
+    private droppedAt;
+    /** When the call that could not reconnect dropped: the next call takes it up again. */
+    private lostAt;
+    private reconnectTries;
+    private reconnectTimer;
     constructor(options: LiveClientOptions);
     private setStatus;
+    /** Still opening: the first connect, or a try at taking a dropped call up again. */
+    private get opening();
     private setConnectPhase;
     private clearConnectTimeout;
     private detachWebSocket;
+    private clearReconnect;
     private teardownConnection;
     /** End the session on `error`: a failed connect or a lost socket. */
     private failSession;
     private createSocket;
+    /**
+     * Start a call. After one that dropped and could not reconnect, it takes that call up again,
+     * so the agent neither greets twice nor forgets; after a hang-up it is a new call.
+     */
     connect(): Promise<void>;
+    /** A lost call that cannot be taken up again, its handle refused or expired, starts as a new one. */
+    private startOver;
+    /** The first message: the call's values, and the handle and time away when it takes a dropped call up again. */
+    private openMessage;
+    /** The socket closed while the call was open: take it up again if the provider gave a handle, else end. */
+    private handleDrop;
+    private scheduleReconnect;
+    /** A try at opening failed: a reconnect waits and tries again, a first connect fails the call. */
+    private failOpen;
+    private open;
     private ensureAudioContext;
     private activateMicrophone;
     private sendMicFrame;
@@ -115,6 +151,8 @@ export declare class LiveSessionClient {
     private processServerEnvelope;
     private enqueueServerMessage;
     private handleReadyEnvelope;
+    /** The call is open, first or again: the page's context catches up with what changed meanwhile. */
+    private becomeReady;
     private handleExecuteToolResultEnvelope;
     private tryHandleControlEnvelope;
     private collectInboundTurn;
@@ -134,7 +172,17 @@ export declare class LiveSessionClient {
     cancelPlayback(): void;
     toggleMute(): boolean;
     sendText(text: string): void;
-    sendContext(text: string): void;
+    /**
+     * The call's slot values. An open call is taken up again with them when the provider gave a
+     * handle; with none, the next call has them.
+     */
+    setSlots(slots: Record<string, string> | undefined): void;
+    /**
+     * Replace the page's context package. In a call it reaches the agent as
+     * background, with no reply; before one, it goes with the call's opening.
+     */
+    setContext(context: unknown): void;
+    private flushContext;
     sendVideo(data: string, mimeType?: string): void;
     disconnect(): void;
     private cleanupAudio;