@theoremjs/react 0.0.0-stage → 0.3.1

package/client/live/live-captions.js

@@ -0,0 +1,174 @@
+import { foldTurnEvents } from '@theoremjs/agents/interface';
+export function emptyLiveCaptionState() {
+    return { turns: [], interimUser: '', interimAgent: '', working: false };
+}
+/** Keep this call's lines so the next call starts below a divider. A call with no lines adds nothing. */
+export function stashLiveCaptionCall(past, turns) {
+    if (turns.length === 0)
+        return past;
+    return [...past, [...turns]];
+}
+function mergeInterimWithFinal(interim, final) {
+    if (!interim)
+        return final;
+    if (!final)
+        return interim;
+    if (final.startsWith(interim) || interim.startsWith(final)) {
+        return final.length >= interim.length ? final : interim;
+    }
+    return appendTurnText(interim, final);
+}
+function appendTurnText(existing, chunk) {
+    if (!existing)
+        return chunk;
+    if (!chunk)
+        return existing;
+    const needsSpace = !existing.endsWith(' ') &&
+        !chunk.startsWith(' ') &&
+        !/^\s/.test(chunk) &&
+        !/^[\s.,!?;:]/.test(chunk);
+    return needsSpace ? `${existing} ${chunk}` : `${existing}${chunk}`;
+}
+function nextTurnId(state) {
+    return `${String(Date.now())}-${String(state.turns.length)}`;
+}
+/**
+ * Fold streaming ASR deltas into turn-by-turn caption lines.
+ * Final chunks for the same role append to the open turn; role switches start a new turn.
+ */
+export function applyLiveTranscript(state, text, isUser, interim, options) {
+    if (!text)
+        return state;
+    if (interim) {
+        return isUser ? { ...state, interimUser: text } : { ...state, interimAgent: text };
+    }
+    const role = isUser ? 'user' : 'agent';
+    const turns = [...state.turns];
+    const last = turns.at(-1);
+    const interimText = isUser ? state.interimUser : state.interimAgent;
+    if (!options?.forceNew && last?.role === role) {
+        turns[turns.length - 1] = {
+            ...last,
+            text: appendTurnText(last.text, text),
+        };
+    }
+    else {
+        const committed = options?.forceNew ? text : mergeInterimWithFinal(interimText, text);
+        turns.push({
+            id: nextTurnId(state),
+            role,
+            text: committed,
+        });
+    }
+    return {
+        ...state,
+        turns,
+        interimUser: isUser ? '' : state.interimUser,
+        interimAgent: isUser ? state.interimAgent : '',
+    };
+}
+function isWorkEvent(event) {
+    return event.type === 'thought' || event.type === 'tool' || event.type === 'citation';
+}
+/** The work that made the call `callId`: where the call's later events go. -1 when no work made it. */
+function callWorkIndex(turns, callId) {
+    return turns.findIndex((turn) => turn.role === 'work' &&
+        turn.events.some((made) => made.type === 'tool' && made.tool.callId === callId));
+}
+/** Which line takes `event`: `turns.length` starts a new one, -1 leaves the event out. */
+function workIndex(turns, event) {
+    if (event.type === 'tool' && event.tool.phase !== undefined) {
+        // why: The call view's dialog asks a gate's question; the transcript's card would ask it twice.
+        return event.tool.phase === 'gate' ? -1 : callWorkIndex(turns, event.tool.callId);
+    }
+    const open = turns.at(-1)?.role === 'work';
+    // why: The thought guard releases a held blank tail after the speech; it is not work of its own.
+    if (event.type === 'thought' && !(open ? event.text : event.text.trim()))
+        return -1;
+    return open ? turns.length - 1 : turns.length;
+}
+/**
+ * Fold a live turn event into the captions. A thought, a tool call, or a citation joins the open
+ * work, else starts it; a tool call's later events go to the work that made the call. `done` ends
+ * the reply.
+ */
+export function applyLiveTurnEvent(state, event) {
+    if (event.type === 'done')
+        return state.working ? { ...state, working: false } : state;
+    if (!isWorkEvent(event))
+        return state;
+    const at = workIndex(state.turns, event);
+    if (at === -1)
+        return state;
+    const turns = [...state.turns];
+    const work = turns[at];
+    turns[at] =
+        work?.role === 'work'
+            ? { ...work, events: [...work.events, event] }
+            : { id: nextTurnId(state), role: 'work', events: [event] };
+    return { ...state, turns, working: true };
+}
+/** A line as the chat's blocks: speech is one, work is what the chat folds its events into. */
+function turnBlocks(turn) {
+    if (turn.role === 'work')
+        return foldTurnEvents(turn.events, { idPrefix: turn.id });
+    return [{ id: turn.id, kind: turn.role === 'user' ? 'user-text' : 'text', text: turn.text }];
+}
+/** How much of a broken-off line the next one must repeat to count as starting it again. */
+const RESTART_MATCH = 24;
+function sharedStart(a, b) {
+    let length = 0;
+    while (length < a.length && length < b.length && a[length] === b[length])
+        length += 1;
+    return length;
+}
+/** The later line opens with the same words as the earlier one, or with all of it when it is short. */
+function restarts(earlier, later) {
+    const start = earlier.trim();
+    return sharedStart(start, later.trim()) >= Math.min(start.length, RESTART_MATCH);
+}
+/**
+ * Drops speech the agent broke off and began again in the same reply, which a
+ * tool call mid-sentence makes it do: the line that starts over replaces it.
+ */
+function withoutRestarts(lines) {
+    return lines.filter((line, index) => {
+        if (line.role !== 'agent')
+            return true;
+        const after = lines.slice(index + 1);
+        const nextUser = after.findIndex((later) => later.role === 'user');
+        const reply = nextUser === -1 ? after : after.slice(0, nextUser);
+        return !reply.some((later) => later.role === 'agent' && restarts(line.text, later.text));
+    });
+}
+/** Captions as the chat's transcript blocks: past calls, this call, then the lines still being heard. */
+export function liveCaptionTranscript(pastCalls, { turns, interimUser, interimAgent }) {
+    const current = [...turns];
+    if (interimUser)
+        current.push({ id: 'interim-user', role: 'user', text: interimUser });
+    if (interimAgent)
+        current.push({ id: 'interim-agent', role: 'agent', text: interimAgent });
+    const calls = [
+        ...pastCalls.map((call, index) => call.map((turn) => ({ ...turn, id: `${String(index)}:${turn.id}` }))),
+        current,
+    ]
+        .map((call) => withoutRestarts(call).flatMap(turnBlocks))
+        .filter((call) => call.length > 0);
+    return {
+        blocks: calls.flat(),
+        callStarts: calls.slice(1).map((call) => call[0].id),
+    };
+}
+/** The agent is mid-reply: a line of its speech is still arriving, or it is working. */
+export function liveCaptionStreaming({ working, interimAgent }) {
+    return interimAgent !== '' || working;
+}
+/** Clear streaming partials when a live turn completes or is interrupted. */
+export function clearLiveCaptionInterim(state) {
+    if (!state.interimUser && !state.interimAgent)
+        return state;
+    return { ...state, interimUser: '', interimAgent: '' };
+}
+export function latestLiveCaptionTurnId(state) {
+    return state.turns.at(-1)?.id ?? null;
+}

package/client/live/live-mic-forward.d.ts

@@ -0,0 +1,30 @@
+import type { ToolCallEvent, TurnEventOf } from '@theoremjs/agents';
+/** Whether a mic frame should be sent upstream given mute / socket / barge-in gates. */
+export declare function shouldForwardMicFrame(args: {
+    isMuted: boolean;
+    socketOpen: boolean;
+    modelPlaying: boolean;
+    rms: number;
+    bargeInRmsWhileSpeaking: number;
+}): boolean;
+/** Classify live evidence transcription into a UI transcript callback payload. */
+export declare function liveTranscriptFromEvidence(event: TurnEventOf<'evidence'>): {
+    text: string;
+    isUser: boolean;
+    interim: boolean;
+} | null;
+export type LiveToolCallDraft = {
+    operationId?: string;
+    id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+};
+/**
+ * Fold a tool turn event into cancelled-id / runnable-call lists. The model's
+ * call is runnable; a failure in the same batch (a name or arguments the
+ * provider could not use) is not: the session answered the model with it.
+ */
+export declare function applyLiveToolTurnEvent(tool: ToolCallEvent, accum: {
+    cancelledToolIds: Set<string>;
+    toolCalls: LiveToolCallDraft[];
+}): void;

package/client/live/live-mic-forward.js

@@ -0,0 +1,46 @@
+/** Whether a mic frame should be sent upstream given mute / socket / barge-in gates. */
+export function shouldForwardMicFrame(args) {
+    if (args.isMuted || !args.socketOpen)
+        return false;
+    if (args.modelPlaying && args.rms < args.bargeInRmsWhileSpeaking)
+        return false;
+    return true;
+}
+/** Classify live evidence transcription into a UI transcript callback payload. */
+export function liveTranscriptFromEvidence(event) {
+    const { evidence, text } = event;
+    if (!text)
+        return null;
+    if (evidence.kind !== 'input_transcription' && evidence.kind !== 'output_transcription')
+        return null;
+    return {
+        text,
+        isUser: evidence.kind === 'input_transcription',
+        interim: evidence.interim === true,
+    };
+}
+/**
+ * Fold a tool turn event into cancelled-id / runnable-call lists. The model's
+ * call is runnable; a failure in the same batch (a name or arguments the
+ * provider could not use) is not: the session answered the model with it.
+ */
+export function applyLiveToolTurnEvent(tool, accum) {
+    if (tool.phase === undefined) {
+        accum.toolCalls.push({
+            id: tool.callId,
+            name: tool.name,
+            arguments: tool.arguments,
+            ...(tool.operationId ? { operationId: tool.operationId } : {}),
+        });
+        return;
+    }
+    if (tool.phase === 'cancel') {
+        accum.cancelledToolIds.add(tool.callId);
+        return;
+    }
+    if (tool.phase !== 'error')
+        return;
+    const failed = accum.toolCalls.findIndex((draft) => draft.id === tool.callId);
+    if (failed !== -1)
+        accum.toolCalls.splice(failed, 1);
+}

package/client/live/live-page-tool.d.ts

@@ -0,0 +1,34 @@
+/** The page's result for a call: the tool's output schema checks it, then the model reads it. */
+export type PageToolAnswer = {
+    output: unknown;
+};
+/** One page tool: what the page does for a call the model made, and what it tells the model. */
+export type PageTool = (args: Record<string, unknown>, call: {
+    callId: string;
+    operationId: string;
+}) => PageToolAnswer | Promise<PageToolAnswer>;
+/**
+ * The page's tools, by name: each answers a tool the profile declares with
+ * `answeredBy: 'page'`. Any other tool runs on the relay, gates and all.
+ */
+export type PageTools = Record<string, PageTool>;
+/** What a host gives a live call. */
+export type LiveCallOptions = {
+    /** The value chosen for each of the profile's `inputs.slots`, read as the call starts. */
+    slots?: Record<string, string>;
+    /**
+     * What the page tells the agent: any JSON. It goes with the call's opening, and
+     * again whenever it changes, as background the agent reads without replying.
+     */
+    context?: unknown;
+    pageTools?: PageTools;
+};
+/**
+ * Where the page's tools and the profile's disagree, for the builder: a page
+ * tool with no handler here leaves the model unanswered, and a handler for a
+ * tool the profile does not give the page never runs.
+ */
+export declare function pageToolMismatch(declared: readonly string[], handled: readonly string[]): {
+    unanswered: string[];
+    unused: string[];
+};

package/client/live/live-page-tool.js

@@ -0,0 +1,11 @@
+/**
+ * Where the page's tools and the profile's disagree, for the builder: a page
+ * tool with no handler here leaves the model unanswered, and a handler for a
+ * tool the profile does not give the page never runs.
+ */
+export function pageToolMismatch(declared, handled) {
+    return {
+        unanswered: declared.filter((name) => !handled.includes(name)),
+        unused: handled.filter((name) => !declared.includes(name)),
+    };
+}

package/client/live/live-state.d.ts

@@ -0,0 +1,13 @@
+export type LiveSessionStatus = 'disconnected' | 'connecting'
+/** The call dropped and the client is taking it up again. */
+ | 'reconnecting' | 'ready' | 'listening' | 'speaking' | 'working' | 'error';
+export type LiveConnectPhase = 'socket' | 'microphone';
+/** What a live call is doing, for its status line; the UI words it. */
+export type LiveState = 'calling_tool' | 'connecting' | 'reconnecting' | 'requesting_mic' | 'speaking' | 'connected' | 'muted' | 'listening' | 'error' | 'ended';
+export declare function liveState(args: {
+    status: LiveSessionStatus;
+    connectPhase: LiveConnectPhase | null;
+    toolName: string | null;
+    isMuted: boolean;
+    voiceEnabled?: boolean;
+}): LiveState;

package/client/live/live-state.js

@@ -0,0 +1,22 @@
+export function liveState(args) {
+    if (args.status === 'reconnecting')
+        return 'reconnecting';
+    if (args.toolName)
+        return 'calling_tool';
+    if (args.connectPhase === 'socket')
+        return 'connecting';
+    if (args.connectPhase === 'microphone')
+        return 'requesting_mic';
+    if (args.status === 'speaking')
+        return 'speaking';
+    if (args.status === 'listening' || args.status === 'ready') {
+        if (args.voiceEnabled === false)
+            return 'connected';
+        return args.isMuted ? 'muted' : 'listening';
+    }
+    if (args.status === 'connecting')
+        return 'connecting';
+    if (args.status === 'error')
+        return 'error';
+    return 'ended';
+}

package/client/live/live-tool.d.ts

@@ -0,0 +1,11 @@
+import type { ToolGate } from '@theoremjs/agents/kernel';
+import type { ToolGateResolution } from '../tool-resume.ts';
+/** A gate the user is asked to answer, for the model's call `callId`. */
+export type LiveToolGatePrompt = {
+    callId: string;
+    toolName: string;
+    input: Record<string, unknown>;
+    gate: ToolGate;
+};
+/** The user's answer to a live gate, or `withdrawn`: the model cancelled the call, so nothing is asked. */
+export type LiveGateAnswer = ToolGateResolution | 'withdrawn';

package/client/live/live-tool.js

@@ -0,0 +1 @@
+export {};

package/client/live/live-video.d.ts

@@ -0,0 +1,10 @@
+export type LiveFacingMode = 'user' | 'environment';
+export type LiveVideoCapture = {
+    stop: () => void;
+    readonly video: HTMLVideoElement;
+    facingMode: () => LiveFacingMode;
+    flip: () => Promise<LiveFacingMode>;
+};
+export declare function startLiveVideoCapture(onFrame: (base64: string) => void, options?: {
+    facingMode?: LiveFacingMode;
+}): Promise<LiveVideoCapture>;

package/client/live/live-video.js

@@ -0,0 +1,71 @@
+import { TheoremError } from '@theoremjs/agents';
+const JPEG_QUALITY = 0.62;
+const FRAME_INTERVAL_MS = 400;
+function openCamera(facing) {
+    return navigator.mediaDevices.getUserMedia({
+        video: { facingMode: { ideal: facing }, width: { ideal: 640 }, height: { ideal: 480 } },
+        audio: false,
+    });
+}
+export function startLiveVideoCapture(onFrame, options) {
+    const initialFacing = options?.facingMode ?? 'user';
+    return openCamera(initialFacing).then((stream) => {
+        const video = document.createElement('video');
+        video.srcObject = stream;
+        video.muted = true;
+        video.playsInline = true;
+        video.setAttribute('playsinline', '');
+        const canvas = document.createElement('canvas');
+        const context = canvas.getContext('2d');
+        if (!context) {
+            for (const track of stream.getTracks())
+                track.stop();
+            throw new TheoremError('unsupported', 'no 2d canvas for live video'); // labels-exempt: internal diagnostic
+        }
+        let timer = null;
+        let stopped = false;
+        let state = { stream, facing: initialFacing };
+        const capture = () => {
+            if (stopped || video.videoWidth === 0)
+                return;
+            canvas.width = video.videoWidth;
+            canvas.height = video.videoHeight;
+            context.drawImage(video, 0, 0);
+            const dataUrl = canvas.toDataURL('image/jpeg', JPEG_QUALITY);
+            const base64 = dataUrl.split(',')[1];
+            if (base64)
+                onFrame(base64);
+        };
+        const startTimer = () => {
+            if (timer || stopped)
+                return;
+            timer = setInterval(capture, FRAME_INTERVAL_MS);
+        };
+        void video.play().then(startTimer);
+        return {
+            video,
+            facingMode: () => state.facing,
+            flip: async () => {
+                if (stopped)
+                    return state.facing;
+                const nextFacing = state.facing === 'user' ? 'environment' : 'user';
+                const nextStream = await openCamera(nextFacing);
+                for (const track of state.stream.getTracks())
+                    track.stop();
+                state = { stream: nextStream, facing: nextFacing };
+                video.srcObject = nextStream;
+                await video.play();
+                return nextFacing;
+            },
+            stop: () => {
+                stopped = true;
+                if (timer)
+                    clearInterval(timer);
+                timer = null;
+                video.srcObject = null;
+                for (const track of state.stream.getTracks())
+                    track.stop();
+            },
+        };
+    });
+}

package/client/live/run-live-tool-call.d.ts

@@ -0,0 +1,22 @@
+import type { ExecuteToolOnRelay } from '../live-messages.ts';
+import type { PageTools } from './live-page-tool.ts';
+import type { LiveGateAnswer, LiveToolGatePrompt } from './live-tool.ts';
+/**
+ * Live provider tool-call handler: the page answers the call if it is the
+ * page's; else run the model's call on the relay; at a gate, ask the user and
+ * send their decision. The session answers the model
+ * however the call ends; its tool events tell the user. A call the model
+ * cancels while its gate is open is gone: nothing is sent.
+ */
+export declare function runLiveToolCall(args: {
+    executeToolOnRelay: ExecuteToolOnRelay;
+    /** The host's page tools, if it has any. */
+    pageTools?: PageTools;
+    name: string;
+    toolArgs: Record<string, unknown>;
+    callId: string;
+    operationId: string;
+    sessionPermissions: string[];
+    setSessionPermissions: (next: string[]) => void;
+    waitForGateDecision: (prompt: LiveToolGatePrompt) => Promise<LiveGateAnswer>;
+}): Promise<void>;

package/client/live/run-live-tool-call.js

@@ -0,0 +1,70 @@
+import { continueGatedToolInvocation } from "../tool-resume.js";
+/**
+ * Live provider tool-call handler: the page answers the call if it is the
+ * page's; else run the model's call on the relay; at a gate, ask the user and
+ * send their decision. The session answers the model
+ * however the call ends; its tool events tell the user. A call the model
+ * cancels while its gate is open is gone: nothing is sent.
+ */
+export async function runLiveToolCall(args) {
+    const { executeToolOnRelay, name, toolArgs, callId, operationId } = args;
+    let sessionPermissions = args.sessionPermissions;
+    // why: A name from the model is looked up among the host's own keys only.
+    const pageTool = args.pageTools && Object.hasOwn(args.pageTools, name) ? args.pageTools[name] : undefined;
+    let step = await executeToolOnRelay({ callId, operationId });
+    const dispatched = new Set();
+    while (step.status !== 'settled') {
+        if (step.status === 'dispatched') {
+            if (dispatched.has(step.grantId))
+                return;
+            dispatched.add(step.grantId);
+            const grantId = step.grantId;
+            const input = step.arguments;
+            const answer = pageTool
+                ? await Promise.resolve()
+                    .then(() => pageTool(input, { callId, operationId }))
+                    .catch(() => undefined)
+                : undefined;
+            step = await executeToolOnRelay({
+                callId,
+                operationId,
+                grantId,
+                ...(answer ? { output: answer.output } : { unanswered: true }),
+            });
+            continue;
+        }
+        const { gate } = step;
+        if (gate.kind === 'page') {
+            await executeToolOnRelay({ callId, operationId, decision: 'deny' });
+            return;
+        }
+        const resolution = await args.waitForGateDecision({
+            callId,
+            toolName: name,
+            input: gate.preparedArguments ?? toolArgs,
+            gate,
+        });
+        if (resolution === 'withdrawn')
+            return;
+        const reply = continueGatedToolInvocation({
+            toolName: name,
+            gate,
+            sessionPermissions,
+            resolution,
+        });
+        if (reply.decision === 'deny') {
+            // why: The session settles the refusal; its tool event tells the user.
+            await executeToolOnRelay({ callId, operationId, decision: 'deny' });
+            return;
+        }
+        sessionPermissions = reply.sessionPermissions;
+        args.setSessionPermissions(reply.sessionPermissions);
+        // why: Signed in: a typed key goes once, with the approval; after an OAuth callback there is none.
+        step = await executeToolOnRelay({
+            callId,
+            decision: 'approve',
+            operationId,
+            ...(reply.secret !== undefined ? { secret: reply.secret } : {}),
+        });
+    }
+}

package/client/live/use-live-caption-log.d.ts

@@ -0,0 +1,17 @@
+import type { TurnEvent } from '@theoremjs/agents';
+import { type LiveCaptionState, type LiveCaptionTurn } from './live-captions.ts';
+/**
+ * The captions a live call shows: this call's lines, and earlier calls kept
+ * when it is started again. Speech arrives through `applyTranscript`, and the
+ * call's turn events (thoughts, tool calls, the reply's end) through
+ * `applyTurnEvent`. A typed send is `noteSentText`, which always starts its own line.
+ */
+export declare function useLiveCaptionLog(): {
+    captions: LiveCaptionState;
+    pastCalls: LiveCaptionTurn[][];
+    applyTranscript: (text: string, isUser: boolean, interim?: boolean) => void;
+    applyTurnEvent: (event: TurnEvent) => void;
+    noteSentText: (text: string) => void;
+    beginNextCall: () => void;
+    clear: () => void;
+};

package/client/live/use-live-caption-log.js

@@ -0,0 +1,51 @@
+import { useCallback, useRef, useState } from 'react';
+import { applyLiveTranscript, applyLiveTurnEvent, emptyLiveCaptionState, stashLiveCaptionCall, } from "./live-captions.js";
+/**
+ * The captions a live call shows: this call's lines, and earlier calls kept
+ * when it is started again. Speech arrives through `applyTranscript`, and the
+ * call's turn events (thoughts, tool calls, the reply's end) through
+ * `applyTurnEvent`. A typed send is `noteSentText`, which always starts its own line.
+ */
+export function useLiveCaptionLog() {
+    const [captions, setCaptions] = useState(emptyLiveCaptionState);
+    const [pastCalls, setPastCalls] = useState([]);
+    const captionsRef = useRef(captions);
+    captionsRef.current = captions;
+    const applyTranscript = useCallback((text, isUser, interim) => {
+        const next = applyLiveTranscript(captionsRef.current, text, isUser, interim);
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const applyTurnEvent = useCallback((event) => {
+        const next = applyLiveTurnEvent(captionsRef.current, event);
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const noteSentText = useCallback((text) => {
+        const next = applyLiveTranscript(captionsRef.current, text, true, false, { forceNew: true });
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const beginNextCall = useCallback(() => {
+        const previous = captionsRef.current.turns;
+        setPastCalls((calls) => stashLiveCaptionCall(calls, previous));
+        const next = emptyLiveCaptionState();
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    const clear = useCallback(() => {
+        setPastCalls([]);
+        const next = emptyLiveCaptionState();
+        captionsRef.current = next;
+        setCaptions(next);
+    }, []);
+    return {
+        captions,
+        pastCalls,
+        applyTranscript,
+        applyTurnEvent,
+        noteSentText,
+        beginNextCall,
+        clear,
+    };
+}