@theoremjs/react 0.0.0-stage → 0.3.0

package/client/live-messages.d.ts

@@ -0,0 +1,110 @@
+/**
+ * The live wire between the browser and a host's relay: the messages the live
+ * client sends (a relay reads them with `parseLiveClientMessage`) and the envelopes it reads
+ * back, each checked against its schema.
+ *
+ * @module
+ */
+import { type GateDecision, type ToolGate, type TraceRecord, type TurnEvent, z } from '@theoremjs/agents';
+import { type HostErrorBody } from './transport.ts';
+import { type MalformedEvent, type UnsupportedEvent } from './wire-line.ts';
+/** What the live client sends its relay, besides the host's own `openMessage`. */
+export type LiveClientMessage = {
+    type: 'audio';
+    data: string;
+} | {
+    type: 'video';
+    data: string;
+    mimeType: string;
+} | {
+    type: 'text';
+    text: string;
+}
+/** Background the model reads without replying: the page the visitor is on, say. */
+ | {
+    type: 'context';
+    text: string;
+}
+/** Run a call the model made, by its id; the live session holds its name, input and gate. */
+ | {
+    type: 'executeTool';
+    callId: string;
+    decision?: GateDecision;
+    input?: unknown;
+    secret?: string;
+    /** What the browser did for a tool that runs there; a relay hands it to the tool as `host.clientOutput`. */
+    output?: unknown;
+};
+/** The live client's messages, as `parseLiveClientMessage` checks them for a relay. */
+export declare const liveClientMessageSchema: z.ZodType<LiveClientMessage>;
+/** The envelopes a relay sends the live client. */
+export type LiveServerEnvelope = {
+    type: 'ready';
+    profile?: string;
+    sessionId?: string;
+}
+/** The session's events; one of a kind this client does not know arrives as `unsupported`, one that fails its check as `malformed`. */
+ | {
+    type: 'events';
+    events: (TurnEvent | UnsupportedEvent | MalformedEvent)[];
+}
+/** A trace record the session wrote, from a relay that delivers its traces (the playground). */
+ | {
+    type: 'trace';
+    record: TraceRecord;
+}
+/** The relay's error body, read as a host error. */
+ | ({
+    type: 'error';
+} & HostErrorBody)
+/**
+ * The relay's answer to the browser's `executeTool`: the session settled the
+ * call (its tool events carry what happened), holds it on a gate, or refused
+ * the message (the relay's error body, read as a host error).
+ */
+ | {
+    type: 'executeToolResult';
+    callId: string;
+    status: 'settled';
+} | {
+    type: 'executeToolResult';
+    callId: string;
+    status: 'gated';
+    gate: ToolGate;
+} | {
+    type: 'executeToolResult';
+    callId: string;
+    status: 'refused';
+    body: HostErrorBody;
+};
+/**
+ * One envelope's text from the relay, checked: an envelope or event of a kind
+ * this client does not know is `unsupported`, and one that fails its check is
+ * `malformed`. A malformed event leaves the envelope's other events standing.
+ */
+export declare function parseLiveServerEnvelope(text: string): LiveServerEnvelope | UnsupportedEvent | MalformedEvent;
+/** What the session did with an `executeTool`: settled the call, or holds it on a gate. */
+export type LiveToolStep = {
+    status: 'settled';
+} | {
+    status: 'gated';
+    gate: ToolGate;
+};
+/**
+ * Ask the relay to run `LiveSession.executeTool` for a call the model made.
+ * `decision` answers its gate; `input` is the user's edit to an approval;
+ * `secret` is the key the user typed at a sign-in gate, sent once: the
+ * session makes it the credential for the gate's slot. `output` is the
+ * browser's result for a tool that runs in the page (see `browserToolHandler`).
+ */
+export type ExecuteToolOnRelay = (args: Omit<Extract<LiveClientMessage, {
+    type: 'executeTool';
+}>, 'type'>) => Promise<LiveToolStep>;
+export type LiveSocket = Pick<WebSocket, 'readyState' | 'binaryType' | 'send' | 'close' | 'onopen' | 'onmessage' | 'onclose' | 'onerror'>;
+export type LiveConnection = ({
+    profile: string;
+} | {
+    openMessage: Record<string, unknown>;
+}) & {
+    createSocket?: () => LiveSocket;
+};

package/client/live-messages.js

@@ -0,0 +1,67 @@
+/**
+ * The live wire between the browser and a host's relay: the messages the live
+ * client sends (a relay reads them with `parseLiveClientMessage`) and the envelopes it reads
+ * back, each checked against its schema.
+ *
+ * @module
+ */
+import { GATE_DECISIONS, toolGateSchema, traceRecordSchema, TURN_EVENT_SCHEMAS, z, } from '@theoremjs/agents';
+import { hostErrorBodySchema } from "./transport.js";
+import { readWireLine, readWireValue, } from "./wire-line.js";
+/** The longest context message a relay accepts. */
+const LIVE_CONTEXT_MAX_CHARS = 2000;
+const liveClientMessage = z.discriminatedUnion('type', [
+    z.object({ type: z.literal('audio'), data: z.string() }),
+    z.object({ type: z.literal('video'), data: z.string(), mimeType: z.string(),
+    }),
+    z.object({ type: z.literal('text'), text: z.string() }),
+    z.object({ type: z.literal('context'), text: z.string().max(LIVE_CONTEXT_MAX_CHARS) }),
+    z.object({
+        type: z.literal('executeTool'),
+        callId: z.string().min(1),
+        decision: z.enum(GATE_DECISIONS).optional(),
+        input: z.unknown().optional(),
+        secret: z.string().optional(),
+        output: z.unknown().optional(),
+    }),
+]);
+true;
+/** The live client's messages, as `parseLiveClientMessage` checks them for a relay. */
+export const liveClientMessageSchema = liveClientMessage;
+const liveServerEnvelopeKinds = {
+    ready: z.object({ type: z.literal('ready'), profile: z.string().optional(), sessionId: z.string().optional(),
+    }),
+    events: z.object({ type: z.literal('events'), events: z.array(z.unknown()) }),
+    trace: z.object({ type: z.literal('trace'), record: traceRecordSchema }),
+    error: z.object({ type: z.literal('error') }).and(hostErrorBodySchema),
+    executeToolResult: z.discriminatedUnion('status', [
+        z.object({ type: z.literal('executeToolResult'), callId: z.string(), status: z.literal('settled'),
+        }),
+        z.object({
+            type: z.literal('executeToolResult'),
+            callId: z.string(),
+            status: z.literal('gated'),
+            gate: toolGateSchema,
+        }),
+        z.object({
+            type: z.literal('executeToolResult'),
+            callId: z.string(),
+            status: z.literal('refused'),
+            body: hostErrorBodySchema,
+        }),
+    ]),
+};
+true;
+const liveServerEnvelopeLines = liveServerEnvelopeKinds;
+/**
+ * One envelope's text from the relay, checked: an envelope or event of a kind
+ * this client does not know is `unsupported`, and one that fails its check is
+ * `malformed`. A malformed event leaves the envelope's other events standing.
+ */
+export function parseLiveServerEnvelope(text) {
+    const envelope = readWireLine(liveServerEnvelopeLines, text);
+    if (envelope.type !== 'events')
+        return envelope;
+    return { type: 'events', events: envelope.events.map((event) => readWireValue(TURN_EVENT_SCHEMAS, event)),
+    };
+}

package/client/mic-capture.d.ts

@@ -0,0 +1,4 @@
+/** The processor name `worklets/mic-capture.js` registers, for `AudioWorkletNode`. */
+export declare const MIC_CAPTURE_PROCESSOR = "mic-capture-processor";
+/** The worklet module for `audioWorklet.addModule`; bundlers ship it as an asset. */
+export declare function micCaptureWorkletUrl(): URL;

package/client/mic-capture.js

@@ -0,0 +1,6 @@
+/** The processor name `worklets/mic-capture.js` registers, for `AudioWorkletNode`. */
+export const MIC_CAPTURE_PROCESSOR = 'mic-capture-processor';
+/** The worklet module for `audioWorklet.addModule`; bundlers ship it as an asset. */
+export function micCaptureWorkletUrl() {
+    return new URL('./worklets/mic-capture.js', import.meta.url);
+}

package/client/oauth-popup.d.ts

@@ -0,0 +1,26 @@
+/**
+ * The OAuth sign-in popup's word back to the chat that opened it.
+ *
+ * The host's callback route saves the token on the server, then its page calls
+ * `notifyOAuthComplete(slot)`. The chat takes the message only from the popup
+ * it opened, on its own origin, for the slot its gate waits for, and resumes
+ * the gate: no token ever reaches the browser.
+ *
+ * @module
+ */
+/**
+ * Tell the chat that opened this popup that `slot`'s token is saved. Call it on
+ * the callback page once the credential is in the host's credential store. On a
+ * page no chat opened, it does nothing.
+ */
+export declare function notifyOAuthComplete(slot: string): void;
+/** The message is `popup`'s, from `origin` (the chat page's own), saying `slot` is signed in. */
+export declare function isOAuthComplete(event: {
+    data: unknown;
+    source: unknown;
+    origin: string;
+}, expected: {
+    popup: unknown;
+    slot: string;
+    origin: string;
+}): boolean;

package/client/oauth-popup.js

@@ -0,0 +1,30 @@
+/**
+ * The OAuth sign-in popup's word back to the chat that opened it.
+ *
+ * The host's callback route saves the token on the server, then its page calls
+ * `notifyOAuthComplete(slot)`. The chat takes the message only from the popup
+ * it opened, on its own origin, for the slot its gate waits for, and resumes
+ * the gate: no token ever reaches the browser.
+ *
+ * @module
+ */
+import { isRecord } from '@theoremjs/agents/kernel';
+const OAUTH_COMPLETE = 'theorem.oauth_complete';
+/**
+ * Tell the chat that opened this popup that `slot`'s token is saved. Call it on
+ * the callback page once the credential is in the host's credential store. On a
+ * page no chat opened, it does nothing.
+ */
+export function notifyOAuthComplete(slot) {
+    const opener = Reflect.get(globalThis, 'opener');
+    if (!isRecord(opener) || typeof opener.postMessage !== 'function')
+        return;
+    opener.postMessage({ type: OAUTH_COMPLETE, slot }, globalThis.location.origin);
+}
+/** The message is `popup`'s, from `origin` (the chat page's own), saying `slot` is signed in. */
+export function isOAuthComplete(event, expected) {
+    if (event.source !== expected.popup || event.origin !== expected.origin)
+        return false;
+    const data = event.data;
+    return isRecord(data) && data.type === OAUTH_COMPLETE && data.slot === expected.slot;
+}

package/client/pcm-downsample.d.ts

@@ -0,0 +1,4 @@
+/** Resample Float32 audio buffer from inputRate to outputRate and convert to Int16 PCM. */
+export declare function downsampleAndConvertToInt16(inputData: Float32Array, inputRate: number, outputRate?: number): Int16Array;
+/** Decode little-endian PCM16 bytes to Float32 samples in [-1, 1]. */
+export declare function pcm16BytesToFloat32(bytes: Uint8Array<ArrayBufferLike>): Float32Array;

package/client/pcm-downsample.js

@@ -0,0 +1,45 @@
+/** Resample Float32 audio buffer from inputRate to outputRate and convert to Int16 PCM. */
+export function downsampleAndConvertToInt16(inputData, inputRate, outputRate = 16000) {
+    if (inputRate === outputRate) {
+        return float32ToInt16(inputData);
+    }
+    const ratio = inputRate / outputRate;
+    const newLength = Math.round(inputData.length / ratio);
+    const result = new Int16Array(newLength);
+    let offsetResult = 0;
+    let offsetInput = 0;
+    while (offsetResult < result.length) {
+        const nextOffsetInput = Math.round((offsetResult + 1) * ratio);
+        let accum = 0;
+        let count = 0;
+        for (let i = offsetInput; i < nextOffsetInput && i < inputData.length; i++) {
+            accum += inputData[i] ?? 0;
+            count++;
+        }
+        result[offsetResult] = floatSampleToInt16(count > 0 ? accum / count : 0);
+        offsetResult++;
+        offsetInput = nextOffsetInput;
+    }
+    return result;
+}
+function float32ToInt16(inputData) {
+    const result = new Int16Array(inputData.length);
+    for (let i = 0; i < inputData.length; i++) {
+        result[i] = floatSampleToInt16(inputData[i] ?? 0);
+    }
+    return result;
+}
+function floatSampleToInt16(val) {
+    const s = Math.max(-1, Math.min(1, val));
+    return s < 0 ? s * 0x8000 : s * 0x7fff;
+}
+/** Decode little-endian PCM16 bytes to Float32 samples in [-1, 1]. */
+export function pcm16BytesToFloat32(bytes) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const float32 = new Float32Array(bytes.byteLength / 2);
+    for (let i = 0; i < float32.length; i++) {
+        const sample = view.getInt16(i * 2, true);
+        float32[i] = sample / (sample < 0 ? 0x8000 : 0x7fff);
+    }
+    return float32;
+}

package/client/run-commit.d.ts

@@ -0,0 +1,61 @@
+import type { TheoremError, TurnBlob, TurnEvent } from '@theoremjs/agents';
+import { type ComposerProfileInterface, type InterfaceTurnSession, type TranscriptBlock, type UserTurnHistoryMedia } from '@theoremjs/agents/interface';
+import { type TurnFailure } from './failure.ts';
+import type { TheoremTransport, TurnEventSink } from './transport.ts';
+/** The turn is over: every event it streamed, gate resumes included, enters history once. */
+export declare function commitCompletedTurn(session: InterfaceTurnSession, events: TurnEvent[], media?: UserTurnHistoryMedia): InterfaceTurnSession;
+export declare function toTurnMedia(encodedAttachments?: TurnBlob[], encodedVoice?: TurnBlob[]): UserTurnHistoryMedia;
+/** Where a run shows itself as it streams: the reply so far, and each line the stream left out. */
+export type StreamView = {
+    blocks(blocks: TranscriptBlock[]): void;
+    /** A `malformed` line: left out, and the reply goes on. */
+    skipped(error: TheoremError): void;
+};
+export declare function continueAfterTool(args: {
+    iface: ComposerProfileInterface;
+    transport: TheoremTransport;
+    session: InterfaceTurnSession;
+    view: StreamView;
+    seedEvents: TurnEvent[];
+}): Promise<{
+    ok: true;
+    session: InterfaceTurnSession;
+    assistantBlocks: TranscriptBlock[];
+} | TurnFailure>;
+/** A turn that failed partway, committed as far as it got. */
+export declare function failTurnStream(args: {
+    session: InterfaceTurnSession;
+    events: TurnEvent[];
+    media: UserTurnHistoryMedia;
+    failure: TurnFailure;
+}): TurnFailure;
+/**
+ * An answer to `callId`'s gate that failed. Once the call settled, the reply
+ * commits as far as it got, or waits on its next gate; before, the reply waits
+ * on the gate as it did (the host puts the call back), and the answer can go again.
+ */
+export declare function failGateAnswer(args: {
+    /** The paused session the answer went out from. */
+    session: InterfaceTurnSession;
+    callId: string;
+    /** The paused reply's events, then the answer's as far as they came. */
+    events: TurnEvent[];
+    failure: TurnFailure;
+}): TurnFailure;
+export declare function finalizeTurnStream(args: {
+    session: InterfaceTurnSession;
+    events: TurnEvent[];
+    media: UserTurnHistoryMedia;
+}): InterfaceTurnSession;
+export declare function streamFoldedTurn(args: {
+    iface: ComposerProfileInterface;
+    view: StreamView;
+    /** Seeded by the caller, and filled as the stream delivers. */
+    events: TurnEvent[];
+    stream: (onEvent: TurnEventSink) => Promise<void>;
+}): Promise<TurnEvent[]>;
+/** `blocks` with its last `turn-done` carrying the reply's work. */
+export declare function stampWorked(blocks: TranscriptBlock[], worked: {
+    workedMs: number;
+    endedAt: number;
+}): TranscriptBlock[];

package/client/run-commit.js

@@ -0,0 +1,124 @@
+import { appendAssistantEventsToHistory, appendUserDraftToHistory, applyTurnEventsToSession, gatedToolFromEvents, settlesToolCall, } from '@theoremjs/agents/interface';
+import { turnFailure } from "./failure.js";
+import { buildTurnRequest, foldAssistantTurn, turnInputFromSession } from "./turn-client.js";
+/** The turn's history before its reply: a turn not yet started adds the user's message. */
+function turnBaseHistory(session, media) {
+    return session.pendingUserDraft
+        ? appendUserDraftToHistory(session.history, session.pendingUserDraft, media)
+        : session.history;
+}
+/** The turn is over: every event it streamed, gate resumes included, enters history once. */
+export function commitCompletedTurn(session, events, media = {}) {
+    return {
+        ...applyTurnEventsToSession(session, events),
+        history: appendAssistantEventsToHistory(turnBaseHistory(session, media), events),
+        gatedTool: null,
+        assistantEvents: [],
+        pendingUserDraft: null,
+        toolSnapshot: undefined,
+        promotedToolIds: [],
+    };
+}
+/**
+ * The turn waits at a gate. History stops at the user's message and the events
+ * wait in `assistantEvents`, so the gated step enters history whole once it settles.
+ */
+function pauseTurn(session, events, media = {}) {
+    return {
+        ...applyTurnEventsToSession(session, events),
+        history: turnBaseHistory(session, media),
+        gatedTool: gatedToolFromEvents(events),
+        assistantEvents: [...events],
+        pendingUserDraft: null,
+    };
+}
+export function toTurnMedia(encodedAttachments, encodedVoice) {
+    return {
+        ...(encodedAttachments?.length ? { attachments: encodedAttachments } : {}),
+        ...(encodedVoice?.length ? { voice: encodedVoice } : {}),
+    };
+}
+/**
+ * A failed turn with the conversation it leaves: the user's message and what the
+ * reply got through (its text and the tool calls that settled), so the next turn
+ * picks up from there. A stopped reply leaves the transcript, so only the message stays.
+ */
+function withFailedTurnSession(failure, commit, events, seedEvents = []) {
+    return { ...failure, session: commit(failure.aborted ? seedEvents : events) };
+}
+/**
+ * Streams into `events`, which the caller owns, so a stream that fails still
+ * leaves what it delivered. An `unsupported` line is the host's to read on its
+ * own transport, and a `malformed` one goes to `view.skipped`: neither enters
+ * the turn, its history or its transcript.
+ */
+async function streamFoldedEvents(stream, view, iface, events) {
+    await stream((event) => {
+        if (event.type === 'unsupported')
+            return;
+        if (event.type === 'malformed') {
+            view.skipped(event.error);
+            return;
+        }
+        events.push(event);
+        view.blocks(foldAssistantTurn(iface, events));
+    });
+    return events;
+}
+export async function continueAfterTool(args) {
+    const events = [...args.seedEvents];
+    // The model reads the turn so far: the settled gate and everything before it.
+    const sent = {
+        ...args.session,
+        history: appendAssistantEventsToHistory(args.session.history, args.seedEvents),
+    };
+    try {
+        await streamFoldedEvents((onEvent) => args.transport.turn(buildTurnRequest(args.iface, sent, turnInputFromSession(sent)), onEvent), args.view, args.iface, events);
+        if (gatedToolFromEvents(events)) {
+            return {
+                ok: true,
+                session: pauseTurn(args.session, events),
+                assistantBlocks: foldAssistantTurn(args.iface, events),
+            };
+        }
+        return {
+            ok: true,
+            session: commitCompletedTurn(args.session, events),
+            assistantBlocks: foldAssistantTurn(args.iface, events),
+        };
+    }
+    catch (err) {
+        return withFailedTurnSession(turnFailure(err, args.iface.lexicon), (kept) => commitCompletedTurn(args.session, kept), events, args.seedEvents);
+    }
+}
+/** A turn that failed partway, committed as far as it got. */
+export function failTurnStream(args) {
+    return withFailedTurnSession(args.failure, (kept) => commitCompletedTurn(args.session, kept, args.media), args.events);
+}
+/**
+ * An answer to `callId`'s gate that failed. Once the call settled, the reply
+ * commits as far as it got, or waits on its next gate; before, the reply waits
+ * on the gate as it did (the host puts the call back), and the answer can go again.
+ */
+export function failGateAnswer(args) {
+    const answered = args.events.slice(args.session.assistantEvents.length);
+    if (!answered.some((event) => settlesToolCall(event, args.callId)))
+        return args.failure;
+    return { ...args.failure, session: finalizeTurnStream({ session: args.session, events: args.events, media: {} }) };
+}
+export function finalizeTurnStream(args) {
+    if (gatedToolFromEvents(args.events)) {
+        return pauseTurn(args.session, args.events, args.media);
+    }
+    return commitCompletedTurn(args.session, args.events, args.media);
+}
+export function streamFoldedTurn(args) {
+    return streamFoldedEvents(args.stream, args.view, args.iface, args.events);
+}
+/** `blocks` with its last `turn-done` carrying the reply's work. */
+export function stampWorked(blocks, worked) {
+    const last = blocks.findLastIndex((block) => block.kind === 'turn-done');
+    if (last < 0)
+        return blocks;
+    return blocks.map((block, index) => (index === last ? { ...block, ...worked } : block));
+}

package/client/run-session.d.ts

@@ -0,0 +1,80 @@
+import { type ComposerProfileInterface, type InterfaceTurnSession, type TranscriptBlock, type UserTurnDraft } from '@theoremjs/agents/interface';
+import { type TurnFailure } from './failure.ts';
+import { type StreamView } from './run-commit.ts';
+import type { TheoremTransport } from './transport.ts';
+import { type ToolGateResolution } from './tool-resume.ts';
+export type { TurnFailure } from './failure.ts';
+export type { StreamView } from './run-commit.ts';
+export type StreamTurnSuccess = {
+    ok: true;
+    session: InterfaceTurnSession;
+    userBlocks: TranscriptBlock[];
+    assistantBlocks: TranscriptBlock[];
+};
+export type StreamInterfaceTurnBaseArgs = {
+    iface: ComposerProfileInterface;
+    transport: TheoremTransport;
+    session: InterfaceTurnSession;
+    view: StreamView;
+    /**
+     * Fires once the message posts, before its reply streams, with the blocks
+     * the transcript gains: a walked-away reply's settled blocks, then the
+     * message's own (with media preview data).
+     */
+    onUserBlocks?: (blocks: TranscriptBlock[]) => void;
+    signal?: AbortSignal;
+    turnId?: string;
+    /**
+     * Send while the reply waits on gates: the message walks away from every
+     * waiting call in its own request, and the host settles each one cancelled
+     * ahead of the reply. `workedMs` is the paused reply's work so far, which
+     * its settled blocks keep. Without it, a waiting session refuses the send
+     * (`session.gate_pending`).
+     */
+    walkAway?: {
+        workedMs: number;
+    };
+};
+export type StreamInterfaceTurnArgs = StreamInterfaceTurnBaseArgs & {
+    text: string;
+    pendingFiles: readonly File[];
+    pendingVoice: readonly File[];
+};
+export type StreamInterfaceDraftTurnArgs = StreamInterfaceTurnBaseArgs & {
+    draft: UserTurnDraft;
+};
+export declare function streamInterfaceTurn(args: StreamInterfaceTurnArgs): Promise<StreamTurnSuccess | TurnFailure>;
+/** Send a turn from an already-encoded pending draft (queue drain / send now). */
+export declare function streamInterfaceDraftTurn(args: StreamInterfaceDraftTurnArgs): Promise<StreamTurnSuccess | TurnFailure>;
+/**
+ * Send the user's answer to the gate the session waits on. The host settles
+ * the call either way (an approval runs it; a refusal records it declined),
+ * so the model and the trace read what the user chose.
+ */
+export declare function resumeInterfaceTool(args: {
+    iface: ComposerProfileInterface;
+    transport: TheoremTransport;
+    session: InterfaceTurnSession;
+    resolution: ToolGateResolution;
+    view: StreamView;
+}): Promise<{
+    ok: true;
+    session: InterfaceTurnSession;
+    assistantBlocks: TranscriptBlock[];
+} | TurnFailure>;
+export declare function applyTurnResultToTranscript(args: {
+    blocks: TranscriptBlock[];
+    streamBlocks: TranscriptBlock[];
+    session: InterfaceTurnSession;
+    userBlocks?: TranscriptBlock[];
+    assistantBlocks: TranscriptBlock[];
+    /** The reply's work so far, stamped on its latest `turn-done` so the time stays with the blocks. */
+    worked?: {
+        workedMs: number;
+        endedAt: number;
+    };
+}): {
+    blocks: TranscriptBlock[];
+    streamBlocks: TranscriptBlock[];
+    session: InterfaceTurnSession;
+};