@theoremjs/agents 0.0.0-stage → 0.3.1

package/esm/src/providers/google/interactions/steps.js

@@ -0,0 +1,200 @@
+import { asRecord, nonEmptyString } from '../../../kernel/engine/record.js';
+import { reportedTokens, usageCount } from '../../../kernel/engine/usage.js';
+import { turnStopFromInteractionStatus } from '../../../kernel/stop.js';
+import { groundingFromSteps } from '../grounding.js';
+import { byModality, groundingCounts, modalityCounts } from '../usage.js';
+/* probed 2026-09-23:
+ * Interactions steps and deltas read only the shapes recorded from the wire
+ * (probes 23/09/2026, gemini-3.8-flash, gemini-3.1-pro-preview, image, and TTS
+ * models, streamed and buffered):
+ *
+ * - `text` deltas carry `text`; `thought_summary` deltas carry
+ *   `content: { type: 'text', text }`; buffered `thought` steps carry
+ *   `summary[]` of the same text blocks.
+ * - `image` / `audio` deltas and `model_output` content blocks carry
+ *   `mime_type` + `data`. Audio also reports `sample_rate` / `channels`; the
+ *   buffered mime already states them (`audio/l16; rate=24000; channels=1`),
+ *   the streamed one does not (`audio/l16`), so they are folded into the mime.
+ * - `code_execution_call` carries `id` and `arguments: { language, code }`;
+ *   `code_execution_result` carries `call_id`, `result`, and `is_error`. Each
+ *   arrives whole, in one delta (5 577 characters of code in one probe).
+ */
+function thoughtText(content) {
+    const block = asRecord(content);
+    return block?.type === 'text' && typeof block.text === 'string' ? block.text : '';
+}
+function interactionsMime(rec) {
+    const mime = rec.mime_type;
+    if (typeof mime !== 'string' || !mime) {
+        return undefined;
+    }
+    const params = mime.toLowerCase();
+    const extra = [];
+    if (typeof rec.sample_rate === 'number' && !/;\s*rate=/.test(params)) {
+        extra.push(`rate=${String(rec.sample_rate)}`);
+    }
+    if (typeof rec.channels === 'number' && !/;\s*channels=/.test(params)) {
+        extra.push(`channels=${String(rec.channels)}`);
+    }
+    return extra.length > 0 ? `${mime}; ${extra.join('; ')}` : mime;
+}
+function interactionsMedia(rec) {
+    const mimeType = interactionsMime(rec);
+    const { data } = rec;
+    if (!mimeType || typeof data !== 'string' || !data) {
+        return [];
+    }
+    return [{ type: 'media', media: { mimeType, data } }];
+}
+function textEvent(type, text) {
+    return text ? [{ type, text }] : [];
+}
+function isCodeExecutionType(type) {
+    return type === 'code_execution_call' || type === 'code_execution_result';
+}
+function isGoogleBuiltinStepType(type) {
+    return type.startsWith('google_') || type === 'url_context_call' || type === 'url_context_result';
+}
+function rawStepEvidence(raw) {
+    return {
+        type: 'evidence',
+        evidence: { provider: 'google', kind: 'provider_step', step: String(raw.type ?? ''), raw },
+    };
+}
+function codeExecutionEvidence(raw) {
+    if (raw.type === 'code_execution_result') {
+        const callId = nonEmptyString(raw.call_id);
+        return {
+            type: 'evidence',
+            evidence: {
+                provider: 'google',
+                kind: 'code_execution_result',
+                raw,
+                ...(typeof raw.result === 'string' ? { result: raw.result } : {}),
+                ...(typeof raw.is_error === 'boolean' ? { isError: raw.is_error } : {}),
+                ...(callId ? { callId } : {}),
+            },
+        };
+    }
+    const args = asRecord(raw.arguments);
+    const id = nonEmptyString(raw.id);
+    if (raw.type !== 'code_execution_call' || !id || typeof args?.code !== 'string') {
+        return rawStepEvidence(raw);
+    }
+    return {
+        type: 'evidence',
+        evidence: {
+            provider: 'google',
+            kind: 'code_execution_call',
+            raw,
+            code: args.code,
+            ...(typeof args.language === 'string' ? { language: args.language } : {}),
+            id,
+        },
+    };
+}
+function eventsFromDelta(deltaValue) {
+    const delta = asRecord(deltaValue);
+    if (!delta) {
+        return [];
+    }
+    switch (delta.type) {
+        case 'thought_summary':
+            return textEvent('thought', thoughtText(delta.content));
+        case 'text':
+            return textEvent('text', typeof delta.text === 'string' ? delta.text : '');
+        case 'image':
+        case 'audio':
+            return interactionsMedia(delta);
+        default:
+            return [];
+    }
+}
+function eventsFromThoughtStep(step) {
+    const summary = Array.isArray(step.summary) ? step.summary : [];
+    return summary.flatMap((block) => textEvent('thought', thoughtText(block)));
+}
+function eventsFromModelOutputStep(step) {
+    const content = Array.isArray(step.content) ? step.content : [];
+    return content.flatMap((block) => {
+        const rec = asRecord(block);
+        if (rec?.type === 'text') {
+            return textEvent('text', typeof rec.text === 'string' ? rec.text : '');
+        }
+        if (rec?.type === 'image' || rec?.type === 'audio') {
+            return interactionsMedia(rec);
+        }
+        return [];
+    });
+}
+/**
+ * Google reports thought and tool-use tokens beside input and output; the
+ * OpenTelemetry meanings fold them in. Probes (gemini-3.8-flash, 22/09/2026):
+ * `total_tokens` = input + output + thought + tool use, and cached sits inside input.
+ * For inputs Google converts first (Markdown, Python, mono `audio/L16`, …) input
+ * comes back 0 while `total_tokens` holds the full sum, so input is derived from it.
+ */
+function interactionsUsageTokens(raw) {
+    const usage = asRecord(raw);
+    if (!usage)
+        return undefined;
+    const output = usageCount(usage.total_output_tokens);
+    const thought = usageCount(usage.total_thought_tokens) ?? 0;
+    const toolUse = usageCount(usage.total_tool_use_tokens) ?? 0;
+    const total = usageCount(usage.total_tokens);
+    const derived = total === undefined ? undefined : total - (output ?? 0) - thought - toolUse;
+    const read = usageCount(usage.total_input_tokens) || (derived && derived > 0 ? derived : 0);
+    return reportedTokens({
+        input: read ? read + toolUse : undefined,
+        output: output === undefined ? undefined : output + thought,
+        thinking: thought,
+        toolUse,
+        cached: usageCount(usage.total_cached_tokens),
+        byModality: byModality(modalityCounts(usage.input_tokens_by_modality, 'tokens'), modalityCounts(usage.output_tokens_by_modality, 'tokens')),
+        grounding: groundingCounts(usage.grounding_tool_count),
+    });
+}
+function extractTokenEvent(interaction) {
+    const usage = interactionsUsageTokens(interaction.usage);
+    if (!usage) {
+        return undefined;
+    }
+    const interactionId = typeof interaction.id === 'string' ? interaction.id : undefined;
+    return {
+        type: 'tokens',
+        tokens: usage,
+        ...(interactionId ? { interactionId } : {}),
+    };
+}
+function eventsFromInteractionEnd(interaction) {
+    const events = [];
+    const tokenEvent = extractTokenEvent(interaction);
+    if (tokenEvent)
+        events.push(tokenEvent);
+    // why: Only a buffered body carries `steps[]`; streamed grounding arrives on deltas.
+    const { steps } = interaction;
+    if (Array.isArray(steps))
+        events.push(...groundingFromSteps(steps));
+    const done = doneFromInteractionStatus(interaction);
+    if (done)
+        events.push(done);
+    return events;
+}
+function interactionResponse(interaction) {
+    const id = nonEmptyString(interaction.id);
+    const model = nonEmptyString(interaction.model);
+    if (!id && !model)
+        return undefined;
+    return { ...(id ? { id } : {}), ...(model ? { model } : {}) };
+}
+function doneFromInteractionStatus(interaction) {
+    const { status } = interaction;
+    if (typeof status !== 'string')
+        return undefined;
+    return {
+        type: 'done',
+        stop: turnStopFromInteractionStatus(status),
+        ...(typeof interaction.id === 'string' ? { interactionId: interaction.id } : {}),
+    };
+}
+export { codeExecutionEvidence, eventsFromDelta, eventsFromInteractionEnd, eventsFromModelOutputStep, eventsFromThoughtStep, extractTokenEvent, interactionResponse, isCodeExecutionType, isGoogleBuiltinStepType, rawStepEvidence, };

package/esm/src/providers/google/interactions/stream.d.ts

@@ -0,0 +1,23 @@
+import type { ModelEvent, ModelRequest, TurnResponse } from '../../../kernel/types.js';
+export interface StreamFold {
+    text: string;
+    /** Open `function_call`, `code_execution_*`, and builtin steps by `index`. */
+    steps: Map<number, Record<string, unknown>>;
+    sawMedia: boolean;
+    /** A `done` came out: the interaction reported a status. */
+    sawDone: boolean;
+    /** Identity named so far: `interaction.created` names it before any output, so a cut call still has it. */
+    response?: TurnResponse;
+    /** The signature of the last thought, until the `function_call` that follows it takes it. */
+    thoughtSignature?: string;
+    /** Function calls read so far: a signature names its call by this position. */
+    calls: number;
+}
+export declare function newStreamFold(): StreamFold;
+export declare function eventsFromStep(step: Record<string, unknown>, fold: StreamFold): ModelEvent[];
+export declare function openStepEvents(fold: StreamFold): ModelEvent[];
+export declare function foldPayload(payload: Record<string, unknown>, fold: StreamFold): ModelEvent[];
+export declare function foldBody(body: Record<string, unknown>, fold: StreamFold): ModelEvent[];
+export declare function finalizeStructured(req: ModelRequest, fold: StreamFold): Generator<ModelEvent>;
+/** What the profile asked for and the model did not return; even an empty turn fails. */
+export declare function missingMediaError(req: ModelRequest, fold: StreamFold): ModelEvent | undefined;

package/esm/src/providers/google/interactions/stream.js

@@ -0,0 +1,230 @@
+import { TheoremError, toErrorEvent } from '../../../guardrails/error.js';
+import { asRecord } from '../../../kernel/engine/record.js';
+import { pcmMediaAsWav } from '../../shared/pcm.js';
+import { foldResponse } from '../../shared/response-identity.js';
+import { structuredEvent } from '../../shared/structured-output.js';
+import { toolCallEvents } from '../../shared/tool-args.js';
+import { toolDraftEvents } from '../../shared/tool-draft.js';
+import { readGeminiApiError } from '../api-error.js';
+import { groundingFromDelta } from '../grounding.js';
+import { codeExecutionEvidence, eventsFromDelta, eventsFromInteractionEnd, eventsFromModelOutputStep, eventsFromThoughtStep, interactionResponse, isCodeExecutionType, isGoogleBuiltinStepType, rawStepEvidence, } from './steps.js';
+export function newStreamFold() {
+    return { text: '', steps: new Map(), sawMedia: false, sawDone: false, calls: 0 };
+}
+function isMergedStepType(type) {
+    return type === 'function_call' || isCodeExecutionType(type) || isGoogleBuiltinStepType(type);
+}
+function nonEmptyString(value) {
+    return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+/**
+ * Hold a thought's signature for the call that follows it. The model thinks
+ * once before a step's calls, so the signature arrives on a `thought` step:
+ * whole when buffered, as a `thought_signature` delta when streamed (probe
+ * 25/09/2026, gemini-3.1-flash-lite). Calls carried an empty `signature`.
+ */
+function holdThoughtSignature(signature, fold) {
+    const held = nonEmptyString(signature);
+    if (held)
+        fold.thoughtSignature = held;
+}
+/**
+ * The signature a `function_call` replays with: its own when it carries a
+ * non-empty one, else the held thought's. The first call takes the held one;
+ * its siblings share that thought and carry none.
+ */
+function takeThoughtSignature(step, fold) {
+    const signature = nonEmptyString(step.signature) ?? fold.thoughtSignature;
+    fold.thoughtSignature = undefined;
+    return signature;
+}
+export function eventsFromStep(step, fold) {
+    const type = String(step.type ?? '');
+    if (type === 'function_call') {
+        const id = typeof step.id === 'string' ? step.id : undefined;
+        const name = typeof step.name === 'string' ? step.name : '';
+        const signature = takeThoughtSignature(step, fold);
+        const call = fold.calls++;
+        return [
+            ...(signature
+                ? [
+                    {
+                        type: 'reasoning',
+                        data: { call, signature },
+                        origin: { protocol: 'geminiInteractions' },
+                    },
+                ]
+                : []),
+            ...toolCallEvents({ id, name }, step.arguments),
+        ];
+    }
+    if (isCodeExecutionType(type)) {
+        return [codeExecutionEvidence(step)];
+    }
+    if (isGoogleBuiltinStepType(type)) {
+        return [rawStepEvidence(step)];
+    }
+    if (type === 'thought') {
+        holdThoughtSignature(step.signature, fold);
+        return eventsFromThoughtStep(step);
+    }
+    if (type === 'model_output') {
+        return eventsFromModelOutputStep(step);
+    }
+    return [];
+}
+function stepIndex(payload) {
+    return typeof payload.index === 'number' ? payload.index : 0;
+}
+function foldStepStart(payload, fold) {
+    const step = asRecord(payload.step);
+    if (!step)
+        return;
+    const type = String(step.type ?? '');
+    if (type === 'thought') {
+        holdThoughtSignature(step.signature, fold);
+    }
+    if (isMergedStepType(type)) {
+        fold.steps.set(stepIndex(payload), { ...step });
+    }
+}
+function foldStepDelta(payload, fold) {
+    const delta = asRecord(payload.delta);
+    if (!delta) {
+        return [];
+    }
+    const open = fold.steps.get(stepIndex(payload));
+    if (!open) {
+        if (delta.type === 'thought_signature') {
+            holdThoughtSignature(delta.signature, fold);
+        }
+        return eventsFromDelta(delta);
+    }
+    if (delta.type === 'arguments_delta') {
+        // why: Replaces the `arguments: {}` placeholder from `step.start` with the streamed JSON string.
+        const sofar = typeof open.arguments === 'string' ? open.arguments : '';
+        const args = sofar + (typeof delta.arguments === 'string' ? delta.arguments : '');
+        open.arguments = args;
+        if (open.type !== 'function_call')
+            return [];
+        // why: The call at `step.stop` goes out under the `id` and `name` that `step.start` gave.
+        return toolDraftEvents({
+            id: nonEmptyString(open.id),
+            name: typeof open.name === 'string' ? open.name : '',
+            args,
+            drafted: sofar.length,
+        });
+    }
+    Object.assign(open, delta);
+    return [];
+}
+function foldStepStop(payload, fold) {
+    const index = stepIndex(payload);
+    const step = fold.steps.get(index);
+    if (!step) {
+        return [];
+    }
+    fold.steps.delete(index);
+    return eventsFromStep(step, fold);
+}
+function partialStepEvidence(step) {
+    const event = isCodeExecutionType(String(step.type ?? ''))
+        ? codeExecutionEvidence(step)
+        : rawStepEvidence(step);
+    return { ...event, evidence: { ...event.evidence, partial: true } };
+}
+export function openStepEvents(fold) {
+    const events = [...fold.steps.values()].map(partialStepEvidence);
+    fold.steps.clear();
+    return events;
+}
+function delivered(events, fold) {
+    return events.map((ev) => {
+        if (ev.type === 'done') {
+            fold.sawDone = true;
+        }
+        if (ev.type === 'text' && ev.text) {
+            fold.text += ev.text;
+        }
+        if (ev.type !== 'media' || !ev.media) {
+            return ev;
+        }
+        fold.sawMedia = true;
+        return { ...ev, media: pcmMediaAsWav(ev.media) };
+    });
+}
+function eventsFromStreamRow(payload, fold) {
+    if (payload.eventType === 'sse_unparsed') {
+        // why: Every observed Interactions row is a JSON object; anything else is a wire change.
+        return [
+            toErrorEvent(new TheoremError('bad_response', 'Interactions stream row was not a JSON object')),
+        ];
+    }
+    const apiError = readGeminiApiError(payload);
+    if (apiError) {
+        return [toErrorEvent(apiError)];
+    }
+    const events = groundingFromDelta(payload);
+    switch (payload.event_type) {
+        case 'step.start':
+            foldStepStart(payload, fold);
+            break;
+        case 'step.delta':
+            events.push(...foldStepDelta(payload, fold));
+            break;
+        case 'step.stop':
+            events.push(...foldStepStop(payload, fold));
+            break;
+        case 'interaction.created': {
+            const interaction = asRecord(payload.interaction);
+            if (interaction)
+                events.push(...identityEvents(interaction, fold));
+            break;
+        }
+        case 'interaction.completed': {
+            const interaction = asRecord(payload.interaction);
+            if (interaction) {
+                events.push(...identityEvents(interaction, fold), ...eventsFromInteractionEnd(interaction));
+            }
+            break;
+        }
+    }
+    return events;
+}
+function identityEvents(interaction, fold) {
+    const identity = foldResponse(fold.response, interactionResponse(interaction));
+    fold.response = identity.known;
+    return identity.event ? [identity.event] : [];
+}
+export function foldPayload(payload, fold) {
+    return delivered(eventsFromStreamRow(payload, fold), fold);
+}
+export function foldBody(body, fold) {
+    const steps = Array.isArray(body.steps) ? body.steps : [];
+    const fromSteps = steps.flatMap((value) => {
+        const step = asRecord(value);
+        return step ? eventsFromStep(step, fold) : [];
+    });
+    return delivered([...identityEvents(body, fold), ...fromSteps, ...eventsFromInteractionEnd(body)], fold);
+}
+export function* finalizeStructured(req, fold) {
+    if (!req.structured || !fold.text) {
+        return;
+    }
+    yield structuredEvent(fold.text);
+}
+function isVoiceProfile(req) {
+    return Boolean(req.speech);
+}
+/** What the profile asked for and the model did not return; even an empty turn fails. */
+export function missingMediaError(req, fold) {
+    if (fold.sawMedia)
+        return undefined;
+    if (isVoiceProfile(req)) {
+        return toErrorEvent(new TheoremError('bad_response', 'speech audio was not returned by the model'));
+    }
+    if (req.image) {
+        return toErrorEvent(new TheoremError('bad_response', 'no image returned from image generation'));
+    }
+    return undefined;
+}

package/esm/src/providers/google/live/framing.d.ts

@@ -0,0 +1,61 @@
+import type { InteractionPart, ModelEvent, ModelRequest, TurnHistoryMessage, TurnTokens, WireFunctionTool } from '../../../kernel/types.js';
+export declare function buildGeminiLiveWebSocketUrl(apiKey: string): string;
+/** `parametersJsonSchema` takes the JSON Schema as it is; `parameters` takes only Google's OpenAPI subset. */
+export declare function wireFunctionDeclaration(decl: WireFunctionTool): Record<string, unknown>;
+/**
+ * Builtins are their own tool entries (`{ googleSearch: {} }`), functions share
+ * one `functionDeclarations` entry. Which builtins a model takes is the API's
+ * answer (probe 23/09/2026: a model without one closes the socket with 1007).
+ */
+/** A live session's setup as Gemini frames it: the request, plus the handle it resumes from. */
+export type LiveSetupRequest = ModelRequest & {
+    sessionResumptionHandle?: string;
+};
+export declare function buildGeminiLiveSetupMessage(req: LiveSetupRequest): Record<string, unknown>;
+export declare function buildGeminiLiveClientContent(history: TurnHistoryMessage[]): Record<string, unknown> | null;
+export declare function buildGeminiLiveContext(text: string): Record<string, unknown>;
+export declare function buildGeminiLiveRealtimeInput(input: InteractionPart): Record<string, unknown>;
+export declare function buildGeminiLiveToolResponse(id: string, name: string, output: unknown, parts?: InteractionPart[]): Record<string, unknown>;
+export declare function buildGeminiLiveToolResponses(responses: Array<{
+    id: string;
+    name: string;
+    output: unknown;
+    parts?: InteractionPart[];
+}>): Record<string, unknown>;
+/** What one outbound frame gives the model to read; setup and control frames give none. */
+export declare function liveFrameInput(frame: Record<string, unknown>): TurnHistoryMessage[];
+export type ParsedLiveMessage = {
+    ok: true;
+    value: Record<string, unknown>;
+} | {
+    ok: false;
+    reason: 'empty' | 'malformed';
+};
+export declare function parseGeminiLiveMessage(raw: unknown): ParsedLiveMessage;
+/**
+ * Live sends one usage row per model response, at its `turnComplete`, covering that
+ * response alone; the prompt count grows because each response re-reads the session.
+ * Probe (gemini-3.1-flash-live-preview, 22/09/2026): `thoughtsTokenCount` sits outside
+ * `responseTokenCount` and `totalTokenCount`, so it is added to output.
+ * `toolUsePromptTokenCount` (not seen in that probe) is added to input, as on Interactions.
+ */
+export declare function extractLiveUsageTokens(metadata: Record<string, unknown>): TurnTokens | undefined;
+export interface LiveFold {
+    /** Tool call names by id, for the cancel that names only ids. */
+    calls: Map<string, string>;
+}
+export declare function newLiveFold(): LiveFold;
+/**
+ * `goAway.timeLeft` → milliseconds. It is a protobuf Duration, which JSON
+ * encodes as seconds with an `s` suffix (`"10s"`, `"1.5s"`).
+ */
+export declare function parseGoAwayTimeLeftMs(timeLeft: unknown): number | undefined;
+export type LiveInteractionStatus = 'IN_PROGRESS' | 'IDLE';
+/**
+ * `serverContent.interactionStatus`, sent beside `turnComplete` by models that
+ * keep working after a turn (gemini-3.8-live-extended-thinking, probe
+ * 23/09/2026: `IN_PROGRESS` on each intermediate `turnComplete` of a tool
+ * flow, `IDLE` on the last).
+ */
+export declare function readLiveInteractionStatus(message: Record<string, unknown>): LiveInteractionStatus | undefined;
+export declare function foldGeminiLiveServerMessage(message: Record<string, unknown> | null | undefined, fold: LiveFold): ModelEvent[];