mojentic 1.2.4 → 1.5.0

package/dist/realtime/session.js

@@ -0,0 +1,715 @@
+"use strict";
+/**
+ * Realtime session — the stateful surface exposed by {@link RealtimeVoiceBroker}.
+ *
+ * Owns the socket lifetime, demultiplexes raw server events into a
+ * vendor-neutral {@link RealtimeEvent} stream, and drives parallel tool
+ * execution per response turn.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RealtimeSession = void 0;
+exports.encodeBase64Pcm16 = encodeBase64Pcm16;
+exports.buildSessionUpdate = buildSessionUpdate;
+const crypto_1 = require("crypto");
+const error_1 = require("../error");
+const tools_1 = require("../llm/tools");
+const config_1 = require("./config");
+const TOOL_BATCH_SOURCE = 'RealtimeVoiceBroker';
+function makeChannel() {
+    const queue = [];
+    const waiters = [];
+    let done = false;
+    return {
+        push: (value) => {
+            if (done)
+                return;
+            const waiter = waiters.shift();
+            if (waiter) {
+                waiter({ value, done: false });
+            }
+            else {
+                queue.push(value);
+            }
+        },
+        end: () => {
+            if (done)
+                return;
+            done = true;
+            while (waiters.length > 0) {
+                const waiter = waiters.shift();
+                waiter?.({ value: undefined, done: true });
+            }
+        },
+        iter: async function* () {
+            while (true) {
+                if (queue.length > 0) {
+                    yield queue.shift();
+                    continue;
+                }
+                if (done)
+                    return;
+                const next = await new Promise((resolve) => {
+                    waiters.push(resolve);
+                });
+                if (next.done)
+                    return;
+                yield next.value;
+            }
+        },
+    };
+}
+/**
+ * Decode a base64 string into an `Int16Array` of PCM samples (little-endian).
+ */
+function decodeBase64Pcm16(b64) {
+    const buf = Buffer.from(b64, 'base64');
+    // Buffer is a Node Uint8Array view; copy into an Int16Array of the right length.
+    const samples = new Int16Array(buf.length / 2);
+    for (let i = 0; i < samples.length; i++) {
+        // eslint-disable-next-line security/detect-object-injection -- bounded index
+        samples[i] = buf.readInt16LE(i * 2);
+    }
+    return samples;
+}
+/**
+ * Encode an `Int16Array` or `Uint8Array` of PCM samples as base64.
+ */
+function encodeBase64Pcm16(frame) {
+    if (frame instanceof Int16Array) {
+        const buf = Buffer.alloc(frame.length * 2);
+        for (let i = 0; i < frame.length; i++) {
+            // eslint-disable-next-line security/detect-object-injection -- bounded index
+            buf.writeInt16LE(frame[i], i * 2);
+        }
+        return buf.toString('base64');
+    }
+    return Buffer.from(frame).toString('base64');
+}
+/**
+ * Build a vendor-specific `session.update` payload from a vendor-neutral
+ * config, matching the OpenAI Realtime GA shape:
+ *
+ * - `session.type: "realtime"` is required.
+ * - Vendor-neutral `modalities` maps to GA `output_modalities`.
+ * - Voice + output audio format live under `session.audio.output`.
+ * - Turn detection, transcription, and input audio format live under
+ *   `session.audio.input`.
+ * - `temperature` is no longer accepted on GA — silently dropped.
+ * - `maxResponseOutputTokens` maps to GA `max_output_tokens`.
+ */
+function buildSessionUpdate(config) {
+    const modalities = config.modalities ?? config_1.REALTIME_DEFAULTS.modalities;
+    const turnDetection = config.turnDetection ?? config_1.REALTIME_DEFAULTS.turnDetection;
+    // GA splits modalities into output-only.
+    const outputModalities = modalities.includes('audio') ? ['audio'] : ['text'];
+    const audioInput = {
+        format: encodeAudioFormat(config.inputAudioFormat ?? config_1.REALTIME_DEFAULTS.inputAudioFormat),
+        turn_detection: encodeTurnDetection(turnDetection),
+    };
+    if (config.inputAudioTranscription === false) {
+        audioInput.transcription = null;
+    }
+    else if (config.inputAudioTranscription !== undefined) {
+        audioInput.transcription = config.inputAudioTranscription;
+    }
+    const audioOutput = {
+        format: encodeAudioFormat(config.outputAudioFormat ?? config_1.REALTIME_DEFAULTS.outputAudioFormat),
+    };
+    if (config.voice !== undefined)
+        audioOutput.voice = config.voice;
+    const session = {
+        type: 'realtime',
+        output_modalities: outputModalities,
+        audio: { input: audioInput, output: audioOutput },
+        tool_choice: encodeToolChoice(config.toolChoice ?? config_1.REALTIME_DEFAULTS.toolChoice),
+    };
+    if (config.instructions !== undefined)
+        session.instructions = config.instructions;
+    if (config.maxResponseOutputTokens !== undefined) {
+        session.max_output_tokens = config.maxResponseOutputTokens;
+    }
+    if (config.tools && config.tools.length > 0) {
+        session.tools = config.tools.map((tool) => {
+            const d = tool.descriptor();
+            return {
+                type: 'function',
+                name: d.function.name,
+                description: d.function.description,
+                parameters: d.function.parameters,
+            };
+        });
+    }
+    if (config.providerExtras) {
+        Object.assign(session, config.providerExtras);
+    }
+    return { type: 'session.update', session };
+}
+function encodeAudioFormat(fmt) {
+    switch (fmt) {
+        case 'pcm16':
+            return { type: 'audio/pcm', rate: 24000 };
+        case 'g711_ulaw':
+            return { type: 'audio/pcmu' };
+        case 'g711_alaw':
+            return { type: 'audio/pcma' };
+    }
+}
+function encodeTurnDetection(td) {
+    if (td === 'none')
+        return null;
+    if (td === 'server_vad')
+        return { type: 'server_vad' };
+    if (td === 'semantic_vad')
+        return { type: 'semantic_vad' };
+    if (typeof td === 'object' && td.type === 'semantic_vad') {
+        const cfg = td;
+        return stripUndefined({
+            type: 'semantic_vad',
+            eagerness: cfg.eagerness,
+            create_response: cfg.createResponse,
+            interrupt_response: cfg.interruptResponse,
+        });
+    }
+    const cfg = td;
+    return stripUndefined({
+        type: 'server_vad',
+        threshold: cfg.threshold,
+        prefix_padding_ms: cfg.prefixPaddingMs,
+        silence_duration_ms: cfg.silenceDurationMs,
+        create_response: cfg.createResponse,
+        interrupt_response: cfg.interruptResponse,
+        idle_timeout_ms: cfg.idleTimeoutMs,
+    });
+}
+function stripUndefined(obj) {
+    const out = {};
+    for (const [k, v] of Object.entries(obj)) {
+        // eslint-disable-next-line security/detect-object-injection -- k comes from Object.entries on a controlled literal
+        if (v !== undefined)
+            out[k] = v;
+    }
+    return out;
+}
+function encodeToolChoice(choice) {
+    if (choice === undefined)
+        return 'auto';
+    if (typeof choice === 'string')
+        return choice;
+    return { type: 'function', name: choice.name };
+}
+/**
+ * Stateful realtime session handle.
+ *
+ * Constructed by {@link RealtimeVoiceBroker}; users don't instantiate this
+ * directly.
+ */
+class RealtimeSession {
+    gatewaySession;
+    config;
+    tools;
+    toolRunner;
+    tracer;
+    correlationId;
+    normalizedChannel = makeChannel();
+    rawChannel = makeChannel();
+    audioChannel = makeChannel();
+    pumpPromise;
+    pendingBatches = new Set();
+    currentTurn;
+    currentResponseId;
+    closed = false;
+    currentInstructions;
+    constructor(gatewaySession, options) {
+        this.gatewaySession = gatewaySession;
+        this.config = options.config;
+        this.tools = options.config.tools ?? [];
+        this.toolRunner = options.toolRunner ?? new tools_1.ParallelToolRunner();
+        this.tracer = options.tracer;
+        this.correlationId = options.correlationId ?? (0, crypto_1.randomUUID)();
+        this.currentInstructions = options.config.instructions;
+        this.normalizedChannel.push({
+            kind: 'session_opened',
+            sessionId: gatewaySession.sessionId,
+        });
+        this.pumpPromise = this.pump();
+    }
+    /**
+     * Initialise the session by sending the `session.update`. Called once by
+     * the broker before returning the session to the caller.
+     */
+    async initialise() {
+        const payload = buildSessionUpdate(this.config);
+        return this.gatewaySession.sendEvent(payload);
+    }
+    /** Vendor-neutral event stream. Terminates when the session closes. */
+    events() {
+        return this.normalizedChannel.iter();
+    }
+    /** Raw server events for power users / debugging. */
+    rawEvents() {
+        return this.rawChannel.iter();
+    }
+    /** Async generator yielding PCM frames from the assistant. */
+    audioOutput() {
+        return this.audioChannel.iter();
+    }
+    /**
+     * Send a text-mode user message. Use this in text-only sessions and tests.
+     */
+    async sendText(text) {
+        const create = await this.gatewaySession.sendEvent({
+            type: 'conversation.item.create',
+            item: {
+                type: 'message',
+                role: 'user',
+                content: [{ type: 'input_text', text }],
+            },
+        });
+        if (!(0, error_1.isOk)(create))
+            return create;
+        return this.gatewaySession.sendEvent({ type: 'response.create' });
+    }
+    /**
+     * Pipe an async iterable of PCM frames into the input audio buffer.
+     *
+     * Frames are encoded as base64 pcm16 and appended to the server VAD buffer.
+     * Returns once the iterable completes or the session closes.
+     */
+    async sendAudio(stream) {
+        try {
+            for await (const frame of stream) {
+                if (this.closed)
+                    return (0, error_1.Ok)(undefined);
+                const append = await this.gatewaySession.sendEvent({
+                    type: 'input_audio_buffer.append',
+                    audio: encodeBase64Pcm16(frame),
+                });
+                if (!(0, error_1.isOk)(append))
+                    return append;
+            }
+            return (0, error_1.Ok)(undefined);
+        }
+        catch (err) {
+            return (0, error_1.Err)(err instanceof Error ? err : new Error(String(err)));
+        }
+    }
+    /**
+     * Manually commit the input audio buffer (only meaningful with
+     * `turnDetection: 'none'`) and request a response.
+     */
+    async commitAudio() {
+        const commit = await this.gatewaySession.sendEvent({
+            type: 'input_audio_buffer.commit',
+        });
+        if (!(0, error_1.isOk)(commit))
+            return commit;
+        return this.gatewaySession.sendEvent({ type: 'response.create' });
+    }
+    /**
+     * Cancel the in-flight response, abort any in-flight tool execution, and
+     * fire an `interrupted` event with `reason: 'manual'`.
+     */
+    async interrupt() {
+        return this.cancelCurrentTurn('manual');
+    }
+    /**
+     * Update the instructions used by future assistant turns. The change is
+     * sent immediately via a `session.update`.
+     */
+    async updateInstructions(instructions) {
+        this.currentInstructions = instructions;
+        return this.gatewaySession.sendEvent({
+            type: 'session.update',
+            session: { instructions },
+        });
+    }
+    /** Close the session, dispose the socket, and end all event streams. */
+    async close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        if (this.currentTurn && !this.currentTurn.cancelled) {
+            this.currentTurn.cancelled = true;
+            this.currentTurn.toolAbort.abort();
+        }
+        await Promise.allSettled(Array.from(this.pendingBatches));
+        await this.gatewaySession.close();
+        this.normalizedChannel.push({ kind: 'session_closed', reason: 'client' });
+        this.normalizedChannel.end();
+        this.rawChannel.end();
+        this.audioChannel.end();
+        await this.pumpPromise.catch(() => {
+            /* swallow — close should never throw */
+        });
+    }
+    /** Symbol.asyncDispose support for `await using` syntax. */
+    [Symbol.asyncDispose]() {
+        return this.close();
+    }
+    /** Effective instructions used in the most recent `session.update`. */
+    getInstructions() {
+        return this.currentInstructions;
+    }
+    // ---------------------------------------------------------------------------
+    // Event pump
+    // ---------------------------------------------------------------------------
+    async pump() {
+        const emit = (event) => this.normalizedChannel.push(event);
+        try {
+            for await (const raw of this.gatewaySession.events()) {
+                this.rawChannel.push(raw);
+                await this.handleServerEvent(raw, emit);
+                if (this.closed)
+                    break;
+            }
+        }
+        finally {
+            if (!this.closed) {
+                this.normalizedChannel.push({ kind: 'session_closed', reason: 'server' });
+                this.normalizedChannel.end();
+                this.rawChannel.end();
+                this.audioChannel.end();
+                this.closed = true;
+            }
+        }
+    }
+    async handleServerEvent(raw, emit) {
+        switch (raw.type) {
+            case 'session.created':
+                // session_opened already emitted on construction; ignore.
+                return;
+            case 'session.updated':
+                emit({ kind: 'session_updated', config: { instructions: this.currentInstructions } });
+                return;
+            case 'input_audio_buffer.speech_started': {
+                const ev = raw;
+                emit({ kind: 'user_speech_started', atMs: ev.audio_start_ms ?? Date.now() });
+                // Barge-in: cancel any in-flight assistant response.
+                if (this.currentTurn && !this.currentTurn.cancelled) {
+                    await this.cancelCurrentTurn('barge_in');
+                }
+                return;
+            }
+            case 'input_audio_buffer.speech_stopped': {
+                const ev = raw;
+                emit({ kind: 'user_speech_stopped', atMs: ev.audio_end_ms ?? Date.now() });
+                return;
+            }
+            case 'conversation.item.input_audio_transcription.delta': {
+                const ev = raw;
+                emit({ kind: 'user_transcript_delta', itemId: ev.item_id, delta: ev.delta });
+                return;
+            }
+            case 'conversation.item.input_audio_transcription.completed': {
+                const ev = raw;
+                emit({ kind: 'user_transcript', itemId: ev.item_id, text: ev.transcript });
+                return;
+            }
+            case 'response.created': {
+                const ev = raw;
+                const turnId = ev.response.id;
+                this.currentResponseId = turnId;
+                this.currentTurn = {
+                    turnId,
+                    calls: new Map(),
+                    textBuffer: '',
+                    transcriptBuffer: '',
+                    cancelled: false,
+                    toolAbort: new AbortController(),
+                };
+                emit({ kind: 'assistant_turn_started', turnId });
+                return;
+            }
+            case 'response.output_item.added': {
+                const ev = raw;
+                if (ev.item.type === 'function_call' && ev.item.call_id && ev.item.name) {
+                    const turn = this.currentTurn;
+                    if (!turn)
+                        return;
+                    turn.calls.set(ev.item.call_id, {
+                        callId: ev.item.call_id,
+                        itemId: ev.item.id,
+                        name: ev.item.name,
+                        argsBuffer: '',
+                        done: false,
+                    });
+                    emit({
+                        kind: 'tool_call_started',
+                        turnId: turn.turnId,
+                        callId: ev.item.call_id,
+                        name: ev.item.name,
+                    });
+                }
+                return;
+            }
+            case 'response.function_call_arguments.delta': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                const call = turn?.calls.get(ev.call_id);
+                if (call) {
+                    call.argsBuffer += ev.delta;
+                }
+                emit({ kind: 'tool_call_args_delta', callId: ev.call_id, delta: ev.delta });
+                return;
+            }
+            case 'response.function_call_arguments.done': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                const call = turn?.calls.get(ev.call_id);
+                if (call) {
+                    call.argsBuffer = ev.arguments || call.argsBuffer;
+                    call.done = true;
+                }
+                return;
+            }
+            case 'response.text.delta':
+            case 'response.output_text.delta': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                if (!turn)
+                    return;
+                turn.textBuffer += ev.delta;
+                emit({ kind: 'assistant_text_delta', turnId: turn.turnId, delta: ev.delta });
+                return;
+            }
+            case 'response.text.done':
+            case 'response.output_text.done': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                if (!turn)
+                    return;
+                emit({ kind: 'assistant_text', turnId: turn.turnId, text: ev.text });
+                return;
+            }
+            case 'response.audio_transcript.delta':
+            case 'response.output_audio_transcript.delta': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                if (!turn)
+                    return;
+                turn.transcriptBuffer += ev.delta;
+                emit({ kind: 'assistant_transcript_delta', turnId: turn.turnId, delta: ev.delta });
+                return;
+            }
+            case 'response.audio_transcript.done':
+            case 'response.output_audio_transcript.done': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                if (!turn)
+                    return;
+                emit({ kind: 'assistant_transcript', turnId: turn.turnId, text: ev.transcript });
+                return;
+            }
+            case 'response.audio.delta':
+            case 'response.output_audio.delta': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                if (!turn)
+                    return;
+                const pcm = decodeBase64Pcm16(ev.delta);
+                this.audioChannel.push(pcm);
+                emit({ kind: 'assistant_audio_delta', turnId: turn.turnId, pcm });
+                return;
+            }
+            case 'response.done': {
+                const ev = raw;
+                const turn = this.currentTurn;
+                if (!turn || turn.turnId !== ev.response.id) {
+                    this.currentTurn = undefined;
+                    return;
+                }
+                const usage = ev.response.usage;
+                const usageOut = usage
+                    ? {
+                        promptTokens: usage.input_tokens,
+                        completionTokens: usage.output_tokens,
+                        totalTokens: usage.total_tokens,
+                        extras: {
+                            input_token_details: usage.input_token_details,
+                            output_token_details: usage.output_token_details,
+                        },
+                    }
+                    : undefined;
+                emit({ kind: 'assistant_turn_completed', turnId: turn.turnId, usage: usageOut });
+                if (turn.calls.size > 0) {
+                    // Fire-and-track: the pump must stay responsive to barge-in /
+                    // server events while tools are running. We keep `currentTurn`
+                    // set during execution so a late `speech_started` or manual
+                    // `interrupt()` can still abort the batch.
+                    const batchPromise = this.runToolBatchForTurn(turn, emit).catch((err) => {
+                        emit({
+                            kind: 'error',
+                            error: err instanceof Error ? err : new Error(String(err)),
+                            recoverable: true,
+                        });
+                    });
+                    this.pendingBatches.add(batchPromise);
+                    batchPromise.finally(() => {
+                        this.pendingBatches.delete(batchPromise);
+                        if (this.currentTurn === turn) {
+                            this.currentTurn = undefined;
+                            this.currentResponseId = undefined;
+                        }
+                    });
+                }
+                else {
+                    this.currentTurn = undefined;
+                    this.currentResponseId = undefined;
+                }
+                return;
+            }
+            case 'rate_limits.updated': {
+                const ev = raw;
+                const first = ev.rate_limits[0] ?? {};
+                const reset = first.reset_seconds ?? 0;
+                emit({
+                    kind: 'rate_limited',
+                    resetMs: Math.round(reset * 1000),
+                    details: { rate_limits: ev.rate_limits },
+                });
+                return;
+            }
+            case 'error': {
+                const ev = raw;
+                // Suppress the expected race when `response.cancel` lands after the
+                // response has already completed server-side — common during barge-in.
+                if (/no active response/i.test(ev.error.message)) {
+                    return;
+                }
+                emit({
+                    kind: 'error',
+                    error: new Error(ev.error.message),
+                    recoverable: ev.error.type !== 'session_error',
+                });
+                return;
+            }
+            default:
+                // Unknown event types stay in rawEvents() only.
+                return;
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // Tool batch execution
+    // ---------------------------------------------------------------------------
+    async runToolBatchForTurn(turn, emit) {
+        const calls = Array.from(turn.calls.values()).filter((c) => c.done || c.argsBuffer.length > 0);
+        const executions = [];
+        for (const call of calls) {
+            try {
+                const args = (call.argsBuffer ? JSON.parse(call.argsBuffer) : {});
+                call.parsedArgs = args;
+                executions.push({ id: call.callId, name: call.name, args });
+            }
+            catch (err) {
+                const error = err instanceof Error ? err : new Error(String(err));
+                emit({ kind: 'tool_call_failed', callId: call.callId, name: call.name, error });
+            }
+        }
+        if (executions.length === 0) {
+            return;
+        }
+        const batchStart = Date.now();
+        const outcomes = await this.toolRunner.runBatch(executions, this.tools, {
+            signal: turn.toolAbort.signal,
+            correlationId: this.correlationId,
+            source: TOOL_BATCH_SOURCE,
+            onCallStart: (call) => {
+                emit({
+                    kind: 'tool_call_dispatched',
+                    callId: call.id,
+                    name: call.name,
+                    args: call.args,
+                });
+            },
+            onCallComplete: (outcome) => {
+                if (this.tracer) {
+                    const args = executions.find((e) => e.id === outcome.id)?.args ?? {};
+                    this.tracer.recordToolCall(outcome.name, args, outcome.ok ? outcome.result : { error: outcome.error.message }, TOOL_BATCH_SOURCE, outcome.durationMs, this.correlationId, 'RealtimeSession.toolBatch');
+                }
+                if (outcome.ok) {
+                    emit({
+                        kind: 'tool_call_completed',
+                        callId: outcome.id,
+                        name: outcome.name,
+                        result: outcome.result,
+                    });
+                }
+                else {
+                    emit({
+                        kind: 'tool_call_failed',
+                        callId: outcome.id,
+                        name: outcome.name,
+                        error: outcome.error,
+                    });
+                }
+            },
+        });
+        if (this.tracer) {
+            const ok = outcomes.filter((o) => o.ok).length;
+            const fail = outcomes.length - ok;
+            this.tracer.recordToolBatch((0, crypto_1.randomUUID)(), executions.map((e) => e.name), ok, fail, Date.now() - batchStart, this.correlationId, 'RealtimeSession.toolBatch');
+        }
+        const policy = this.config.onInterrupt ?? config_1.REALTIME_DEFAULTS.onInterrupt;
+        const toSubmit = this.selectOutputsToSubmit(turn, outcomes, policy);
+        const submittedIds = [];
+        for (const outcome of toSubmit) {
+            const output = serialiseOutput(outcome);
+            const send = await this.gatewaySession.sendEvent({
+                type: 'conversation.item.create',
+                item: {
+                    type: 'function_call_output',
+                    call_id: outcome.id,
+                    output,
+                },
+            });
+            if ((0, error_1.isOk)(send)) {
+                submittedIds.push(outcome.id);
+            }
+        }
+        if (submittedIds.length > 0) {
+            emit({ kind: 'tool_batch_submitted', turnId: turn.turnId, callIds: submittedIds });
+            // Trigger the model's follow-up response.
+            await this.gatewaySession.sendEvent({ type: 'response.create' });
+        }
+    }
+    selectOutputsToSubmit(turn, outcomes, policy) {
+        if (!turn.cancelled)
+            return outcomes;
+        if (policy === 'submit')
+            return outcomes;
+        if (policy === 'drop')
+            return [];
+        // submit-completed-only: only outcomes that wrapped up before abort signal landed.
+        return outcomes.filter((o) => o.ok);
+    }
+    async cancelCurrentTurn(reason) {
+        const turn = this.currentTurn;
+        if (!turn || turn.cancelled) {
+            return (0, error_1.Ok)(undefined);
+        }
+        turn.cancelled = true;
+        turn.toolAbort.abort();
+        this.normalizedChannel.push({ kind: 'interrupted', turnId: turn.turnId, reason });
+        if (!this.currentResponseId)
+            return (0, error_1.Ok)(undefined);
+        return this.gatewaySession.sendEvent({ type: 'response.cancel' });
+    }
+}
+exports.RealtimeSession = RealtimeSession;
+function serialiseOutput(outcome) {
+    if (outcome.ok) {
+        return safeJsonStringify(outcome.result);
+    }
+    return JSON.stringify({ error: outcome.error.message });
+}
+function safeJsonStringify(value) {
+    if (typeof value === 'string')
+        return value;
+    try {
+        return JSON.stringify(value);
+    }
+    catch (err) {
+        return JSON.stringify({ error: 'serialisation_failed', detail: String(err) });
+    }
+}
+//# sourceMappingURL=session.js.map