@komaa/standin-sdk 0.1.0

package/dist/plugins/openai/agent.js

@@ -0,0 +1,264 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+/**
+ * One OpenAI Realtime session, as a socket.
+ *
+ * The one place in the SDK where the wire rate and the model rate disagree.
+ * StandIn speaks PCM16 at 16 kHz; the Realtime API speaks PCM at 24 kHz and
+ * nothing else. This file owns that conversion in both directions, so the
+ * handler above it and the call below it both stay at their own rate and
+ * neither has to think about it.
+ *
+ * The Python SDK's `standin.audio.resample_pcm16` is the same resampler, driven
+ * by the same conformance vectors, so a call sounds identical whichever
+ * language the plugin is written in.
+ */
+import { WebSocket } from "ws";
+import { REALTIME_SAMPLE_RATE_HZ, resamplePcm16 } from "../../audio.js";
+import { toolSchemas } from "../../callTools.js";
+import { logger } from "../../log.js";
+import { SAMPLE_RATE_HZ } from "../../protocol.js";
+const CONNECT_TIMEOUT_MS = 10_000;
+/** A stalled socket must not accumulate an unbounded queue of caller audio. */
+const MAX_SEND_BUFFER_BYTES = 1024 * 1024;
+/**
+ * The call capabilities every session gets, declared as tools.
+ *
+ * The list is the SDK's, in the Realtime API's shape, so every provider offers
+ * the same agent and a new capability needs no edit here.
+ */
+export const BUILT_IN_TOOLS = toolSchemas("openai");
+/** The names above, for the shadowing check. */
+export const BUILT_IN_TOOL_NAMES = new Set(BUILT_IN_TOOLS.map((tool) => tool.name));
+/**
+ * Normalise a remote MCP tool entry for the session's tools array.
+ *
+ * The Realtime API dials the server itself and runs those tools server-side, so
+ * nothing here executes at call time. Approval defaults to never because a live
+ * voice call has no approval interface, and a pending approval is a caller
+ * listening to silence.
+ */
+export function mcpTool(entry) {
+    if (typeof entry.server_label !== "string" ||
+        typeof entry.server_url !== "string") {
+        throw new Error("an MCP tool entry needs server_label and server_url");
+    }
+    return { require_approval: "never", ...entry, type: "mcp" };
+}
+/** Compose the effective instructions: your prompt, the caller, and tool guidance. */
+export function buildInstructions(base, caller) {
+    return [
+        base.trim(),
+        `The caller's name is ${caller.callerName}. Their organization id is ` +
+            `${caller.tenantId}. This is an ${caller.direction} call.`,
+        "Use the end_call tool when the conversation is over. Use the look tool when the " +
+            "caller refers to something on their camera or shared screen. Use show_image to put " +
+            "a picture on your video tile, and express to show an emotion on your avatar.",
+    ].join("\n\n");
+}
+/** Per-call session configuration, the Realtime equivalent of a conversation init. */
+export function buildSessionUpdate(config, instructions, extraTools = []) {
+    const input = {
+        format: { type: "audio/pcm", rate: REALTIME_SAMPLE_RATE_HZ },
+        // interrupt_response cancels the in-flight response server-side the moment
+        // the caller speaks. The handler mirrors that to the call, which is what
+        // actually stops the bot talking.
+        turn_detection: {
+            type: config.vadType,
+            create_response: true,
+            interrupt_response: true,
+        },
+    };
+    if (config.transcriptionModel)
+        input.transcription = { model: config.transcriptionModel };
+    const output = {
+        format: { type: "audio/pcm", rate: REALTIME_SAMPLE_RATE_HZ },
+    };
+    if (config.voice)
+        output.voice = config.voice;
+    return {
+        type: "session.update",
+        session: {
+            type: "realtime",
+            output_modalities: ["audio"],
+            instructions,
+            audio: { input, output },
+            tools: [...BUILT_IN_TOOLS, ...extraTools],
+            tool_choice: "auto",
+        },
+    };
+}
+/** The socket to one OpenAI Realtime session. */
+export class AgentSocket {
+    #ws;
+    #pendingBytes = 0;
+    #dropped = 0;
+    #lastDropWarning = 0;
+    #closedByUs = false;
+    constructor() { }
+    /** Open the session, retrying once on a transient failure. */
+    static async connect(config, onMessage, 
+    /** Agent audio, already resampled DOWN to the wire rate. */
+    onAudio, onClose) {
+        const socket = new AgentSocket();
+        const open = async () => {
+            const url = `wss://${config.host}/v1/realtime?model=${encodeURIComponent(config.model)}`;
+            const ws = new WebSocket(url, {
+                headers: { authorization: `Bearer ${config.apiKey}` },
+                maxPayload: 16 * 1024 * 1024,
+            });
+            await new Promise((resolve, reject) => {
+                const fail = setTimeout(() => {
+                    ws.terminate();
+                    reject(new Error("the OpenAI Realtime socket did not open in time"));
+                }, CONNECT_TIMEOUT_MS);
+                fail.unref?.();
+                ws.once("open", () => {
+                    clearTimeout(fail);
+                    resolve();
+                });
+                ws.once("error", (err) => {
+                    clearTimeout(fail);
+                    reject(err);
+                });
+            });
+            return ws;
+        };
+        try {
+            socket.#ws = await open();
+        }
+        catch (err) {
+            logger.warn(`standin: OpenAI connect failed (${String(err)}); retrying once`);
+            await new Promise((resolve) => setTimeout(resolve, 250));
+            socket.#ws = await open();
+        }
+        socket.#listen(onMessage, onAudio, onClose);
+        return socket;
+    }
+    #listen(onMessage, onAudio, onClose) {
+        const ws = this.#ws;
+        if (ws === undefined)
+            return;
+        ws.on("message", (raw) => {
+            let message;
+            try {
+                message = JSON.parse(raw.toString());
+            }
+            catch {
+                logger.warn("standin: OpenAI sent an unparseable frame; dropping");
+                return;
+            }
+            if (typeof message !== "object" || message === null)
+                return;
+            const typed = message;
+            if (typeof typed.type !== "string")
+                return;
+            // The hot path first: audio deltas are resampled here so nothing above
+            // this line ever sees the model's rate.
+            if (typed.type === "response.output_audio.delta" &&
+                typeof typed.delta === "string") {
+                const atModelRate = Buffer.from(typed.delta, "base64");
+                if (atModelRate.length > 0) {
+                    const atWireRate = resamplePcm16(atModelRate, REALTIME_SAMPLE_RATE_HZ, SAMPLE_RATE_HZ);
+                    void Promise.resolve(onAudio(atWireRate)).catch((err) => {
+                        logger.error(`standin: handling OpenAI audio failed: ${String(err)}`);
+                    });
+                }
+                return;
+            }
+            void Promise.resolve(onMessage(typed)).catch((err) => {
+                logger.error(`standin: handling OpenAI ${typed.type} failed: ${String(err)}`);
+            });
+        });
+        ws.on("close", (code, reason) => {
+            void onClose(code, reason.toString());
+        });
+        ws.on("error", (err) => {
+            logger.warn(`standin: the OpenAI Realtime socket failed: ${String(err)}`);
+        });
+    }
+    get isOpen() {
+        return this.#ws !== undefined && this.#ws.readyState === WebSocket.OPEN;
+    }
+    #send(message, droppable = false) {
+        const ws = this.#ws;
+        if (ws === undefined || ws.readyState !== WebSocket.OPEN)
+            return;
+        const payload = JSON.stringify(message);
+        if (droppable && this.#pendingBytes > MAX_SEND_BUFFER_BYTES) {
+            this.#dropped += 1;
+            const now = Date.now();
+            if (now - this.#lastDropWarning >= 1000) {
+                logger.warn(`standin: OpenAI send backpressure, dropped ${this.#dropped} chunk(s)`);
+                this.#lastDropWarning = now;
+                this.#dropped = 0;
+            }
+            return;
+        }
+        this.#pendingBytes += payload.length;
+        ws.send(payload, () => {
+            this.#pendingBytes -= payload.length;
+        });
+    }
+    sendSessionUpdate(update) {
+        this.#send(update);
+    }
+    /** Caller audio at the wire rate, resampled up to the model's rate. */
+    sendAudio(pcm) {
+        const atModelRate = resamplePcm16(pcm, SAMPLE_RATE_HZ, REALTIME_SAMPLE_RATE_HZ);
+        this.#send({
+            type: "input_audio_buffer.append",
+            audio: atModelRate.toString("base64"),
+        }, true);
+    }
+    /**
+     * Background context, with no response requested, so the model sees it on its
+     * next turn rather than interrupting to acknowledge it.
+     */
+    sendContext(text) {
+        this.#send({
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "user",
+                content: [{ type: "input_text", text }],
+            },
+        });
+    }
+    /** An interrupting user turn, which is how a goodbye gets spoken. */
+    sendUserTurn(text) {
+        this.#send({
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "user",
+                content: [{ type: "input_text", text }],
+            },
+        });
+        this.#send({ type: "response.create" });
+    }
+    cancelResponse() {
+        this.#send({ type: "response.cancel" });
+    }
+    /**
+     * Answer a tool call, then ask for the turn that speaks the answer. Without
+     * the second message the model has the result and says nothing about it.
+     */
+    sendToolResult(callId, output) {
+        this.#send({
+            type: "conversation.item.create",
+            item: { type: "function_call_output", call_id: callId, output },
+        });
+        this.#send({ type: "response.create" });
+    }
+    /** Close the session. Safe to call twice. */
+    async aclose() {
+        if (this.#closedByUs)
+            return;
+        this.#closedByUs = true;
+        const ws = this.#ws;
+        if (ws !== undefined && ws.readyState === WebSocket.OPEN)
+            ws.close(1000, "session-end");
+    }
+}
+//# sourceMappingURL=agent.js.map

package/dist/plugins/openai/agent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../src/plugins/openai/agent.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAE/B;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAE/B,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACxE,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,IAAI,GAAG,IAAI,CAAC;AAY1C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GACzB,WAAW,CAAC,QAAQ,CAAC,CAAC;AAExB,gDAAgD;AAChD,MAAM,CAAC,MAAM,mBAAmB,GAAwB,IAAI,GAAG,CAC7D,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAc,CAAC,CAClD,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,UAAU,OAAO,CACrB,KAA8B;IAE9B,IACE,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QACtC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,EACpC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC9D,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,MAAqB;IACnE,OAAO;QACL,IAAI,CAAC,IAAI,EAAE;QACX,wBAAwB,MAAM,CAAC,UAAU,6BAA6B;YACpE,GAAG,MAAM,CAAC,QAAQ,gBAAgB,MAAM,CAAC,SAAS,QAAQ;QAC5D,kFAAkF;YAChF,qFAAqF;YACrF,8EAA8E;KACjF,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjB,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,kBAAkB,CAChC,MAAoB,EACpB,YAAoB,EACpB,aAA6C,EAAE;IAE/C,MAAM,KAAK,GAA4B;QACrC,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,uBAAuB,EAAE;QAC5D,2EAA2E;QAC3E,yEAAyE;QACzE,kCAAkC;QAClC,cAAc,EAAE;YACd,IAAI,EAAE,MAAM,CAAC,OAAO;YACpB,eAAe,EAAE,IAAI;YACrB,kBAAkB,EAAE,IAAI;SACzB;KACF,CAAC;IACF,IAAI,MAAM,CAAC,kBAAkB;QAC3B,KAAK,CAAC,aAAa,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,kBAAkB,EAAE,CAAC;IAE7D,MAAM,MAAM,GAA4B;QACtC,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,uBAAuB,EAAE;KAC7D,CAAC;IACF,IAAI,MAAM,CAAC,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAE9C,OAAO;QACL,IAAI,EAAE,gBAAgB;QACtB,OAAO,EAAE;YACP,IAAI,EAAE,UAAU;YAChB,iBAAiB,EAAE,CAAC,OAAO,CAAC;YAC5B,YAAY;YACZ,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;YACxB,KAAK,EAAE,CAAC,GAAG,cAAc,EAAE,GAAG,UAAU,CAAC;YACzC,WAAW,EAAE,MAAM;SACpB;KACF,CAAC;AACJ,CAAC;AAeD,iDAAiD;AACjD,MAAM,OAAO,WAAW;IACtB,GAAG,CAAwB;IAC3B,aAAa,GAAG,CAAC,CAAC;IAClB,QAAQ,GAAG,CAAC,CAAC;IACb,gBAAgB,GAAG,CAAC,CAAC;IACrB,WAAW,GAAG,KAAK,CAAC;IAEpB,gBAAuB,CAAC;IAExB,8DAA8D;IAC9D,MAAM,CAAC,KAAK,CAAC,OAAO,CAClB,MAAoB,EACpB,SAA0D;IAC1D,4DAA4D;IAC5D,OAA8C,EAC9C,OAA+D;QAE/D,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,KAAK,IAAwB,EAAE;YAC1C,MAAM,GAAG,GAAG,SAAS,MAAM,CAAC,IAAI,sBAAsB,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACzF,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,EAAE;gBACrD,UAAU,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;aAC7B,CAAC,CAAC;YACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC3B,EAAE,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC,CAAC;gBACvE,CAAC,EAAE,kBAAkB,CAAC,CAAC;gBACvB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;gBACf,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE;oBACnB,YAAY,CAAC,IAAI,CAAC,CAAC;oBACnB,OAAO,EAAE,CAAC;gBACZ,CAAC,CAAC,CAAC;gBACH,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;oBACvB,YAAY,CAAC,IAAI,CAAC,CAAC;oBACnB,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YACH,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CACT,mCAAmC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CACjE,CAAC;YACF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;QAC5B,CAAC;QACD,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,CACL,SAA0D,EAC1D,OAA8C,EAC9C,OAA+D;QAE/D,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC;QACpB,IAAI,EAAE,KAAK,SAAS;YAAE,OAAO;QAE7B,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAW,EAAE,EAAE;YAC/B,IAAI,OAAgB,CAAC;YACrB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;gBACnE,OAAO;YACT,CAAC;YACD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI;gBAAE,OAAO;YAC5D,MAAM,KAAK,GAAG,OAAkC,CAAC;YACjD,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO;YAE3C,uEAAuE;YACvE,wCAAwC;YACxC,IACE,KAAK,CAAC,IAAI,KAAK,6BAA6B;gBAC5C,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,EAC/B,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBACvD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3B,MAAM,UAAU,GAAG,aAAa,CAC9B,WAAW,EACX,uBAAuB,EACvB,cAAc,CACf,CAAC;oBACF,KAAK,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;wBAC/D,MAAM,CAAC,KAAK,CACV,0CAA0C,MAAM,CAAC,GAAG,CAAC,EAAE,CACxD,CAAC;oBACJ,CAAC,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO;YACT,CAAC;YACD,KAAK,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,KAAqB,CAAC,CAAC,CAAC,KAAK,CAC1D,CAAC,GAAY,EAAE,EAAE;gBACf,MAAM,CAAC,KAAK,CACV,4BAA4B,KAAK,CAAC,IAAc,YAAY,MAAM,CAAC,GAAG,CAAC,EAAE,CAC1E,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;YAC9B,KAAK,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;QACH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACrB,MAAM,CAAC,IAAI,CAAC,+CAA+C,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,OAAgC,EAAE,SAAS,GAAG,KAAK;QACvD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC;QACpB,IAAI,EAAE,KAAK,SAAS,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI;YAAE,OAAO;QACjE,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,SAAS,IAAI,IAAI,CAAC,aAAa,GAAG,qBAAqB,EAAE,CAAC;YAC5D,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,GAAG,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CACT,8CAA8C,IAAI,CAAC,QAAQ,WAAW,CACvE,CAAC;gBACF,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC;gBAC5B,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACpB,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;QACrC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;QACvC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,iBAAiB,CAAC,MAA+B;QAC/C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,uEAAuE;IACvE,SAAS,CAAC,GAAW;QACnB,MAAM,WAAW,GAAG,aAAa,CAC/B,GAAG,EACH,cAAc,EACd,uBAAuB,CACxB,CAAC;QACF,IAAI,CAAC,KAAK,CACR;YACE,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC;SACtC,EACD,IAAI,CACL,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,IAAY;QACtB,IAAI,CAAC,KAAK,CAAC;YACT,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACJ,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aACxC;SACF,CAAC,CAAC;IACL,CAAC;IAED,qEAAqE;IACrE,YAAY,CAAC,IAAY;QACvB,IAAI,CAAC,KAAK,CAAC;YACT,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACJ,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aACxC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,cAAc;QACZ,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACH,cAAc,CAAC,MAAc,EAAE,MAAc;QAC3C,IAAI,CAAC,KAAK,CAAC;YACT,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;SAChE,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,6CAA6C;IAC7C,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC;QACpB,IAAI,EAAE,KAAK,SAAS,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI;YACtD,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAClC,CAAC;CACF"}

package/dist/plugins/openai/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/plugins/openai/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/plugins/openai/cli.ts"],"names":[],"mappings":""}

package/dist/plugins/openai/cli.js

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+/** `npx standin-openai` */
+import { serve } from "./index.js";
+serve().catch((err) => {
+    console.error(String(err));
+    process.exitCode = 1;
+});
+//# sourceMappingURL=cli.js.map

package/dist/plugins/openai/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/plugins/openai/cli.ts"],"names":[],"mappings":";AACA,oCAAoC;AACpC,+BAA+B;AAE/B,2BAA2B;AAE3B,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;IAC7B,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}

package/dist/plugins/openai/config.d.ts

@@ -0,0 +1,29 @@
+export declare const DEFAULT_INSTRUCTIONS: string;
+/** Everything the OpenAI plugin needs, resolved once per worker. */
+export interface OpenAIConfig {
+    /** `OPENAI_API_KEY`. Never logged, never sent to StandIn. */
+    apiKey: string;
+    /** `OPENAI_REALTIME_MODEL`. */
+    model: string;
+    /** `OPENAI_REALTIME_HOST`. */
+    host: string;
+    /** `OPENAI_VOICE`, or the model's default when unset. */
+    voice?: string;
+    /** `OPENAI_INSTRUCTIONS`: the agent's base prompt. */
+    instructions: string;
+    /**
+     * `OPENAI_VAD_TYPE`: `server_vad` hears silence, `semantic_vad` hears
+     * finished thoughts. Semantic interrupts less often mid-sentence.
+     */
+    vadType: "server_vad" | "semantic_vad";
+    /** `OPENAI_TRANSCRIPTION_MODEL`, to get transcripts alongside the audio. */
+    transcriptionModel?: string;
+    /**
+     * `OPENAI_LOG_TRANSCRIPTS`. Off by default, and gated a second time on the
+     * call actually being recorded.
+     */
+    logTranscripts: boolean;
+}
+/** Read the configuration, or throw naming the variable that is missing. */
+export declare function openAIConfigFromEnv(): OpenAIConfig;
+//# sourceMappingURL=config.d.ts.map

package/dist/plugins/openai/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../src/plugins/openai/config.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,oBAAoB,QAGd,CAAC;AAEpB,oEAAoE;AACpE,MAAM,WAAW,YAAY;IAC3B,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,yDAAyD;IACzD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,OAAO,EAAE,YAAY,GAAG,cAAc,CAAC;IACvC,4EAA4E;IAC5E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,cAAc,EAAE,OAAO,CAAC;CACzB;AASD,4EAA4E;AAC5E,wBAAgB,mBAAmB,IAAI,YAAY,CA0BlD"}

package/dist/plugins/openai/config.js

@@ -0,0 +1,37 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+/** What the OpenAI Realtime plugin reads from the environment. */
+import { StandInError } from "../../errors.js";
+/** Refuse a host that is not OpenAI or Azure: the API key travels to it. */
+const ALLOWED_HOST_SUFFIXES = [".openai.com", ".azure.com"];
+export const DEFAULT_INSTRUCTIONS = "You are a helpful voice assistant on a live Microsoft Teams call. You are speaking " +
+    "aloud: keep replies short, natural and conversational, and never use markdown, " +
+    "lists or emoji.";
+function required(name) {
+    const value = (process.env[name] ?? "").trim();
+    if (!value)
+        throw new StandInError(`${name} is required to answer calls with OpenAI`);
+    return value;
+}
+/** Read the configuration, or throw naming the variable that is missing. */
+export function openAIConfigFromEnv() {
+    const host = (process.env.OPENAI_REALTIME_HOST ?? "").trim() || "api.openai.com";
+    if (!ALLOWED_HOST_SUFFIXES.some((suffix) => host.endsWith(suffix))) {
+        throw new StandInError(`OPENAI_REALTIME_HOST must be an openai.com or azure.com host, got ${host}`);
+    }
+    const vad = (process.env.OPENAI_VAD_TYPE ?? "").trim() || "semantic_vad";
+    if (vad !== "server_vad" && vad !== "semantic_vad") {
+        throw new StandInError(`OPENAI_VAD_TYPE must be server_vad or semantic_vad, got ${vad}`);
+    }
+    return {
+        apiKey: required("OPENAI_API_KEY"),
+        model: (process.env.OPENAI_REALTIME_MODEL ?? "").trim() || "gpt-realtime",
+        host,
+        voice: (process.env.OPENAI_VOICE ?? "").trim() || undefined,
+        instructions: (process.env.OPENAI_INSTRUCTIONS ?? "").trim() || DEFAULT_INSTRUCTIONS,
+        vadType: vad,
+        transcriptionModel: (process.env.OPENAI_TRANSCRIPTION_MODEL ?? "").trim() || undefined,
+        logTranscripts: process.env.OPENAI_LOG_TRANSCRIPTS === "true",
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/plugins/openai/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../../src/plugins/openai/config.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAE/B,kEAAkE;AAElE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,4EAA4E;AAC5E,MAAM,qBAAqB,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;AAE5D,MAAM,CAAC,MAAM,oBAAoB,GAC/B,qFAAqF;IACrF,iFAAiF;IACjF,iBAAiB,CAAC;AA4BpB,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,IAAI,CAAC,KAAK;QACR,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,0CAA0C,CAAC,CAAC;IAC5E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,mBAAmB;IACjC,MAAM,IAAI,GACR,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,gBAAgB,CAAC;IACtE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACnE,MAAM,IAAI,YAAY,CACpB,qEAAqE,IAAI,EAAE,CAC5E,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,cAAc,CAAC;IACzE,IAAI,GAAG,KAAK,YAAY,IAAI,GAAG,KAAK,cAAc,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CACpB,2DAA2D,GAAG,EAAE,CACjE,CAAC;IACJ,CAAC;IACD,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,gBAAgB,CAAC;QAClC,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,cAAc;QACzE,IAAI;QACJ,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS;QAC3D,YAAY,EACV,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,oBAAoB;QACxE,OAAO,EAAE,GAAG;QACZ,kBAAkB,EAChB,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS;QACpE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,MAAM;KAC9D,CAAC;AACJ,CAAC"}

package/dist/plugins/openai/handler.d.ts

@@ -0,0 +1,65 @@
+/**
+ * The relay: one Microsoft Teams call on one side, one OpenAI Realtime session
+ * on the other.
+ *
+ * Speech to speech, so there is no transcription step in the middle and the
+ * model hears the caller's voice rather than a transcript of it. The one thing
+ * this plugin must get right that the others do not is the rate: the
+ * Realtime API speaks 24 kHz and the call speaks 16 kHz, and `agent.ts` owns
+ * that conversion so nothing here has to think about it.
+ *
+ * The call capabilities come from `standin/callTools`, so a capability added to
+ * the SDK reaches this plugin without an edit. Add your own with
+ * {@link CustomTool} and they are declared the same way.
+ */
+import type { CallHandler, CallSession } from "../../handler.js";
+import { FrameDescriber, type VideoFrame } from "../../vision.js";
+import { type AgentMessage, type AgentPort } from "./agent.js";
+import { type OpenAIConfig } from "./config.js";
+/** What a custom tool is told about the call it is running inside. */
+export interface ToolContext {
+    call: CallSession;
+    /** Whether the call is being recorded. Gate anything that stores what the caller showed. */
+    recording: boolean;
+}
+/** A function of your own, which the model calls and your code answers. */
+export interface CustomTool {
+    name: string;
+    description: string;
+    parameters?: Record<string, unknown>;
+    handler: (params: Record<string, unknown>, ctx: ToolContext) => string | Promise<string>;
+}
+/** How the handler reaches OpenAI. Replaceable so tests need no network. */
+export type Connect = (config: OpenAIConfig, onMessage: (message: AgentMessage) => void | Promise<void>, onAudio: (pcm: Buffer) => void | Promise<void>, onClose: (code: number, reason: string) => void | Promise<void>) => Promise<AgentPort>;
+/** Options for {@link OpenAIHandler}. */
+export interface OpenAIHandlerOptions {
+    config?: OpenAIConfig;
+    tools?: CustomTool[];
+    /** Remote MCP entries, normalised with `mcpTool`. Run server-side by OpenAI. */
+    mcpTools?: Array<Record<string, unknown>>;
+    describer?: FrameDescriber;
+    connect?: Connect;
+}
+/** One Microsoft Teams call answered by one OpenAI Realtime session. */
+export declare class OpenAIHandler implements CallHandler {
+    #private;
+    constructor(options?: OpenAIHandlerOptions);
+    onStart(session: CallSession): Promise<void>;
+    onCallerAudio(pcm: Buffer): Promise<void>;
+    /**
+     * Context arrives as a user item with no response requested, so the model
+     * reads it on its next turn instead of interrupting to acknowledge it.
+     */
+    /**
+     * Keep a short history, so a question about a slide already gone can still be
+     * answered. Only kept while the call is recorded.
+     */
+    onVideoFrame(frame: VideoFrame): Promise<void>;
+    onContext(text: string): Promise<void>;
+    /** Cancel whatever is being said, then ask for the goodbye. */
+    onGoodbye(text: string): Promise<void>;
+    aclose(): Promise<void>;
+    /** Run one tool and return what the model is told. Exposed for tests. */
+    dispatch(name: string, params: Record<string, unknown>): Promise<string>;
+}
+//# sourceMappingURL=handler.d.ts.map

package/dist/plugins/openai/handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../src/plugins/openai/handler.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAOjE,OAAO,EAAE,cAAc,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAElE,OAAO,EAKL,KAAK,YAAY,EACjB,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAuB,KAAK,YAAY,EAAE,MAAM,aAAa,CAAC;AAQrE,sEAAsE;AACtE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,WAAW,CAAC;IAClB,4FAA4F;IAC5F,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,2EAA2E;AAC3E,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,CACP,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,GAAG,EAAE,WAAW,KACb,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/B;AAED,4EAA4E;AAC5E,MAAM,MAAM,OAAO,GAAG,CACpB,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC1D,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC9C,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAC5D,OAAO,CAAC,SAAS,CAAC,CAAC;AAExB,yCAAyC;AACzC,MAAM,WAAW,oBAAoB;IACnC,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,wEAAwE;AACxE,qBAAa,aAAc,YAAW,WAAW;;gBAmBnC,OAAO,GAAE,oBAAyB;IA+BxC,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiE5C,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS/C;;;OAGG;IACH;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS5C,+DAA+D;IACzD,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAStC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAsJ7B,yEAAyE;IACnE,QAAQ,CACZ,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,OAAO,CAAC,MAAM,CAAC;CAkBnB"}