@komaa/standin-sdk 0.1.0

package/dist/vision.js

@@ -0,0 +1,301 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+/**
+ * The vision lane: what the caller shows you, and what you show back.
+ *
+ * A Microsoft Teams call carries more than voice. StandIn samples the caller's
+ * camera and their screen share and forwards single JPEG frames, and it will
+ * draw an image you send onto the bot's own tile. This module is both halves of
+ * that: {@link parseVideoFrame} reads what arrives, {@link displayImage} builds
+ * what goes back.
+ *
+ * Frames arrive **sparsely and best-effort**. StandIn drops a frame rather than
+ * queueing it when the socket is busy, so this is not a video stream and must
+ * not be treated as one. The useful shape is the one every provider plugin
+ * ends up with: keep the latest frame per source and send it to a vision model
+ * only when something asks to look. {@link CallSession.latestVideoFrame} does
+ * that for you, so a plugin that only wants on-demand vision implements
+ * no callback at all.
+ *
+ * {@link FrameDescriber} is the other way round, and the one most voice
+ * providers need: a speech-to-speech model that hears but cannot see gets a
+ * sentence of text instead of a picture. The frame goes to a vision model of
+ * your choosing, transiently, and only the description comes back.
+ *
+ * Identical in shape to the Python SDK's `standin.vision`, translated to TS
+ * naming.
+ */
+import { createHash } from "node:crypto";
+import { encode } from "./protocolRuntime.js";
+import { TYPE_DISPLAY_FRAME, TYPE_DISPLAY_IMAGE } from "./protocol.js";
+/** The two things a caller can show: their camera, or their screen share. */
+export const VIDEO_SOURCES = ["camera", "screenshare"];
+/** What StandIn will draw on the bot tile. JPEG or PNG, nothing else. */
+export const DISPLAY_IMAGE_MIME_TYPES = ["image/jpeg", "image/png"];
+/**
+ * One wire message is bounded at 2 MB by both SDKs, and base64 costs a third on
+ * top of the raw bytes. Refusing an oversized image here names the real problem,
+ * rather than letting the service close the socket mid-call.
+ */
+export const MAX_IMAGE_BYTES = 1_400_000;
+function text(value) {
+    if (typeof value !== "string")
+        return undefined;
+    const trimmed = value.trim();
+    return trimmed === "" ? undefined : trimmed;
+}
+function positiveInt(value) {
+    return typeof value === "number" && Number.isInteger(value) && value > 0
+        ? value
+        : undefined;
+}
+function decodeStrict(dataBase64) {
+    const data = Buffer.from(dataBase64, "base64");
+    // Buffer.from never throws on bad input, it silently discards what it cannot
+    // read. Re-encoding is the only way to know the bytes about to reach a
+    // provider really are the payload that was sent.
+    return data.length > 0 && data.toString("base64") === dataBase64
+        ? data
+        : undefined;
+}
+/**
+ * Read a `video.frame`, or return `undefined` if it is unusable.
+ *
+ * Never throws. A frame that fails any check is a frame to drop: the call is
+ * healthy, the caller is still talking, and one malformed image is not worth
+ * ending a conversation over. That is the same leniency the rest of the wire
+ * contract is built on, where a receiver ignores what it cannot use.
+ */
+export function parseVideoFrame(msg) {
+    const source = text(msg.source);
+    if (source !== "camera" && source !== "screenshare")
+        return undefined;
+    const dataBase64 = msg.dataBase64;
+    if (typeof dataBase64 !== "string" || dataBase64 === "")
+        return undefined;
+    const data = decodeStrict(dataBase64);
+    if (data === undefined)
+        return undefined;
+    const width = positiveInt(msg.width);
+    const height = positiveInt(msg.height);
+    if (width === undefined || height === undefined)
+        return undefined;
+    const ts = msg.ts;
+    const mime = text(msg.mime) ?? "image/jpeg";
+    return {
+        source,
+        ts: typeof ts === "number" && Number.isInteger(ts) && ts >= 0 ? ts : 0,
+        width,
+        height,
+        mime,
+        dataBase64,
+        participantId: text(msg.participantId),
+        participantName: text(msg.participantName),
+        data,
+        dataUrl: `data:${mime};base64,${dataBase64}`,
+    };
+}
+function encodeImage(image, mime, label) {
+    if (!DISPLAY_IMAGE_MIME_TYPES.includes(mime)) {
+        throw new Error(`${label} mime must be one of ${DISPLAY_IMAGE_MIME_TYPES.join(", ")}, got ${mime}`);
+    }
+    let size;
+    let dataBase64;
+    if (typeof image === "string") {
+        // Already base64: measure the decoded size, because that is what the 2 MB
+        // envelope actually bounds.
+        const decoded = decodeStrict(image);
+        if (decoded === undefined)
+            throw new Error(`${label} data is not valid base64`);
+        size = decoded.length;
+        dataBase64 = image;
+    }
+    else {
+        size = image.length;
+        dataBase64 = image.toString("base64");
+    }
+    if (size === 0)
+        throw new Error(`${label} carries no image data`);
+    if (size > MAX_IMAGE_BYTES) {
+        throw new Error(`${label} is ${size} bytes, over the ${MAX_IMAGE_BYTES} limit`);
+    }
+    return dataBase64;
+}
+/**
+ * Build a `display.image`: show the caller a still, then return to the avatar.
+ *
+ * `image` is a Buffer or an already-base64 string.
+ */
+export function displayImage(image, options = {}) {
+    const mime = options.mime ?? "image/jpeg";
+    const message = {
+        type: TYPE_DISPLAY_IMAGE,
+        dataBase64: encodeImage(image, mime, "display.image"),
+        mime,
+        // The wire reserves a timeline anchor this lane does not use. Both SDKs
+        // send 0 rather than one omitting it, so a single conformance vector
+        // covers both and neither can drift.
+        ts: 0,
+    };
+    if (options.durationMs !== undefined && options.durationMs > 0) {
+        message.durationMs = options.durationMs;
+    }
+    if (options.mode)
+        message.mode = options.mode;
+    if (options.caption)
+        message.caption = options.caption;
+    return encode(message);
+}
+/**
+ * Build a `display.frame`: one frame of continuous avatar video.
+ *
+ * Latest wins. There is no handshake, the first frames start the stream and
+ * silence ends it, and a sender under backpressure MUST drop frames rather than
+ * buffer them, exactly as it does for hot-path audio.
+ *
+ * `ts` belongs to the sender's own media timeline, the same one its outbound
+ * audio is stamped on, so the two streams share a clock.
+ */
+export function displayFrame(seq, ts, image, options = {}) {
+    const mime = options.mime ?? "image/jpeg";
+    const message = {
+        type: TYPE_DISPLAY_FRAME,
+        seq,
+        ts,
+        mime,
+        dataBase64: encodeImage(image, mime, "display.frame"),
+    };
+    if (options.width !== undefined)
+        message.width = options.width;
+    if (options.height !== undefined)
+        message.height = options.height;
+    return encode(message);
+}
+/** Hard bound on the vision round trip. The caller hears this as silence. */
+const DESCRIBE_TIMEOUT_MS = 20_000;
+/** Enough for a sentence or two read aloud. A voice agent cannot relay an essay. */
+const DESCRIBE_MAX_TOKENS = 300;
+/**
+ * Turn a frame into a sentence, using a vision model you choose.
+ *
+ * Most speech-to-speech providers hear but cannot see. This is what lets one
+ * answer "what is on my screen?": the frame goes to any OpenAI-compatible
+ * chat-completions endpoint that accepts image input (OpenAI, Azure OpenAI,
+ * Ollama, vLLM, whatever you run), and what comes back is text the agent can say
+ * out loud.
+ *
+ * The frame is sent for inference and not stored, which is the difference
+ * between this and uploading it into a provider's own conversation history.
+ *
+ * Deliberately NOT put through the guard in `fetch.ts`: this URL is yours, set
+ * by you in the environment, and a vision model on localhost is a normal way to
+ * run one. That is the opposite of an image URL a model chose.
+ */
+/**
+ * A short, stable fingerprint of one frame.
+ *
+ * For asking "is this the same screen as last time?" without keeping the
+ * picture. A hash of the encoded form is enough: two encodes of an unchanged
+ * screen are byte-identical.
+ */
+export function frameDigest(dataBase64) {
+    return createHash("sha256")
+        .update(dataBase64 ?? "", "ascii")
+        .digest("hex")
+        .slice(0, 32);
+}
+/** Who is showing this, when the wire said. */
+export function frameOwner(frame) {
+    const name = (frame.participantName ?? "").trim();
+    return name === "" ? undefined : name;
+}
+/**
+ * Who to say it is when nobody was named.
+ *
+ * Attribution that degrades rather than vanishing: "a participant's screen" is
+ * worth more to a model than an unlabelled picture.
+ */
+export function fallbackOwner(source) {
+    return source === "screenshare" ? "a participant" : "the caller";
+}
+/** The sentence that goes beside a frame, so a model knows whose it is. */
+export function frameCaption(owner) {
+    return owner === "a participant"
+        ? `screen shared by ${owner}`
+        : `camera of ${owner}`;
+}
+export class FrameDescriber {
+    url;
+    model;
+    apiKey;
+    constructor(options) {
+        this.url = options.url;
+        this.model = options.model;
+        this.apiKey = options.apiKey;
+    }
+    /**
+     * Build one from `STANDIN_VISION_API_URL` and `STANDIN_VISION_MODEL`.
+     *
+     * Returns `undefined` when they are not set, which is the signal a
+     * plugin uses to tell an agent that looking is not available here.
+     */
+    static fromEnv() {
+        const url = (process.env.STANDIN_VISION_API_URL ?? "").trim();
+        const model = (process.env.STANDIN_VISION_MODEL ?? "").trim();
+        if (!url || !model)
+            return undefined;
+        const apiKey = (process.env.STANDIN_VISION_API_KEY ?? "").trim();
+        return new FrameDescriber({ url, model, apiKey: apiKey || undefined });
+    }
+    /**
+     * Ask the model about one frame. Throws on anything that goes wrong, so a
+     * caller can hand the reason back to the agent that asked.
+     */
+    async describe(frame, question) {
+        const who = frameOwner(frame) ?? fallbackOwner(frame.source);
+        const seeing = frameCaption(who);
+        const headers = {
+            "content-type": "application/json",
+        };
+        if (this.apiKey)
+            headers.authorization = `Bearer ${this.apiKey}`;
+        const controller = new AbortController();
+        const deadline = setTimeout(() => controller.abort(), DESCRIBE_TIMEOUT_MS);
+        try {
+            const response = await fetch(this.url, {
+                method: "POST",
+                headers,
+                signal: controller.signal,
+                body: JSON.stringify({
+                    model: this.model,
+                    max_tokens: DESCRIBE_MAX_TOKENS,
+                    messages: [
+                        {
+                            role: "user",
+                            content: [
+                                {
+                                    type: "text",
+                                    text: `This is a live frame from a Microsoft Teams call (${seeing}). ` +
+                                        `Answer concisely, for a voice agent to say out loud. Question: ${question}`,
+                                },
+                                { type: "image_url", image_url: { url: frame.dataUrl } },
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                throw new Error(`the vision model returned HTTP ${response.status}`);
+            }
+            const data = (await response.json());
+            const text = (data.choices?.[0]?.message?.content ?? "").trim();
+            if (!text)
+                throw new Error("the vision model returned nothing");
+            return text;
+        }
+        finally {
+            clearTimeout(deadline);
+        }
+    }
+}
+//# sourceMappingURL=vision.js.map

package/dist/vision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision.js","sourceRoot":"","sources":["../src/vision.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAE/B;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEvE,6EAA6E;AAC7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,QAAQ,EAAE,aAAa,CAAU,CAAC;AAKhE,yEAAyE;AACzE,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,YAAY,EAAE,WAAW,CAAU,CAAC;AAQ7E;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,SAAS,CAAC;AAsCzC,SAAS,IAAI,CAAC,KAAc;IAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AAC9C,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QACtE,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,UAAkB;IACtC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/C,6EAA6E;IAC7E,uEAAuE;IACvE,iDAAiD;IACjD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,UAAU;QAC9D,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAC7B,GAA4B;IAE5B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,aAAa;QAAE,OAAO,SAAS,CAAC;IAEtE,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;IAClC,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IAC1E,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAEzC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAElE,MAAM,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;IAClB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC;IAC5C,OAAO;QACL,MAAM;QACN,EAAE,EAAE,OAAO,EAAE,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACtE,KAAK;QACL,MAAM;QACN,IAAI;QACJ,UAAU;QACV,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC;QACtC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC;QAC1C,IAAI;QACJ,OAAO,EAAE,QAAQ,IAAI,WAAW,UAAU,EAAE;KAC7C,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,KAAsB,EACtB,IAAY,EACZ,KAAa;IAEb,IAAI,CAAE,wBAA8C,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,wBAAwB,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CACnF,CAAC;IACJ,CAAC;IACD,IAAI,IAAY,CAAC;IACjB,IAAI,UAAkB,CAAC;IACvB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,0EAA0E;QAC1E,4BAA4B;QAC5B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,OAAO,KAAK,SAAS;YACvB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC;QACvD,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;QACtB,UAAU,GAAG,KAAK,CAAC;IACrB,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC;QACpB,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,IAAI,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAC;IAClE,IAAI,IAAI,GAAG,eAAe,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,OAAO,IAAI,oBAAoB,eAAe,QAAQ,CAC/D,CAAC;IACJ,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAcD;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAsB,EACtB,UAA+B,EAAE;IAEjC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,YAAY,CAAC;IAC1C,MAAM,OAAO,GAA4B;QACvC,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,eAAe,CAAC;QACrD,IAAI;QACJ,wEAAwE;QACxE,qEAAqE;QACrE,qCAAqC;QACrC,EAAE,EAAE,CAAC;KACN,CAAC;IACF,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;QAC/D,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC9C,IAAI,OAAO,CAAC,OAAO;QAAE,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACvD,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC;AACzB,CAAC;AAYD;;;;;;;;;GASG;AACH,MAAM,UAAU,YAAY,CAC1B,GAAW,EACX,EAAU,EACV,KAAsB,EACtB,UAA+B,EAAE;IAEjC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,YAAY,CAAC;IAC1C,MAAM,OAAO,GAA4B;QACvC,IAAI,EAAE,kBAAkB;QACxB,GAAG;QACH,EAAE;QACF,IAAI;QACJ,UAAU,EAAE,WAAW,CAAC,KAAK,EAAE,IAAI,EAAE,eAAe,CAAC;KACtD,CAAC;IACF,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC/D,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAClE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC;AACzB,CAAC;AAED,6EAA6E;AAC7E,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAEnC,oFAAoF;AACpF,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAYhC;;;;;;;;;;;;;;;GAeG;AACH;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,UAAkB;IAC5C,OAAO,UAAU,CAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,UAAU,IAAI,EAAE,EAAE,OAAO,CAAC;SACjC,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,+CAA+C;AAC/C,MAAM,UAAU,UAAU,CAAC,KAAiB;IAC1C,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;AACxC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,MAAc;IAC1C,OAAO,MAAM,KAAK,aAAa,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC;AACnE,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,OAAO,KAAK,KAAK,eAAe;QAC9B,CAAC,CAAC,oBAAoB,KAAK,EAAE;QAC7B,CAAC,CAAC,aAAa,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,OAAO,cAAc;IAChB,GAAG,CAAS;IACZ,KAAK,CAAS;IACd,MAAM,CAAqB;IAEpC,YAAY,OAA8B;QACxC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,OAAO;QACZ,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9D,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QACrC,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjE,OAAO,IAAI,cAAc,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAiB,EAAE,QAAgB;QAChD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,OAAO,GAA2B;YACtC,cAAc,EAAE,kBAAkB;SACnC,CAAC;QACF,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,aAAa,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;QAEjE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,mBAAmB,CAAC,CAAC;QAC3E,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACrC,MAAM,EAAE,MAAM;gBACd,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,mBAAmB;oBAC/B,QAAQ,EAAE;wBACR;4BACE,IAAI,EAAE,MAAM;4BACZ,OAAO,EAAE;gCACP;oCACE,IAAI,EAAE,MAAM;oCACZ,IAAI,EACF,qDAAqD,MAAM,KAAK;wCAChE,kEAAkE,QAAQ,EAAE;iCAC/E;gCACD,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE;6BACzD;yBACF;qBACF;iBACF,CAAC;aACH,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,kCAAkC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;YACF,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAChE,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAChE,OAAO,IAAI,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;CACF"}

package/dist/visionTools.d.ts

@@ -0,0 +1,237 @@
+import type { CallSession } from "./handler.js";
+import { type DisplayImageMode, type FrameDescriber, type VideoFrame, type VideoSource } from "./vision.js";
+/** The only two ways a picture can be put on the tile. */
+export declare const DISPLAY_MODES: readonly ["fullscreen", "overlay"];
+/** How long each picture of a slideshow stays up before the next one. */
+export declare const SLIDESHOW_HOLD_MS = 4000;
+/**
+ * How much longer than the gap each picture is held for.
+ *
+ * Without it the tile blanks for the moment between one picture expiring and
+ * the next arriving, and the caller sees a flicker rather than a slideshow.
+ */
+export declare const SLIDESHOW_OVERLAP_MS = 500;
+/**
+ * How many pictures one slideshow may hold.
+ *
+ * A model handed a folder will pass the whole folder. Ten at four seconds is
+ * already most of a minute of a live call spent looking at pictures.
+ */
+export declare const MAX_SLIDESHOW_IMAGES = 10;
+/** How long a page has to render before the caller is told it did not. */
+export declare const PAGE_RENDER_TIMEOUT_MS = 45000;
+/**
+ * How long a rendered page stays on the tile. Longer than a chart: a page is
+ * read rather than glanced at.
+ */
+export declare const PAGE_DISPLAY_MS = 15000;
+/**
+ * The display mode this value means, or the default.
+ *
+ * One rule, shared by both SDKs and every plugin, because the value comes from
+ * a model: `"pip"`, `"inset"`, `"full"` and nothing at all are all things a
+ * model will say, and none of them is a mode.
+ */
+export declare function normalizeDisplayMode(value: unknown, fallback?: DisplayImageMode): DisplayImageMode | undefined;
+/**
+ * A filename to show beside a picture.
+ *
+ * Taken from the source when it looks like a filename and nothing else. The
+ * string came from a model steered by whoever is on the call, and it is about
+ * to be shown to them.
+ */
+export declare function displayImageName(pathOrUrl: string, mime: string): string;
+/** The last picture the caller actually saw. */
+export declare class ShownImage {
+    readonly image: Buffer | string;
+    readonly mime: string;
+    readonly name: string;
+    readonly atMs: number;
+    constructor(image: Buffer | string, mime: string, name: string, atMs: number);
+    asBase64(): string;
+}
+/**
+ * Renders one page to bytes plus a mime type. Supplied by a plugin whose host
+ * already runs a browser; the core never gains one.
+ */
+export type PageRenderer = (url: string) => Promise<{
+    bytes: Buffer | string;
+    mime: string;
+}>;
+/** One picture in a slideshow. Bytes, base64, or an https URL. */
+export interface ShowItem {
+    image: Buffer | string;
+    mime?: string;
+    name?: string;
+}
+/**
+ * A ceiling on how often one call may spend on vision.
+ *
+ * A model that can look can look in a loop, and each look is a paid inference
+ * over somebody's screen. A sliding window rather than a total, so a long call
+ * is not punished for having been long.
+ *
+ * Spending returns a token and refunding takes that token back. Two tool calls
+ * can overlap, and a refund that simply dropped "the most recent charge" would
+ * refund the wrong one and let the budget drift upward under exactly the load it
+ * exists to bound.
+ */
+export declare class VisionBudget {
+    #private;
+    readonly maxPerMinute: number;
+    constructor(maxPerMinute?: number);
+    /** Take one look's worth of budget, or undefined when there is none left. */
+    tryConsume(): string | undefined;
+    /**
+     * How much of the window only an explicit look may spend.
+     *
+     * Ambient vision spends on every scene change, which is exactly the load that
+     * would leave a caller's own "look at this" with nothing left. The reserve is
+     * what the ambient lane cannot touch.
+     */
+    get reserve(): number;
+    /**
+     * Take one look's worth, from the ambient lane only.
+     *
+     * Refused once the window is down to the reserve. Refunded through the same
+     * {@link refund}, with the same token, so a failed ambient push and a failed
+     * explicit look are given back the same way.
+     */
+    tryConsumeAmbient(): string | undefined;
+    /** Give back a charge whose look never happened. Idempotent. */
+    refund(token: string): void;
+    /** Looks charged in the current window. */
+    get spent(): number;
+}
+/**
+ * A short history of what the caller showed.
+ *
+ * The call session keeps the LATEST frame per source, which answers "what am I
+ * looking at now". This answers "what was on that slide a moment ago".
+ *
+ * Bounded, and **gated on the call being recorded**. Keeping a history of
+ * somebody's screen is a materially different promise from glancing at it once,
+ * and the recording is the thing that told them their call is being kept.
+ */
+export declare class KeyframeStore {
+    #private;
+    constructor(capacity?: number);
+    /** Keep this frame, if the call is being recorded. Returns whether it was kept. */
+    offer(frame: VideoFrame, recording: boolean): boolean;
+    /** Frames kept so far, oldest first. */
+    recent(source?: VideoSource): VideoFrame[];
+    /** Forget everything. Called on teardown. */
+    clear(): void;
+    get size(): number;
+}
+/** One beat of a walkthrough: something to say, optionally something to show. */
+export interface WalkthroughStep {
+    /** The line spoken before the image appears. */
+    say: string;
+    image?: Buffer | string;
+    mime?: string;
+    caption?: string;
+}
+/**
+ * Says one line and resolves when the caller has heard it. Supplied by the
+ * plugin, because "finished speaking" is a thing only the provider knows.
+ */
+export type Speaker = (text: string) => Promise<void>;
+/** Options for {@link VisionTools}. */
+export interface VisionToolsOptions {
+    describer?: FrameDescriber;
+    budget?: VisionBudget;
+    keyframes?: KeyframeStore;
+    /**
+     * What to use when the model says nothing. Left out, no mode is sent at all,
+     * so the service's own default applies rather than one chosen here.
+     */
+    defaultDisplayMode?: string;
+}
+/**
+ * The capabilities, bound to one call.
+ *
+ * Every method returns a sentence for a model to read out, including when it
+ * failed. None of them throw.
+ */
+export declare class VisionTools {
+    #private;
+    readonly budget: VisionBudget;
+    readonly keyframes: KeyframeStore;
+    constructor(session: CallSession, options?: VisionToolsOptions);
+    /**
+     * The picture the caller can see, if any.
+     *
+     * Recorded only after a send actually returned, so "send me that" attaches
+     * what they saw rather than what was attempted. One slot, replaced each time:
+     * a list would be a growing copy of everything shown on the call.
+     */
+    get lastShown(): ShownImage | undefined;
+    /** The slideshow now running, if any. Await it to let one finish. */
+    get slideshow(): Promise<void> | undefined;
+    /** Forget what was shown and stop any slideshow. Call this on teardown. */
+    reset(): Promise<void>;
+    /**
+     * Answer a question about what the caller is showing.
+     *
+     * Uses the newest frame, preferring the screen share, because an agent asked
+     * to look is nearly always being asked about what is being shown rather than
+     * who is showing it.
+     */
+    look(question?: string, source?: string): Promise<string>;
+    /**
+     * Answer about a frame the caller has already moved past.
+     *
+     * Only possible when the call is being recorded, because that is the only time
+     * frames are kept at all.
+     */
+    lookBack(question?: string): Promise<string>;
+    /** Put an image on the bot's video tile. */
+    show(image: Buffer | string, mime?: string, caption?: string, durationMs?: number, display?: string, name?: string): Promise<string>;
+    /**
+     * Fetch an image the model chose, and show it.
+     *
+     * The URL comes from a model steered by whoever is on the call, so it goes
+     * through the SDK's guard: public hosts only, and the address re-checked at
+     * connect time.
+     */
+    showUrl(url: string, caption?: string, display?: string): Promise<string>;
+    /**
+     * Put a web page on the tile, as a picture of it.
+     *
+     * The core has no browser and must never gain one. `render` is supplied by a
+     * plugin whose host already runs one, and it returns bytes rather than a
+     * path: reading a file chosen downstream of whoever is on the call is not a
+     * primitive this belongs in.
+     *
+     * The guard runs HERE, before the renderer is reached, and it runs even when
+     * that renderer is a browser advertising private-network protection of its
+     * own. Such a browser assumes whoever wrote the URL already has a shell on
+     * the machine. Here the URL was written by a model being steered by a
+     * stranger, which is exactly the case that relaxation lets through.
+     */
+    showPage(url: string, caption?: string, render?: PageRenderer, timeoutMs?: number): Promise<string>;
+    /**
+     * Show several pictures in turn, without waiting for all of them.
+     *
+     * The FIRST one goes before this resolves, so the model can say "here it is"
+     * and be right. The rest are paced from a detached chain: a model that waits
+     * out a ten-picture slideshow before speaking leaves the caller in silence for
+     * most of a minute.
+     *
+     * Never throws. The sentence says what is on screen now and what follows.
+     */
+    showMany(items: readonly ShowItem[], caption?: string, display?: string, holdMs?: number): Promise<string>;
+    /**
+     * Say and show several things in order, pausing for each.
+     *
+     * The pacing is here; the SPEAKING is not. Only the provider knows when a line
+     * has finished being said, so `speak` is supplied by the plugin and awaited
+     * before the next beat begins. Without that, a walkthrough talks over itself.
+     *
+     * `interrupted` is checked between beats. A caller who cuts in should stop the
+     * tour, and the plugin is the only thing that knows they did.
+     */
+    walkthrough(steps: readonly WalkthroughStep[], speak: Speaker, interrupted?: () => boolean, display?: string): Promise<string>;
+}
+//# sourceMappingURL=visionTools.d.ts.map

package/dist/visionTools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"visionTools.d.ts","sourceRoot":"","sources":["../src/visionTools.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,WAAW,EAEjB,MAAM,aAAa,CAAC;AAWrB,0DAA0D;AAC1D,eAAO,MAAM,aAAa,oCAAqC,CAAC;AAEhE,yEAAyE;AACzE,eAAO,MAAM,iBAAiB,OAAQ,CAAC;AAEvC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,MAAM,CAAC;AAExC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAIvC,0EAA0E;AAC1E,eAAO,MAAM,sBAAsB,QAAS,CAAC;AAE7C;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAYtC;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,OAAO,EACd,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,gBAAgB,GAAG,SAAS,CAO9B;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAQxE;AAED,gDAAgD;AAChD,qBAAa,UAAU;IAEnB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM;gBAHZ,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM;IAGvB,QAAQ,IAAI,MAAM;CAKnB;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,CACzB,GAAG,EAAE,MAAM,KACR,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAqBvD,kEAAkE;AAClE,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,YAAY;;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;gBAGlB,YAAY,SAAI;IAI5B,6EAA6E;IAC7E,UAAU,IAAI,MAAM,GAAG,SAAS;IAYhC;;;;;;OAMG;IACH,IAAI,OAAO,IAAI,MAAM,CAGpB;IAED;;;;;;OAMG;IACH,iBAAiB,IAAI,MAAM,GAAG,SAAS;IAMvC,gEAAgE;IAChE,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI3B,2CAA2C;IAC3C,IAAI,KAAK,IAAI,MAAM,CAGlB;CACF;AAED;;;;;;;;;GASG;AACH,qBAAa,aAAa;;gBAMZ,QAAQ,SAAK;IAIzB,mFAAmF;IACnF,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO;IAerD,wCAAwC;IACxC,MAAM,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,UAAU,EAAE;IAM1C,6CAA6C;IAC7C,KAAK,IAAI,IAAI;IAKb,IAAI,IAAI,IAAI,MAAM,CAEjB;CACF;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC9B,gDAAgD;IAChD,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,MAAM,OAAO,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEtD,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;GAKG;AACH,qBAAa,WAAW;;IAGtB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;gBAOtB,OAAO,EAAE,WAAW,EAAE,OAAO,GAAE,kBAAuB;IAQlE;;;;;;OAMG;IACH,IAAI,SAAS,IAAI,UAAU,GAAG,SAAS,CAEtC;IAED,qEAAqE;IACrE,IAAI,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAEzC;IAED,2EAA2E;IACrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B;;;;;;OAMG;IACG,IAAI,CAAC,QAAQ,SAAK,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAmC3D;;;;;OAKG;IACG,QAAQ,CAAC,QAAQ,SAAK,GAAG,OAAO,CAAC,MAAM,CAAC;IAyB9C,4CAA4C;IACtC,IAAI,CACR,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,IAAI,SAAe,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,CAAC;IA8BlB;;;;;;OAMG;IACG,OAAO,CACX,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;IAkBlB;;;;;;;;;;;;;OAaG;IACG,QAAQ,CACZ,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,YAAY,EACrB,SAAS,SAAyB,GACjC,OAAO,CAAC,MAAM,CAAC;IAiClB;;;;;;;;;OASG;IACG,QAAQ,CACZ,KAAK,EAAE,SAAS,QAAQ,EAAE,EAC1B,OAAO,CAAC,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,SAAoB,GACzB,OAAO,CAAC,MAAM,CAAC;IAkIlB;;;;;;;;;OASG;IACG,WAAW,CACf,KAAK,EAAE,SAAS,eAAe,EAAE,EACjC,KAAK,EAAE,OAAO,EACd,WAAW,CAAC,EAAE,MAAM,OAAO,EAC3B,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;CA8BnB"}