@komaa/standin-sdk 0.1.0

package/dist/startup.js

@@ -0,0 +1,112 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+/**
+ * What arrived before your agent was ready to hear it.
+ *
+ * A call starts the moment StandIn dials. Your agent starts a little later: a
+ * socket has to open, a room has to be joined, a session has to be configured.
+ * Everything the caller does in that gap still arrives, and if nothing holds it,
+ * it is gone.
+ *
+ * Two things arrive in that gap and both matter:
+ *
+ * **The caller's first words.** People start talking the instant the call
+ * connects, and often the first thing they say is the reason they called. Drop
+ * it and the agent opens by asking a question that was already answered.
+ *
+ * **The first context.** The "there are three people here, stay quiet unless
+ * addressed" sentence and the recording-status change both land within the first
+ * moment of a meeting. Drop those and a group-call gate never engages and a
+ * recording gate never opens.
+ *
+ * This exists because every plugin was solving half of it. Of the nine plugins
+ * in this SDK, two held both, and five held one and silently lost the other.
+ *
+ * Bounded on purpose. A socket that never opens must not grow a buffer for the
+ * length of the call, so the oldest entries are dropped rather than the newest.
+ *
+ * Identical in shape to the Python SDK's `standin.startup`.
+ */
+/**
+ * About four seconds of speech at the wire's frame size. Enough to hold an
+ * opening sentence, far short of enough to hide a socket that never opened.
+ */
+export const MAX_PENDING_AUDIO = 200;
+/** Context sentences are rare and each one is small. This is generous. */
+export const MAX_PENDING_CONTEXT = 20;
+/**
+ * Holds caller audio and call context until the agent can take them.
+ *
+ * Order is preserved within each lane, and audio is released before context,
+ * because the provider needs the caller's words in the order they were said and
+ * the context is a note about the call rather than part of the conversation.
+ */
+export class StartupBuffer {
+    #maxAudio;
+    #maxContext;
+    #audio = [];
+    #context = [];
+    #holding = true;
+    #droppedAudio = 0;
+    #droppedContext = 0;
+    constructor(maxAudio = MAX_PENDING_AUDIO, maxContext = MAX_PENDING_CONTEXT) {
+        this.#maxAudio = Math.max(1, maxAudio);
+        this.#maxContext = Math.max(1, maxContext);
+    }
+    /** Whether the agent is still being set up. */
+    get holding() {
+        return this.#holding;
+    }
+    /**
+     * How much was lost to the bounds.
+     *
+     * Worth logging when it is not zero: it means the agent took long enough to
+     * start that the caller outran it.
+     */
+    get dropped() {
+        return { audio: this.#droppedAudio, context: this.#droppedContext };
+    }
+    /** Hold one frame of the caller's voice. */
+    audio(pcm) {
+        if (pcm.length === 0)
+            return;
+        this.#audio.push(pcm);
+        if (this.#audio.length > this.#maxAudio) {
+            this.#audio.shift();
+            this.#droppedAudio += 1;
+        }
+    }
+    /** Hold one line of call context. */
+    context(text) {
+        if (!text)
+            return;
+        this.#context.push(text);
+        if (this.#context.length > this.#maxContext) {
+            this.#context.shift();
+            this.#droppedContext += 1;
+        }
+    }
+    /**
+     * Hand everything over, in order, and stop holding.
+     *
+     * Both callbacks may be sync or async, because a provider's send is often
+     * fire-and-forget. Safe to call twice: the second call releases nothing.
+     */
+    async release(sendAudio, sendContext) {
+        this.#holding = false;
+        const audio = this.#audio.splice(0);
+        const context = this.#context.splice(0);
+        for (const frame of audio)
+            await sendAudio?.(frame);
+        for (const line of context)
+            await sendContext?.(line);
+        return { audio: audio.length, context: context.length };
+    }
+    /** Throw it away. For a call that ended before the agent was ready. */
+    discard() {
+        this.#holding = false;
+        this.#audio = [];
+        this.#context = [];
+    }
+}
+//# sourceMappingURL=startup.js.map

package/dist/startup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"startup.js","sourceRoot":"","sources":["../src/startup.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAE/B;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAErC,0EAA0E;AAC1E,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAEtC;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IACf,SAAS,CAAS;IAClB,WAAW,CAAS;IAC7B,MAAM,GAAa,EAAE,CAAC;IACtB,QAAQ,GAAa,EAAE,CAAC;IACxB,QAAQ,GAAG,IAAI,CAAC;IAChB,aAAa,GAAG,CAAC,CAAC;IAClB,eAAe,GAAG,CAAC,CAAC;IAEpB,YAAY,QAAQ,GAAG,iBAAiB,EAAE,UAAU,GAAG,mBAAmB;QACxE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,+CAA+C;IAC/C,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACH,IAAI,OAAO;QACT,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC;IACtE,CAAC;IAED,4CAA4C;IAC5C,KAAK,CAAC,GAAW;QACf,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,OAAO,CAAC,IAAY;QAClB,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO,CACX,SAAiD,EACjD,WAAoD;QAEpD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxC,KAAK,MAAM,KAAK,IAAI,KAAK;YAAE,MAAM,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC;QACpD,KAAK,MAAM,IAAI,IAAI,OAAO;YAAE,MAAM,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC;QACtD,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IAC1D,CAAC;IAED,uEAAuE;IACvE,OAAO;QACL,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;IACrB,CAAC;CACF"}

package/dist/tile.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Putting your agent's own face on the bot's video tile.
+ *
+ * StandIn renders an avatar by default. When your agent already produces video
+ * of its own - an avatar worker, a rendered face, a camera - this streams that
+ * onto the tile instead, as a continuous run of `display.frame` messages.
+ *
+ * The awkward part is not sending frames. It is sending them at a rate that does
+ * not hurt the call, and that is what {@link TileStream} owns:
+ *
+ * **Latest wins, and each frame goes at most once.** Frames are offered into a
+ * single slot, never a queue. A ticker takes whatever is newest and sends it.
+ * That means a source producing faster than the wire drops the middle frames
+ * rather than falling behind, and a source that STOPS producing goes quiet
+ * rather than repeating one stale frame forever. Silence is how a stream ends.
+ *
+ * **The timestamp is the audio clock.** `ts` comes from
+ * {@link CallSession.mediaTimeMs}, the same timeline the outbound audio is
+ * stamped on. A wall clock keeps ticking through listening silence while the
+ * audio clock does not, so a wall-clock stamp makes the two streams look like
+ * they are drifting apart when they are in step.
+ *
+ * **Video yields to audio.** The budget here is tighter than the audio one on
+ * purpose. Both streams share a socket, and a caller forgives a dropped frame
+ * far more readily than a break in the voice.
+ *
+ * Encoding is yours to supply or ours to find. Pass an `encoder` and the stream
+ * uses it; pass nothing and it looks for sharp. If neither is there, the tile
+ * relay stays off with one line in the log and **the call is unaffected**.
+ *
+ * Identical in shape to the Python SDK's `standin.tile`.
+ */
+import type { CallSession } from "./handler.js";
+/**
+ * The tile size frames are encoded to. Shipping an avatar's native resolution
+ * only spends bandwidth on pixels the tile will not show.
+ */
+export declare const TILE_WIDTH = 640;
+export declare const TILE_HEIGHT = 360;
+/**
+ * A sender-side sanity clamp, not a protocol limit. A talking-head tile gains
+ * nothing above this, and a higher rate only spends local CPU on encoding and
+ * base64.
+ */
+export declare const MAX_TILE_FPS = 20;
+/** What an encoder is: packed RGB in, JPEG bytes out. */
+export type Encoder = (rgb: Buffer, width: number, height: number) => Promise<Buffer>;
+/**
+ * Find an encoder, or return undefined having said why.
+ *
+ * sharp is an optional peer precisely because most deployments never put their
+ * own video on the tile. A missing encoder is a tile relay that does not run,
+ * which is a smaller problem than a dependency every install pays for.
+ */
+export declare function jpegEncoder(): Promise<Encoder | undefined>;
+/** Options for {@link TileStream}. */
+export interface TileStreamOptions {
+    /** Frames per second, clamped to {@link MAX_TILE_FPS}. */
+    fps?: number;
+    /** Supply your own, or leave it and sharp is looked for. */
+    encoder?: Encoder;
+    /** Outbound bytes past which a frame is dropped rather than sent. */
+    maxBufferedBytes?: number;
+}
+/**
+ * A paced run of `display.frame` messages onto the bot's video tile.
+ *
+ * Built by a plugin that has video, driven by whatever produces it:
+ *
+ * ```ts
+ * const tile = new TileStream(session, { fps: 12 });
+ * await tile.start();
+ * tile.offerRgb(rgb, width, height);   // as often as you like
+ * await tile.aclose();
+ * ```
+ *
+ * {@link offerRgb} and {@link offerJpeg} never block and never throw. They are
+ * meant to be called from a drain loop that must not be slowed down by the wire.
+ */
+export declare class TileStream {
+    #private;
+    constructor(session: CallSession, options?: TileStreamOptions);
+    /** How many frames have reached the tile. */
+    get framesSent(): number;
+    /** How many were dropped for backpressure. A healthy call has some. */
+    get framesDropped(): number;
+    /** Offer packed RGB. Replaces whatever was waiting. */
+    offerRgb(rgb: Buffer, width: number, height: number): void;
+    /**
+     * Offer an already-encoded frame, skipping the encoder entirely.
+     *
+     * For a source that hands you JPEG already. Nothing is re-encoded, and no
+     * encoder needs to be installed.
+     */
+    offerJpeg(jpeg: Buffer, width?: number, height?: number): void;
+    /** Begin sending. Returns once armed; the pacing runs in the background. */
+    start(): Promise<void>;
+    /** Stop sending. Safe to call twice, and on every teardown path. */
+    aclose(): Promise<void>;
+}
+//# sourceMappingURL=tile.d.ts.map

package/dist/tile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tile.d.ts","sourceRoot":"","sources":["../src/tile.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGhD;;;GAGG;AACH,eAAO,MAAM,UAAU,MAAM,CAAC;AAC9B,eAAO,MAAM,WAAW,MAAM,CAAC;AAQ/B;;;;GAIG;AACH,eAAO,MAAM,YAAY,KAAK,CAAC;AAK/B,yDAAyD;AACzD,MAAM,MAAM,OAAO,GAAG,CACpB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,MAAM,CAAC,CAAC;AAErB;;;;;;GAMG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,OAAO,GAAG,SAAS,CAAC,CA6BhE;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAChC,0DAA0D;IAC1D,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,UAAU;;gBAiBT,OAAO,EAAE,WAAW,EAAE,OAAO,GAAE,iBAAsB;IAWjE,6CAA6C;IAC7C,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,uEAAuE;IACvE,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED,uDAAuD;IACvD,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAM1D;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,SAAa,EAAE,MAAM,SAAc,GAAG,IAAI;IAMvE,4EAA4E;IACtE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAa5B,oEAAoE;IAC9D,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAoE9B"}

package/dist/tile.js

@@ -0,0 +1,195 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+import { logger } from "./log.js";
+/**
+ * The tile size frames are encoded to. Shipping an avatar's native resolution
+ * only spends bandwidth on pixels the tile will not show.
+ */
+export const TILE_WIDTH = 640;
+export const TILE_HEIGHT = 360;
+/**
+ * Encoder quality. Chosen where a talking head still looks right and the frame
+ * still fits comfortably inside the wire envelope.
+ */
+const JPEG_QUALITY = 58;
+/**
+ * A sender-side sanity clamp, not a protocol limit. A talking-head tile gains
+ * nothing above this, and a higher rate only spends local CPU on encoding and
+ * base64.
+ */
+export const MAX_TILE_FPS = 20;
+/** Tighter than the audio buffer cap, deliberately. See the module comment. */
+const VIDEO_BACKPRESSURE_BYTES = 320 * 1024;
+/**
+ * Find an encoder, or return undefined having said why.
+ *
+ * sharp is an optional peer precisely because most deployments never put their
+ * own video on the tile. A missing encoder is a tile relay that does not run,
+ * which is a smaller problem than a dependency every install pays for.
+ */
+export async function jpegEncoder() {
+    try {
+        // A variable specifier, so the compiler does not require the optional
+        // module to be present; it is resolved only at runtime when enabled.
+        const specifier = "sharp";
+        const mod = (await import(specifier)).default;
+        return async (rgb, width, height) => mod(rgb, { raw: { width, height, channels: 3 } })
+            .resize(TILE_WIDTH, TILE_HEIGHT, { fit: "fill" })
+            .jpeg({ quality: JPEG_QUALITY })
+            .toBuffer();
+    }
+    catch {
+        logger.warn("standin: the avatar tile relay needs sharp to encode frames (npm install sharp); " +
+            "the relay is off and audio is unaffected");
+        return undefined;
+    }
+}
+/**
+ * A paced run of `display.frame` messages onto the bot's video tile.
+ *
+ * Built by a plugin that has video, driven by whatever produces it:
+ *
+ * ```ts
+ * const tile = new TileStream(session, { fps: 12 });
+ * await tile.start();
+ * tile.offerRgb(rgb, width, height);   // as often as you like
+ * await tile.aclose();
+ * ```
+ *
+ * {@link offerRgb} and {@link offerJpeg} never block and never throw. They are
+ * meant to be called from a drain loop that must not be slowed down by the wire.
+ */
+export class TileStream {
+    #session;
+    #fps;
+    #periodMs;
+    #maxBuffered;
+    #encoder;
+    #resolvedEncoder = false;
+    /** The single newest frame awaiting a send. Never a queue. */
+    #latest;
+    #ticker;
+    #encoding = false;
+    #closed = false;
+    #sent = 0;
+    #dropped = 0;
+    #lastDropLog = 0;
+    constructor(session, options = {}) {
+        this.#session = session;
+        this.#fps = Math.max(1, Math.min(Math.trunc(options.fps ?? 12), MAX_TILE_FPS));
+        this.#periodMs = Math.max(1, Math.round(1000 / this.#fps));
+        this.#encoder = options.encoder;
+        this.#maxBuffered = options.maxBufferedBytes ?? VIDEO_BACKPRESSURE_BYTES;
+    }
+    /** How many frames have reached the tile. */
+    get framesSent() {
+        return this.#sent;
+    }
+    /** How many were dropped for backpressure. A healthy call has some. */
+    get framesDropped() {
+        return this.#dropped;
+    }
+    /** Offer packed RGB. Replaces whatever was waiting. */
+    offerRgb(rgb, width, height) {
+        if (!this.#closed && rgb.length > 0) {
+            this.#latest = { data: rgb, width, height, jpeg: false };
+        }
+    }
+    /**
+     * Offer an already-encoded frame, skipping the encoder entirely.
+     *
+     * For a source that hands you JPEG already. Nothing is re-encoded, and no
+     * encoder needs to be installed.
+     */
+    offerJpeg(jpeg, width = TILE_WIDTH, height = TILE_HEIGHT) {
+        if (!this.#closed && jpeg.length > 0) {
+            this.#latest = { data: jpeg, width, height, jpeg: true };
+        }
+    }
+    /** Begin sending. Returns once armed; the pacing runs in the background. */
+    async start() {
+        if (this.#ticker !== undefined || this.#closed)
+            return;
+        if (this.#encoder === undefined && !this.#resolvedEncoder) {
+            this.#resolvedEncoder = true;
+            this.#encoder = await jpegEncoder();
+        }
+        this.#ticker = setInterval(() => void this.#tick(), this.#periodMs);
+        this.#ticker.unref?.();
+        logger.info(`standin: avatar tile relay armed at ${this.#fps} fps, ${TILE_WIDTH}x${TILE_HEIGHT}`);
+    }
+    /** Stop sending. Safe to call twice, and on every teardown path. */
+    async aclose() {
+        this.#closed = true;
+        this.#latest = undefined;
+        if (this.#ticker !== undefined) {
+            clearInterval(this.#ticker);
+            this.#ticker = undefined;
+        }
+    }
+    async #tick() {
+        if (this.#closed || this.#latest === undefined || this.#encoding)
+            return;
+        const frame = this.#latest;
+        // Consume the slot. Each offered frame is sent at most once, so a source
+        // that goes quiet leaves a silent wire rather than a frozen repeat.
+        this.#latest = undefined;
+        if (this.#overBudget())
+            return;
+        let data = frame.data;
+        let { width, height } = frame;
+        if (!frame.jpeg) {
+            const encoder = this.#encoder;
+            if (encoder === undefined)
+                return;
+            this.#encoding = true;
+            try {
+                data = await encoder(frame.data, frame.width, frame.height);
+                width = TILE_WIDTH;
+                height = TILE_HEIGHT;
+            }
+            catch (err) {
+                logger.warn(`standin: encoding an avatar frame failed: ${String(err)}`);
+                return;
+            }
+            finally {
+                this.#encoding = false;
+            }
+            if (this.#closed)
+                return;
+            // Re-check after the encode yielded: audio may have filled the socket
+            // while we were off the loop, and a video frame must not be what starves
+            // the voice.
+            if (this.#overBudget())
+                return;
+        }
+        this.#sent += 1;
+        try {
+            // The session owns the sequence and the timestamp, the same way it owns
+            // them for audio, so the two streams share one clock.
+            await this.#session.sendTileFrame(data, width, height);
+        }
+        catch {
+            // A dying socket is the close handler's business, not the ticker's.
+        }
+    }
+    #overBudget() {
+        let buffered = 0;
+        try {
+            buffered = this.#session.bufferedBytes;
+        }
+        catch {
+            buffered = 0;
+        }
+        if (buffered <= this.#maxBuffered)
+            return false;
+        this.#dropped += 1;
+        const now = Date.now();
+        if (now - this.#lastDropLog >= 5000) {
+            logger.info(`standin: avatar tile is dropping frames to protect the audio (${this.#dropped} so far)`);
+            this.#lastDropLog = now;
+        }
+        return true;
+    }
+}
+//# sourceMappingURL=tile.js.map

package/dist/tile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tile.js","sourceRoot":"","sources":["../src/tile.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAoC/B,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,GAAG,CAAC;AAC9B,MAAM,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC;AAE/B;;;GAGG;AACH,MAAM,YAAY,GAAG,EAAE,CAAC;AAExB;;;;GAIG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,EAAE,CAAC;AAE/B,+EAA+E;AAC/E,MAAM,wBAAwB,GAAG,GAAG,GAAG,IAAI,CAAC;AAS5C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,IAAI,CAAC;QACH,sEAAsE;QACtE,qEAAqE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC;QAC1B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,OAWrC,CAAC;QACF,OAAO,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAClC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;aAC9C,MAAM,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;aAChD,IAAI,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;aAC/B,QAAQ,EAAE,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,CAAC,IAAI,CACT,mFAAmF;YACjF,0CAA0C,CAC7C,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAYD;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,UAAU;IACZ,QAAQ,CAAc;IACtB,IAAI,CAAS;IACb,SAAS,CAAS;IAClB,YAAY,CAAS;IAC9B,QAAQ,CAAsB;IAC9B,gBAAgB,GAAG,KAAK,CAAC;IACzB,8DAA8D;IAC9D,OAAO,CACsE;IAC7E,OAAO,CAA6B;IACpC,SAAS,GAAG,KAAK,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAChB,KAAK,GAAG,CAAC,CAAC;IACV,QAAQ,GAAG,CAAC,CAAC;IACb,YAAY,GAAG,CAAC,CAAC;IAEjB,YAAY,OAAoB,EAAE,UAA6B,EAAE;QAC/D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAClB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,YAAY,CAAC,CACtD,CAAC;QACF,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,gBAAgB,IAAI,wBAAwB,CAAC;IAC3E,CAAC;IAED,6CAA6C;IAC7C,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,uEAAuE;IACvE,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,uDAAuD;IACvD,QAAQ,CAAC,GAAW,EAAE,KAAa,EAAE,MAAc;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAC3D,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,IAAY,EAAE,KAAK,GAAG,UAAU,EAAE,MAAM,GAAG,WAAW;QAC9D,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACvD,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1D,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CACT,uCAAuC,IAAI,CAAC,IAAI,SAAS,UAAU,IAAI,WAAW,EAAE,CACrF,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QACzB,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QACzE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3B,yEAAyE;QACzE,oEAAoE;QACpE,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;QACzB,IAAI,IAAI,CAAC,WAAW,EAAE;YAAE,OAAO;QAE/B,IAAI,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACtB,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC9B,IAAI,OAAO,KAAK,SAAS;gBAAE,OAAO;YAClC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC5D,KAAK,GAAG,UAAU,CAAC;gBACnB,MAAM,GAAG,WAAW,CAAC;YACvB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,6CAA6C,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACxE,OAAO;YACT,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACzB,CAAC;YACD,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO;YACzB,sEAAsE;YACtE,yEAAyE;YACzE,aAAa;YACb,IAAI,IAAI,CAAC,WAAW,EAAE;gBAAE,OAAO;QACjC,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC;YACH,wEAAwE;YACxE,sDAAsD;YACtD,MAAM,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,oEAAoE;QACtE,CAAC;IACH,CAAC;IAED,WAAW;QACT,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;QACzC,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC;QACD,IAAI,QAAQ,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,KAAK,CAAC;QAChD,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CACT,iEAAiE,IAAI,CAAC,QAAQ,UAAU,CACzF,CAAC;YACF,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;QAC1B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/version.d.ts

@@ -0,0 +1,2 @@
+export declare const VERSION = "0.1.0";
+//# sourceMappingURL=version.d.ts.map

package/dist/version.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,OAAO,UAAU,CAAC"}

package/dist/version.js

@@ -0,0 +1,4 @@
+// Copyright (c) 2026 Komaa DigiTech
+// SPDX-License-Identifier: MIT
+export const VERSION = "0.1.0";
+//# sourceMappingURL=version.js.map

package/dist/version.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.js","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":"AAAA,oCAAoC;AACpC,+BAA+B;AAE/B,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}

package/dist/vision.d.ts

@@ -0,0 +1,160 @@
+/** The two things a caller can show: their camera, or their screen share. */
+export declare const VIDEO_SOURCES: readonly ["camera", "screenshare"];
+/** Which lane a frame came from. */
+export type VideoSource = (typeof VIDEO_SOURCES)[number];
+/** What StandIn will draw on the bot tile. JPEG or PNG, nothing else. */
+export declare const DISPLAY_IMAGE_MIME_TYPES: readonly ["image/jpeg", "image/png"];
+/** A MIME type StandIn will draw. */
+export type DisplayImageMime = (typeof DISPLAY_IMAGE_MIME_TYPES)[number];
+/** `"fullscreen"` replaces the tile; `"overlay"` draws a picture-in-picture inset. */
+export type DisplayImageMode = "fullscreen" | "overlay";
+/**
+ * One wire message is bounded at 2 MB by both SDKs, and base64 costs a third on
+ * top of the raw bytes. Refusing an oversized image here names the real problem,
+ * rather than letting the service close the socket mid-call.
+ */
+export declare const MAX_IMAGE_BYTES = 1400000;
+/**
+ * One sampled frame of what the caller is showing.
+ *
+ * `participantId` and `participantName` are best-effort and absent for guest and
+ * anonymous participants, so a group-call prompt that says who is sharing must
+ * tolerate not knowing.
+ */
+export interface VideoFrame {
+    /** Which lane this came from. */
+    readonly source: VideoSource;
+    /** Capture time in milliseconds. */
+    readonly ts: number;
+    /** Pixel width, already downscaled by StandIn before sending. */
+    readonly width: number;
+    /** Pixel height. */
+    readonly height: number;
+    /** Image MIME type. StandIn sends `image/jpeg`. */
+    readonly mime: string;
+    /**
+     * The image, base64-encoded, exactly as it arrived.
+     *
+     * Kept in this form because it is the form most providers want back: a `data:`
+     * URL for a vision model costs one template string from here, while
+     * {@link VideoFrame.data} costs a decode.
+     */
+    readonly dataBase64: string;
+    /** Whose frame this is, when StandIn could tell. */
+    readonly participantId?: string;
+    /** Display name matching {@link VideoFrame.participantId}. */
+    readonly participantName?: string;
+    /** The decoded image bytes, for an API that uploads a file. */
+    readonly data: Buffer;
+    /** The frame as a `data:` URL, which is what most vision models take. */
+    readonly dataUrl: string;
+}
+/**
+ * Read a `video.frame`, or return `undefined` if it is unusable.
+ *
+ * Never throws. A frame that fails any check is a frame to drop: the call is
+ * healthy, the caller is still talking, and one malformed image is not worth
+ * ending a conversation over. That is the same leniency the rest of the wire
+ * contract is built on, where a receiver ignores what it cannot use.
+ */
+export declare function parseVideoFrame(msg: Record<string, unknown>): VideoFrame | undefined;
+/** Options for {@link displayImage}. */
+export interface DisplayImageOptions {
+    /** `image/jpeg` (the default) or `image/png`. */
+    mime?: string;
+    /** How long to show it. StandIn applies its own default when omitted. */
+    durationMs?: number;
+    /** `fullscreen` (StandIn's default) or `overlay`. */
+    mode?: DisplayImageMode;
+    /** Short label drawn along the bottom of the image. */
+    caption?: string;
+}
+/**
+ * Build a `display.image`: show the caller a still, then return to the avatar.
+ *
+ * `image` is a Buffer or an already-base64 string.
+ */
+export declare function displayImage(image: Buffer | string, options?: DisplayImageOptions): string;
+/** Options for {@link displayFrame}. */
+export interface DisplayFrameOptions {
+    /** Frame encoding; senders send `image/jpeg`. */
+    mime?: string;
+    /** Source pixel width (informational). */
+    width?: number;
+    /** Source pixel height (informational). */
+    height?: number;
+}
+/**
+ * Build a `display.frame`: one frame of continuous avatar video.
+ *
+ * Latest wins. There is no handshake, the first frames start the stream and
+ * silence ends it, and a sender under backpressure MUST drop frames rather than
+ * buffer them, exactly as it does for hot-path audio.
+ *
+ * `ts` belongs to the sender's own media timeline, the same one its outbound
+ * audio is stamped on, so the two streams share a clock.
+ */
+export declare function displayFrame(seq: number, ts: number, image: Buffer | string, options?: DisplayFrameOptions): string;
+/** How to reach the vision model. */
+export interface FrameDescriberOptions {
+    /** Chat-completions endpoint, for example `https://api.openai.com/v1/chat/completions`. */
+    url: string;
+    /** The vision model to ask. */
+    model: string;
+    /** Sent as a bearer token when set. A local endpoint usually needs none. */
+    apiKey?: string;
+}
+/**
+ * Turn a frame into a sentence, using a vision model you choose.
+ *
+ * Most speech-to-speech providers hear but cannot see. This is what lets one
+ * answer "what is on my screen?": the frame goes to any OpenAI-compatible
+ * chat-completions endpoint that accepts image input (OpenAI, Azure OpenAI,
+ * Ollama, vLLM, whatever you run), and what comes back is text the agent can say
+ * out loud.
+ *
+ * The frame is sent for inference and not stored, which is the difference
+ * between this and uploading it into a provider's own conversation history.
+ *
+ * Deliberately NOT put through the guard in `fetch.ts`: this URL is yours, set
+ * by you in the environment, and a vision model on localhost is a normal way to
+ * run one. That is the opposite of an image URL a model chose.
+ */
+/**
+ * A short, stable fingerprint of one frame.
+ *
+ * For asking "is this the same screen as last time?" without keeping the
+ * picture. A hash of the encoded form is enough: two encodes of an unchanged
+ * screen are byte-identical.
+ */
+export declare function frameDigest(dataBase64: string): string;
+/** Who is showing this, when the wire said. */
+export declare function frameOwner(frame: VideoFrame): string | undefined;
+/**
+ * Who to say it is when nobody was named.
+ *
+ * Attribution that degrades rather than vanishing: "a participant's screen" is
+ * worth more to a model than an unlabelled picture.
+ */
+export declare function fallbackOwner(source: string): string;
+/** The sentence that goes beside a frame, so a model knows whose it is. */
+export declare function frameCaption(owner: string): string;
+export declare class FrameDescriber {
+    readonly url: string;
+    readonly model: string;
+    readonly apiKey: string | undefined;
+    constructor(options: FrameDescriberOptions);
+    /**
+     * Build one from `STANDIN_VISION_API_URL` and `STANDIN_VISION_MODEL`.
+     *
+     * Returns `undefined` when they are not set, which is the signal a
+     * plugin uses to tell an agent that looking is not available here.
+     */
+    static fromEnv(): FrameDescriber | undefined;
+    /**
+     * Ask the model about one frame. Throws on anything that goes wrong, so a
+     * caller can hand the reason back to the agent that asked.
+     */
+    describe(frame: VideoFrame, question: string): Promise<string>;
+}
+//# sourceMappingURL=vision.d.ts.map

package/dist/vision.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../src/vision.ts"],"names":[],"mappings":"AAkCA,6EAA6E;AAC7E,eAAO,MAAM,aAAa,oCAAqC,CAAC;AAEhE,oCAAoC;AACpC,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzD,yEAAyE;AACzE,eAAO,MAAM,wBAAwB,sCAAuC,CAAC;AAE7E,qCAAqC;AACrC,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzE,sFAAsF;AACtF,MAAM,MAAM,gBAAgB,GAAG,YAAY,GAAG,SAAS,CAAC;AAExD;;;;GAIG;AACH,eAAO,MAAM,eAAe,UAAY,CAAC;AAEzC;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,iCAAiC;IACjC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,oBAAoB;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,mDAAmD;IACnD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,oDAAoD;IACpD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,8DAA8D;IAC9D,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,+DAA+D;IAC/D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAwBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC3B,UAAU,GAAG,SAAS,CA2BxB;AAmCD,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC,iDAAiD;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,IAAI,CAAC,EAAE,gBAAgB,CAAC;IACxB,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,OAAO,GAAE,mBAAwB,GAChC,MAAM,CAiBR;AAED,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC,iDAAiD;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,GAAG,EAAE,MAAM,EACX,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,OAAO,GAAE,mBAAwB,GAChC,MAAM,CAYR;AAQD,qCAAqC;AACrC,MAAM,WAAW,qBAAqB;IACpC,2FAA2F;IAC3F,GAAG,EAAE,MAAM,CAAC;IACZ,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;;;;GAeG;AACH;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,+CAA+C;AAC/C,wBAAgB,UAAU,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,GAAG,SAAS,CAGhE;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIlD;AAED,qBAAa,cAAc;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;gBAExB,OAAO,EAAE,qBAAqB;IAM1C;;;;;OAKG;IACH,MAAM,CAAC,OAAO,IAAI,cAAc,GAAG,SAAS;IAQ5C;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CA+CrE"}