@nodetool-ai/runtime 0.7.0-rc.23 → 0.8.1

package/dist/providers/video-frame-fallback.js

@@ -0,0 +1,155 @@
+/**
+ * Read a video with a vision model that has no video content part.
+ *
+ * Most chat APIs are OpenAI-compatible and have nowhere to put a clip, so a
+ * `video` part reaching them used to be a hard refusal — even when the model
+ * behind the endpoint reads images perfectly well. This samples the clip into
+ * stills with ffmpeg and puts them on the message as ordinary image parts,
+ * labelled with their timestamps so the model can reason about order and
+ * timing.
+ *
+ * It is a downgrade, and it says so: the header text names the frame count and
+ * the sample rate, so a model that needs continuous motion or the audio track
+ * can report that it is missing rather than inventing it. A provider that
+ * reads video natively (`supportsVideoInput`) never comes through here.
+ */
+import { safeFetch } from "./safe-url.js";
+import { sampleVideoFrames } from "./video-frames.js";
+import { createLogger } from "@nodetool-ai/config";
+const log = createLogger("nodetool:video-frame-fallback");
+/**
+ * Whether the fallback is on. `NODETOOL_VIDEO_FRAME_FALLBACK=0` turns it off
+ * and restores the old refusal, for a caller that would rather fail than
+ * silently pay for a lossy read.
+ */
+export function videoFrameFallbackEnabled() {
+    const raw = globalThis.process?.env?.["NODETOOL_VIDEO_FRAME_FALLBACK"];
+    return raw !== "0" && raw !== "false";
+}
+/** Whether any message carries a `video` content part. */
+function messagesContainVideo(messages) {
+    return messages.some((m) => Array.isArray(m.content) && m.content.some((c) => c.type === "video"));
+}
+function decodeBase64(b64) {
+    return new Uint8Array(Buffer.from(b64, "base64"));
+}
+/** The bytes behind a `data:` URI. */
+function parseDataUri(uri) {
+    const comma = uri.indexOf(",");
+    if (comma < 0)
+        throw new Error("Invalid data URI");
+    return decodeBase64(uri.slice(comma + 1));
+}
+/**
+ * Source a video part's bytes: inline data first, then its URI. The container
+ * type is not read off the ref — ffmpeg sniffs it from the file itself, which
+ * is right more often than a `mimeType` the caller guessed.
+ */
+async function videoContentBytes(video, options) {
+    if (typeof video.data === "string" && video.data.length > 0) {
+        return video.data.startsWith("data:")
+            ? parseDataUri(video.data)
+            : decodeBase64(video.data);
+    }
+    if (video.data instanceof Uint8Array && video.data.length > 0) {
+        return video.data;
+    }
+    if (!video.uri)
+        return new Uint8Array();
+    const resolved = video.uri.startsWith("data:")
+        ? video.uri
+        : ((await options.resolveUri?.(video.uri)) ?? video.uri);
+    if (resolved.startsWith("data:"))
+        return parseDataUri(resolved);
+    const resp = await safeFetch(resolved, options.signal ? { signal: options.signal } : undefined);
+    if (!resp.ok)
+        throw new Error(`Failed to fetch video: ${resp.status}`);
+    return new Uint8Array(await resp.arrayBuffer());
+}
+function round(value, digits = 1) {
+    return value.toFixed(digits).replace(/\.0+$/, "");
+}
+/** The header that tells the model what it is looking at, and what it is not. */
+export function frameHeaderText(sample) {
+    const count = sample.frames.length;
+    const duration = sample.durationSec !== null ? `${round(sample.durationSec)}s ` : "";
+    const rate = sample.fps >= 1
+        ? `${round(sample.fps, 2)} frames/second`
+        : `one frame every ${round(1 / sample.fps, 1)}s`;
+    const cut = sample.truncated
+        ? ` Sampling stopped at the frame limit, so the end of the clip is not shown.`
+        : "";
+    return (`[The following ${count} still frame${count === 1 ? "" : "s"} were ` +
+        `sampled from a ${duration}video at ${rate}, in order. They are stills: ` +
+        `there is no audio and no motion between them.${cut}]`);
+}
+/** One frame as an image part, preceded by its timestamp. */
+function framePartsFor(sample) {
+    const parts = [];
+    for (const [index, frame] of sample.frames.entries()) {
+        parts.push({
+            type: "text",
+            text: `Frame ${index + 1} at ${round(frame.timeSec)}s:`
+        });
+        parts.push({
+            type: "image_url",
+            image: {
+                uri: `data:${frame.mimeType};base64,${Buffer.from(frame.data).toString("base64")}`,
+                mimeType: frame.mimeType
+            }
+        });
+    }
+    return parts;
+}
+/**
+ * Replace every `video` content part with sampled frames. Messages with no
+ * video are returned untouched (same object identity), so this is cheap to
+ * call on every request.
+ */
+export async function expandVideoContentAsFrames(messages, options = {}) {
+    if (!messagesContainVideo(messages))
+        return messages;
+    const out = [];
+    for (const message of messages) {
+        if (!Array.isArray(message.content) ||
+            !message.content.some((c) => c.type === "video")) {
+            out.push(message);
+            continue;
+        }
+        const parts = [];
+        for (const part of message.content) {
+            if (part.type !== "video") {
+                parts.push(part);
+                continue;
+            }
+            const bytes = await videoContentBytes(part.video, options);
+            if (bytes.length === 0) {
+                parts.push({
+                    type: "text",
+                    text: "[A video was attached but its bytes could not be read.]"
+                });
+                continue;
+            }
+            const sample = await sampleVideoFrames(bytes, {
+                signal: options.signal
+            });
+            if (sample.frames.length === 0) {
+                throw new Error(`ffmpeg decoded no frames from the attached video, so ` +
+                    `${options.provider ?? "this provider"} has nothing to read. The ` +
+                    `file may not be a video, or may use a codec this ffmpeg build ` +
+                    `does not decode.`);
+            }
+            log.debug("Sampled a video into frames", {
+                provider: options.provider,
+                frames: sample.frames.length,
+                fps: sample.fps,
+                durationSec: sample.durationSec
+            });
+            parts.push({ type: "text", text: frameHeaderText(sample) });
+            parts.push(...framePartsFor(sample));
+        }
+        out.push({ ...message, content: parts });
+    }
+    return out;
+}
+//# sourceMappingURL=video-frame-fallback.js.map

package/dist/providers/video-frame-fallback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-frame-fallback.js","sourceRoot":"","sources":["../../src/providers/video-frame-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAyB,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,GAAG,GAAG,YAAY,CAAC,+BAA+B,CAAC,CAAC;AAU1D;;;;GAIG;AACH,MAAM,UAAU,yBAAyB;IACvC,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,+BAA+B,CAAC,CAAC;IACvE,OAAO,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,OAAO,CAAC;AACxC,CAAC;AAED,0DAA0D;AAC1D,SAAS,oBAAoB,CAAC,QAA4B;IACxD,OAAO,QAAQ,CAAC,IAAI,CAClB,CAAC,CAAC,EAAE,EAAE,CACJ,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CACxE,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,sCAAsC;AACtC,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACnD,OAAO,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,iBAAiB,CAC9B,KAAmC,EACnC,OAAkC;IAElC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YACnC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;YAC1B,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,YAAY,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9D,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,GAAG;QAAE,OAAO,IAAI,UAAU,EAAE,CAAC;IAExC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;QAC5C,CAAC,CAAC,KAAK,CAAC,GAAG;QACX,CAAC,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEhE,MAAM,IAAI,GAAG,MAAM,SAAS,CAC1B,QAAQ,EACR,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CACxD,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACvE,OAAO,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,KAAK,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;AACpD,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,eAAe,CAAC,MAAwB;IACtD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;IACnC,MAAM,QAAQ,GACZ,MAAM,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACtE,MAAM,IAAI,GACR,MAAM,CAAC,GAAG,IAAI,CAAC;QACb,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,gBAAgB;QACzC,CAAC,CAAC,mBAAmB,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC;IACrD,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS;QAC1B,CAAC,CAAC,4EAA4E;QAC9E,CAAC,CAAC,EAAE,CAAC;IACP,OAAO,CACL,kBAAkB,KAAK,eAAe,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ;QACpE,kBAAkB,QAAQ,YAAY,IAAI,+BAA+B;QACzE,gDAAgD,GAAG,GAAG,CACvD,CAAC;AACJ,CAAC;AAED,6DAA6D;AAC7D,SAAS,aAAa,CAAC,MAAwB;IAC7C,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,SAAS,KAAK,GAAG,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI;SACxD,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE;gBACL,GAAG,EAAE,QAAQ,KAAK,CAAC,QAAQ,WAAW,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gBAClF,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACzB;SACF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,QAAmB,EACnB,OAAO,GAA8B,EAAE;IAEvC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAAE,OAAO,QAAQ,CAAC;IAErD,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;YAC/B,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,EAChD,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAClC,IAA4B,CAAC,KAAK,EACnC,OAAO,CACR,CAAC;YACF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,yDAAyD;iBAChE,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE;gBAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CACb,uDAAuD;oBACrD,GAAG,OAAO,CAAC,QAAQ,IAAI,eAAe,4BAA4B;oBAClE,gEAAgE;oBAChE,kBAAkB,CACrB,CAAC;YACJ,CAAC;YACD,GAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE;gBACvC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;gBAC5B,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC,CAAC,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/providers/video-frames.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Sample a video into still frames with ffmpeg.
+ *
+ * This is what lets a vision model that cannot read a clip still answer
+ * questions about one: the bytes are decoded here, a bounded number of frames
+ * come back as JPEGs, and the caller sends them as ordinary image content.
+ *
+ * The argv is ours, not a model's — the only input path is a temp file this
+ * module wrote — so the containment `packages/agents/src/host-binary-guard.ts`
+ * applies to a model-supplied argv is not needed. `-protocol_whitelist file`
+ * and `-nostdin` still go in: an mp4 can name an external opener, and a child
+ * that inherits the server's stdin outlives the call.
+ */
+/** A single sampled frame, JPEG-encoded. */
+export type SampledFrame = {
+    data: Uint8Array;
+    mimeType: string;
+    /** Seconds into the clip, derived from the sample rate. */
+    timeSec: number;
+};
+export type VideoFrameSample = {
+    frames: SampledFrame[];
+    /** Clip length from ffprobe, or null when the probe did not answer. */
+    durationSec: number | null;
+    /** Frames per second actually requested of ffmpeg. */
+    fps: number;
+    /**
+     * True when the frame cap stopped the sample before the end of the clip.
+     * Only reachable with an unknown duration: once ffprobe answers, the rate is
+     * chosen to fit the budget across the whole clip and the cap never bites.
+     */
+    truncated: boolean;
+};
+export type SampleVideoFramesOptions = {
+    /** Hard cap on frames returned. */
+    maxFrames?: number;
+    /** Ceiling on the sample rate; a short clip is sampled no denser than this. */
+    maxFps?: number;
+    /** Longest edge of each frame, in pixels. Frames are never upscaled. */
+    maxDimension?: number;
+    signal?: AbortSignal;
+};
+/** ffmpeg or ffprobe is not installed. Names the binary and the way out. */
+export declare class FrameSamplingUnavailableError extends Error {
+    readonly binary: string;
+    constructor(binary: string);
+}
+/** Read a video duration from its bytes without decoding frames. */
+export declare function probeVideoDurationSeconds(bytes: Uint8Array, signal: AbortSignal): Promise<number | null>;
+/**
+ * The sample rate that spreads `maxFrames` across the whole clip without
+ * exceeding `maxFps`. An unknown duration samples at the ceiling and lets the
+ * frame cap stop it.
+ */
+export declare function frameRateFor(durationSec: number | null, maxFrames: number, maxFps: number): number;
+/**
+ * Decode `bytes` and return up to `maxFrames` JPEG stills spread across the
+ * clip. Throws {@link FrameSamplingUnavailableError} when ffmpeg is absent.
+ */
+export declare function sampleVideoFrames(bytes: Uint8Array, options?: SampleVideoFramesOptions): Promise<VideoFrameSample>;
+//# sourceMappingURL=video-frames.d.ts.map

package/dist/providers/video-frames.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-frames.d.ts","sourceRoot":"","sources":["../../src/providers/video-frames.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,4CAA4C;AAC5C,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,uEAAuE;IACvE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,sDAAsD;IACtD,GAAG,EAAE,MAAM,CAAC;IACZ;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,mCAAmC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,4EAA4E;AAC5E,qBAAa,6BAA8B,SAAQ,KAAK;IACtD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,YAAY,MAAM,EAAE,MAAM,EASzB;CACF;AAwHD,oEAAoE;AACpE,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAuBxB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,EAC1B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,GACb,MAAM,CAMR;AAYD;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,gBAAgB,CAAC,CA4D3B"}

package/dist/providers/video-frames.js

@@ -0,0 +1,208 @@
+/**
+ * Sample a video into still frames with ffmpeg.
+ *
+ * This is what lets a vision model that cannot read a clip still answer
+ * questions about one: the bytes are decoded here, a bounded number of frames
+ * come back as JPEGs, and the caller sends them as ordinary image content.
+ *
+ * The argv is ours, not a model's — the only input path is a temp file this
+ * module wrote — so the containment `packages/agents/src/host-binary-guard.ts`
+ * applies to a model-supplied argv is not needed. `-protocol_whitelist file`
+ * and `-nostdin` still go in: an mp4 can name an external opener, and a child
+ * that inherits the server's stdin outlives the call.
+ */
+import { importNodeBuiltin } from "@nodetool-ai/config";
+import { withSpan } from "../tracing-helpers.js";
+import { mp4DurationSeconds } from "./video-duration.js";
+/** ffmpeg or ffprobe is not installed. Names the binary and the way out. */
+export class FrameSamplingUnavailableError extends Error {
+    binary;
+    constructor(binary) {
+        super(`${binary} is not installed or not on PATH. Reading a video with a ` +
+            `provider that has no video content part samples frames with ffmpeg ` +
+            `first — install ffmpeg (the Package Manager UI can do this), or use ` +
+            `a provider whose chat models read video natively (gemini).`);
+        this.name = "FrameSamplingUnavailableError";
+        this.binary = binary;
+    }
+}
+const DEFAULT_MAX_FRAMES = 16;
+const DEFAULT_MAX_FPS = 1;
+const DEFAULT_MAX_DIMENSION = 768;
+/** ffmpeg `-q:v` for the JPEG encoder (2 = best, 31 = worst). */
+const JPEG_QUALITY = 4;
+const FFMPEG_MAX_BUFFER = 8 * 1024 * 1024;
+/** Read a positive number from the environment, or fall back. */
+function envNumber(name, fallback) {
+    const raw = Number(globalThis.process?.env?.[name]);
+    return Number.isFinite(raw) && raw > 0 ? raw : fallback;
+}
+/** Frame-sampling limits, env-overridable. Read per call so a test can vary them. */
+function frameSamplingLimits() {
+    return {
+        maxFrames: Math.floor(envNumber("NODETOOL_VIDEO_FRAME_MAX_FRAMES", DEFAULT_MAX_FRAMES)),
+        maxFps: envNumber("NODETOOL_VIDEO_FRAME_MAX_FPS", DEFAULT_MAX_FPS),
+        maxDimension: Math.floor(envNumber("NODETOOL_VIDEO_FRAME_MAX_DIMENSION", DEFAULT_MAX_DIMENSION))
+    };
+}
+async function nodeModules() {
+    const [cp, fs, os, path] = await Promise.all([
+        importNodeBuiltin("node:child_process"),
+        importNodeBuiltin("node:fs/promises"),
+        importNodeBuiltin("node:os"),
+        importNodeBuiltin("node:path")
+    ]);
+    if (!cp || !fs || !os || !path) {
+        throw new Error("Video frame sampling requires Node built-in modules");
+    }
+    return { cp, fs, os, path };
+}
+/** Whether a spawn failed because the binary is not on PATH. */
+function isSpawnEnoent(err) {
+    return (typeof err === "object" &&
+        err !== null &&
+        err.code === "ENOENT");
+}
+async function run(cp, binary, args, signal) {
+    return await new Promise((resolve, reject) => {
+        cp.execFile(binary, args, { maxBuffer: FFMPEG_MAX_BUFFER, signal, windowsHide: true }, (error, stdout, stderr) => {
+            if (error) {
+                if (isSpawnEnoent(error)) {
+                    reject(new FrameSamplingUnavailableError(binary));
+                    return;
+                }
+                reject(error);
+                return;
+            }
+            resolve({ stdout: String(stdout), stderr: String(stderr) });
+        });
+    });
+}
+/** Clip length in seconds, or null when ffprobe cannot say. */
+async function probeDuration(cp, file, signal) {
+    try {
+        const { stdout } = await run(cp, "ffprobe", [
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "csv=p=0",
+            file
+        ], signal);
+        const seconds = Number(stdout.trim().split("\n")[0]);
+        return Number.isFinite(seconds) && seconds > 0 ? seconds : null;
+    }
+    catch (error) {
+        if (signal?.aborted)
+            throw error;
+        // A missing binary is actionable and must reach the caller. A probe that
+        // ran and failed (an odd container, no format header) is not: the sample
+        // falls back to the ceiling rate below.
+        if (error instanceof FrameSamplingUnavailableError)
+            throw error;
+        return null;
+    }
+}
+/** Read a video duration from its bytes without decoding frames. */
+export async function probeVideoDurationSeconds(bytes, signal) {
+    signal.throwIfAborted();
+    const mp4Duration = mp4DurationSeconds(bytes);
+    if (mp4Duration !== null) {
+        signal.throwIfAborted();
+        return mp4Duration;
+    }
+    return withSpan("media.probe_video_duration", { "media.byte_length": bytes.byteLength }, async () => {
+        const { cp, fs, os, path } = await nodeModules();
+        const dir = await fs.mkdtemp(path.join(os.tmpdir(), "nodetool-vduration-"));
+        const input = path.join(dir, "input.bin");
+        try {
+            await fs.writeFile(input, bytes, { signal });
+            return await probeDuration(cp, input, signal);
+        }
+        finally {
+            // Cleanup failure must not replace the duration result or probe error.
+            await fs.rm(dir, { recursive: true, force: true }).catch(() => { });
+        }
+    });
+}
+/**
+ * The sample rate that spreads `maxFrames` across the whole clip without
+ * exceeding `maxFps`. An unknown duration samples at the ceiling and lets the
+ * frame cap stop it.
+ */
+export function frameRateFor(durationSec, maxFrames, maxFps) {
+    if (durationSec === null || durationSec <= 0)
+        return maxFps;
+    // The first frame lands at t=0, so N frames need N-1 intervals to reach the
+    // end of the clip. Sampling at N/duration instead leaves the tail unseen.
+    const spread = maxFrames > 1 ? (maxFrames - 1) / durationSec : 1 / durationSec;
+    return Math.min(maxFps, Math.max(spread, 0.01));
+}
+/**
+ * Fit inside a square of `maxDimension` without upscaling. ffmpeg's
+ * `force_original_aspect_ratio=decrease` would enlarge a smaller frame, which
+ * costs tokens and adds no detail.
+ */
+function scaleFilter(maxDimension) {
+    const m = maxDimension;
+    return `scale='if(gt(iw,ih),min(${m},iw),-2)':'if(gt(iw,ih),-2,min(${m},ih))'`;
+}
+/**
+ * Decode `bytes` and return up to `maxFrames` JPEG stills spread across the
+ * clip. Throws {@link FrameSamplingUnavailableError} when ffmpeg is absent.
+ */
+export async function sampleVideoFrames(bytes, options = {}) {
+    const limits = frameSamplingLimits();
+    const maxFrames = Math.max(1, options.maxFrames ?? limits.maxFrames);
+    const maxFps = options.maxFps ?? limits.maxFps;
+    const maxDimension = options.maxDimension ?? limits.maxDimension;
+    const { cp, fs, os, path } = await nodeModules();
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), "nodetool-vframes-"));
+    const input = path.join(dir, "input.bin");
+    const outDir = path.join(dir, "frames");
+    try {
+        await fs.writeFile(input, bytes);
+        await fs.mkdir(outDir);
+        const durationSec = await probeDuration(cp, input, options.signal);
+        const fps = frameRateFor(durationSec, maxFrames, maxFps);
+        await run(cp, "ffmpeg", [
+            "-nostdin",
+            "-y",
+            "-v",
+            "error",
+            "-protocol_whitelist",
+            "file",
+            "-i",
+            input,
+            "-vf",
+            `fps=${fps.toFixed(6)},${scaleFilter(maxDimension)}`,
+            "-frames:v",
+            String(maxFrames),
+            "-q:v",
+            String(JPEG_QUALITY),
+            path.join(outDir, "frame_%04d.jpg")
+        ], options.signal);
+        const names = (await fs.readdir(outDir))
+            .filter((n) => n.startsWith("frame_") && n.endsWith(".jpg"))
+            .sort();
+        const frames = [];
+        for (const [index, name] of names.entries()) {
+            frames.push({
+                data: new Uint8Array(await fs.readFile(path.join(outDir, name))),
+                mimeType: "image/jpeg",
+                timeSec: index / fps
+            });
+        }
+        return {
+            frames,
+            durationSec,
+            fps,
+            truncated: durationSec === null && frames.length >= maxFrames
+        };
+    }
+    finally {
+        await fs.rm(dir, { recursive: true, force: true }).catch(() => { });
+    }
+}
+//# sourceMappingURL=video-frames.js.map

package/dist/providers/video-frames.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-frames.js","sourceRoot":"","sources":["../../src/providers/video-frames.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAkCzD,4EAA4E;AAC5E,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IAC7C,MAAM,CAAS;IACxB,YAAY,MAAc;QACxB,KAAK,CACH,GAAG,MAAM,2DAA2D;YAClE,qEAAqE;YACrE,sEAAsE;YACtE,4DAA4D,CAC/D,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,iEAAiE;AACjE,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAE1C,iEAAiE;AACjE,SAAS,SAAS,CAAC,IAAY,EAAE,QAAgB;IAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC1D,CAAC;AAED,qFAAqF;AACrF,SAAS,mBAAmB;IAG1B,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,KAAK,CACnB,SAAS,CAAC,iCAAiC,EAAE,kBAAkB,CAAC,CACjE;QACD,MAAM,EAAE,SAAS,CAAC,8BAA8B,EAAE,eAAe,CAAC;QAClE,YAAY,EAAE,IAAI,CAAC,KAAK,CACtB,SAAS,CAAC,oCAAoC,EAAE,qBAAqB,CAAC,CACvE;KACF,CAAC;AACJ,CAAC;AAID,KAAK,UAAU,WAAW;IAMxB,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC3C,iBAAiB,CACf,oBAAoB,CACrB;QACD,iBAAiB,CAAoC,kBAAkB,CAAC;QACxE,iBAAiB,CAA2B,SAAS,CAAC;QACtD,iBAAiB,CAA6B,WAAW,CAAC;KAC3D,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AAC9B,CAAC;AAED,gEAAgE;AAChE,SAAS,aAAa,CAAC,GAAY;IACjC,OAAO,CACL,OAAO,GAAG,KAAK,QAAQ;QACvB,GAAG,KAAK,IAAI;QACX,GAA0B,CAAC,IAAI,KAAK,QAAQ,CAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,GAAG,CAChB,EAAuC,EACvC,MAA4B,EAC5B,IAAc,EACd,MAAoB;IAEpB,OAAO,MAAM,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACvD,EAAE,CAAC,QAAQ,CACT,MAAM,EACN,IAAI,EACJ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,EAC3D,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,6BAA6B,CAAC,MAAM,CAAC,CAAC,CAAC;oBAClD,OAAO;gBACT,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,CAAC;gBACd,OAAO;YACT,CAAC;YACD,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,+DAA+D;AAC/D,KAAK,UAAU,aAAa,CAC1B,EAAuC,EACvC,IAAY,EACZ,MAAoB;IAEpB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAC1B,EAAE,EACF,SAAS,EACT;YACE,IAAI;YACJ,OAAO;YACP,eAAe;YACf,iBAAiB;YACjB,KAAK;YACL,SAAS;YACT,IAAI;SACL,EACD,MAAM,CACP,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,KAAK,CAAC;QACjC,yEAAyE;QACzE,yEAAyE;QACzE,wCAAwC;QACxC,IAAI,KAAK,YAAY,6BAA6B;YAAE,MAAM,KAAK,CAAC;QAChE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,KAAiB,EACjB,MAAmB;IAEnB,MAAM,CAAC,cAAc,EAAE,CAAC;IACxB,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,CAAC,cAAc,EAAE,CAAC;QACxB,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,OAAO,QAAQ,CACb,4BAA4B,EAC5B,EAAE,mBAAmB,EAAE,KAAK,CAAC,UAAU,EAAE,EACzC,KAAK,IAAI,EAAE;QACT,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,WAAW,EAAE,CAAC;QACjD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,qBAAqB,CAAC,CAAC,CAAC;QAC5E,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7C,OAAO,MAAM,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAChD,CAAC;gBAAS,CAAC;YACT,uEAAuE;YACvE,MAAM,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrE,CAAC;IACH,CAAC,CACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAC1B,WAA0B,EAC1B,SAAiB,EACjB,MAAc;IAEd,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5D,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;IAC/E,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,YAAoB;IACvC,MAAM,CAAC,GAAG,YAAY,CAAC;IACvB,OAAO,2BAA2B,CAAC,kCAAkC,CAAC,QAAQ,CAAC;AACjF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAiB,EACjB,OAAO,GAA6B,EAAE;IAEtC,MAAM,MAAM,GAAG,mBAAmB,EAAE,CAAC;IACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;IAEjE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,WAAW,EAAE,CAAC;IACjD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,MAAM,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEvB,MAAM,WAAW,GAAG,MAAM,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACnE,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAEzD,MAAM,GAAG,CACP,EAAE,EACF,QAAQ,EACR;YACE,UAAU;YACV,IAAI;YACJ,IAAI;YACJ,OAAO;YACP,qBAAqB;YACrB,MAAM;YACN,IAAI;YACJ,KAAK;YACL,KAAK;YACL,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC,EAAE;YACpD,WAAW;YACX,MAAM,CAAC,SAAS,CAAC;YACjB,MAAM;YACN,MAAM,CAAC,YAAY,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC;SACpC,EACD,OAAO,CAAC,MAAM,CACf,CAAC;QAEF,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;aAC3D,IAAI,EAAE,CAAC;QACV,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;gBAChE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,KAAK,GAAG,GAAG;aACrB,CAAC,CAAC;QACL,CAAC;QACD,OAAO;YACL,MAAM;YACN,WAAW;YACX,GAAG;YACH,SAAS,EAAE,WAAW,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,SAAS;SAC9D,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrE,CAAC;AACH,CAAC"}

package/dist/providers/video-mime.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Detect a video container from its leading magic bytes.
+ *
+ * Used when a video reaches a provider without a reliable Content-Type — e.g.
+ * asset bytes read from storage.
+ */
+/** Identify a video container; `null` when nothing matches. */
+export declare function sniffVideoMimeOrNull(bytes: Uint8Array): string | null;
+/**
+ * As {@link sniffVideoMimeOrNull}, defaulting to mp4 — the container every
+ * multimodal provider accepts.
+ */
+export declare function sniffVideoMime(bytes: Uint8Array): string;
+//# sourceMappingURL=video-mime.d.ts.map

package/dist/providers/video-mime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-mime.d.ts","sourceRoot":"","sources":["../../src/providers/video-mime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,+DAA+D;AAC/D,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,GAAG,IAAI,CAuCrE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAExD"}

package/dist/providers/video-mime.js

@@ -0,0 +1,44 @@
+/**
+ * Detect a video container from its leading magic bytes.
+ *
+ * Used when a video reaches a provider without a reliable Content-Type — e.g.
+ * asset bytes read from storage.
+ */
+/** Identify a video container; `null` when nothing matches. */
+export function sniffVideoMimeOrNull(bytes) {
+    // ISO base media file format: <4-byte size> "ftyp" — mp4, m4v and mov all
+    // use it; the brand that follows separates QuickTime from mp4.
+    if (bytes[4] === 0x66 &&
+        bytes[5] === 0x74 &&
+        bytes[6] === 0x79 &&
+        bytes[7] === 0x70) {
+        const brand = String.fromCharCode(bytes[8] ?? 0, bytes[9] ?? 0, bytes[10] ?? 0, bytes[11] ?? 0);
+        return brand === "qt  " ? "video/quicktime" : "video/mp4";
+    }
+    // EBML header — Matroska and WebM share it; WebM is the only one Gemini takes.
+    if (bytes[0] === 0x1a &&
+        bytes[1] === 0x45 &&
+        bytes[2] === 0xdf &&
+        bytes[3] === 0xa3) {
+        return "video/webm";
+    }
+    // "RIFF" ... "AVI "
+    if (bytes[0] === 0x52 &&
+        bytes[1] === 0x49 &&
+        bytes[2] === 0x46 &&
+        bytes[3] === 0x46 &&
+        bytes[8] === 0x41 &&
+        bytes[9] === 0x56 &&
+        bytes[10] === 0x49) {
+        return "video/x-msvideo";
+    }
+    return null;
+}
+/**
+ * As {@link sniffVideoMimeOrNull}, defaulting to mp4 — the container every
+ * multimodal provider accepts.
+ */
+export function sniffVideoMime(bytes) {
+    return sniffVideoMimeOrNull(bytes) ?? "video/mp4";
+}
+//# sourceMappingURL=video-mime.js.map

package/dist/providers/video-mime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-mime.js","sourceRoot":"","sources":["../../src/providers/video-mime.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,+DAA+D;AAC/D,MAAM,UAAU,oBAAoB,CAAC,KAAiB;IACpD,0EAA0E;IAC1E,+DAA+D;IAC/D,IACE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EACjB,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAC/B,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EACb,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EACb,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EACd,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CACf,CAAC;QACF,OAAO,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,WAAW,CAAC;IAC5D,CAAC;IACD,+EAA+E;IAC/E,IACE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EACjB,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,oBAAoB;IACpB,IACE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;QACjB,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,EAClB,CAAC;QACD,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,OAAO,oBAAoB,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC;AACpD,CAAC"}

package/dist/providers/video-to-audio.d.ts

@@ -0,0 +1,34 @@
+import { z } from "zod";
+import type { BaseProvider } from "./base-provider.js";
+import type { EncodedAudioResult } from "./types.js";
+export declare const VIDEO_TO_AUDIO_TASK = "video_to_audio";
+/** Source times are seconds in the original asset, before timeline retiming. */
+export declare const VideoToAudioRequest: z.ZodReadonly<z.ZodObject<{
+    task: z.ZodLiteral<"video_to_audio">;
+    model: z.ZodReadonly<z.ZodObject<{
+        id: z.ZodString;
+        provider: z.ZodString;
+        supportedTasks: z.ZodReadonly<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>;
+    source: z.ZodReadonly<z.ZodObject<{
+        assetId: z.ZodString;
+        durationSeconds: z.ZodNumber;
+        startSeconds: z.ZodNumber;
+        endSeconds: z.ZodNumber;
+    }, z.core.$strict>>;
+    sceneContext: z.ZodString;
+}, z.core.$strict>>;
+export type VideoToAudioRequest = z.infer<typeof VideoToAudioRequest>;
+export type VideoToAudioParams = VideoToAudioRequest & {
+    readonly signal?: AbortSignal;
+};
+/**
+ * Validate before provider dispatch. `video` contains the original source asset.
+ * The provider must condition on the captured window and return encoded audio
+ * starting at that window's zero, with its duration. Retiming and timeline
+ * placement belong to a future caller. This function never applies the result.
+ */
+export declare function requestVideoToAudio(provider: Pick<BaseProvider, "provider" | "getCapabilities" | "getAvailableVideoModels" | "videoToAudio">, video: Uint8Array, input: unknown, options?: {
+    readonly signal?: AbortSignal;
+}): Promise<EncodedAudioResult>;
+//# sourceMappingURL=video-to-audio.d.ts.map

package/dist/providers/video-to-audio.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-to-audio.d.ts","sourceRoot":"","sources":["../../src/providers/video-to-audio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AAEpD,gFAAgF;AAChF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;mBAkCnB,CAAC;AAEd,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AACtE,MAAM,MAAM,kBAAkB,GAAG,mBAAmB,GAAG;IACrD,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CAC/B,CAAC;AAEF;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACvC,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,GAAG,iBAAiB,GAAG,yBAAyB,GAAG,cAAc,CAAC,EACzG,KAAK,EAAE,UAAU,EACjB,KAAK,EAAE,OAAO,EACd,OAAO,GAAE;IAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GAC9C,OAAO,CAAC,kBAAkB,CAAC,CA2B7B"}

package/dist/providers/video-to-audio.js

@@ -0,0 +1,62 @@
+import { z } from "zod";
+export const VIDEO_TO_AUDIO_TASK = "video_to_audio";
+/** Source times are seconds in the original asset, before timeline retiming. */
+export const VideoToAudioRequest = z
+    .strictObject({
+    task: z.literal(VIDEO_TO_AUDIO_TASK),
+    model: z
+        .strictObject({
+        id: z.string().trim().min(1),
+        provider: z.string().trim().min(1),
+        supportedTasks: z
+            .array(z.string())
+            .refine((tasks) => tasks.includes(VIDEO_TO_AUDIO_TASK), "The model must explicitly support video_to_audio")
+            .readonly()
+    })
+        .readonly(),
+    source: z
+        .strictObject({
+        assetId: z.string().trim().min(1),
+        durationSeconds: z.number().finite().positive(),
+        startSeconds: z.number().finite().nonnegative(),
+        endSeconds: z.number().finite().positive()
+    })
+        .refine((source) => source.endSeconds > source.startSeconds, "The source window must have positive duration")
+        .refine((source) => source.endSeconds <= source.durationSeconds, "The source window must fit within the asset duration")
+        .readonly(),
+    sceneContext: z.string().trim().min(1)
+})
+    .readonly();
+/**
+ * Validate before provider dispatch. `video` contains the original source asset.
+ * The provider must condition on the captured window and return encoded audio
+ * starting at that window's zero, with its duration. Retiming and timeline
+ * placement belong to a future caller. This function never applies the result.
+ */
+export async function requestVideoToAudio(provider, video, input, options = {}) {
+    const request = VideoToAudioRequest.parse(input);
+    if (!(video instanceof Uint8Array) || video.byteLength === 0) {
+        throw new Error("video_to_audio requires nonempty source video bytes");
+    }
+    if (request.model.provider !== provider.provider) {
+        throw new Error("The video_to_audio model belongs to a different provider");
+    }
+    if (!provider.getCapabilities().includes(VIDEO_TO_AUDIO_TASK)) {
+        throw new Error(`${provider.provider} does not support video_to_audio`);
+    }
+    options.signal?.throwIfAborted();
+    const model = (await provider.getAvailableVideoModels()).find((candidate) => candidate.id === request.model.id && candidate.provider === provider.provider);
+    if (!model?.supportedTasks?.includes(VIDEO_TO_AUDIO_TASK)) {
+        throw new Error(`Model ${request.model.id} does not support video_to_audio in the provider catalog`);
+    }
+    options.signal?.throwIfAborted();
+    const validatedRequest = {
+        ...request,
+        model: { ...request.model, supportedTasks: [...model.supportedTasks] }
+    };
+    if (options.signal) {
+        return provider.videoToAudio(video, { ...validatedRequest, signal: options.signal });
+    }
+    return provider.videoToAudio(video, validatedRequest);
+}
+//# sourceMappingURL=video-to-audio.js.map

package/dist/providers/video-to-audio.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video-to-audio.js","sourceRoot":"","sources":["../../src/providers/video-to-audio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,MAAM,CAAC,MAAM,mBAAmB,GAAG,gBAAgB,CAAC;AAEpD,gFAAgF;AAChF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC;KACjC,YAAY,CAAC;IACZ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC;IACpC,KAAK,EAAE,CAAC;SACL,YAAY,CAAC;QACZ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAClC,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,MAAM,CACL,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAC9C,kDAAkD,CACnD;aACA,QAAQ,EAAE;KACd,CAAC;SACD,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC;SACN,YAAY,CAAC;QACZ,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACjC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC/C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;QAC/C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC3C,CAAC;SACD,MAAM,CACL,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,YAAY,EACnD,+CAA+C,CAChD;SACA,MAAM,CACL,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,eAAe,EACvD,sDAAsD,CACvD;SACA,QAAQ,EAAE;IACb,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,QAAQ,EAAE,CAAC;AAOd;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,QAAyG,EACzG,KAAiB,EACjB,KAAc,EACd,OAAO,GAAsC,EAAE;IAE/C,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjD,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,kCAAkC,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAC3D,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,SAAS,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAC7F,CAAC;IACF,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,CAAC,KAAK,CAAC,EAAE,0DAA0D,CAAC,CAAC;IACvG,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,gBAAgB,GAAwB;QAC5C,GAAG,OAAO;QACV,KAAK,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,cAAc,EAAE,CAAC,GAAG,KAAK,CAAC,cAAc,CAAC,EAAE;KACvE,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;AACxD,CAAC"}