@yanickxia/agent-vision 0.1.0

package/dist/video.js

@@ -0,0 +1,200 @@
+import { spawn } from "node:child_process";
+import { access, chmod, readFile, readdir, rm, stat } from "node:fs/promises";
+import { constants as fsConstants } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import path from "node:path";
+import { mkdtemp } from "node:fs/promises";
+import { VisionError } from "./errors.js";
+import { downloadRemoteFile } from "./network.js";
+let ffmpegReady;
+async function usableFile(filePath) {
+    try {
+        await access(filePath, process.platform === "win32" ? fsConstants.F_OK : fsConstants.X_OK);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function systemFfmpeg() {
+    const executableName = process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg";
+    for (const directory of (process.env.PATH ?? "").split(path.delimiter)) {
+        if (!directory)
+            continue;
+        const candidate = path.join(directory, executableName);
+        if (await usableFile(candidate))
+            return candidate;
+    }
+    return undefined;
+}
+async function bundledFfmpeg() {
+    try {
+        const imported = await import("@ffmpeg-installer/ffmpeg");
+        const installer = imported.default;
+        return installer?.path;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function ensureFfmpeg() {
+    ffmpegReady ??= (async () => {
+        const configured = process.env.AGENT_VISION_FFMPEG_PATH?.trim();
+        if (configured) {
+            const resolved = path.resolve(configured);
+            if (!(await usableFile(resolved))) {
+                throw new VisionError("FFMPEG_NOT_AVAILABLE", `AGENT_VISION_FFMPEG_PATH is not executable: ${resolved}`);
+            }
+            return resolved;
+        }
+        const ffmpegPath = (await systemFfmpeg()) ?? (await bundledFfmpeg());
+        if (!ffmpegPath) {
+            throw new VisionError("FFMPEG_NOT_AVAILABLE", "The bundled ffmpeg binary is unavailable on this platform");
+        }
+        if (process.platform !== "win32") {
+            try {
+                await chmod(ffmpegPath, 0o755);
+            }
+            catch (error) {
+                throw new VisionError("FFMPEG_NOT_AVAILABLE", "The bundled ffmpeg binary could not be made executable", { cause: error });
+            }
+        }
+        return ffmpegPath;
+    })();
+    return ffmpegReady;
+}
+function expandLocalPath(source) {
+    if (source === "~")
+        return homedir();
+    if (source.startsWith("~/"))
+        return path.join(homedir(), source.slice(2));
+    return path.resolve(source.startsWith("@") ? source.slice(1) : source);
+}
+async function runFfmpeg(args, allowFailure = false, timeoutMs = 120_000) {
+    const command = await ensureFfmpeg();
+    return new Promise((resolve, reject) => {
+        const child = spawn(command, args, { stdio: ["ignore", "ignore", "pipe"] });
+        let stderr = "";
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            child.kill("SIGKILL");
+        }, timeoutMs);
+        child.stderr.setEncoding("utf8");
+        child.stderr.on("data", (chunk) => {
+            if (stderr.length < 1_000_000)
+                stderr += chunk;
+        });
+        child.on("error", (error) => {
+            clearTimeout(timer);
+            reject(new VisionError("FFMPEG_ERROR", "Could not start ffmpeg", { cause: error }));
+        });
+        child.on("close", (code) => {
+            clearTimeout(timer);
+            if (timedOut) {
+                reject(new VisionError("FFMPEG_TIMEOUT", `ffmpeg timed out after ${timeoutMs} ms`));
+                return;
+            }
+            if (code === 0 || allowFailure)
+                resolve(stderr);
+            else
+                reject(new VisionError("FFMPEG_ERROR", stderr.trim().slice(-1000) || `ffmpeg exited with code ${code}`));
+        });
+    });
+}
+async function videoDuration(filePath, timeoutMs) {
+    const output = await runFfmpeg(["-hide_banner", "-i", filePath], true, timeoutMs);
+    const match = output.match(/Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/);
+    if (!match?.[1] || !match[2] || !match[3]) {
+        throw new VisionError("UNSUPPORTED_VIDEO", "Could not determine video duration");
+    }
+    const duration = Number(match[1]) * 3600 + Number(match[2]) * 60 + Number(match[3]);
+    if (!Number.isFinite(duration) || duration <= 0) {
+        throw new VisionError("UNSUPPORTED_VIDEO", "Video duration is invalid");
+    }
+    return duration;
+}
+async function prepareInput(source, workDir, config) {
+    if (/^https?:\/\//i.test(source)) {
+        const destination = path.join(workDir, "input-video");
+        await downloadRemoteFile({
+            url: source,
+            destination,
+            maxBytes: config.maxVideoBytes,
+            timeoutMs: config.timeoutMs,
+            allowPrivateUrls: config.allowPrivateUrls,
+        });
+        return destination;
+    }
+    if (/^[a-z][a-z0-9+.-]*:/i.test(source)) {
+        throw new VisionError("INVALID_SOURCE", "Video source must be a local path or HTTP(S) URL");
+    }
+    const filePath = expandLocalPath(source);
+    let fileStat;
+    try {
+        fileStat = await stat(filePath);
+    }
+    catch (error) {
+        throw new VisionError("FILE_NOT_FOUND", `Video file not found: ${filePath}`, {
+            cause: error,
+        });
+    }
+    if (!fileStat.isFile()) {
+        throw new VisionError("INVALID_SOURCE", `Not a file: ${filePath}`);
+    }
+    if (fileStat.size > config.maxVideoBytes) {
+        throw new VisionError("SOURCE_TOO_LARGE", `Video is ${(fileStat.size / 1024 / 1024).toFixed(1)} MB; limit is ${(config.maxVideoBytes / 1024 / 1024).toFixed(1)} MB`);
+    }
+    return filePath;
+}
+export async function extractVideoFrames(options) {
+    if (!Number.isInteger(options.maxFrames) || options.maxFrames < 1 || options.maxFrames > 16) {
+        throw new VisionError("INVALID_ARGUMENT", "maxFrames must be an integer from 1 to 16");
+    }
+    const prefix = path.join(options.tempRoot ?? tmpdir(), "agent-vision-");
+    const workDir = await mkdtemp(prefix);
+    try {
+        const inputPath = await prepareInput(options.source.trim(), workDir, options.config);
+        const duration = await videoDuration(inputPath, options.config.timeoutMs);
+        const timestamps = Array.from({ length: options.maxFrames }, (_, index) => ((index + 0.5) * duration) / options.maxFrames);
+        for (const [index, timestamp] of timestamps.entries()) {
+            const outputPath = path.join(workDir, `frame-${String(index + 1).padStart(3, "0")}.jpg`);
+            await runFfmpeg([
+                "-hide_banner",
+                "-loglevel",
+                "error",
+                "-ss",
+                timestamp.toFixed(3),
+                "-i",
+                inputPath,
+                "-frames:v",
+                "1",
+                "-vf",
+                "scale=1600:1600:force_original_aspect_ratio=decrease",
+                "-q:v",
+                "3",
+                "-y",
+                outputPath,
+            ], false, options.config.timeoutMs);
+        }
+        const frameNames = (await readdir(workDir))
+            .filter((name) => /^frame-\d+\.jpg$/.test(name))
+            .sort();
+        if (frameNames.length === 0) {
+            throw new VisionError("FFMPEG_ERROR", "ffmpeg did not produce any video frames");
+        }
+        return Promise.all(frameNames.map(async (name, index) => {
+            const timestamp = timestamps[index] ?? 0;
+            const buffer = await readFile(path.join(workDir, name));
+            return {
+                dataUrl: `data:image/jpeg;base64,${buffer.toString("base64")}`,
+                label: `frame at ${timestamp.toFixed(2)} seconds`,
+                timestampSeconds: timestamp,
+            };
+        }));
+    }
+    finally {
+        await rm(workDir, { recursive: true, force: true });
+    }
+}
+//# sourceMappingURL=video.js.map

package/dist/video.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video.js","sourceRoot":"","sources":["../src/video.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,IAAI,WAAW,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAOlD,IAAI,WAAwC,CAAC;AAE7C,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,MAAM,CACV,QAAQ,EACR,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CACnE,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY;IACzB,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC9E,KAAK,MAAM,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,SAAS;YAAE,SAAS;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACvD,IAAI,MAAM,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IACpD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,aAAa;IAC1B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAwC,CAAC;QACpE,OAAO,SAAS,EAAE,IAAI,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,WAAW,KAAK,CAAC,KAAK,IAAI,EAAE;QAC1B,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;QAChE,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAClC,MAAM,IAAI,WAAW,CACnB,sBAAsB,EACtB,+CAA+C,QAAQ,EAAE,CAC1D,CAAC;YACJ,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,MAAM,UAAU,GAAG,CAAC,MAAM,YAAY,EAAE,CAAC,IAAI,CAAC,MAAM,aAAa,EAAE,CAAC,CAAC;QACrE,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,WAAW,CACnB,sBAAsB,EACtB,2DAA2D,CAC5D,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,WAAW,CACnB,sBAAsB,EACtB,wDAAwD,EACxD,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC,EAAE,CAAC;IACL,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,OAAO,EAAE,CAAC;IACrC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,IAAc,EACd,YAAY,GAAG,KAAK,EACpB,SAAS,GAAG,OAAO;IAEnB,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAC;IACrC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5E,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxB,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC;QACjD,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,IAAI,WAAW,CAAC,cAAc,EAAE,wBAAwB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,WAAW,CAAC,gBAAgB,EAAE,0BAA0B,SAAS,KAAK,CAAC,CAAC,CAAC;gBACpF,OAAO;YACT,CAAC;YACD,IAAI,IAAI,KAAK,CAAC,IAAI,YAAY;gBAAE,OAAO,CAAC,MAAM,CAAC,CAAC;;gBAC3C,MAAM,CAAC,IAAI,WAAW,CAAC,cAAc,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,2BAA2B,IAAI,EAAE,CAAC,CAAC,CAAC;QAChH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,SAAiB;IAC9D,MAAM,MAAM,GAAG,MAAM,SAAS,CAC5B,CAAC,cAAc,EAAE,IAAI,EAAE,QAAQ,CAAC,EAChC,IAAI,EACJ,SAAS,CACV,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACtE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,WAAW,CAAC,mBAAmB,EAAE,oCAAoC,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,WAAW,CAAC,mBAAmB,EAAE,2BAA2B,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAc,EACd,OAAe,EACf,MAAoB;IAEpB,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACtD,MAAM,kBAAkB,CAAC;YACvB,GAAG,EAAE,MAAM;YACX,WAAW;YACX,QAAQ,EAAE,MAAM,CAAC,aAAa;YAC9B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,OAAO,WAAW,CAAC;IACrB,CAAC;IACD,IAAI,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,WAAW,CAAC,gBAAgB,EAAE,kDAAkD,CAAC,CAAC;IAC9F,CAAC;IACD,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,WAAW,CAAC,gBAAgB,EAAE,yBAAyB,QAAQ,EAAE,EAAE;YAC3E,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACvB,MAAM,IAAI,WAAW,CAAC,gBAAgB,EAAE,eAAe,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;QACzC,MAAM,IAAI,WAAW,CACnB,kBAAkB,EAClB,YAAY,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,aAAa,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAC1H,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAKxC;IACC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG,EAAE,EAAE,CAAC;QAC5F,MAAM,IAAI,WAAW,CAAC,kBAAkB,EAAE,2CAA2C,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACrF,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1E,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAC3B,EAAE,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE,EAC7B,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,OAAO,CAAC,SAAS,CAC7D,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACtD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YACzF,MAAM,SAAS,CACb;gBACE,cAAc;gBACd,WAAW;gBACX,OAAO;gBACP,KAAK;gBACL,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpB,IAAI;gBACJ,SAAS;gBACT,WAAW;gBACX,GAAG;gBACH,KAAK;gBACL,sDAAsD;gBACtD,MAAM;gBACN,GAAG;gBACH,IAAI;gBACJ,UAAU;aACX,EACD,KAAK,EACL,OAAO,CAAC,MAAM,CAAC,SAAS,CACzB,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;aACxC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;aAC/C,IAAI,EAAE,CAAC;QACV,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,WAAW,CAAC,cAAc,EAAE,yCAAyC,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,OAAO,CAAC,GAAG,CAChB,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;YACxD,OAAO;gBACL,OAAO,EAAE,0BAA0B,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;gBAC9D,KAAK,EAAE,YAAY,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU;gBACjD,gBAAgB,EAAE,SAAS;aAC5B,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;AACH,CAAC"}

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@yanickxia/agent-vision",
+  "version": "0.1.0",
+  "description": "A small MCP server and Agent Skill for image and video understanding.",
+  "type": "module",
+  "main": "./dist/cli.js",
+  "bin": {
+    "agent-vision": "dist/cli.js"
+  },
+  "files": [
+    "dist/",
+    "skills/",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "test": "npm run build && tsx --test test/*.test.ts",
+    "prepare": "npm run build",
+    "prepack": "npm run typecheck && npm run build"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "keywords": [
+    "mcp",
+    "agent-skills",
+    "vision",
+    "image-understanding",
+    "video-understanding",
+    "openai-compatible"
+  ],
+  "author": "Yanick Xia",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yanickxia/agent-vision.git"
+  },
+  "bugs": {
+    "url": "https://github.com/yanickxia/agent-vision/issues"
+  },
+  "homepage": "https://github.com/yanickxia/agent-vision#readme",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "dependencies": {
+    "@ffmpeg-installer/ffmpeg": "^1.1.0",
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "zod": "^3.25.76"
+  },
+  "devDependencies": {
+    "@types/node": "^22.18.0",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.2"
+  }
+}

package/skills/agent-vision/SKILL.md

@@ -0,0 +1,64 @@
+---
+name: agent-vision
+description: Analyze images, screenshots, diagrams, and videos with the agent-vision MCP tools. Use for image questions, OCR, UI inspection, visual debugging, screen recordings, video summaries, and timeline analysis.
+license: MIT
+compatibility: Requires the agent-vision MCP server or Node.js 20+ for the npx fallback, plus an OpenAI-compatible vision API.
+---
+
+# agent-vision
+
+Use this skill when the user asks you to inspect an image or video. The MCP
+server is the execution layer; this skill explains when and how to call it.
+
+## Choose the tool
+
+| Input | Tool |
+|---|---|
+| Screenshot, photo, diagram, UI, scanned text | `analyze_image` |
+| Screen recording, demo, clip, animation timeline | `analyze_video` |
+
+Do not call a vision tool for a text-only task.
+
+## Calling rules
+
+1. Pass the user's original question as `prompt`. A focused question is better
+   than a generic request for a description.
+2. Prefer an absolute local path. HTTP(S) URLs are also supported. Images may
+   additionally be supplied as a `data:image/...;base64,...` URI.
+3. For videos, keep the default `max_frames=8` unless the user needs a faster
+   result or a denser timeline. The allowed range is 1–16.
+4. If the result is uncertain, say so. Never invent visual details after a
+   failed tool call.
+5. Retry at most once for a transient API or timeout error. Configuration,
+   unsupported-format, and missing-file errors require correction, not retries.
+
+## Examples
+
+```json
+analyze_image({
+  "source": "/absolute/path/to/error.png",
+  "prompt": "Read the error message and identify the likely cause."
+})
+```
+
+```json
+analyze_video({
+  "source": "/absolute/path/to/demo.mp4",
+  "prompt": "Summarize the UI flow in chronological order.",
+  "max_frames": 8
+})
+```
+
+## npx fallback
+
+If the MCP tools are not available but shell execution is allowed, run:
+
+```bash
+npx -y @yanickxia/agent-vision image "/absolute/path/to/image.png" --prompt "user question"
+npx -y @yanickxia/agent-vision video "/absolute/path/to/video.mp4" --prompt "user question" --frames 8
+```
+
+Before the npm package is available, replace the package name with
+`github:yanickxia/agent-vision`.
+
+Configuration details are in `references/configuration.md`.

package/skills/agent-vision/references/configuration.md

@@ -0,0 +1,26 @@
+# Configuration
+
+Required for normal use:
+
+```bash
+export AGENT_VISION_BASE_URL="https://your-openai-compatible-endpoint/v1"
+export AGENT_VISION_API_KEY="your-key"
+export AGENT_VISION_MODEL="your-vision-model"
+```
+
+`AGENT_VISION_API_KEY` may be omitted for a trusted local endpoint that does
+not require authentication. The server also recognizes `VISION_*` and
+`OPENAI_*` aliases.
+
+Useful optional variables:
+
+- `AGENT_VISION_MAX_TOKENS` (default `4096`)
+- `AGENT_VISION_TIMEOUT_MS` (default `120000`)
+- `AGENT_VISION_MAX_IMAGE_MB` (default `20`)
+- `AGENT_VISION_MAX_VIDEO_MB` (default `200`)
+- `AGENT_VISION_HEADERS` (JSON object of extra HTTP headers)
+- `AGENT_VISION_ALLOW_PRIVATE_URLS=true` (allow private image/video source URLs)
+- `AGENT_VISION_FFMPEG_PATH` (override ffmpeg; otherwise PATH, then bundled binary)
+
+The configured API endpoint itself may be private; the private-URL guard only
+applies to user-provided image and video source URLs.