@wrenautomation/read 0.0.0-stage → 0.29.0

package/dist/video/url.js

@@ -0,0 +1,255 @@
+/**
+ * A video or podcast on the web read as one markdown document (`VideoRead`). Four
+ * readers: a YouTube video from its captions and chapters (yt-dlp) plus a video model's read of
+ * the screen; a YouTube video from its URL alone (the model hears and reads it: no download, no
+ * home IP); any other video yt-dlp fetches (`watchFile`: an ear and an eye); a podcast
+ * episode from its audio URL (fetched, then `hearAudio`). Everything that takes a URL or
+ * spawns yt-dlp lives here; the reading over a file is in `watch.ts` and `read.ts`. The video
+ * model is Gemini (`geminiVideo`): the one that takes a YouTube URL. Every caller passes its own models
+ * and tools. Nothing here touches a database.
+ */
+import { createHash } from "node:crypto";
+import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { readMedia, SHOWN } from "./gemini.js";
+import { CLIP_S, captionsBody, hearAudio, isoDate, real, } from "./read.js";
+import { clock, runFor, TOOL_MS } from "./tools.js";
+import { renderWatched, watchFile } from "./watch.js";
+/** yt-dlp's download of a reel or video: 540p on the short side, remuxed to mp4. */
+export async function download(url, ytDlp, dir) {
+    const [cmd, ...pre] = ytDlp.split(/\s+/);
+    const { stdout } = await runFor(cmd, [
+        ...pre,
+        ...["-j", "--no-simulate", "--no-warnings", "--no-playlist"],
+        // `res` is the short side, so a vertical reel gets 540x960, not its 1080x1920.
+        ...["-S", "res:540", "-f", "bv*+ba/b", "--remux-video", "mp4"],
+        ...["-o", join(dir, "video.%(ext)s"), url],
+    ], TOOL_MS, { maxBuffer: 256 * 1024 * 1024 });
+    const file = (await readdir(dir)).find((f) => f.endsWith(".mp4"));
+    if (!file)
+        throw new Error(`${url}: yt-dlp wrote no mp4`);
+    return { file: join(dir, file), info: JSON.parse(stdout) };
+}
+/**
+ * Watch a video: a URL yt-dlp fetches first (`ytDlp` is the command, split on spaces:
+ * `uvx yt-dlp`), or a file on disk, which goes straight to `watchFile`. A URL's yt-dlp
+ * metadata comes back as `info`; a duration ffmpeg could not read falls back to it.
+ */
+export async function watch(input, opts) {
+    if (!/^https?:\/\//.test(input))
+        return watchFile(input, opts);
+    const dir = await mkdtemp(join(tmpdir(), "read-watch-"));
+    try {
+        const { file, info } = await download(input, opts.ytDlp ?? "yt-dlp", dir);
+        const watched = await watchFile(file, opts);
+        return { ...watched, durationS: watched.durationS || info.duration || 0, info };
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+}
+const captionTracks = (info) => [info.subtitles?.en, info.automatic_captions?.en, info.automatic_captions?.["en-orig"]]
+    .flatMap((t) => t ?? [])
+    .filter((t) => t.ext === "json3");
+const SHOWN_ASK = (start, paraphrase) => `${SHOWN(start, paraphrase)} Markdown. If nothing is shown, reply with the single word none.`;
+/** A YouTube page as a video part; `clip` reads only that stretch, at half a frame a second. */
+const youtubePart = (page, start, durationS) => ({
+    type: "video",
+    data: new URL(page),
+    mediaType: "video/mp4",
+    ...(start === null
+        ? {}
+        : { clip: { startS: start, endS: Math.min(start + CLIP_S, durationS), fps: 0.5 } }),
+});
+// Low resolution reads a shown document fine at a quarter of the tokens.
+const LOW = { resolution: "low" };
+/**
+ * What the video shows on screen (documents, slides, tables, templates), read by `model` straight
+ * from the YouTube URL in `CLIP_S` clips. One markdown block per clip, timestamps absolute.
+ * Empty with no model.
+ */
+async function screenText(url, durationS, model) {
+    if (!model)
+        return "";
+    const clips = Array.from({ length: Math.ceil(durationS / CLIP_S) }, (_, i) => i * CLIP_S);
+    const read = async (start) => {
+        try {
+            return await readMedia(model, [youtubePart(url, start, durationS)], (paraphrase) => SHOWN_ASK(start, paraphrase), LOW);
+        }
+        catch (err) {
+            // One unread clip must not lose the other 23: leave a marker and carry on.
+            const why = err instanceof Error ? err.message : String(err);
+            // The document is written all the same: its text keeps the marker below at this time.
+            console.warn(`${url} ${clock(start)}: a clip left unread by ${model.id} (${why})`);
+            return `[${clock(start)}] (not read: ${why})`;
+        }
+    };
+    const out = [];
+    // 4 clips at a time; limits are per key, so go wider if it ever drags.
+    for (let i = 0; i < clips.length; i += 4) {
+        const texts = await Promise.all(clips.slice(i, i + 4).map(read));
+        out.push(...texts.filter(real));
+    }
+    return out.join("\n\n");
+}
+/**
+ * A YouTube video from its captions and chapters, through `yt-dlp` (`ytDlp` is the command,
+ * split on spaces: `uvx yt-dlp`). With a `video` model, an `## On screen` section follows: what
+ * the video showed, read from the frames, since the captions carry only the speech.
+ */
+export async function readYoutube(url, opts) {
+    const fetchFn = opts.fetchFn ?? fetch;
+    const [cmd, ...pre] = opts.ytDlp.split(/\s+/);
+    const { stdout } = await runFor(cmd, [...pre, "-J", "--no-warnings", "--no-playlist", url], TOOL_MS, { maxBuffer: 256 * 1024 * 1024 });
+    const info = JSON.parse(stdout);
+    const tracks = captionTracks(info);
+    if (!tracks.length)
+        throw new Error(`${url}: no English captions (manual or auto) to read`);
+    // YouTube throttles single caption URLs (429); the same text sits behind several, so try each.
+    let json;
+    let last = 0;
+    for (const t of tracks) {
+        const res = await fetchFn(t.url);
+        last = res.status;
+        if (!res.ok)
+            continue;
+        json = (await res.json());
+        break;
+    }
+    if (!json)
+        throw new Error(`${url}: captions fetch HTTP ${last} on all ${tracks.length} tracks`);
+    let body = captionsBody(info, json);
+    const screen = await screenText(url, info.duration ?? 0, opts.video);
+    if (screen)
+        body += `\n## On screen\n\nWhat the video showed (documents, slides, tables, visuals), read from the frames; the transcript above has only the speech.\n\n${screen}\n`;
+    const uploaded = isoDate(info.upload_date);
+    return {
+        kind: "youtube",
+        id: info.id,
+        title: info.title,
+        url: info.webpage_url ?? `https://www.youtube.com/watch?v=${info.id}`,
+        ...(info.channel ? { channel: info.channel } : {}),
+        ...(uploaded ? { uploaded } : {}),
+        ...(info.duration ? { duration: Math.round(info.duration) } : {}),
+        ...(info.thumbnail ? { thumbnail: info.thumbnail } : {}),
+        body,
+    };
+}
+const SPEECH = "Transcribe the speech in this video: every word spoken, in paragraphs, each starting with its [m:ss] video time. Fix only obvious mishearings. Markdown, no heading. If nobody speaks, reply with the single word none.";
+/**
+ * A YouTube video read from its URL alone, no download and no home IP: `model` hears the speech
+ * and reads the screen in `CLIP_S` clips (one clip when the length isn't known). The title,
+ * channel and thumbnail come from YouTube's oEmbed. For a server; a machine with yt-dlp and a
+ * home IP reads the captions instead (`readYoutube`).
+ */
+export async function readYoutubeByUrl(url, opts) {
+    const fetchFn = opts.fetchFn ?? fetch;
+    const id = /(?:v=|youtu\.be\/|shorts\/|live\/)([\w-]{11})/.exec(url)?.[1];
+    if (!id)
+        throw new Error(`${url}: not a YouTube video`);
+    const page = `https://www.youtube.com/watch?v=${id}`;
+    const meta = (await fetchFn(`https://www.youtube.com/oembed?format=json&url=${encodeURIComponent(page)}`)
+        .then((r) => (r.ok ? r.json() : {}))
+        .catch(() => ({})));
+    const duration = opts.durationS ?? 0;
+    const starts = duration
+        ? Array.from({ length: Math.ceil(duration / CLIP_S) }, (_, i) => i * CLIP_S)
+        : [null];
+    const readClip = async (start) => {
+        const media = [youtubePart(page, start, duration)];
+        const label = `${opts.model.id} ${id} ${clock(start ?? 0)}`;
+        const [said, screen] = await Promise.all([
+            readMedia(opts.model, media, SPEECH, LOW).catch((err) => {
+                throw new Error(`${label} speech: ${err instanceof Error ? err.message : err}`, {
+                    cause: err,
+                });
+            }),
+            // The screen is extra: a clip it cannot read keeps its speech.
+            readMedia(opts.model, media, (p) => SHOWN_ASK(start ?? 0, p), LOW).catch((err) => {
+                console.warn(`${label} screen: unread, ${err instanceof Error ? err.message : err}`);
+                return "";
+            }),
+        ]);
+        return { said, screen };
+    };
+    const clips = [];
+    // 4 clips at a time, kept in order: a key fleet spreads them.
+    for (let i = 0; i < starts.length; i += 4)
+        clips.push(...(await Promise.all(starts.slice(i, i + 4).map(readClip))));
+    const speech = clips.map((c) => c.said).filter(real);
+    const shown = clips.map((c) => c.screen).filter(real);
+    return {
+        kind: "youtube",
+        id,
+        title: meta.title?.trim() || page,
+        url: page,
+        ...(meta.author_name ? { channel: meta.author_name } : {}),
+        ...(duration ? { duration: Math.round(duration) } : {}),
+        ...(meta.thumbnail_url ? { thumbnail: meta.thumbnail_url } : {}),
+        body: `## Speech\n\n${speech.join("\n\n") || "(no speech)"}\n${shown.length ? `\n## On screen\n\n${shown.join("\n\n")}\n` : ""}`,
+    };
+}
+/**
+ * Any other video `yt-dlp` fetches (an Instagram reel, TikTok, X). No captions there, so
+ * `watch` hears the audio and reads the frames (its `video` model adds a read of the whole
+ * screen when given). The body holds the caption, then Speech, On screen and Cuts.
+ */
+export async function readVideo(url, opts) {
+    const watched = await watch(url, opts);
+    const info = watched.info;
+    const caption = info.description?.trim() ?? "";
+    const by = info.channel ?? info.uploader;
+    const uploaded = isoDate(info.upload_date);
+    const duration = info.duration ?? watched.durationS;
+    return {
+        kind: (info.extractor_key ?? "video").toLowerCase(),
+        id: info.id,
+        title: caption.split("\n")[0]?.slice(0, 100) || info.title,
+        url: info.webpage_url ?? url,
+        ...(by ? { channel: by } : {}),
+        ...(uploaded ? { uploaded } : {}),
+        ...(duration ? { duration: Math.round(duration) } : {}),
+        ...(info.thumbnail ? { thumbnail: info.thumbnail } : {}),
+        body: `${caption ? `## Caption\n\n${caption}\n\n` : ""}${renderWatched(watched)}\n`,
+    };
+}
+const AUDIO_MAX = 600_000_000;
+/**
+ * A podcast episode heard from its audio URL: downloaded to a temp file, then
+ * `hearAudio` (ffmpeg clips, `model` transcribes each). `url` is the episode's page; `title` and
+ * `creator` are what its feed said. The id is a hash of the audio URL.
+ */
+export async function readAudio(audioUrl, opts) {
+    const fetchFn = opts.fetchFn ?? fetch;
+    const dir = await mkdtemp(join(tmpdir(), "read-audio-"));
+    try {
+        const res = await fetchFn(audioUrl, { signal: AbortSignal.timeout(600_000) });
+        if (!res.ok || !res.body)
+            throw new Error(`${audioUrl}: HTTP ${res.status}`);
+        if (Number(res.headers.get("content-length") ?? 0) > AUDIO_MAX)
+            throw new Error(`${audioUrl}: over ${AUDIO_MAX / 1e6} MB`);
+        const whole = join(dir, "episode");
+        await writeFile(whole, Buffer.from(await res.arrayBuffer()));
+        const id = createHash("sha1").update(audioUrl).digest("hex").slice(0, 12);
+        const heard = await hearAudio(whole, {
+            model: opts.model,
+            ...(opts.ffmpeg ? { ffmpeg: opts.ffmpeg } : {}),
+            ...(opts.clipS ? { clipS: opts.clipS } : {}),
+            label: id,
+        });
+        return {
+            kind: "podcast",
+            id,
+            title: opts.title?.trim() || opts.url,
+            url: opts.url,
+            ...(opts.creator ? { channel: opts.creator } : {}),
+            ...(heard.duration ? { duration: heard.duration } : {}),
+            body: heard.speech,
+        };
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+}
+//# sourceMappingURL=url.js.map

package/dist/video/url.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.js","sourceRoot":"","sources":["../../src/video/url.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EACL,MAAM,EACN,YAAY,EACZ,SAAS,EACT,OAAO,EAEP,IAAI,GAEL,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,EAAE,aAAa,EAAmC,SAAS,EAAE,MAAM,YAAY,CAAC;AAevF,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,KAAa,EACb,GAAW;IAEX,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAA0B,CAAC;IAClE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAC7B,GAAG,EACH;QACE,GAAG,GAAG;QACN,GAAG,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,CAAC;QAC5D,+EAA+E;QAC/E,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,eAAe,EAAE,KAAK,CAAC;QAC9D,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,EAAE,GAAG,CAAC;KAC3C,EACD,OAAO,EACP,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,CACjC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uBAAuB,CAAC,CAAC;IAC1D,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAc,EAAE,CAAC;AAC1E,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,KAAa,EACb,IAAuC;IAEvC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC/D,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IACzD,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC1E,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC5C,OAAO,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;IAClF,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED,MAAM,aAAa,GAAG,CAAC,IAAe,EAAE,EAAE,CACxC,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,kBAAkB,EAAE,EAAE,EAAE,IAAI,CAAC,kBAAkB,EAAE,CAAC,SAAS,CAAC,CAAC;KACpF,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;KACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC;AAEtC,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,UAAmB,EAAE,EAAE,CACvD,GAAG,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,kEAAkE,CAAC;AAEhG,gGAAgG;AAChG,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,KAAoB,EAAE,SAAiB,EAAQ,EAAE,CAAC,CAAC;IACpF,IAAI,EAAE,OAAO;IACb,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC;IACnB,SAAS,EAAE,WAAW;IACtB,GAAG,CAAC,KAAK,KAAK,IAAI;QAChB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;CACtF,CAAC,CAAC;AAEH,yEAAyE;AACzE,MAAM,GAAG,GAAG,EAAE,UAAU,EAAE,KAAK,EAAW,CAAC;AAE3C;;;;GAIG;AACH,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,SAAiB,EACjB,KAA+B;IAE/B,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACtB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAC1F,MAAM,IAAI,GAAG,KAAK,EAAE,KAAa,EAAmB,EAAE;QACpD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,CACpB,KAAK,EACL,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,EACpC,CAAC,UAAU,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,EAC5C,GAAG,CACJ,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,2EAA2E;YAC3E,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,sFAAsF;YACtF,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,2BAA2B,KAAK,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,CAAC;YACnF,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,gBAAgB,GAAG,GAAG,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IACF,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,uEAAuE;IACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACjE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,IAAqE;IAErE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;IACtC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAA0B,CAAC;IACvE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAC7B,GAAG,EACH,CAAC,GAAG,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,CAAC,EACrD,OAAO,EACP,EAAE,SAAS,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,CACjC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAc,CAAC;IAC7C,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,gDAAgD,CAAC,CAAC;IAC5F,+FAA+F;IAC/F,IAAI,IAAuB,CAAC;IAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC;QAClB,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,SAAS;QACtB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAU,CAAC;QACnC,MAAM;IACR,CAAC;IACD,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,yBAAyB,IAAI,WAAW,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;IACjG,IAAI,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACrE,IAAI,MAAM;QACR,IAAI,IAAI,oJAAoJ,MAAM,IAAI,CAAC;IACzK,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC3C,OAAO;QACL,IAAI,EAAE,SAAS;QACf,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,GAAG,EAAE,IAAI,CAAC,WAAW,IAAI,mCAAmC,IAAI,CAAC,EAAE,EAAE;QACrE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GACV,yNAAyN,CAAC;AAE5N;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,IAKC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;IACtC,MAAM,EAAE,GAAG,+CAA+C,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1E,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,uBAAuB,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,mCAAmC,EAAE,EAAE,CAAC;IACrD,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CACzB,kDAAkD,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAC7E;SACE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACnC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAqE,CAAC;IAC1F,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,QAAQ;QACrB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;QAC5E,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACX,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAoB,EAAE,EAAE;QAC9C,MAAM,KAAK,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5D,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBAC/D,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,YAAY,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC9E,KAAK,EAAE,GAAG;iBACX,CAAC,CAAC;YACL,CAAC,CAAC;YACF,+DAA+D;YAC/D,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACxF,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;gBACrF,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;SACH,CAAC,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC1B,CAAC,CAAC;IACF,MAAM,KAAK,GAAuC,EAAE,CAAC;IACrD,8DAA8D;IAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACtD,OAAO;QACL,IAAI,EAAE,SAAS;QACf,EAAE;QACF,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,IAAI;QACjC,GAAG,EAAE,IAAI;QACT,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,IAAI,EAAE,gBAAgB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,aAAa,KACxD,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,qBAAqB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAC/D,EAAE;KACH,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAAW,EACX,IAAsC;IAEtC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAiB,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,SAAS,CAAC;IACpD,OAAO;QACL,IAAI,EAAE,CAAC,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE;QACnD,EAAE,EAAE,IAAI,CAAC,EAAE;QACX,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK;QAC1D,GAAG,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG;QAC5B,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,iBAAiB,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI;KACpF,CAAC;AACJ,CAAC;AAED,MAAM,SAAS,GAAG,WAAW,CAAC;AAE9B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,IAUC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IACzD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,UAAU,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7E,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS;YAC5D,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,UAAU,SAAS,GAAG,GAAG,KAAK,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACnC,MAAM,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC7D,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE;YACnC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,KAAK,EAAE,EAAE;SACV,CAAC,CAAC;QACH,OAAO;YACL,IAAI,EAAE,SAAS;YACf,EAAE;YACF,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,GAAG;YACrC,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,IAAI,EAAE,KAAK,CAAC,MAAM;SACnB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;AACH,CAAC"}

package/dist/video/watch.d.ts

@@ -0,0 +1,94 @@
+import { type Model } from "@wrenautomation/models";
+import { type Ear } from "@wrenautomation/models/hear";
+export declare const COHERE_VISION_MODEL = "command-a-vision-07-2025";
+/** The text model for asks that need no picture (`video look --with cohere`). */
+export declare const COHERE_TEXT_MODEL = "command-a-03-2025";
+export interface Frame {
+    /** Seconds into the video, from the ffmpeg frame grid. */
+    at: number;
+    text: string | null;
+}
+/** PySceneDetect's detectors: `hash` is the one films trust, `adaptive` the second signal. */
+export declare const SCENE_DETECTORS: readonly ["hash", "adaptive"];
+export type SceneDetector = (typeof SCENE_DETECTORS)[number];
+export interface Watched {
+    durationS: number;
+    /** Scene cuts ffmpeg found, in seconds. */
+    cuts: number[];
+    /** PySceneDetect's cuts by detector; only with `sceneDetect`. */
+    cutsBy?: Record<SceneDetector, number[]>;
+    /** Whether the file has a sound track at all. */
+    audio: boolean;
+    /** What the ear heard; "" when nobody speaks or speech was not asked for. */
+    speech: string;
+    frames: Frame[];
+    /** The video model's read of the whole screen, when one was given and it answered. */
+    whole?: string;
+}
+export interface WatchOptions {
+    /** Reads the frames, `BATCH` images a call (`cohereEye`). */
+    eye: Model;
+    /** Hears the speech (`cohereEar` from models). None: no transcript. */
+    ear?: Ear;
+    /** Optional: reads the whole video for what it shows (`geminiVideo`). It must take video. */
+    video?: Model | null;
+    ffmpeg?: string;
+    /** The PySceneDetect command (split on spaces: `uvx scenedetect`); set, its cuts come too (`cutsBy`). */
+    sceneDetect?: string;
+    /** False: skip the transcription (the caller has the words already). */
+    speech?: boolean;
+    /** ISO 639-1 code of the spoken language. */
+    language?: string;
+    /** Seconds between frames; stretched so a long video stays under `MAX_FRAMES`. */
+    everyS?: number;
+}
+/** Cohere's vision model on a key ring, waiting out a busy ring. */
+export declare const cohereEye: (keys: readonly string[], o?: {
+    fetch?: typeof fetch;
+}) => Model;
+/** Cohere's Transcribe on a key ring, waiting out a busy ring. */
+export declare const cohereSpeech: (keys: readonly string[], o?: {
+    fetch?: typeof fetch;
+}) => Ear;
+/** `0:03.2`, a mark with a tenth of a second. */
+export declare function stamp(s: number): string;
+type Shot = {
+    at: number;
+    jpeg: Buffer;
+};
+/**
+ * On-screen text for each frame, `BATCH` frames a call, `BATCHES_AT_ONCE` calls at a time. A
+ * batch that cannot be asked at all leaves its frames unread (and a warning); all of them
+ * failing throws.
+ */
+export declare function readScreen(shots: Shot[], eye: Model): Promise<Frame[]>;
+/** Cut times from a `list-scenes` CSV (`--skip-cuts`: the header, then one row a scene). Each scene after the first starts on a cut. */
+export declare function parseScenes(csv: string): number[];
+/** PySceneDetect's cuts by one detector. ffmpeg's `scene` misses a fade and a card swap on black; `hash` finds them (designs/studies/2026-10-25-launch-films.md). */
+export declare function sceneDetectCuts(file: string, detector: SceneDetector, cmd: string, dir: string): Promise<number[]>;
+/** Shot changes: the frames `select` keeps are logged by `showinfo`, on stderr. */
+export declare function sceneCuts(file: string, ffmpeg: string): Promise<number[]>;
+/** Watch a video file on disk (a URL is fetched first by `watch` in `url.ts`). */
+export declare function watchFile(input: string, opts: WatchOptions): Promise<Watched>;
+/** The frames with something to say, each under its ffmpeg frame time; the unread ones say so. */
+export declare function onScreenLines(w: Pick<Watched, "frames">): string[];
+export declare const cutsLine: (w: Pick<Watched, "cuts">) => string;
+/** `read`, `unread`: how many frames the model returned a line for. */
+export declare function frameCounts(w: Pick<Watched, "frames">): {
+    read: number;
+    unread: number;
+};
+/** The `## Speech`, `## On screen` and `## Cuts` sections; the CLI prints it and a SOP source holds it. */
+/** What a film's facts are read from later: the length, the sound track, every detector's cuts. */
+export interface WatchFacts {
+    duration_s: number;
+    audio: boolean;
+    cuts_by: {
+        ffmpeg: number[];
+    } & Partial<Record<SceneDetector, number[]>>;
+}
+export declare const watchFacts: (w: Pick<Watched, "durationS" | "audio" | "cuts" | "cutsBy">) => WatchFacts;
+/** The facts fence of a rendered page, when it has one. */
+export declare function readWatchFacts(md: string): WatchFacts | null;
+export declare function renderWatched(w: Watched): string;
+export {};