@wrenautomation/falcon 0.0.0-stage → 0.3.0

package/dist/text/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/text/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,WAAW,CAAC"}

package/dist/text/text.d.ts

@@ -0,0 +1,8 @@
+/** Postgres text refuses NUL; a PDF glyph with no text mapping comes out as one. */
+export declare const withoutNul: (text: string) => string;
+/** Tidy lines: trimmed, inner runs of spaces collapsed, at most one blank line between blocks. */
+export declare function tidy(text: string): string;
+/** HTML as readable text: blocks become lines, table cells stay on their row. */
+export declare function htmlText(html: string): string;
+/** A PDF's text, pages joined; null when it cannot be read (scanned, encrypted, broken). */
+export declare function pdfText(bytes: Uint8Array): Promise<string | null>;

package/dist/text/text.js

@@ -0,0 +1,38 @@
+import { decodeHtml } from "./entities.js";
+/** Postgres text refuses NUL; a PDF glyph with no text mapping comes out as one. */
+export const withoutNul = (text) => text.replaceAll("\u0000", " ");
+/** Tidy lines: trimmed, inner runs of spaces collapsed, at most one blank line between blocks. */
+export function tidy(text) {
+    return withoutNul(text)
+        .replace(/\r\n?/g, "\n")
+        .split("\n")
+        .map((line) => line.replace(/(?:[ \t\u00a0\u2007\u200b\u200c\u2060]|\u200d|\u034f)+/g, " ").trim())
+        .join("\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/** HTML as readable text: blocks become lines, table cells stay on their row. */
+export function htmlText(html) {
+    const text = html
+        .replace(/<!--[\s\S]*?-->/g, "")
+        .replace(/<(head|style|script|title)\b[\s\S]*?<\/\1\s*>/gi, "")
+        .replace(/<br\s*\/?>/gi, "\n")
+        .replace(/<\/(p|div|tr|li|h[1-6]|table|section|header|footer|blockquote)\s*>/gi, "\n")
+        .replace(/<\/(td|th)\s*>/gi, " \t ")
+        .replace(/<[^>]+>/g, "");
+    return tidy(decodeHtml(text));
+}
+/** A PDF's text, pages joined; null when it cannot be read (scanned, encrypted, broken). */
+export async function pdfText(bytes) {
+    try {
+        const { extractText, getDocumentProxy } = await import("unpdf");
+        const pdf = await getDocumentProxy(new Uint8Array(bytes));
+        const { text } = await extractText(pdf, { mergePages: true });
+        const out = tidy(text);
+        return out || null;
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=text.js.map

package/dist/text/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/text/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,oFAAoF;AACpF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAE3E,kGAAkG;AAClG,MAAM,UAAU,IAAI,CAAC,IAAY;IAC/B,OAAO,UAAU,CAAC,IAAI,CAAC;SACpB,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,IAAI,CAAC,OAAO,CAAC,yDAAyD,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CACpF;SACA,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,IAAI,GAAG,IAAI;SACd,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,iDAAiD,EAAE,EAAE,CAAC;SAC9D,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC;SAC7B,OAAO,CAAC,sEAAsE,EAAE,IAAI,CAAC;SACrF,OAAO,CAAC,kBAAkB,EAAE,MAAM,CAAC;SACnC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAiB;IAC7C,IAAI,CAAC;QACH,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QAChE,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1D,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,GAAG,IAAI,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/video/gemini.d.ts

@@ -0,0 +1,35 @@
+/**
+ * What to read off the frames; shared by YouTube clips and downloaded videos. Gemini won't repeat
+ * text it knows from the web (RECITATION); `paraphrase` asks for what that text says instead.
+ */
+export declare const SHOWN: (start: number, paraphrase: boolean) => string;
+/**
+ * One Gemini call. Keys rotate on 429/403, a busy flash (503) hands to lite, backing off 2s
+ * doubling to 1 min. A call that stalls or drops is a refusal too. After 24 refusals, or
+ * `askMs` (6 minutes), it returns `unread` with the last one.
+ */
+export declare function askGemini(body: unknown, keys: readonly string[], fetchFn: typeof fetch, label: string, limits?: {
+    callMs?: number;
+    askMs?: number;
+    pause?: (ms: number) => Promise<void>;
+}): Promise<{
+    text: string;
+} | {
+    unread: string;
+}>;
+/** The on-screen ask, asked again as a paraphrase when Gemini won't recite. */
+export declare function askShown(body: (paraphrase: boolean) => unknown, keys: readonly string[], fetchFn: typeof fetch, label: string): Promise<{
+    text: string;
+} | {
+    unread: string;
+}>;
+export declare const INLINE_MAX = 14000000;
+/**
+ * A video re-encoded to fit `INLINE_MAX`: short side 360, 15 fps, mono speech, and a bitrate
+ * that spends 85% of the cap over its length (four minutes when yt-dlp didn't say).
+ */
+export declare function shrink(src: string, out: string, info: {
+    duration?: number;
+    width?: number;
+    height?: number;
+}, ffmpeg?: string): Promise<Buffer>;

package/dist/video/gemini.js

@@ -0,0 +1,103 @@
+/**
+ * Gemini, as the video readers ask it: key and model rotation, the on-screen ask, and the 360p copy
+ * that fits inline. Own file so `watch.ts` and the SOP reader both import it.
+ */
+import { readFile } from "node:fs/promises";
+import { clock, runFor, TOOL_MS } from "./tools.js";
+const GEMINI = (model) => `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`;
+// Rotated per try: a busy (503) or reciting model hands the ask to the next.
+const GEMINI_MODELS = ["gemini-3.8-flash", "gemini-3.5-flash", "gemini-3.5-flash-lite"];
+/**
+ * What to read off the frames; shared by YouTube clips and downloaded videos. Gemini won't repeat
+ * text it knows from the web (RECITATION); `paraphrase` asks for what that text says instead.
+ */
+export const SHOWN = (start, paraphrase) => `Transcribe in full, verbatim, every document, list, table, slide, template or other text the speaker shows on screen in this clip, each under a [h:mm:ss] heading of the video time it appears (the clip starts at ${clock(start)}). For every image, screenshot, chart, diagram or UI on screen, extract the information it carries, not what it looks like: the text in it, setting names and their values, numbers, prices, limits, labels, before and after states, and what is highlighted, circled, crossed out, ticked or pointed at. Write each as a line starting with its [h:mm:ss] then [visual], stating the fact as a reader would use it ("Instantly warmup settings: daily limit 10, slow ramp on, +1 per day, reply rate 95%"), not "a screenshot of the settings page". Skip visuals that carry no information (decoration, layout, gestures, transitions) and repeats of anything already captured. Skip the speaker's face, chat windows, b-roll and stock footage, and anything already shown earlier in the clip.${paraphrase ? " Where a shown passage is published text (a post, article, page), give its first line word for word, then every point it makes in your own words." : ""}`;
+let nextKey = 0;
+/** One Gemini call gives up after this long: a stalled socket would wait out undici's 5 minutes, per try. */
+const GEMINI_CALL_MS = 180_000;
+/** One ask, its retries and backoffs together, gives up after this long. */
+const GEMINI_ASK_MS = 360_000;
+/**
+ * One Gemini call. Keys rotate on 429/403, a busy flash (503) hands to lite, backing off 2s
+ * doubling to 1 min. A call that stalls or drops is a refusal too. After 24 refusals, or
+ * `askMs` (6 minutes), it returns `unread` with the last one.
+ */
+export async function askGemini(body, keys, fetchFn, label, limits = {}) {
+    const until = Date.now() + (limits.askMs ?? GEMINI_ASK_MS);
+    const pause = limits.pause ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    let last = "";
+    let wait = 2000;
+    let recited = 0;
+    for (let i = 0; i < 24 && Date.now() < until; i++) {
+        const key = keys[nextKey++ % keys.length];
+        const model = GEMINI_MODELS[i % GEMINI_MODELS.length];
+        let res;
+        let json;
+        try {
+            res = await fetchFn(GEMINI(model), {
+                method: "POST",
+                headers: { "content-type": "application/json", "x-goog-api-key": key },
+                body: JSON.stringify(body),
+                signal: AbortSignal.timeout(limits.callMs ?? GEMINI_CALL_MS),
+            });
+            json = (await res.json());
+        }
+        catch (err) {
+            last = `${model} ${err instanceof Error ? err.message : String(err)}`;
+            await pause(wait);
+            wait = Math.min(wait * 2, 60_000);
+            continue;
+        }
+        const text = (json.candidates?.[0]?.content?.parts ?? [])
+            .map((p) => p.text ?? "")
+            .join("")
+            .trim();
+        if (text)
+            return { text };
+        const reason = json.candidates?.[0]?.finishReason;
+        // Recitation is per model and try, not per key: no backoff, and three strikes go back to the caller.
+        if (reason === "RECITATION") {
+            if (++recited === 3)
+                return { unread: "RECITATION" };
+            continue;
+        }
+        // A 200 with no text (blocked, cut off) is a refusal too: try the next key and model.
+        last = res.ok
+            ? `${model} empty, ${reason ?? "no candidate"}`
+            : `${model} HTTP ${res.status} ${json.error?.message ?? ""}`.trim();
+        if (!res.ok && ![401, 402, 403, 429, 500, 503].includes(res.status))
+            throw new Error(`gemini ${label}: ${last}`);
+        await pause(wait);
+        wait = Math.min(wait * 2, 60_000);
+    }
+    return { unread: last || "timed out" };
+}
+/** The on-screen ask, asked again as a paraphrase when Gemini won't recite. */
+export async function askShown(body, keys, fetchFn, label) {
+    const r = await askGemini(body(false), keys, fetchFn, label);
+    return "unread" in r && r.unread === "RECITATION"
+        ? askGemini(body(true), keys, fetchFn, label)
+        : r;
+}
+// Gemini takes 20 MB a request inline, and base64 adds a third.
+export const INLINE_MAX = 14_000_000;
+/**
+ * A video re-encoded to fit `INLINE_MAX`: short side 360, 15 fps, mono speech, and a bitrate
+ * that spends 85% of the cap over its length (four minutes when yt-dlp didn't say).
+ */
+export async function shrink(src, out, info, ffmpeg = "ffmpeg") {
+    const seconds = info.duration && info.duration > 0 ? info.duration : 240;
+    const audio = 48_000;
+    const video = Math.max(120_000, Math.floor((INLINE_MAX * 8 * 0.85) / seconds - audio));
+    const tall = (info.height ?? 1) >= (info.width ?? 0);
+    await runFor(ffmpeg, [
+        ...["-y", "-loglevel", "error", "-i", src],
+        ...["-vf", tall ? "scale=360:-2" : "scale=-2:360", "-r", "15"],
+        ...["-c:v", "libx264", "-preset", "veryfast", "-b:v", String(video)],
+        ...["-maxrate", String(video), "-bufsize", String(video * 2)],
+        ...["-c:a", "aac", "-b:a", String(audio), "-ac", "1"],
+        ...["-movflags", "+faststart", out],
+    ], TOOL_MS);
+    return readFile(out);
+}
+//# sourceMappingURL=gemini.js.map

package/dist/video/gemini.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini.js","sourceRoot":"","sources":["../../src/video/gemini.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,EAAE,CAC/B,2DAA2D,KAAK,kBAAkB,CAAC;AACrF,6EAA6E;AAC7E,MAAM,aAAa,GAAG,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,uBAAuB,CAAC,CAAC;AAExF;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,UAAmB,EAAE,EAAE,CAC1D,sNAAsN,KAAK,CAAC,KAAK,CAAC,uwBAAuwB,UAAU,CAAC,CAAC,CAAC,mJAAmJ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AAEnpC,IAAI,OAAO,GAAG,CAAC,CAAC;AAEhB,6GAA6G;AAC7G,MAAM,cAAc,GAAG,OAAO,CAAC;AAC/B,4EAA4E;AAC5E,MAAM,aAAa,GAAG,OAAO,CAAC;AAE9B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAa,EACb,IAAuB,EACvB,OAAqB,EACrB,KAAa,EACb,SAAqF,EAAE;IAEvF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAW,CAAC;QACpD,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAAW,CAAC;QAChE,IAAI,GAAa,CAAC;QAClB,IAAI,IAGH,CAAC;QACF,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBACjC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,GAAG,EAAE;gBACtE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,cAAc,CAAC;aAC7D,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAgB,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,GAAG,KAAK,IAAI,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAClB,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;YAClC,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;aACtD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;aACxB,IAAI,CAAC,EAAE,CAAC;aACR,IAAI,EAAE,CAAC;QACV,IAAI,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC;QAClD,qGAAqG;QACrG,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;YAC5B,IAAI,EAAE,OAAO,KAAK,CAAC;gBAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;YACrD,SAAS;QACX,CAAC;QACD,sFAAsF;QACtF,IAAI,GAAG,GAAG,CAAC,EAAE;YACX,CAAC,CAAC,GAAG,KAAK,WAAW,MAAM,IAAI,cAAc,EAAE;YAC/C,CAAC,CAAC,GAAG,KAAK,SAAS,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QACtE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,IAAI,WAAW,EAAE,CAAC;AACzC,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAsC,EACtC,IAAuB,EACvB,OAAqB,EACrB,KAAa;IAEb,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAC7D,OAAO,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,YAAY;QAC/C,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC;QAC7C,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,gEAAgE;AAChE,MAAM,CAAC,MAAM,UAAU,GAAG,UAAU,CAAC;AAErC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,GAAW,EACX,GAAW,EACX,IAA4D,EAC5D,MAAM,GAAG,QAAQ;IAEjB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;IACzE,MAAM,KAAK,GAAG,MAAM,CAAC;IACrB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,CACV,MAAM,EACN;QACE,GAAG,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC;QAC1C,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC;QAC9D,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACpE,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC7D,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC;QACrD,GAAG,CAAC,WAAW,EAAE,YAAY,EAAE,GAAG,CAAC;KACpC,EACD,OAAO,CACR,CAAC;IACF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;AACvB,CAAC"}

package/dist/video/index.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Video: `watch` (ffmpeg facts, Cohere speech and on-screen text, Gemini when a key works) over a
+ * file on disk, and the `VideoRead` shape with the captions renderer and the audio hearer
+ * (`read.ts`). Every caller passes its own keys and tools. Fetching is weaver's. No database here.
+ */
+export * from "./gemini.js";
+export * from "./read.js";
+export * from "./shots.js";
+export * from "./tools.js";
+export * from "./watch.js";
+/** What `yt-dlp -J` prints; only what the readers use. */
+export interface VideoInfo {
+    id: string;
+    title: string;
+    channel?: string;
+    upload_date?: string;
+    webpage_url?: string;
+    duration?: number;
+    /** Its thumbnail's address. */
+    thumbnail?: string;
+    chapters?: {
+        start_time: number;
+        end_time: number;
+        title: string;
+    }[] | null;
+    subtitles?: Record<string, {
+        ext: string;
+        url: string;
+    }[]>;
+    automatic_captions?: Record<string, {
+        ext: string;
+        url: string;
+    }[]>;
+}

package/dist/video/index.js

@@ -0,0 +1,11 @@
+/**
+ * Video: `watch` (ffmpeg facts, Cohere speech and on-screen text, Gemini when a key works) over a
+ * file on disk, and the `VideoRead` shape with the captions renderer and the audio hearer
+ * (`read.ts`). Every caller passes its own keys and tools. Fetching is weaver's. No database here.
+ */
+export * from "./gemini.js";
+export * from "./read.js";
+export * from "./shots.js";
+export * from "./tools.js";
+export * from "./watch.js";
+//# sourceMappingURL=index.js.map

package/dist/video/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/video/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC"}

package/dist/video/read.d.ts

@@ -0,0 +1,52 @@
+import type { VideoInfo } from "./index.js";
+/** What a reader gives back: where the piece is, and its markdown sections. */
+export interface VideoRead {
+    /** The platform, lower case: `youtube`, `instagram`, `tiktok`, `podcast` (yt-dlp's extractor key). */
+    kind: string;
+    /** Its id on the platform; a podcast's is a hash of its audio URL. */
+    id: string;
+    title: string;
+    url: string;
+    channel?: string;
+    /** YYYY-MM-DD. */
+    uploaded?: string;
+    /** Seconds, rounded. */
+    duration?: number;
+    thumbnail?: string;
+    /** The sections as markdown, headings at `##`: one per chapter, or Caption, Speech, On screen, Cuts. */
+    body: string;
+}
+/** YouTube's json3 caption track; only what is read. */
+export interface Json3 {
+    events: {
+        tStartMs: number;
+        segs?: {
+            utf8: string;
+        }[];
+    }[];
+}
+/** Seconds of video per model call. */
+export declare const CLIP_S = 600;
+/** yt-dlp's `YYYYMMDD` as an ISO date. */
+export declare const isoDate: (d: string | undefined) => string | undefined;
+/** A model's "none": nothing shown or said. */
+export declare const real: (s: string) => boolean | "";
+/** Captions as markdown: a heading per chapter, a `[m:ss]` marker every minute so a reader can cite a moment. */
+export declare function captionsBody(info: VideoInfo, track: Json3): string;
+/**
+ * A podcast episode heard from its audio file on disk: cut by ffmpeg into `CLIP_S` mono clips
+ * small enough to send inline (about 2 MB each), each transcribed by Gemini, 4 at a time, times
+ * running on across clips. The clips are written beside the file and left for the caller's
+ * cleanup. `label` names the calls in errors.
+ */
+export declare function hearAudio(file: string, opts: {
+    geminiKeys: readonly string[];
+    fetchFn?: typeof fetch;
+    ffmpeg?: string;
+    /** Seconds per clip; `CLIP_S` unless a test cuts a short tone. */
+    clipS?: number;
+    label?: string;
+}): Promise<{
+    speech: string;
+    duration: number;
+}>;

package/dist/video/read.js

@@ -0,0 +1,99 @@
+/**
+ * A video or podcast read as one markdown document: the `VideoRead` shape, the captions renderer
+ * (a YouTube json3 track to a heading per chapter) and the audio hearer (a file cut by ffmpeg into
+ * clips, each transcribed by Gemini). Everything here takes text or a file on disk; fetching the
+ * captions or the audio is weaver's (`@wrenautomation/weaver/video`). Every caller passes its own
+ * keys and tools. Nothing here touches a database.
+ */
+import { readdir, readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { askGemini } from "./gemini.js";
+import { clock, run } from "./tools.js";
+const MARK_EVERY_S = 60;
+/** Seconds of video per model call. */
+export const CLIP_S = 600;
+/** yt-dlp's `YYYYMMDD` as an ISO date. */
+export const isoDate = (d) => d?.replace(/^(\d{4})(\d{2})(\d{2})$/, "$1-$2-$3");
+/** A model's "none": nothing shown or said. */
+export const real = (s) => s && s.toLowerCase() !== "none";
+/** Captions as markdown: a heading per chapter, a `[m:ss]` marker every minute so a reader can cite a moment. */
+export function captionsBody(info, track) {
+    const chapters = info.chapters?.length
+        ? info.chapters
+        : [{ start_time: 0, end_time: info.duration ?? Number.POSITIVE_INFINITY, title: info.title }];
+    const words = track.events
+        .filter((e) => e.segs?.length)
+        .map((e) => ({
+        t: e.tStartMs / 1000,
+        text: (e.segs ?? [])
+            .map((s) => s.utf8)
+            .join("")
+            .replace(/\s+/g, " ")
+            .trim(),
+    }))
+        .filter((w) => w.text);
+    return chapters
+        .map((c) => {
+        const lines = [];
+        let mark = Number.NEGATIVE_INFINITY;
+        for (const w of words) {
+            if (w.t < c.start_time || w.t >= c.end_time)
+                continue;
+            if (w.t - mark >= MARK_EVERY_S) {
+                mark = w.t;
+                lines.push(`\n[${clock(w.t)}]`);
+            }
+            lines.push(w.text);
+        }
+        return `## [${clock(c.start_time)}] ${c.title}\n${lines.join(" ").trim()}\n`;
+    })
+        .join("\n");
+}
+const HEARD = (start) => `Transcribe the speech in this podcast audio: every word spoken, in paragraphs, each starting with its time in the episode as [m:ss], or [h:mm:ss] past the hour (this clip starts at ${clock(start)}), and the speaker's name in bold when it's known. Fix only obvious mishearings. Leave out ad reads. Markdown, no heading. If nobody speaks, reply with the single word none.`;
+/**
+ * A podcast episode heard from its audio file on disk: cut by ffmpeg into `CLIP_S` mono clips
+ * small enough to send inline (about 2 MB each), each transcribed by Gemini, 4 at a time, times
+ * running on across clips. The clips are written beside the file and left for the caller's
+ * cleanup. `label` names the calls in errors.
+ */
+export async function hearAudio(file, opts) {
+    if (!opts.geminiKeys.length)
+        throw new Error(`${file}: needs Gemini keys to transcribe`);
+    const clipS = opts.clipS ?? CLIP_S;
+    const fetchFn = opts.fetchFn ?? fetch;
+    const ffmpeg = opts.ffmpeg ?? "ffmpeg";
+    const label = opts.label ?? file;
+    const dir = dirname(file);
+    await run(ffmpeg, [
+        ...["-hide_banner", "-loglevel", "error", "-i", file, "-vn", "-ac", "1", "-ar", "16000"],
+        ...["-c:a", "libopus", "-b:a", "24k", "-f", "segment", "-segment_time", String(clipS)],
+        ...["-reset_timestamps", "1", join(dir, "clip%03d.ogg")],
+    ], { maxBuffer: 16 * 1024 * 1024 });
+    const clips = (await readdir(dir)).filter((f) => f.startsWith("clip")).sort();
+    if (!clips.length)
+        throw new Error(`${file}: ffmpeg cut no clips`);
+    const hear = async (clip, i) => {
+        const data = (await readFile(join(dir, clip))).toString("base64");
+        const said = await askGemini({
+            contents: [
+                {
+                    parts: [{ inlineData: { mimeType: "audio/ogg", data } }, { text: HEARD(i * clipS) }],
+                },
+            ],
+        }, opts.geminiKeys, fetchFn, `${label} ${clock(i * clipS)} speech`);
+        if ("unread" in said)
+            throw new Error(`gemini ${label} ${clock(i * clipS)}: unread, last ${said.unread}`);
+        return real(said.text) ? said.text : "";
+    };
+    const speech = [];
+    for (let i = 0; i < clips.length; i += 4)
+        speech.push(...(await Promise.all(clips.slice(i, i + 4).map((f, j) => hear(f, i + j)))));
+    const { stdout } = await run(ffmpeg.replace(/ffmpeg$/, "ffprobe"), [
+        ...["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file],
+    ]).catch(() => ({ stdout: "" }));
+    return {
+        speech: `## Speech\n\n${speech.filter(Boolean).join("\n\n") || "(no speech)"}\n`,
+        duration: Math.round(Number(stdout.trim()) || 0),
+    };
+}
+//# sourceMappingURL=read.js.map

package/dist/video/read.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.js","sourceRoot":"","sources":["../../src/video/read.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAyBxC,MAAM,YAAY,GAAG,EAAE,CAAC;AACxB,uCAAuC;AACvC,MAAM,CAAC,MAAM,MAAM,GAAG,GAAG,CAAC;AAE1B,0CAA0C;AAC1C,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAqB,EAAsB,EAAE,CACnE,CAAC,EAAE,OAAO,CAAC,yBAAyB,EAAE,UAAU,CAAC,CAAC;AAEpD,+CAA+C;AAC/C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;AAEnE,iHAAiH;AACjH,MAAM,UAAU,YAAY,CAAC,IAAe,EAAE,KAAY;IACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM;QACpC,CAAC,CAAC,IAAI,CAAC,QAAQ;QACf,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,iBAAiB,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAChG,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM;SACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC;SAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,CAAC,EAAE,CAAC,CAAC,QAAQ,GAAG,IAAI;QACpB,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;aACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,IAAI,CAAC,EAAE,CAAC;aACR,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,IAAI,EAAE;KACV,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACpC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ;gBAAE,SAAS;YACtD,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,YAAY,EAAE,CAAC;gBAC/B,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;gBACX,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,OAAO,OAAO,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC;IAC/E,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE,CAC9B,wLAAwL,KAAK,CAAC,KAAK,CAAC,+KAA+K,CAAC;AAEtX;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,IAOC;IAED,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,mCAAmC,CAAC,CAAC;IACzF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,MAAM,GAAG,CACP,MAAM,EACN;QACE,GAAG,CAAC,cAAc,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC;QACxF,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACtF,GAAG,CAAC,mBAAmB,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;KACzD,EACD,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAChC,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,uBAAuB,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,KAAK,EAAE,IAAY,EAAE,CAAS,EAAE,EAAE;QAC7C,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,SAAS,CAC1B;YACE,QAAQ,EAAE;gBACR;oBACE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC;iBACrF;aACF;SACF,EACD,IAAI,CAAC,UAAU,EACf,OAAO,EACP,GAAG,KAAK,IAAI,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,SAAS,CACtC,CAAC;QACF,IAAI,QAAQ,IAAI,IAAI;YAClB,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,kBAAkB,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1C,CAAC,CAAC;IACF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE;QACjE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC;KAC/E,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IACjC,OAAO;QACL,MAAM,EAAE,gBAAgB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,aAAa,IAAI;QAChF,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC;KACjD,CAAC;AACJ,CAAC"}

package/dist/video/shots.d.ts

@@ -0,0 +1,33 @@
+/** The labels asked, each with its closed set. */
+export declare const LABELS: readonly ["type", "move", "in"];
+export type Label = (typeof LABELS)[number];
+export type LabelSets = Record<Label, readonly string[]>;
+export type ShotLabels = Record<Label, string | null>;
+/** A shot to label: its start and length in seconds. */
+export interface ShotSpan {
+    t: number;
+    len: number;
+}
+export declare const labelAsk: (sets: LabelSets, first: boolean) => string;
+/** The model's answer, held to the sets: an answer outside one is null. */
+export declare function parseLabels(answer: string, sets: LabelSets): ShotLabels;
+/** The frame times for a shot: just before it (not for the first), its start, middle and end. */
+export declare function shotTimes(s: ShotSpan): number[];
+/**
+ * Each shot's labels, one vision call a shot, `AT_ONCE` at a time. A shot whose call fails keeps
+ * null labels and a warning; every call failing throws.
+ */
+export declare function labelShots(file: string, shots: readonly ShotSpan[], sets: LabelSets, o: {
+    cohereKeys: readonly string[];
+    ffmpeg?: string;
+    bright?: boolean;
+    fetchFn?: typeof fetch;
+    pause?: (ms: number) => Promise<void>;
+}): Promise<ShotLabels[]>;
+/** Each label's share of hand labels the machine matched; a hand label that is null is skipped. */
+export declare function labelMatch(hand: readonly Partial<ShotLabels>[], machine: readonly ShotLabels[]): Record<Label, {
+    n: number;
+    hits: number;
+    share: number;
+    misses: [string, string | null][];
+}>;

package/dist/video/shots.js

@@ -0,0 +1,151 @@
+/**
+ * Shot labels by vision (designs/2026-10-25-competitor-analyzer.md, Read): for each shot of a film,
+ * frames at its start, middle and end, and one just before it starts, go to Cohere's vision
+ * model, which picks the shot's type, camera move and way in from closed sets the caller gives.
+ * An answer outside a set is null, never guessed. The caller decides which labels it trusts: a
+ * label ships only once it matches a hand seed at 80% or better (`labelMatch`).
+ */
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runFor, TOOL_MS } from "./tools.js";
+import { COHERE_VISION_MODEL, cohereChat } from "./watch.js";
+/** The labels asked, each with its closed set. */
+export const LABELS = ["type", "move", "in"];
+/** Calls at a time: the vision model's per-minute limit refuses 3 at once on a film. */
+const AT_ONCE = 1;
+/** A per-minute cap: wait it out this long, at most `CAP_WAITS` times a shot. */
+const CAP_WAIT_MS = 61_000;
+const CAP_WAITS = 3;
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** `fn`, again after a minute each time Cohere's per-minute cap refuses it. */
+async function waitOutCap(fn, pause) {
+    for (let i = 0;; i++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            if (i >= CAP_WAITS || !/HTTP 429/.test(String(err)))
+                throw err;
+            await pause(CAP_WAIT_MS);
+        }
+    }
+}
+/** Dark films read as nothing; this lifts the shadows before the model sees them. */
+const BRIGHTEN = "eq=gamma=1.6:contrast=1.15";
+const MEANS = {
+    type: "what the shot shows: ui (an app screen), b-roll (real-world footage), graphic (motion design, no app), logo, text-card (words alone on a plain ground), talking-head (a person speaking to camera)",
+    move: "how the camera moves within the shot: static, push (in), pull (out), pan, track (slides along), orbit (circles the subject), roll (tilts the frame or the plane), rack-focus, macro (extreme close-up), ui-zoom (zooms into part of a screen)",
+    in: "how the shot begins after the one before: cut, dissolve (the two overlap), fade (from or through black or white), whip (a fast blur)",
+};
+export const labelAsk = (sets, first) => `These frames are from one shot of a produced product film, in order: ${first ? "its start" : "the last moment of the shot before it, then its start"}, its middle and its end. Label the shot.
+${LABELS.map((l) => `- ${l}: ${MEANS[l]}. One of: ${sets[l].join(", ")}.`).join("\n")}
+Answer JSON only: {${LABELS.map((l) => `"${l}": "..."`).join(", ")}}. Use null for a label you cannot tell.`;
+/** The model's answer, held to the sets: an answer outside one is null. */
+export function parseLabels(answer, sets) {
+    const json = /\{[\s\S]*\}/.exec(answer)?.[0];
+    let got = {};
+    try {
+        got = json ? JSON.parse(json) : {};
+    }
+    catch {
+        got = {};
+    }
+    const pick = (l) => {
+        const v = typeof got[l] === "string" ? got[l].trim().toLowerCase() : null;
+        return v && sets[l].includes(v) ? v : null;
+    };
+    return { type: pick("type"), move: pick("move"), in: pick("in") };
+}
+/** The frame times for a shot: just before it (not for the first), its start, middle and end. */
+export function shotTimes(s) {
+    const edge = Math.min(0.3, s.len * 0.15);
+    const r = (x) => Math.round(x * 100) / 100;
+    return [
+        ...(s.t > 0 ? [r(Math.max(0, s.t - 0.2))] : []),
+        r(s.t + edge),
+        r(s.t + s.len / 2),
+        r(s.t + s.len - edge),
+    ];
+}
+async function frameAt(file, at, out, ffmpeg, bright) {
+    const scale = "scale='if(gt(iw,ih),-2,360)':'if(gt(iw,ih),360,-2)'";
+    await runFor(ffmpeg, [
+        ...["-nostdin", "-hide_banner", "-loglevel", "error", "-ss", String(at), "-i", file, "-an"],
+        ...["-frames:v", "1", "-vf", bright ? `${scale},${BRIGHTEN}` : scale, "-q:v", "4", out],
+    ], TOOL_MS);
+    return readFile(out);
+}
+/**
+ * Each shot's labels, one vision call a shot, `AT_ONCE` at a time. A shot whose call fails keeps
+ * null labels and a warning; every call failing throws.
+ */
+export async function labelShots(file, shots, sets, o) {
+    const ffmpeg = o.ffmpeg ?? "ffmpeg";
+    const dir = await mkdtemp(join(tmpdir(), "falcon-shots-"));
+    const none = { type: null, move: null, in: null };
+    try {
+        const out = [];
+        let failed = 0;
+        for (let i = 0; i < shots.length; i += AT_ONCE) {
+            const group = shots.slice(i, i + AT_ONCE);
+            const done = await Promise.allSettled(group.map((s, j) => waitOutCap(() => labelOne(s, i + j), o.pause ?? sleep)));
+            for (const r of done) {
+                if (r.status === "fulfilled")
+                    out.push(r.value);
+                else {
+                    failed++;
+                    console.warn(`shot ${out.length + 1}: ${r.reason}; labels left empty`);
+                    out.push(none);
+                }
+            }
+        }
+        if (shots.length && failed === shots.length)
+            throw new Error("every shot's label call failed");
+        return out;
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+    async function labelOne(s, n) {
+        const times = shotTimes(s);
+        const jpegs = await Promise.all(times.map((at, k) => frameAt(file, at, join(dir, `${n}-${k}.jpg`), ffmpeg, o.bright ?? true)));
+        const answer = await cohereChat(o.cohereKeys, {
+            model: COHERE_VISION_MODEL,
+            messages: [
+                {
+                    role: "user",
+                    content: [
+                        { type: "text", text: labelAsk(sets, s.t <= 0) },
+                        ...jpegs.map((b) => ({
+                            type: "image_url",
+                            image_url: { url: `data:image/jpeg;base64,${b.toString("base64")}` },
+                        })),
+                    ],
+                },
+            ],
+        }, {
+            label: `shot ${n + 1}`,
+            ...(o.fetchFn ? { fetchFn: o.fetchFn } : {}),
+            ...(o.pause ? { pause: o.pause } : {}),
+        });
+        return parseLabels(answer, sets);
+    }
+}
+/** Each label's share of hand labels the machine matched; a hand label that is null is skipped. */
+export function labelMatch(hand, machine) {
+    return Object.fromEntries(LABELS.map((l) => {
+        const pairs = hand.flatMap((h, i) => h[l] ? [[h[l], machine[i]?.[l] ?? null]] : []);
+        const hits = pairs.filter(([a, b]) => a === b).length;
+        return [
+            l,
+            {
+                n: pairs.length,
+                hits,
+                share: pairs.length ? hits / pairs.length : 0,
+                misses: pairs.filter(([a, b]) => a !== b),
+            },
+        ];
+    }));
+}
+//# sourceMappingURL=shots.js.map

package/dist/video/shots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shots.js","sourceRoot":"","sources":["../../src/video/shots.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7D,kDAAkD;AAClD,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAU,CAAC;AAWtD,wFAAwF;AACxF,MAAM,OAAO,GAAG,CAAC,CAAC;AAClB,iFAAiF;AACjF,MAAM,WAAW,GAAG,MAAM,CAAC;AAC3B,MAAM,SAAS,GAAG,CAAC,CAAC;AACpB,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAE1E,+EAA+E;AAC/E,KAAK,UAAU,UAAU,CAAI,EAAoB,EAAE,KAAoC;IACrF,KAAK,IAAI,CAAC,GAAG,CAAC,GAAI,CAAC,EAAE,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAAE,MAAM,GAAG,CAAC;YAC/D,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,qFAAqF;AACrF,MAAM,QAAQ,GAAG,4BAA4B,CAAC;AAE9C,MAAM,KAAK,GAA0B;IACnC,IAAI,EAAE,oMAAoM;IAC1M,IAAI,EAAE,+OAA+O;IACrP,EAAE,EAAE,sIAAsI;CAC3I,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAe,EAAE,KAAc,EAAE,EAAE,CAC1D,wEAAwE,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,uDAAuD;EACrJ,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;qBAChE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,0CAA0C,CAAC;AAE7G,2EAA2E;AAC3E,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,IAAe;IACzD,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,GAAG,GAA4B,EAAE,CAAC;IACtC,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAA6B,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,GAAG,EAAE,CAAC;IACX,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAQ,EAAE,EAAE;QACxB,MAAM,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAY,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACtF,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AACpE,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,SAAS,CAAC,CAAW;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;IACnD,OAAO;QACL,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QACb,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC;KACtB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,IAAY,EAAE,EAAU,EAAE,GAAW,EAAE,MAAc,EAAE,MAAe;IAC3F,MAAM,KAAK,GAAG,qDAAqD,CAAC;IACpE,MAAM,MAAM,CACV,MAAM,EACN;QACE,GAAG,CAAC,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC;QAC3F,GAAG,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC;KACxF,EACD,OAAO,CACR,CAAC;IACF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAY,EACZ,KAA0B,EAC1B,IAAe,EACf,CAMC;IAED,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;IACpC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAe,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAiB,EAAE,CAAC;QAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,OAAO,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC;YAC1C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,UAAU,CACnC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,CAC5E,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;oBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;qBAC3C,CAAC;oBACJ,MAAM,EAAE,CAAC;oBACT,OAAO,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,MAAM,qBAAqB,CAAC,CAAC;oBACvE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,IAAI,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC/F,OAAO,GAAG,CAAC;IACb,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,UAAU,QAAQ,CAAC,CAAW,EAAE,CAAS;QAC5C,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAC9F,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,CAAC,CAAC,UAAU,EACZ;YACE,KAAK,EAAE,mBAAmB;YAC1B,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE;wBAChD,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;4BACnB,IAAI,EAAE,WAAW;4BACjB,SAAS,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE;yBACrE,CAAC,CAAC;qBACJ;iBACF;aACF;SACF,EACD;YACE,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE;YACtB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvC,CACF,CAAC;QACF,OAAO,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,UAAU,CACxB,IAAoC,EACpC,OAA8B;IAE9B,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAClC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAW,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAA4B,CAAC,CAAC,CAAC,CAAC,EAAE,CACnF,CAAC;QACF,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACtD,OAAO;YACL,CAAC;YACD;gBACE,CAAC,EAAE,KAAK,CAAC,MAAM;gBACf,IAAI;gBACJ,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC7C,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;aAC1C;SACF,CAAC;IACJ,CAAC,CAAC,CACM,CAAC;AACb,CAAC"}

package/dist/video/tools.d.ts

@@ -0,0 +1,16 @@
+/**
+ * What the source readers share: a child tool run with a time limit, and the clock mark.
+ * Own file so `watch.ts` and any other reader import the same one.
+ */
+import { execFile } from "node:child_process";
+export declare const run: typeof execFile.__promisify__;
+/** yt-dlp's metadata call, or a download and remux of a reel, past this is stuck. */
+export declare const TOOL_MS = 300000;
+/** Run a tool; past `ms` it is killed and the error says which tool and how long. */
+export declare function runFor(cmd: string, args: string[], ms: number, opts?: {
+    maxBuffer?: number;
+}): Promise<{
+    stdout: string;
+    stderr: string;
+}>;
+export declare const clock: (s: number) => string;