opencaptions 1.2.0 → 1.3.0

package/README.md

@@ -1,14 +1,19 @@
 # opencaptions
 
 Animated word-by-word captions for your videos, from Claude Code, Claude Desktop, Cursor or any MCP
-client, or from the command line. It runs on your own [OpenCaptions](https://github.com/leogaudin/opencaptions)
-server: your video goes there and back, and nowhere else.
+client, or from the command line. It runs on your computer, with nothing to sign up for: the speech
+model and the caption engine of [OpenCaptions](https://github.com/leogaudin/opencaptions) run locally,
+and your video stays where it is.
 
 ```bash
-export OPENCAPTIONS_KEY=oc_...          # an API key from your server's Account page
 npx opencaptions caption reel.mp4 --look Bold --language fr
 ```
 
+It needs [FFmpeg](https://ffmpeg.org/download.html), and downloads the speech model (about 700 MB)
+the first time. It transcribes 25 European languages. To use an OpenCaptions server instead (other
+languages, Whisper models, a GPU, the web editor), set `OPENCAPTIONS_KEY` to an API key from its
+Account page, and `OPENCAPTIONS_URL` if it is not at `http://localhost:5173`.
+
 As an MCP server:
 
 ```json
@@ -16,8 +21,7 @@ As an MCP server:
   "mcpServers": {
     "opencaptions": {
       "command": "npx",
-      "args": ["-y", "opencaptions", "mcp"],
-      "env": { "OPENCAPTIONS_KEY": "oc_...", "OPENCAPTIONS_URL": "http://localhost:5173" }
+      "args": ["-y", "opencaptions", "mcp"]
     }
   }
 }
@@ -29,5 +33,4 @@ The tools caption a video, list the looks, change any style setting, read and fi
 again in another format or as captions alone over green, and export SRT or VTT. Setup, every tool and
 every option: [docs.opencaptions.app/guides/agents](https://docs.opencaptions.app/guides/agents/).
 
-Needs Node.js 20.10 or later and a running OpenCaptions server
-([quick start](https://docs.opencaptions.app/getting-started/quick-start/)). AGPL-3.0-only.
+Needs Node.js 20.10 or later. AGPL-3.0-only.

package/dist/captioner.js

@@ -0,0 +1,31 @@
+/**
+ * The actions the command line and the MCP server offer, on a server or on this computer. With
+ * neither OPENCAPTIONS_URL nor OPENCAPTIONS_KEY set, the work is done here (`local/`); otherwise on
+ * the server they name. A project is an id on a server and a file beside the video here.
+ */
+import { captionVideo, defaultOutput, exportSubtitles, fixWords, getProject, readTranscript, resume, saveVideo, setStyle, } from "./captions.js";
+import { api, connectionFromEnv } from "./client.js";
+export function serverCaptioner(client) {
+    return {
+        caption: (request, progress, deadline) => captionVideo(client, request, progress, deadline),
+        resume: (ticket, progress, deadline) => resume(client, ticket, progress, deadline),
+        setStyle: (id, look, changes) => setStyle(client, id, look, changes),
+        readTranscript: (id) => readTranscript(client, id),
+        fixWords: (id, fixes) => fixWords(client, id, fixes),
+        save: async (id, options, progress, deadline) => {
+            const stem = defaultOutput((await getProject(client, id)).title, options.captions_only);
+            return saveVideo(client, id, { ...options, stem }, progress, deadline);
+        },
+        exportSubtitles: async (id, kind, output) => exportSubtitles(client, id, kind, output ?? defaultOutput((await getProject(client, id)).title)),
+    };
+}
+/** Whether the environment names a server; without one, the work is done on this computer. */
+export function wantsServer(env = process.env) {
+    return Boolean(env.OPENCAPTIONS_URL?.trim() || env.OPENCAPTIONS_KEY?.trim());
+}
+export async function captionerFromEnv(env = process.env) {
+    if (wantsServer(env))
+        return serverCaptioner(api(connectionFromEnv(env)));
+    const { localCaptioner } = await import("./local/index.js");
+    return localCaptioner();
+}

package/dist/captions.js

@@ -1,7 +1,8 @@
 /**
  * What the command line and the MCP server do, over the server's API: open a video as a project,
  * transcribe it, give it a look, fix words, and save the captioned video or its subtitles. The server
- * transcribes and draws; nothing here draws a caption.
+ * transcribes and draws. `local/` does the same on this computer, with the pieces here that do not
+ * depend on where it runs (`styled`, `applyFixes`, `viewTranscript`).
  */
 import { createWriteStream, openAsBlob } from "node:fs";
 import { rename, rm, stat, writeFile } from "node:fs/promises";
@@ -9,7 +10,7 @@ import { basename, dirname, extname, join, resolve } from "node:path";
 import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
 import { OpenCaptionsError, unwrap, } from "./client.js";
-import { captionLines, setWord, shuffleVariants } from "./engine.js";
+import { captionLines, mergeWords, setWord, shuffleVariants } from "./engine.js";
 import { findLook, LOOKS } from "./looks.js";
 /** How often a job is asked about, in milliseconds. */
 const pollMs = () => Number(process.env.OPENCAPTIONS_POLL_MS) || 1000;
@@ -99,39 +100,52 @@ export async function waitForJob(client, jobId, progress, deadline = Number.POSI
  */
 export async function setStyle(client, projectId, look, changes = {}) {
     const project = await getProject(client, projectId);
-    // A project never styled draws as the first look, so changes alone go over that.
-    const base = look ? findLook(look).config : (project.style_config ?? LOOKS[0]?.config);
-    const style = { ...base, ...changes };
+    const style = styled(project.style_config, look, changes);
     const transcript = look ? await dealt(project.transcript ?? undefined, style) : undefined;
     return ((await update(client, projectId, { style_config: style, ...(transcript && { transcript }) }))
         .style_config ?? style);
 }
-async function dealt(transcript, style) {
+/** A look with changes over it; without a look the changes go over `current`. */
+export function styled(current, look, changes = {}) {
+    // A project never styled draws as the first look, so changes alone go over that.
+    const base = look ? findLook(look).config : (current ?? LOOKS[0]?.config);
+    return { ...base, ...changes };
+}
+export async function dealt(transcript, style) {
     const count = style.variants?.length ?? 0;
     return transcript && count > 0 ? shuffleVariants(transcript, count) : undefined;
 }
 /** The words with their place in the captions, for checking names and spelling. */
-export async function readTranscript(client, projectId) {
-    const project = await getProject(client, projectId);
-    if (!project.transcript)
+export async function viewTranscript(transcript, style, offsetMs) {
+    if (!transcript)
         throw new OpenCaptionsError("This project has no captions yet.");
-    const words = project.transcript.segments
+    const words = transcript.segments
         .flatMap((s) => s.words)
         .map((w, index) => ({ index, text: w.text, start: w.start }));
-    const lines = await captionLines(project.transcript, project.style_config?.words_per_line ?? 3, project.caption_offset_ms);
+    const lines = await captionLines(transcript, style?.words_per_line ?? 3, offsetMs);
     return { lines, words };
 }
-/** Sets words by their index; empty text removes a word. The indices are the ones read before. */
-export async function fixWords(client, projectId, fixes) {
+export async function readTranscript(client, projectId) {
     const project = await getProject(client, projectId);
-    if (!project.transcript)
+    return viewTranscript(project.transcript, project.style_config, project.caption_offset_ms);
+}
+/** The fixes applied by the engine; the indices are the ones read before. */
+export async function applyFixes(transcript, fixes) {
+    if (!transcript)
         throw new OpenCaptionsError("This project has no captions yet.");
-    // Last first: removing a word moves the ones after it, never the ones before.
-    let transcript = project.transcript;
+    // Last first: removing or joining words moves the ones after them, never the ones before.
+    let fixed = transcript;
     for (const fix of [...fixes].sort((a, b) => b.index - a.index)) {
-        transcript = await setWord(transcript, fix.index, fix.text);
+        fixed =
+            (fix.count ?? 1) > 1
+                ? await mergeWords(fixed, fix.index, fix.count ?? 1, fix.text)
+                : await setWord(fixed, fix.index, fix.text);
     }
-    await update(client, projectId, { transcript });
+    return fixed;
+}
+export async function fixWords(client, projectId, fixes) {
+    const project = await getProject(client, projectId);
+    await update(client, projectId, { transcript: await applyFixes(project.transcript, fixes) });
     return fixes.length;
 }
 function renderRequest(save) {
@@ -249,7 +263,7 @@ async function afterTranscript(client, projectId) {
     if (transcript)
         await update(client, projectId, { transcript });
 }
-function saveOnly(save) {
+export function saveOnly(save) {
     const { format, resolution, frame_rate, captions_only, output, stem } = save;
     return { format, resolution, frame_rate, captions_only, output, stem };
 }

package/dist/cli.js

@@ -5,11 +5,11 @@
  */
 import { readFile } from "node:fs/promises";
 import { parseArgs } from "node:util";
-import { captionVideo, exportSubtitles } from "./captions.js";
-import { api, connectionFromEnv, OpenCaptionsError } from "./client.js";
+import { captionerFromEnv } from "./captioner.js";
+import { OpenCaptionsError } from "./client.js";
 import { describeLook, describeStyle, LOOKS } from "./looks.js";
 import { VERSION } from "./version.js";
-const HELP = `opencaptions ${VERSION}: animated captions for your videos, on your OpenCaptions server.
+const HELP = `opencaptions ${VERSION}: animated captions for your videos.
 
   opencaptions caption <video or link> [options]   transcribe, style and save the captioned video
   opencaptions looks                                the built-in looks
@@ -21,7 +21,7 @@ Options for caption:
   --look <name>            a built-in look (default: Purple Punch)
   --style <json>           style settings over the look, e.g. '{"position_y":0.7}'
   --language <code>        ISO 639-1, or auto (default)
-  --model <id>             a speech model the server offers
+  --model <id>             a speech model the server offers (without one: parakeet-ultra)
   --transcript <file>      use these captions (the API's Transcript JSON) instead of transcribing
   --format <id>            mp4 (default), mp4-hevc, webm or mov
   --resolution <side>      original (default), 2160, 1080 or 720
@@ -29,8 +29,9 @@ Options for caption:
   --captions-only          the captions alone over green, no sound, to key out
   --subtitles <kinds>      also write subtitles: srt, vtt, json, comma separated
 
-The server: OPENCAPTIONS_URL (default http://localhost:5173) and OPENCAPTIONS_KEY, an API key
-from its Account page.`;
+It runs on this computer, with FFmpeg installed; the speech model (700 MB) is downloaded on first
+use. To use an OpenCaptions server instead, set OPENCAPTIONS_KEY (an API key from its Account
+page) and OPENCAPTIONS_URL (default http://localhost:5173).`;
 /** Progress on one line of the terminal, rewritten as it moves. */
 const report = (fraction, message) => {
     if (process.stderr.isTTY) {
@@ -73,8 +74,8 @@ async function caption(args) {
         frame_rate: values.fps,
         captions_only: values["captions-only"],
     };
-    const client = api(connectionFromEnv());
-    const result = await captionVideo(client, request, report);
+    const captioner = await captionerFromEnv();
+    const result = await captioner.caption(request, report);
     if (process.stderr.isTTY)
         process.stderr.write("\r\x1b[2K");
     if (!result.done)
@@ -84,7 +85,7 @@ async function caption(args) {
         if (kind !== "srt" && kind !== "vtt" && kind !== "json") {
             throw new OpenCaptionsError(`Subtitles are srt, vtt or json, not ${kind}.`);
         }
-        console.log(await exportSubtitles(client, result.project_id, kind, result.output.replace(/\.[^.]+$/, "")));
+        console.log(await captioner.exportSubtitles(result.project_id, kind, result.output.replace(/\.[^.]+$/, "")));
     }
 }
 async function main(argv) {

package/dist/engine.js

@@ -1,10 +1,12 @@
 /**
- * The caption engine's edits (apps/engine/src/edit.rs), through the same WebAssembly file the web
- * editor loads: a word is fixed, and a look's word variants dealt, by the code every editor uses,
- * never by a second implementation here.
+ * The caption engine, through the same WebAssembly file the web editor loads: a word is fixed, a
+ * look's word variants dealt, a frame drawn and a file's FFmpeg command made by the code every
+ * editor and the render server use, never by a second implementation here.
  */
-import { readFile } from "node:fs/promises";
+import { readdir, readFile } from "node:fs/promises";
 const WASM = new URL("../engine/opencaptions_engine.wasm", import.meta.url);
+/** The faces the render server bundles (apps/engine/fonts), copied beside the WebAssembly. */
+const FONTS = new URL("../engine/fonts/", import.meta.url);
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
 let loaded;
@@ -23,18 +25,24 @@ function copyIn(x, bytes) {
 function result(x, ok) {
     const text = decoder.decode(new Uint8Array(x.memory.buffer, x.oc_result_ptr(), x.oc_result_len()));
     if (!ok)
-        throw new Error(`the caption engine refused the edit: ${text}`);
+        throw new Error(`the caption engine refused it: ${text}`);
     return JSON.parse(text);
 }
-const json = (x, t) => copyIn(x, encoder.encode(JSON.stringify(t)));
+const json = (x, value) => copyIn(x, encoder.encode(JSON.stringify(value)));
+const text = (x, value) => copyIn(x, encoder.encode(value));
 export async function captionLines(t, wordsPerLine, offsetMs) {
     const x = await exports();
     return result(x, x.oc_caption_lines(...json(x, t), wordsPerLine, offsetMs));
 }
 /** Word `index` (counted across segments) given `text`; empty text removes it. */
-export async function setWord(t, index, text) {
+export async function setWord(t, index, word) {
     const x = await exports();
-    return result(x, x.oc_set_word(...json(x, t), index, ...copyIn(x, encoder.encode(text))));
+    return result(x, x.oc_set_word(...json(x, t), index, ...text(x, word)));
+}
+/** `count` words from `index` replaced by one word `text`, spanning them all. */
+export async function mergeWords(t, index, count, word) {
+    const x = await exports();
+    return result(x, x.oc_merge_words(...json(x, t), index, count, ...text(x, word)));
 }
 /** Deals every word one of `count` looks, or the style's own, as picking such a look does. */
 export async function shuffleVariants(t, count) {
@@ -42,3 +50,65 @@ export async function shuffleVariants(t, count) {
     const seed = crypto.getRandomValues(new Uint32Array(1))[0] ?? 0;
     return result(x, x.oc_shuffle_variants(...json(x, t), count, seed));
 }
+/** The captions as SubRip or WebVTT, cue for cue the caption lines. */
+export async function subtitles(t, wordsPerLine, offsetMs, format) {
+    const x = await exports();
+    return result(x, x.oc_subtitles(...json(x, t), wordsPerLine, offsetMs, format === "srt" ? 0 : 1));
+}
+let bundled;
+/** The bundled faces, registered once, in name order as the render server loads them. */
+function bundledFonts(x) {
+    bundled ??= readdir(FONTS).then(async (files) => {
+        for (const file of files.filter((f) => /\.(ttf|otf)$/i.test(f)).sort()) {
+            x.oc_add_font(...copyIn(x, await readFile(new URL(file, FONTS))));
+        }
+    });
+    return bundled;
+}
+/** Whether a family can be drawn without fetching it. */
+export async function hasFont(family) {
+    const x = await exports();
+    await bundledFonts(x);
+    return x.oc_has_font(...text(x, family)) === 1;
+}
+/** Registers a family fetched for the style (`fonts.ts`). False if it is not a usable font. */
+export async function addFont(family, bytes) {
+    const x = await exports();
+    await bundledFonts(x);
+    return x.oc_add_requested_font(...text(x, family), ...copyIn(x, bytes)) === 1;
+}
+/** A scene laid out once and drawn frame by frame, as the render server draws an export. */
+export class Scene {
+    x;
+    handle;
+    width;
+    constructor(x, handle, width) {
+        this.x = x;
+        this.handle = handle;
+        this.width = width;
+    }
+    static async create(input) {
+        const x = await exports();
+        await bundledFonts(x);
+        const handle = x.oc_scene_new(...json(x, input));
+        if (!handle)
+            result(x, 0);
+        return new Scene(x, handle, input.width);
+    }
+    plan(output) {
+        return result(this.x, this.x.oc_encode_plan(this.handle, ...json(this.x, output)));
+    }
+    /** The `band` (x, y, w, h) of the frame at `t` seconds, as RGBA rows, into `into`. */
+    band(t, [bx, by, bw, bh], into) {
+        this.x.oc_render(this.handle, t);
+        const frame = new Uint8Array(this.x.memory.buffer, this.x.oc_frame_ptr(this.handle));
+        const row = bw * 4;
+        for (let y = 0; y < bh; y++) {
+            const start = ((by + y) * this.width + bx) * 4;
+            into.set(frame.subarray(start, start + row), y * row);
+        }
+    }
+    free() {
+        this.x.oc_scene_free(this.handle);
+    }
+}

package/dist/generated/render_formats.json

@@ -0,0 +1,42 @@
+[
+    {
+        "id": "mp4",
+        "label": "MP4 (H.264)",
+        "codec": "h264",
+        "extension": ".mp4",
+        "mime": "video/mp4",
+        "crf": 15,
+        "pro_res_profile": null,
+        "note": null
+    },
+    {
+        "id": "mp4-hevc",
+        "label": "MP4 (H.265)",
+        "codec": "h265",
+        "extension": ".mp4",
+        "mime": "video/mp4",
+        "crf": 20,
+        "pro_res_profile": null,
+        "note": null
+    },
+    {
+        "id": "webm",
+        "label": "WebM (VP9)",
+        "codec": "vp9",
+        "extension": ".webm",
+        "mime": "video/webm",
+        "crf": 24,
+        "pro_res_profile": null,
+        "note": "VP9 encoding is very slow, expect 5-10x real-time on a modern CPU."
+    },
+    {
+        "id": "mov",
+        "label": "MOV (ProRes)",
+        "codec": "prores",
+        "extension": ".mov",
+        "mime": "video/quicktime",
+        "crf": null,
+        "pro_res_profile": "hq",
+        "note": "ProRes files are very large (~220 Mbps, roughly 1 GB per 40 seconds of video)."
+    }
+]

package/dist/local/cache.js

@@ -0,0 +1,8 @@
+/** Where captioning without a server keeps what it downloads: the speech model and fonts. */
+import { homedir } from "node:os";
+import { join } from "node:path";
+export function cacheDir() {
+    const base = process.env.OPENCAPTIONS_CACHE ||
+        (process.env.XDG_CACHE_HOME ? join(process.env.XDG_CACHE_HOME, "opencaptions") : "");
+    return base || join(homedir(), ".cache", "opencaptions");
+}

package/dist/local/fonts.js

@@ -0,0 +1,81 @@
+/**
+ * The style's font, for captioning without a server: a bundled face when there is one, else the
+ * family from Google Fonts at its weight nearest 800, fetched once into the cache. The same choice
+ * as the API's (`apps/api/app/services/fonts.py`), so a look draws the same on both.
+ */
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { addFont, hasFont } from "../engine.js";
+import { cacheDir } from "./cache.js";
+const CATALOG_URL = "https://fonts.google.com/metadata/fonts";
+const CSS_URL = "https://fonts.googleapis.com/css2";
+// The CSS API picks the format from the client: one it does not take for a browser gets
+// TrueType, which the engine reads and woff2 is not.
+const HEADERS = { "User-Agent": "OpenCaptions" };
+const TARGET_WEIGHT = 800;
+/** The family's upright weight nearest 800; ties go to the heavier one. */
+export function nearestWeight(styles) {
+    const upright = Object.keys(styles)
+        .filter((k) => /^\d+$/.test(k))
+        .map(Number);
+    if (upright.length === 0)
+        return 400;
+    return upright.reduce((best, w) => {
+        const d = Math.abs(w - TARGET_WEIGHT) - Math.abs(best - TARGET_WEIGHT);
+        return d < 0 || (d === 0 && w > best) ? w : best;
+    });
+}
+async function weightOf(family) {
+    const path = join(cacheDir(), "fonts", "catalog.json");
+    let weights;
+    try {
+        weights = JSON.parse(await readFile(path, "utf8"));
+    }
+    catch {
+        const response = await fetch(CATALOG_URL, { headers: HEADERS });
+        if (!response.ok)
+            return undefined;
+        const data = JSON.parse((await response.text()).replace(/^\)\]\}'/, ""));
+        weights = Object.fromEntries(data.familyMetadataList.map((f) => [f.family, nearestWeight(f.fonts ?? {})]));
+        await mkdir(join(cacheDir(), "fonts"), { recursive: true });
+        await writeFile(path, JSON.stringify(weights));
+    }
+    return weights[family];
+}
+async function download(family) {
+    const weight = await weightOf(family);
+    if (weight === undefined)
+        return undefined;
+    const css = await fetch(`${CSS_URL}?family=${encodeURIComponent(`${family}:wght@${weight}`)}`, {
+        headers: HEADERS,
+    });
+    if (!css.ok)
+        return undefined;
+    const url = /url\((https:\/\/fonts\.gstatic\.com\/[^)]+)\)/.exec(await css.text());
+    if (!url?.[1])
+        return undefined;
+    const font = await fetch(url[1], { headers: HEADERS });
+    return font.ok ? new Uint8Array(await font.arrayBuffer()) : undefined;
+}
+/**
+ * Makes `family` drawable. False when it is neither bundled nor to be had: the engine then draws
+ * its default face, as the server does in the same case.
+ */
+export async function ensureFont(family) {
+    if (await hasFont(family))
+        return true;
+    const slug = family.replace(/[^A-Za-z0-9]+/g, "-");
+    const path = join(cacheDir(), "fonts", `${slug}.ttf`);
+    let bytes;
+    try {
+        bytes = await readFile(path);
+    }
+    catch {
+        bytes = await download(family).catch(() => undefined);
+        if (bytes) {
+            await mkdir(join(cacheDir(), "fonts"), { recursive: true });
+            await writeFile(path, bytes);
+        }
+    }
+    return bytes !== undefined && (await addFont(family, bytes));
+}