opencaptions 0.0.0-stage → 1.2.0

package/README.md

@@ -1,3 +1,33 @@
-# Temporary Holding Version
+# opencaptions
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Animated word-by-word captions for your videos, from Claude Code, Claude Desktop, Cursor or any MCP
+client, or from the command line. It runs on your own [OpenCaptions](https://github.com/leogaudin/opencaptions)
+server: your video goes there and back, and nowhere else.
+
+```bash
+export OPENCAPTIONS_KEY=oc_...          # an API key from your server's Account page
+npx opencaptions caption reel.mp4 --look Bold --language fr
+```
+
+As an MCP server:
+
+```json
+{
+  "mcpServers": {
+    "opencaptions": {
+      "command": "npx",
+      "args": ["-y", "opencaptions", "mcp"],
+      "env": { "OPENCAPTIONS_KEY": "oc_...", "OPENCAPTIONS_URL": "http://localhost:5173" }
+    }
+  }
+}
+```
+
+In Claude Code: `/plugin install opencaptions --marketplace leogaudin/opencaptions`.
+
+The tools caption a video, list the looks, change any style setting, read and fix the transcript, save
+again in another format or as captions alone over green, and export SRT or VTT. Setup, every tool and
+every option: [docs.opencaptions.app/guides/agents](https://docs.opencaptions.app/guides/agents/).
+
+Needs Node.js 20.10 or later and a running OpenCaptions server
+([quick start](https://docs.opencaptions.app/getting-started/quick-start/)). AGPL-3.0-only.

package/dist/captions.js

@@ -0,0 +1,267 @@
+/**
+ * What the command line and the MCP server do, over the server's API: open a video as a project,
+ * transcribe it, give it a look, fix words, and save the captioned video or its subtitles. The server
+ * transcribes and draws; nothing here draws a caption.
+ */
+import { createWriteStream, openAsBlob } from "node:fs";
+import { rename, rm, stat, writeFile } from "node:fs/promises";
+import { basename, dirname, extname, join, resolve } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { OpenCaptionsError, unwrap, } from "./client.js";
+import { captionLines, setWord, shuffleVariants } from "./engine.js";
+import { findLook, LOOKS } from "./looks.js";
+/** How often a job is asked about, in milliseconds. */
+const pollMs = () => Number(process.env.OPENCAPTIONS_POLL_MS) || 1000;
+const isUrl = (source) => /^https?:\/\//i.test(source);
+/**
+ * Where a save writes when no output is given: next to the source, marked as captioned, without
+ * the extension, which is the format's.
+ */
+export function defaultOutput(source, captionsOnly = false) {
+    const name = isUrl(source)
+        ? basename(new URL(source).pathname) || "video"
+        : basename(source, extname(source));
+    const stem = name.replace(extname(name), "") || "video";
+    const folder = isUrl(source) ? process.cwd() : dirname(resolve(source));
+    return join(folder, `${stem}.${captionsOnly ? "captions" : "captioned"}`);
+}
+export async function openProject(client, source) {
+    if (isUrl(source)) {
+        const title = basename(new URL(source).pathname) || "Video";
+        return unwrap(await client.POST("/api/v1/projects", {
+            body: { title, video_url: source },
+            bodySerializer: (body) => form({ title: body.title, video_url: source }),
+        })).id;
+    }
+    const path = resolve(source);
+    const file = await stat(path).catch(() => undefined);
+    if (!file?.isFile())
+        throw new OpenCaptionsError(`There is no video at ${path}.`);
+    const video = await openAsBlob(path);
+    return unwrap(await client.POST("/api/v1/projects", {
+        body: { title: basename(path, extname(path)) },
+        bodySerializer: (body) => form({ title: body.title }, ["video", video, basename(path)]),
+    })).id;
+}
+function form(fields, file) {
+    const data = new FormData();
+    for (const [key, value] of Object.entries(fields))
+        data.append(key, value);
+    if (file)
+        data.append(...file);
+    return data;
+}
+export async function getProject(client, projectId) {
+    return unwrap(await client.GET("/api/v1/projects/{project_id}", {
+        params: { path: { project_id: projectId } },
+    }));
+}
+async function update(client, projectId, body) {
+    return unwrap(await client.PATCH("/api/v1/projects/{project_id}", {
+        params: { path: { project_id: projectId } },
+        body,
+    }));
+}
+export async function startTranscription(client, projectId, request) {
+    return unwrap(await client.POST("/api/v1/projects/{project_id}/transcribe", {
+        params: { path: { project_id: projectId } },
+        body: {
+            language: request.language ?? "auto",
+            model: request.model ?? null,
+            provider: request.provider ?? null,
+        },
+    }));
+}
+/**
+ * Waits for a job, reporting as it goes. True once it has finished; false when `deadline` (a time
+ * in milliseconds) came first and it is still going. A job that failed or was cancelled throws.
+ */
+export async function waitForJob(client, jobId, progress, deadline = Number.POSITIVE_INFINITY) {
+    for (;;) {
+        const job = unwrap(await client.GET("/api/v1/jobs/{job_id}", { params: { path: { job_id: jobId } } }));
+        if (job.status === "completed")
+            return true;
+        if (job.status === "failed") {
+            throw new OpenCaptionsError(job.error ?? job.message ?? "The job failed on the server.");
+        }
+        if (job.status === "cancelled")
+            throw new OpenCaptionsError("The job was cancelled.");
+        progress(job.progress, job.message ?? (job.type === "transcription" ? "Transcribing" : "Saving"));
+        if (Date.now() + pollMs() > deadline)
+            return false;
+        await new Promise((done) => setTimeout(done, pollMs()));
+    }
+}
+/**
+ * The project's style set to a look, with changes over it; without a look the changes go over the
+ * style it has. A look with word variants deals them to the words, as picking it in an editor does.
+ */
+export async function setStyle(client, projectId, look, changes = {}) {
+    const project = await getProject(client, projectId);
+    // A project never styled draws as the first look, so changes alone go over that.
+    const base = look ? findLook(look).config : (project.style_config ?? LOOKS[0]?.config);
+    const style = { ...base, ...changes };
+    const transcript = look ? await dealt(project.transcript ?? undefined, style) : undefined;
+    return ((await update(client, projectId, { style_config: style, ...(transcript && { transcript }) }))
+        .style_config ?? style);
+}
+async function dealt(transcript, style) {
+    const count = style.variants?.length ?? 0;
+    return transcript && count > 0 ? shuffleVariants(transcript, count) : undefined;
+}
+/** The words with their place in the captions, for checking names and spelling. */
+export async function readTranscript(client, projectId) {
+    const project = await getProject(client, projectId);
+    if (!project.transcript)
+        throw new OpenCaptionsError("This project has no captions yet.");
+    const words = project.transcript.segments
+        .flatMap((s) => s.words)
+        .map((w, index) => ({ index, text: w.text, start: w.start }));
+    const lines = await captionLines(project.transcript, project.style_config?.words_per_line ?? 3, project.caption_offset_ms);
+    return { lines, words };
+}
+/** Sets words by their index; empty text removes a word. The indices are the ones read before. */
+export async function fixWords(client, projectId, fixes) {
+    const project = await getProject(client, projectId);
+    if (!project.transcript)
+        throw new OpenCaptionsError("This project has no captions yet.");
+    // Last first: removing a word moves the ones after it, never the ones before.
+    let transcript = project.transcript;
+    for (const fix of [...fixes].sort((a, b) => b.index - a.index)) {
+        transcript = await setWord(transcript, fix.index, fix.text);
+    }
+    await update(client, projectId, { transcript });
+    return fixes.length;
+}
+function renderRequest(save) {
+    return {
+        format: save.format ?? "mp4",
+        resolution: save.resolution ?? "original",
+        frame_rate: save.frame_rate ?? "original",
+        green_screen: save.captions_only ?? false,
+    };
+}
+/** Asks for the video; it comes back at once when an identical one was made before. */
+export async function requestVideo(client, projectId, save) {
+    const answer = unwrap(await client.POST("/api/v1/projects/{project_id}/download", {
+        params: { path: { project_id: projectId } },
+        body: renderRequest(save),
+    }));
+    return answer.ready ? undefined : (answer.job_id ?? undefined);
+}
+/**
+ * Writes the finished video to `output`, or to `stem` with the extension of the format the server
+ * made. Returns the path written.
+ */
+export async function downloadVideo(client, projectId, save) {
+    const request = renderRequest(save);
+    const { data, response } = await client.GET("/api/v1/projects/{project_id}/download/{format_id}", {
+        params: {
+            path: { project_id: projectId, format_id: request.format },
+            query: {
+                resolution: request.resolution,
+                frame_rate: request.frame_rate,
+                green_screen: request.green_screen,
+            },
+        },
+        parseAs: "stream",
+    });
+    if (!response.ok || !data)
+        unwrap({ error: await response.json().catch(() => null), response });
+    const named = /filename="[^"]*?(\.[a-z0-9]+)"/i.exec(response.headers.get("content-disposition") ?? "");
+    const output = save.output ?? `${save.stem}${named?.[1] ?? ".mp4"}`;
+    const partial = `${output}.part`;
+    try {
+        await pipeline(Readable.fromWeb(data), createWriteStream(partial));
+        await rename(partial, output);
+    }
+    catch (error) {
+        await rm(partial, { force: true });
+        throw error;
+    }
+    return output;
+}
+/** Saves the captioned video, waiting for it up to `deadline`. */
+export async function saveVideo(client, projectId, save, progress, deadline) {
+    const job = await requestVideo(client, projectId, save);
+    if (job && !(await waitForJob(client, job, progress, deadline))) {
+        return pending({ project_id: projectId, job_id: job, step: "save", save });
+    }
+    progress(1, "Writing the file");
+    return {
+        done: true,
+        project_id: projectId,
+        output: await downloadVideo(client, projectId, save),
+    };
+}
+/** The captions as SRT or VTT, or the transcript as JSON, written next to `output`. */
+export async function exportSubtitles(client, projectId, kind, output) {
+    const path = extname(output) === `.${kind}` ? output : `${output}.${kind}`;
+    const options = { params: { path: { project_id: projectId } }, parseAs: "text" };
+    const { data, response } = kind === "srt"
+        ? await client.GET("/api/v1/projects/{project_id}/export.srt", options)
+        : kind === "vtt"
+            ? await client.GET("/api/v1/projects/{project_id}/export.vtt", options)
+            : await client.GET("/api/v1/projects/{project_id}/export.json", options);
+    if (!response.ok || data === undefined) {
+        unwrap({ error: await response.json().catch(() => null), response });
+    }
+    await writeFile(path, data ?? "");
+    return path;
+}
+function pending(ticket) {
+    const what = ticket.step === "transcribe" ? "Transcribing" : "Saving the video";
+    return { done: false, ticket, message: `${what} is still going on the server.` };
+}
+/** Opens, transcribes, styles and saves: the whole of "caption this video". */
+export async function captionVideo(client, request, progress, deadline) {
+    progress(0, "Sending the video");
+    const projectId = await openProject(client, request.source);
+    const save = {
+        ...request,
+        stem: defaultOutput(request.source, request.captions_only),
+    };
+    if (request.look !== undefined || request.style !== undefined) {
+        await setStyle(client, projectId, request.look, request.style);
+    }
+    if (request.transcript) {
+        await update(client, projectId, { transcript: request.transcript });
+    }
+    else {
+        const job = await startTranscription(client, projectId, request);
+        if (!(await waitForJob(client, job.id, progress, deadline))) {
+            return pending({
+                project_id: projectId,
+                job_id: job.id,
+                step: "transcribe",
+                save: saveOnly(save),
+            });
+        }
+    }
+    await afterTranscript(client, projectId);
+    return saveVideo(client, projectId, saveOnly(save), progress, deadline);
+}
+/** A look's word variants go to the words once there are words to give them to. */
+async function afterTranscript(client, projectId) {
+    const project = await getProject(client, projectId);
+    const transcript = project.style_config && (await dealt(project.transcript ?? undefined, project.style_config));
+    if (transcript)
+        await update(client, projectId, { transcript });
+}
+function saveOnly(save) {
+    const { format, resolution, frame_rate, captions_only, output, stem } = save;
+    return { format, resolution, frame_rate, captions_only, output, stem };
+}
+/** Carries on from where a ticket was left. */
+export async function resume(client, ticket, progress, deadline) {
+    if (!(await waitForJob(client, ticket.job_id, progress, deadline)))
+        return pending(ticket);
+    if (ticket.step === "save") {
+        progress(1, "Writing the file");
+        const output = await downloadVideo(client, ticket.project_id, ticket.save);
+        return { done: true, project_id: ticket.project_id, output };
+    }
+    await afterTranscript(client, ticket.project_id);
+    return saveVideo(client, ticket.project_id, ticket.save, progress, deadline);
+}

package/dist/cli.js

@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+/**
+ * `opencaptions`: captions a video from the command line, or serves the same actions to an agent
+ * over MCP (`opencaptions mcp`).
+ */
+import { readFile } from "node:fs/promises";
+import { parseArgs } from "node:util";
+import { captionVideo, exportSubtitles } from "./captions.js";
+import { api, connectionFromEnv, OpenCaptionsError } from "./client.js";
+import { describeLook, describeStyle, LOOKS } from "./looks.js";
+import { VERSION } from "./version.js";
+const HELP = `opencaptions ${VERSION}: animated captions for your videos, on your OpenCaptions server.
+
+  opencaptions caption <video or link> [options]   transcribe, style and save the captioned video
+  opencaptions looks                                the built-in looks
+  opencaptions style                                every style setting, for --style
+  opencaptions mcp                                  serve these to an agent over MCP (stdio)
+
+Options for caption:
+  -o, --output <path>      where to write the video (default: <name>.captioned next to it)
+  --look <name>            a built-in look (default: Purple Punch)
+  --style <json>           style settings over the look, e.g. '{"position_y":0.7}'
+  --language <code>        ISO 639-1, or auto (default)
+  --model <id>             a speech model the server offers
+  --transcript <file>      use these captions (the API's Transcript JSON) instead of transcribing
+  --format <id>            mp4 (default), mp4-hevc, webm or mov
+  --resolution <side>      original (default), 2160, 1080 or 720
+  --fps <rate>             original (default), 30 or 60
+  --captions-only          the captions alone over green, no sound, to key out
+  --subtitles <kinds>      also write subtitles: srt, vtt, json, comma separated
+
+The server: OPENCAPTIONS_URL (default http://localhost:5173) and OPENCAPTIONS_KEY, an API key
+from its Account page.`;
+/** Progress on one line of the terminal, rewritten as it moves. */
+const report = (fraction, message) => {
+    if (process.stderr.isTTY) {
+        process.stderr.write(`\r\x1b[2K${Math.round(fraction * 100)}% ${message}`);
+    }
+};
+async function caption(args) {
+    const { values, positionals } = parseArgs({
+        args,
+        allowPositionals: true,
+        options: {
+            output: { type: "string", short: "o" },
+            look: { type: "string" },
+            style: { type: "string" },
+            language: { type: "string" },
+            model: { type: "string" },
+            transcript: { type: "string" },
+            format: { type: "string" },
+            resolution: { type: "string" },
+            fps: { type: "string" },
+            "captions-only": { type: "boolean" },
+            subtitles: { type: "string" },
+        },
+    });
+    const source = positionals[0];
+    if (!source)
+        throw new OpenCaptionsError("Which video? opencaptions caption <video or link>");
+    const request = {
+        source,
+        output: values.output,
+        look: values.look,
+        style: values.style ? JSON.parse(values.style) : undefined,
+        language: values.language,
+        model: values.model,
+        transcript: values.transcript
+            ? JSON.parse(await readFile(values.transcript, "utf8"))
+            : undefined,
+        format: values.format,
+        resolution: values.resolution,
+        frame_rate: values.fps,
+        captions_only: values["captions-only"],
+    };
+    const client = api(connectionFromEnv());
+    const result = await captionVideo(client, request, report);
+    if (process.stderr.isTTY)
+        process.stderr.write("\r\x1b[2K");
+    if (!result.done)
+        throw new OpenCaptionsError(result.message);
+    console.log(result.output);
+    for (const kind of (values.subtitles ?? "").split(",").filter(Boolean)) {
+        if (kind !== "srt" && kind !== "vtt" && kind !== "json") {
+            throw new OpenCaptionsError(`Subtitles are srt, vtt or json, not ${kind}.`);
+        }
+        console.log(await exportSubtitles(client, result.project_id, kind, result.output.replace(/\.[^.]+$/, "")));
+    }
+}
+async function main(argv) {
+    const [command, ...rest] = argv;
+    switch (command) {
+        case "caption":
+            return caption(rest);
+        case "looks":
+            console.log(LOOKS.map(describeLook).join("\n"));
+            return;
+        case "style":
+            console.log(describeStyle());
+            return;
+        case "mcp": {
+            const { serve } = await import("./mcp.js");
+            return serve();
+        }
+        case "--version":
+        case "-v":
+            console.log(VERSION);
+            return;
+        default:
+            console.log(HELP);
+            if (command && command !== "help" && command !== "--help" && command !== "-h") {
+                process.exitCode = 2;
+            }
+    }
+}
+main(process.argv.slice(2)).catch((error) => {
+    if (process.stderr.isTTY)
+        process.stderr.write("\r\x1b[2K");
+    console.error(error instanceof OpenCaptionsError ? error.message : error);
+    process.exitCode = 1;
+});

package/dist/client.js

@@ -0,0 +1,56 @@
+/**
+ * The OpenCaptions server, reached through its API with a key. Every call is typed against the API's
+ * generated types (`generated/api.ts`), so a route or a body that drifts from the server fails `tsc`.
+ */
+import createClient, {} from "openapi-fetch";
+/** A failure in words a person (or an agent) can act on. */
+export class OpenCaptionsError extends Error {
+}
+export const DEFAULT_URL = "http://localhost:5173";
+/** The server named by OPENCAPTIONS_URL and OPENCAPTIONS_KEY. */
+export function connectionFromEnv(env = process.env) {
+    const key = env.OPENCAPTIONS_KEY?.trim();
+    if (!key) {
+        throw new OpenCaptionsError("OPENCAPTIONS_KEY is not set. Create an API key on your OpenCaptions server's Account page " +
+            "and set it, with OPENCAPTIONS_URL if the server is not at " +
+            `${DEFAULT_URL}.`);
+    }
+    return { url: (env.OPENCAPTIONS_URL?.trim() || DEFAULT_URL).replace(/\/+$/, ""), key };
+}
+export function api(connection) {
+    const client = createClient({
+        baseUrl: connection.url,
+        headers: { Authorization: `Bearer ${connection.key}` },
+    });
+    client.use({
+        async onError({ error }) {
+            const cause = error instanceof Error ? error.message : String(error);
+            return new OpenCaptionsError(`Cannot reach OpenCaptions at ${connection.url} (${cause}). Is the server running?`);
+        },
+    });
+    return client;
+}
+/** The value of a call, or what the server said was wrong with it. */
+export function unwrap(result) {
+    const { data, error, response } = result;
+    if (response.ok)
+        return data;
+    if (response.status === 401) {
+        throw new OpenCaptionsError("The server refused the key: check OPENCAPTIONS_KEY.");
+    }
+    throw new OpenCaptionsError(describe(error) ?? `The server answered ${response.status}.`);
+}
+function describe(error) {
+    if (typeof error !== "object" || error === null || !("detail" in error))
+        return undefined;
+    const { detail } = error;
+    if (typeof detail === "string")
+        return detail;
+    // A validation error: each problem, and where it is.
+    if (Array.isArray(detail)) {
+        return detail
+            .map((d) => `${(d.loc ?? []).slice(1).join(".")}: ${d.msg}`)
+            .join("; ");
+    }
+    return JSON.stringify(detail);
+}

package/dist/engine.js

@@ -0,0 +1,44 @@
+/**
+ * The caption engine's edits (apps/engine/src/edit.rs), through the same WebAssembly file the web
+ * editor loads: a word is fixed, and a look's word variants dealt, by the code every editor uses,
+ * never by a second implementation here.
+ */
+import { readFile } from "node:fs/promises";
+const WASM = new URL("../engine/opencaptions_engine.wasm", import.meta.url);
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+let loaded;
+function exports() {
+    loaded ??= readFile(WASM).then(async (bytes) => {
+        const { instance } = await WebAssembly.instantiate(bytes, {});
+        return instance.exports;
+    });
+    return loaded;
+}
+function copyIn(x, bytes) {
+    const ptr = x.oc_alloc(bytes.length);
+    new Uint8Array(x.memory.buffer, ptr, bytes.length).set(bytes);
+    return [ptr, bytes.length];
+}
+function result(x, ok) {
+    const text = decoder.decode(new Uint8Array(x.memory.buffer, x.oc_result_ptr(), x.oc_result_len()));
+    if (!ok)
+        throw new Error(`the caption engine refused the edit: ${text}`);
+    return JSON.parse(text);
+}
+const json = (x, t) => copyIn(x, encoder.encode(JSON.stringify(t)));
+export async function captionLines(t, wordsPerLine, offsetMs) {
+    const x = await exports();
+    return result(x, x.oc_caption_lines(...json(x, t), wordsPerLine, offsetMs));
+}
+/** Word `index` (counted across segments) given `text`; empty text removes it. */
+export async function setWord(t, index, text) {
+    const x = await exports();
+    return result(x, x.oc_set_word(...json(x, t), index, ...copyIn(x, encoder.encode(text))));
+}
+/** Deals every word one of `count` looks, or the style's own, as picking such a look does. */
+export async function shuffleVariants(t, count) {
+    const x = await exports();
+    const seed = crypto.getRandomValues(new Uint32Array(1))[0] ?? 0;
+    return result(x, x.oc_shuffle_variants(...json(x, t), count, seed));
+}

package/dist/generated/api.js

@@ -0,0 +1,5 @@
+/**
+ * This file was auto-generated by openapi-typescript.
+ * Do not make direct changes to the file.
+ */
+export {};