@ossclip/core 0.1.36 → 0.1.37

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ossclip/core",
-  "version": "0.1.36",
+  "version": "0.1.37",
   "description": "ossclip's framework-free pipeline: schema, transcription, analysis, cutlist, captions, framing, and the LLM producer",
   "type": "module",
   "license": "MIT",

package/src/config.ts

@@ -199,6 +199,30 @@ export interface OssclipConfig {
    * (`publishConfigured` in the CLI), never coerced.
    */
   postizUrl?: string;
+  /**
+   * Base URL of an OpenAI-compatible transcription server, ending in `/v1`
+   * (Groq: "https://api.groq.com/openai/v1"; a self-hosted speaches:
+   * "http://localhost:8000/v1"). Set, transcription runs REMOTELY instead of
+   * on this machine's CPU — the 2026-09-01 field report from an i3 2nd gen,
+   * where whisper is the dominant cost of a run. Unset, local whisper-cli
+   * stays the default, and `--whisper-backend local` overrides per run.
+   *
+   * Non-secret, the `postizUrl` posture, so it may live here; the API key is
+   * `OSSCLIP_WHISPER_API_KEY` in the ENVIRONMENT only (env.ts's documented
+   * rule) — and OPTIONAL, because self-hosted servers run keyless. Validated
+   * at the consumer (`resolveWhisperBackend` in the CLI), never coerced.
+   */
+  whisperUrl?: string;
+  /**
+   * Model name sent to that server ("whisper-large-v3-turbo" on Groq,
+   * "Systran/faster-whisper-large-v3" on a speaches box). Deliberately NOT
+   * `model`, which names the local ggml file — a remote run must not be able
+   * to send "small.en" to a server that has never heard of it. The default
+   * lives at the consumer (`resolveWhisperBackend`), not in DEFAULTS: an
+   * unset key here must stay unset so nothing writes a remote model name into
+   * a local-only config.
+   */
+  whisperRemoteModel?: string;
   /**
    * `--resolution`'s default for this machine: "auto" (keep what the source
    * has, capped at 2160), "1080" (the built-in default), "1440" or "2160".
@@ -367,6 +391,15 @@ export function resolveConfig(
     // deliberately lives in the environment (publish.ts's
     // `publishConfigured`), so this is only the instance URL.
     postizUrl: fileCfg.postizUrl,
+    // Env spellings ON PURPOSE, unlike postizUrl: the Groq quickstart is
+    // "export two vars and run", and a user trying a free tier for the first
+    // time should not have to hand-edit config.json to do it. Env beats file
+    // so a one-off `OSSCLIP_WHISPER_URL=… ossclip produce` works against a
+    // machine whose config says otherwise. Both are strings straight through
+    // — validated at the consumer (`resolveWhisperBackend`), never coerced,
+    // and the postizUrl lesson above is why they are copied here at all.
+    whisperUrl: env.OSSCLIP_WHISPER_URL ?? fileCfg.whisperUrl,
+    whisperRemoteModel: env.OSSCLIP_WHISPER_REMOTE_MODEL ?? fileCfg.whisperRemoteModel,
     resolution: fileCfg.resolution,
   };
 }

package/src/ingest.ts

@@ -85,6 +85,42 @@ export async function extractAudio(tools: IngestTools, src: string, outWav: stri
   ]);
 }
 
+/**
+ * Pure: ffmpeg args for the compressed remote-upload sidecar (2026-09-01,
+ * the remote-transcription backend). Split from the spawn the same way
+ * `openCommand()` is split from `openInBrowser()` — the codec choice is the
+ * whole decision here, so it must be assertable without an ffmpeg on the box.
+ *
+ * Opus 32 kbps 16 kHz mono: ASR-transparent for speech and ~240 KB/min, so
+ * ~100 minutes of audio fit under Groq's free-tier 25 MB per-file cap. The
+ * PCM wav `extractAudio` writes is 1.92 MB/min and hits that cap at ~13
+ * minutes — unusable for a normal recording session. FLAC was rejected:
+ * lossless buys ASR nothing and only reaches ~25 minutes.
+ */
+export function uploadAudioArgs(wavPath: string, outOgg: string): string[] {
+  return ["-y", "-i", wavPath, "-vn", "-c:a", "libopus", "-b:a", "32k", "-ar", "16000", "-ac", "1", outOgg];
+}
+
+/**
+ * Encode the remote-upload sidecar. libopus is in the bundled static ffmpeg;
+ * a user's own minimal build may lack it, in which case `run` surfaces
+ * ffmpeg's own "Unknown encoder" verbatim — better than a guess at the cause.
+ */
+export async function encodeUploadAudio(tools: IngestTools, wav: string, outOgg: string): Promise<void> {
+  await run(tools.ffmpegPath, uploadAudioArgs(wav, outOgg));
+}
+
+/**
+ * 24 million bytes: Groq's free tier caps a file at "25MB" without saying
+ * whether that is decimal or MiB — 24_000_000 sits under BOTH readings
+ * (25,000,000 and 26,214,336), so a boundary file fails HERE with our message
+ * (naming the ~100-minute ceiling and the local-backend escape hatch) instead
+ * of coming back as somebody else's 413 after the whole upload was paid for.
+ * (24 MiB = 25,165,824 would EXCEED a decimal cap — caught 2026-09-01.)
+ * Chunking is out of scope for v1.
+ */
+export const REMOTE_UPLOAD_MAX_BYTES = 24_000_000;
+
 /**
  * Extract ONE span of an existing wav, same 16 kHz mono PCM shape
  * (2026-08-26, the caption re-alignment pass).

package/src/transcribe/index.ts

@@ -0,0 +1,3 @@
+export * from "./whisper-cli";
+export * from "./provider";
+export * from "./openai-compatible";

package/src/transcribe/openai-compatible.ts

@@ -0,0 +1,199 @@
+import { basename } from "node:path";
+import { z } from "zod/v4";
+import { TranscriptSchema, type Transcript, type Word } from "../schema";
+import { NOISE_TOKEN, normalizeWords, type TranscribeProvider, type TranscribeRequest } from "./provider";
+
+/**
+ * Transcription against any OpenAI-compatible `/v1/audio/transcriptions`
+ * server — Groq's free tier (8h audio/day, `whisper-large-v3-turbo`), a
+ * self-hosted speaches, or anything else speaking that shape.
+ *
+ * Why (2026-09-01 field report): on a weak CPU — an i3 2nd gen — whisper is
+ * the dominant cost of a produce run, minutes of decode per minute of video.
+ * A remote call makes it seconds. Local whisper-cli stays the DEFAULT; this
+ * is opt-in via `whisperUrl` / `OSSCLIP_WHISPER_URL`, and the API key is
+ * optional because self-hosted servers run keyless (unlike publish, where the
+ * key is required).
+ *
+ * Error posture is postiz.ts's, for the same reason: a transcription is the
+ * user's explicit action, so every non-2xx throws with the status and a body
+ * snippet, and there are NO retries — a retry against a metered free tier
+ * silently doubles the quota burn for a failure the user is about to see
+ * anyway.
+ */
+
+/**
+ * `whisperUrl` → the endpoint: trailing slashes dropped,
+ * `/audio/transcriptions` appended unless the user already wrote it
+ * (`postizApiBase`'s mould). The user configures the OpenAI-compatible BASE
+ * ("https://api.groq.com/openai/v1"), which is the URL every provider's
+ * quickstart prints — but someone who pastes the full endpoint must not end
+ * up posting to `/v1/audio/transcriptions/audio/transcriptions`.
+ */
+export function openaiTranscriptionsUrl(baseUrl: string): string {
+  const trimmed = baseUrl.replace(/\/+$/, "");
+  return trimmed.endsWith("/audio/transcriptions") ? trimmed : `${trimmed}/audio/transcriptions`;
+}
+
+export class RemoteTranscribeHttpError extends Error {
+  constructor(
+    readonly url: string,
+    readonly status: number,
+    bodySnippet: string,
+  ) {
+    // Per-status hints, PostizHttpError's mould: the raw status tells a user
+    // nothing about which of the two env vars (or which URL spelling) is
+    // wrong, and this path is reached by people who just pasted a quickstart.
+    const hint =
+      status === 401 || status === 403
+        ? " — the server rejected OSSCLIP_WHISPER_API_KEY (or none was sent — set it in the environment or ~/.ossclip/.env)"
+        : status === 404
+          ? " — no /audio/transcriptions here — whisperUrl should be the OpenAI-compatible base ending in /v1 (e.g. https://api.groq.com/openai/v1)"
+          : status === 413
+            ? " — audio too large for this server — free Groq caps uploads at 25MB; use --whisper-backend local or the dev tier"
+            : status === 429
+              ? " — rate limited (Groq free tier: 8h audio/day)"
+              : "";
+    super(`remote transcription POST ${url} failed: ${status}${hint}${bodySnippet ? `\n${bodySnippet}` : ""}`);
+    this.name = "RemoteTranscribeHttpError";
+  }
+}
+
+/**
+ * `verbose_json` as we consume it. LOOSE on purpose: servers add fields
+ * freely (Groq sends `task`, `duration`, `segments`, `x_groq`), and a strict
+ * object would turn a perfectly good transcription into a parse error the
+ * next time one of them ships a field.
+ */
+const RemoteWordSchema = z.looseObject({
+  word: z.string(),
+  start: z.number(),
+  end: z.number(),
+});
+const VerboseJsonSchema = z.looseObject({
+  language: z.string().optional(),
+  words: z.array(RemoteWordSchema).optional(),
+});
+
+export interface OpenAiCompatibleOptions {
+  /** OpenAI-compatible base, e.g. "https://api.groq.com/openai/v1". */
+  baseUrl: string;
+  model: string;
+  /** Optional: self-hosted servers (speaches, whisper.cpp server) run keyless. */
+  apiKey?: string;
+  /** The postiz test seam — the whole HTTP surface is testable without a network. */
+  fetchImpl?: typeof fetch;
+  /** Per-request cap; an hour of audio takes a while to upload and decode. */
+  timeoutMs?: number;
+}
+
+const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000;
+const BODY_SNIPPET_CHARS = 300;
+
+export function createOpenAiCompatibleProvider(opts: OpenAiCompatibleOptions): TranscribeProvider {
+  const url = openaiTranscriptionsUrl(opts.baseUrl);
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+
+  return {
+    name: "openai-compatible",
+    async transcribe(audioPath: string, req: TranscribeRequest): Promise<Transcript> {
+      // Belt and braces — the CLI refuses this combination earlier, with the
+      // fix named. `/audio/translations` is a different endpoint AND a
+      // different default model upstream, so silently swapping both behind
+      // `--whisper-translate` would be a surprise, not a convenience.
+      if (req.translate === true) {
+        throw new Error(
+          "--whisper-translate needs the local backend (the OpenAI-compatible API translates on a different endpoint and model) — use --whisper-backend local, or drop the flag.",
+        );
+      }
+      const { openAsBlob } = await import("node:fs");
+      // Streams the file into multipart form-data instead of holding it in
+      // memory (postiz.ts's rationale): the upload sidecar is small, but a
+      // span wav or an uncompressed hour is not.
+      const blob = await openAsBlob(audioPath, {
+        type: audioPath.endsWith(".ogg") ? "audio/ogg" : "audio/wav",
+      });
+      const form = new FormData();
+      form.append("file", blob, basename(audioPath));
+      form.append("model", opts.model);
+      form.append("response_format", "verbose_json");
+      // The literal bracketed field name is the wire spelling OpenAI and Groq
+      // accept — it is an array parameter in a multipart body, not a typo.
+      form.append("timestamp_granularities[]", "word");
+      // "auto" is whisper.cpp's vocabulary, not an ISO code: sending it makes
+      // the server reject the request, while OMITTING the field is exactly
+      // what asks for auto-detection.
+      if (req.language !== undefined && req.language !== "auto") form.append("language", req.language);
+      if (req.prompt !== undefined) form.append("prompt", req.prompt);
+
+      const ac = new AbortController();
+      const timer = setTimeout(() => ac.abort(), timeoutMs);
+      let res: Response;
+      try {
+        res = await fetchImpl(url, {
+          method: "POST",
+          headers: opts.apiKey ? { Authorization: `Bearer ${opts.apiKey}` } : {},
+          body: form,
+          signal: ac.signal,
+        });
+      } catch (err) {
+        throw new Error(
+          `remote transcription unreachable at ${url}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      } finally {
+        clearTimeout(timer);
+      }
+      const text = await res.text();
+      if (!res.ok) throw new RemoteTranscribeHttpError(url, res.status, text.slice(0, BODY_SNIPPET_CHARS));
+      let json: unknown;
+      try {
+        json = JSON.parse(text);
+      } catch {
+        throw new Error(
+          `remote transcription answered non-JSON from ${url}: ${text.slice(0, BODY_SNIPPET_CHARS)}`,
+        );
+      }
+      const parsed = VerboseJsonSchema.parse(json);
+      if (!parsed.words || parsed.words.length === 0) {
+        // Two causes, both worth naming: a server that transcribed fine but
+        // has no word-timestamp support (a plain whisper.cpp server), and
+        // near-silent audio, which Groq's turbo model answers wordlessly.
+        // Everything downstream (cuts, captions, zoom) is word-stamp driven,
+        // so a text-only answer is unusable, not a degraded success.
+        throw new Error(
+          `the server answered without word timestamps — it must support response_format=verbose_json with timestamp_granularities[]=word (Groq and speaches do; a plain whisper.cpp server may not), or the audio contained no speech (${url})`,
+        );
+      }
+      // No token merging and no §130 byte repair here: those heal whisper.cpp
+      // `-ml 1` artifacts (one BPE token per segment, split mid-character).
+      // The HTTP API returns whole words with punctuation attached — already
+      // the shape parseWhisperJson works to produce.
+      const words: Word[] = [];
+      for (const w of parsed.words) {
+        const wordText = w.word.trim();
+        if (!wordText || NOISE_TOKEN.test(wordText)) continue;
+        words.push({
+          text: wordText,
+          // Clamped: a server answering -0.01 for the first word would trip
+          // WordSchema's nonnegative and fail the whole run over a rounding
+          // artifact at the very start of the audio.
+          start: Math.max(0, w.start),
+          end: w.end,
+        });
+      }
+      return TranscriptSchema.parse({
+        // The requested code wins — it is what the cache key and the caption
+        // pipeline were told the audio is. Otherwise the server's own answer,
+        // lowercased. NOTE: some servers answer with a full NAME ("english")
+        // rather than a code; no code-sensitive consumer exists today
+        // (captions' RTL check is a Unicode heuristic), so no name→code table.
+        language:
+          req.language !== undefined && req.language !== "auto"
+            ? req.language
+            : parsed.language?.toLowerCase(),
+        words: normalizeWords(words),
+      });
+    },
+  };
+}

package/src/transcribe/provider.ts

@@ -0,0 +1,101 @@
+import type { Transcript, Word } from "../schema";
+
+/**
+ * The seam between ossclip and any transcription backend (2026-09-01, the
+ * weak-CPU field report: whisper is the dominant cost on an i3 2nd gen, and a
+ * free remote tier makes it disappear). Two implementations today —
+ * whisper.cpp on the box (`whisper-cli.ts`) and any OpenAI-compatible
+ * `/v1/audio/transcriptions` server (`openai-compatible.ts`).
+ *
+ * The local path deliberately keeps calling `runWhisper` directly rather than
+ * going through this interface: produce.ts and the edit server inject
+ * `runWhisper` itself as a test seam, and forcing that through a provider
+ * object would churn every stub for no behavior change.
+ */
+export interface TranscribeRequest {
+  /** Resolved language code; "auto" is handled per provider (whisper.cpp
+   *  takes it literally, the HTTP API wants the field OMITTED). */
+  language?: string;
+  /** `whisperPromptFor()` output — the user dictionary as decoder bias. */
+  prompt?: string;
+  /** whisper-cli's TRANSLATE task; the OpenAI-compatible provider rejects it
+   *  (a different endpoint AND a different default model upstream). */
+  translate?: boolean;
+}
+
+export interface TranscribeProvider {
+  name: string;
+  transcribe(audioPath: string, req: TranscribeRequest): Promise<Transcript>;
+}
+
+/** [BLANK_AUDIO], (buzzing), [MUSIC] … — noise markers, not speech. Shared:
+ *  remote servers emit the same bracketed markers whisper.cpp does. */
+export const NOISE_TOKEN = /^[[(].*[\])]$/;
+
+/**
+ * Run length at which a stack of zero-length words at ONE instant stops being
+ * a rounding artifact and becomes a repetition-loop hallucination. Real speech
+ * never emits 8 tokens at a single instant; the field case emitted 118.
+ */
+export const REPETITION_BURST_MIN = 8;
+
+/**
+ * Drop whisper repetition-loop bursts (field case 2026-08-18): an Urdu take
+ * re-decoded a whole phrase as 118 CONSECUTIVE tokens all stamped
+ * `from === to === 31040` — zero length, at one instant. The stamp repair
+ * below then fans such a burst out into 118 fabricated 50ms words marching
+ * forward from 31.04s, so the phrase ships TWICE in the captions (31.04s and
+ * 33.54s) and a fifth of the transcript carries the tell-tale exactly-0.05s
+ * duration. `-mc 0` in whisperArgs is the decoder-side mitigation for the same
+ * failure; it did not prevent this occurrence, and it can never repair an
+ * already-cached transcript.json — hence a parse-side guard too.
+ *
+ * A burst is a MAXIMAL run of consecutive zero-length/inverted words sharing
+ * one `start`. Equality is exact, not epsilon: these stamps are integer
+ * milliseconds divided by 1000, so members of one burst are the same double
+ * bit-for-bit, and a tolerance would only start swallowing real neighbors.
+ * Runs shorter than REPETITION_BURST_MIN fall through untouched — a lone
+ * zero-length stamp is a rounding artifact, not a hallucination. The drop is
+ * silent by design: this function is pure and total, and there is no logging
+ * channel in the parse path to warn on.
+ */
+export function dropRepetitionBursts(words: readonly Word[]): Word[] {
+  const out: Word[] = [];
+  let i = 0;
+  while (i < words.length) {
+    const w = words[i]!;
+    if (w.end > w.start) {
+      out.push(w);
+      i++;
+      continue;
+    }
+    let j = i + 1;
+    while (j < words.length && words[j]!.end <= words[j]!.start && words[j]!.start === w.start) j++;
+    if (j - i < REPETITION_BURST_MIN) for (let k = i; k < j; k++) out.push(words[k]!);
+    i = j;
+  }
+  return out;
+}
+
+/**
+ * The word-stamp hygiene EVERY backend's output goes through, extracted from
+ * the tail of `parseWhisperJson` so the remote provider cannot drift from it
+ * (2026-09-01). Burst drop runs BEFORE the repair, never after: the repair
+ * rewrites every burst member into a distinct monotone stamp, so once it has
+ * run the shared timestamp — the only evidence a burst existed — is gone.
+ *
+ * Copies before mutating, so a caller's array survives the call unchanged;
+ * `parseWhisperJson` builds its words fresh, so this is byte-identical to the
+ * in-place loop it replaces.
+ */
+export function normalizeWords(words: readonly Word[]): Word[] {
+  const kept = dropRepetitionBursts(words).map((w) => ({ ...w }));
+  // Whisper occasionally emits zero-length or inverted stamps; repair minimally.
+  for (let i = 0; i < kept.length; i++) {
+    const w = kept[i]!;
+    if (w.end <= w.start) w.end = w.start + 0.05;
+    const next = kept[i + 1];
+    if (next && next.start < w.end) next.start = w.end;
+  }
+  return kept;
+}

package/src/{transcribe.ts → transcribe/whisper-cli.ts}

@@ -1,6 +1,7 @@
 import { readFile } from "node:fs/promises";
-import { run } from "./exec";
-import type { Transcript, Word } from "./schema";
+import { run } from "../exec";
+import type { Transcript, Word } from "../schema";
+import { NOISE_TOKEN, normalizeWords } from "./provider";
 
 /** Shape of whisper.cpp's `-oj` JSON output (the fields we consume). */
 export interface WhisperJson {
@@ -11,8 +12,6 @@ export interface WhisperJson {
   }>;
 }
 
-const NOISE_TOKEN = /^[[(].*[\])]$/; // [BLANK_AUDIO], (buzzing), [MUSIC] …
-
 /**
  * How many bytes at the END of `bytes` form the start of a multi-byte UTF-8
  * character whose continuation bytes are missing (§130: whisper.cpp `-ml 1`
@@ -78,51 +77,6 @@ function repairSplitSegments(json: WhisperJson): WhisperJson {
   };
 }
 
-/**
- * Run length at which a stack of zero-length words at ONE instant stops being
- * a rounding artifact and becomes a repetition-loop hallucination. Real speech
- * never emits 8 tokens at a single instant; the field case emitted 118.
- */
-export const REPETITION_BURST_MIN = 8;
-
-/**
- * Drop whisper repetition-loop bursts (field case 2026-08-18): an Urdu take
- * re-decoded a whole phrase as 118 CONSECUTIVE tokens all stamped
- * `from === to === 31040` — zero length, at one instant. The stamp repair
- * below then fans such a burst out into 118 fabricated 50ms words marching
- * forward from 31.04s, so the phrase ships TWICE in the captions (31.04s and
- * 33.54s) and a fifth of the transcript carries the tell-tale exactly-0.05s
- * duration. `-mc 0` in whisperArgs is the decoder-side mitigation for the same
- * failure; it did not prevent this occurrence, and it can never repair an
- * already-cached transcript.json — hence a parse-side guard too.
- *
- * A burst is a MAXIMAL run of consecutive zero-length/inverted words sharing
- * one `start`. Equality is exact, not epsilon: these stamps are integer
- * milliseconds divided by 1000, so members of one burst are the same double
- * bit-for-bit, and a tolerance would only start swallowing real neighbors.
- * Runs shorter than REPETITION_BURST_MIN fall through untouched — a lone
- * zero-length stamp is a rounding artifact, not a hallucination. The drop is
- * silent by design: this function is pure and total, and there is no logging
- * channel in the parse path to warn on.
- */
-export function dropRepetitionBursts(words: readonly Word[]): Word[] {
-  const out: Word[] = [];
-  let i = 0;
-  while (i < words.length) {
-    const w = words[i]!;
-    if (w.end > w.start) {
-      out.push(w);
-      i++;
-      continue;
-    }
-    let j = i + 1;
-    while (j < words.length && words[j]!.end <= words[j]!.start && words[j]!.start === w.start) j++;
-    if (j - i < REPETITION_BURST_MIN) for (let k = i; k < j; k++) out.push(words[k]!);
-    i = j;
-  }
-  return out;
-}
-
 const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
 
 /**
@@ -199,18 +153,11 @@ export function parseWhisperJson(json: WhisperJson): Transcript {
     else if (next) next.start = Math.min(next.start, w.start);
     words.splice(i, 1);
   }
-  // BEFORE the repair loop, never after: the repair rewrites every burst
-  // member into a distinct monotone stamp, so once it has run the shared
-  // timestamp — the only evidence a burst existed — is gone.
-  const kept = dropRepetitionBursts(words);
-  // Whisper occasionally emits zero-length or inverted stamps; repair minimally.
-  for (let i = 0; i < kept.length; i++) {
-    const w = kept[i]!;
-    if (w.end <= w.start) w.end = w.start + 0.05;
-    const next = kept[i + 1];
-    if (next && next.start < w.end) next.start = w.end;
-  }
-  return { language: json.result?.language ?? "en", words: kept };
+  // Burst drop + stamp repair now live in `normalizeWords` (provider.ts,
+  // 2026-09-01): the remote backend needs exactly the same two passes in
+  // exactly the same order, and duplicating them is how the two paths would
+  // drift. Behavior here is unchanged — the parser matrix pins it.
+  return { language: json.result?.language ?? "en", words: normalizeWords(words) };
 }
 
 export interface WhisperOptions {