nixamp 0.25.0 → 0.26.0

package/src/server.ts

@@ -16,6 +16,9 @@ import { randomBytes } from "node:crypto";
 import { hostname, networkInterfaces } from "node:os";
 import { spawn, spawnSync } from "node:child_process";
 import { readFileSync } from "node:fs";
+import { Readable } from "node:stream";
+import { pipeline as pipeStream } from "node:stream/promises";
+import { LiveVoice, LIVE_VOICE_LANGUAGES, LIVE_VOICE_MODEL, type VoiceRequest } from "./live-voice.ts";
 import { Connections, type Kind } from "./connections.ts";
 import {
   Broadcaster,
@@ -86,7 +89,7 @@ import { Tickets, needsTicket, ticketFrom, ticketsFromEnv } from "./tickets.ts";
 import { Layouts } from "./layouts.ts";
 import { Rooms } from "./rooms.ts";
 import { Trollbox, TrollboxError, fallbackHandle, roomFor } from "./trollbox.ts";
-import { MAX_BYTES as SPEECH_BYTES, Speech, SpeechError, isWav, languageOf } from "./speech.ts";
+import { MAX_BYTES as SPEECH_BYTES, NATIVE_REVISION, Speech, SpeechError, isWav, languageOf } from "./speech.ts";
 import { Captions } from "./captions.ts";
 import { LANGUAGES, Translator } from "./translate.ts";
 import { StoredTranslations } from "./translate-jobs.ts";
@@ -1350,7 +1353,7 @@ const CORS: Record<string, string> = {
   // paths and takes six commands; binding to 127.0.0.1 is what keeps it shut.
   "access-control-allow-origin": "*",
   "access-control-allow-methods": "GET, POST, PUT, DELETE, OPTIONS",
-  "access-control-allow-headers": "content-type",
+  "access-control-allow-headers": "content-type, authorization",
   "access-control-max-age": "86400",
 };
 
@@ -1548,6 +1551,7 @@ export function joinDocument(shell: string, subject: JoinSubject, site: string):
 }
 
 function json(response: ServerResponse, code: number, body: unknown): void {
+  if (code === 429) response.setHeader("retry-after", "60");
   const text = JSON.stringify(body);
   response.writeHead(code, {
     ...CORS,
@@ -1558,6 +1562,16 @@ function json(response: ServerResponse, code: number, body: unknown): void {
   response.end(text);
 }
 
+/** Stream PCM without waiting for a complete utterance, respecting browser backpressure. */
+async function voiceResponse(response: ServerResponse, upstream: Response): Promise<void> {
+  response.writeHead(upstream.status, {
+    ...CORS, "content-type": upstream.headers.get("content-type") ?? "application/json",
+    "cache-control": "no-store", "x-accel-buffering": "no",
+  });
+  if (!upstream.body) { response.end(); return; }
+  await pipeStream(Readable.fromWeb(upstream.body as import("node:stream/web").ReadableStream), response);
+}
+
 async function readBody(request: IncomingMessage, limit = 64 * 1024): Promise<string> {
   const chunks: Buffer[] = [];
   let size = 0;
@@ -1745,6 +1759,7 @@ export interface HandlerOptions {
   trollbox?: Trollbox;
   /** Speech to text: a line said out loud, heard here. Needs the optional model. */
   speech?: Speech;
+  liveVoice?: LiveVoice;
   /** Translation: texts in another language, by a model here. The same optional library. */
   translator?: Translator;
   /** The transcript store: what was heard, kept under the media's identity. Where the accounts are. */
@@ -2843,6 +2858,45 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
      * room, the words go straight into that room's trollbox as a line by
      * whoever spoke them. The page, the CLI and the MCP tools all come here.
      */
+    if (["/api/v1/speech/voices", "/api/v1/speech/grant", "/api/v1/speech/synthesize", "/api/v1/speech/speakers"].includes(path) && options.accounts) {
+      const controller = new AbortController();
+      response.once("close", () => controller.abort());
+      try {
+        const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
+        if (!guard.check(`voice-ip:${caller}`, { allowed: 600, windowMs: 60_000 }).ok) { json(response, 429, { error: "too many voice requests" }); return; }
+        const service = options.liveVoice;
+        if (!service?.available()) { json(response, 503, { error: "translated audio needs ELEVENLABS_API_KEY on the account server" }); return; }
+        if (path.endsWith("/synthesize") && request.method === "POST") {
+          let body: VoiceRequest;
+          try { body = JSON.parse(await readBody(request, 4096)) as VoiceRequest; }
+          catch { json(response, 400, { error: "send a caption as JSON" }); return; }
+          if (!body || typeof body !== "object") { json(response, 400, { error: "send a caption as JSON" }); return; }
+          const by = await service.authorize(tokenFrom(request.headers), body.channel ?? "", typeof body.text === "string" ? body.text.length : 0);
+          await voiceResponse(response, await service.stream(body, by, controller.signal));
+        } else {
+          const who = await options.accounts.whoIs(tokenFrom(request.headers));
+          if (!who) { json(response, 401, { error: "sign in to use translated audio" }); return; }
+          if (path.endsWith("/voices") && request.method === "GET") {
+            json(response, 200, { voices: await service.voices(), model: LIVE_VOICE_MODEL, languages: [...LIVE_VOICE_LANGUAGES] });
+          } else if (path.endsWith("/speakers") && request.method === "POST") {
+            if (!request.headers["content-type"]?.startsWith("audio/wav")) { json(response, 415, { error: "send a mono 16 kHz WAV" }); return; }
+            const bytes = await readBytes(request, 484_000);
+            json(response, 200, await service.hear(bytes, who.id, controller.signal));
+          } else if (path.endsWith("/grant") && request.method === "POST") {
+            if (!request.headers["content-type"]?.startsWith("application/json")) { json(response, 415, { error: "send JSON" }); return; }
+            let body: { channel?: unknown };
+            try { body = JSON.parse(await readBody(request, 1024)) as typeof body; }
+            catch { json(response, 400, { error: "send a channel as JSON" }); return; }
+            json(response, 200, await service.grant(who.id, typeof body?.channel === "string" ? body.channel : ""));
+          } else json(response, 405, { error: "GET voices or POST an audio request" });
+        }
+      } catch (error) {
+        if (response.headersSent) response.destroy();
+        else json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "translated audio is unavailable" });
+      }
+      return;
+    }
+
     if (path === "/api/v1/speech/transcribe" && options.speech && options.accounts) {
       const speech = options.speech;
       try {
@@ -2872,6 +2926,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
           // Pieces with their timing, for a whole file being written down.
           timestamps: url.searchParams.get("timestamps") === "1",
           by: who.id,
+          ...(url.searchParams.get("live") === "1" ? { deadline: Date.now() + 10_000 } : {}),
         });
         // The language travels back: as told, or as the ear guessed it, so
         // a captioner can say it next time and a transcript can be kept as it.
@@ -3082,7 +3137,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
         return;
       }
       try {
-        json(response, 200, await translator.translate(texts, from, to, { by: who.id }));
+        json(response, 200, await translator.translate(texts, from, to, { by: who.id, ...(url.searchParams.get("live") === "1" ? { deadline: Date.now() + 10_000 } : {}) }));
       } catch (error) {
         if (error instanceof SpeechError) json(response, error.status, { error: error.message });
         else throw error;
@@ -3136,6 +3191,12 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
             json(response, 400, { error: "format is json, srt, vtt or txt" });
             return;
           }
+          // Joining a live must not start translating an entire old transcript.
+          if (url.searchParams.get("cached") === "1") {
+            const saved = await store.get(id, language);
+            json(response, saved ? 200 : 404, saved ? wire(saved) : { error: "no cached transcript" });
+            return;
+          }
           const answer = await options.translations.get(id, language, who.id);
           if (answer.status !== 200 && answer.status !== 202) {
             json(response, answer.status, { error: answer.error });
@@ -4486,6 +4547,31 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
        * started by the first person asking. `captions` is the live stream
        * of lines; `transcript` is the recent ones as JSON, for a poll.
        */
+      if ((action === "voice" || action === "voice-options") && request.method === "GET") {
+        const controller = new AbortController();
+        response.once("close", () => controller.abort());
+        const timeout = setTimeout(() => controller.abort(), 12_000);
+        try {
+          const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
+          if (!guard.check(`channel-voice-ip:${caller}`, { allowed: 60, windowMs: 60_000 }).ok || !guard.check(`channel-voice:${id}`, { allowed: 240, windowMs: 60_000 }).ok) {
+            json(response, 429, { error: "too many translated audio requests" }); return;
+          }
+          if (!channels.has(id)) { json(response, 404, { error: "nothing is playing on that channel" }); return; }
+          if (!options.captions) { json(response, 503, { error: "this server cannot caption" }); return; }
+          const at = action === "voice-options" ? null : Number(url.searchParams.get("at"));
+          const language = languageCode(url.searchParams.get("language"));
+          if ((at !== null && (!url.searchParams.has("at") || !Number.isFinite(at))) || language === null) {
+            json(response, 400, { error: "audio needs a caption time and language" }); return;
+          }
+          const upstream = await options.captions.voiceRequest(id, at, language, url.searchParams.get("voice") ?? "auto", controller.signal, tokenFrom(request.headers));
+          await voiceResponse(response, upstream);
+        } catch (error) {
+          if (response.headersSent) response.destroy();
+          else json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "translated audio is unavailable" });
+        } finally { clearTimeout(timeout); }
+        return;
+      }
+
       if ((action === "captions" || action === "transcript") && request.method === "GET") {
         const captions = options.captions;
         if (!captions) {
@@ -4496,6 +4582,11 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
           json(response, 503, { error: "this server is not signed in to nixamp.com, so it cannot caption; run `nixamp login` on it" });
           return;
         }
+        const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
+        if (!guard.check(`captions:${caller}`, { allowed: 120, windowMs: 60_000 }).ok || !captions.capacity(id)) {
+          json(response, 429, { error: "captioning is at capacity; try again in a moment" });
+          return;
+        }
         if (!channels.has(id)) {
           json(response, 404, { error: "nothing is playing on that channel" });
           return;
@@ -4514,7 +4605,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
           // Asking keeps the captioner up: it stops a minute after the last ask.
           captions.subscribe(id, () => undefined, wanted)?.();
           json(response, 200, {
-            channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted),
+            channel: id, recognitionRevision: NATIVE_REVISION, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted),
           });
           return;
         }
@@ -4535,7 +4626,7 @@ export function createHandler(engine: Engine, options: HandlerOptions) {
         }
         // How far behind the live edge a newcomer's playback starts, so the
         // page can hold each line until its own sound gets there.
-        write("hello", { channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted) });
+        write("hello", { channel: id, recognitionRevision: NATIVE_REVISION, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted) });
         const beat = setInterval(() => response.write(": beat\n\n"), 20_000);
         beat.unref?.();
         const done = (): void => {
@@ -5909,6 +6000,13 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
   // the first person to speak does not wait for the model to arrive; a box
   // without the optional model never says it is ready, and answers 503.
   const speech = accounts && process.env["NIXAMP_STT"] !== "off" ? new Speech() : undefined;
+  const liveVoice = accounts && process.env["NIXAMP_DUBBING"] !== "off" ? new LiveVoice({
+    ...(pool ? { db: pool } : {}),
+    dailyChars: Number(process.env["NIXAMP_DUB_DAILY_CHARS"] ?? 200_000),
+    dailyAudioSeconds: Number(process.env["NIXAMP_DUB_DAILY_AUDIO_SECONDS"] ?? 86_400),
+    userDailyChars: Number(process.env["NIXAMP_DUB_USER_DAILY_CHARS"] ?? 120_000),
+    userDailyAudioSeconds: Number(process.env["NIXAMP_DUB_USER_DAILY_AUDIO_SECONDS"] ?? 43_200),
+  }) : undefined;
   if (speech) {
     void speech.warm().then((ready) => {
       console.error(ready ? `nixamp: hearing with ${speech.model}` : `nixamp: not hearing: ${speech.lastFailure}`);
@@ -6400,6 +6498,7 @@ export async function serve(argv: string[], version = "0.1.0"): Promise<void> {
     ...(rooms ? { rooms } : {}),
     ...(trollbox ? { trollbox } : {}),
     ...(speech ? { speech } : {}),
+    ...(liveVoice ? { liveVoice } : {}),
     ...(translator ? { translator } : {}),
     ...(transcripts ? { transcripts } : {}),
     ...(translations ? { translations } : {}),

package/src/speaker-turns.ts

@@ -0,0 +1,34 @@
+/** Provider speaker IDs belong to a clip. Clients reconcile the overlapping
+ * word timestamps before assigning a voice; these are not people's identities. */
+import { reliableText, type Wav } from "./speech.ts";
+import { voiceProfile, type VoiceProfile } from "./voice-profile.ts";
+
+export interface SpeakerTurn { start: number; end: number; text: string; speaker: string; profile: VoiceProfile; words: { text: string; start: number; end: number }[]; }
+export interface SpeakerTranscript { language: string; seconds: number; turns: SpeakerTurn[]; }
+export interface ScribeResult {
+  language_code?: string;
+  words?: { text: string; start: number; end: number; type: string; speaker_id?: string | null }[];
+}
+const ISO: Record<string, string> = Object.fromEntries("eng:en spa:es deu:de ger:de fra:fr fre:fr por:pt ita:it nld:nl dut:nl swe:sv dan:da fin:fi rus:ru ukr:uk ces:cs cze:cs hun:hu cmn:zh zho:zh ara:ar hin:hi vie:vi ind:id jpn:ja kor:ko pol:pl tur:tr ron:ro bul:bg ell:el nor:no nob:no nno:no".split(" ").map(pair => pair.split(":")));
+
+export function speakerTurns(body: ScribeResult, wav: Wav): SpeakerTranscript {
+  const seconds = wav.samples.length / wav.rate;
+  const language = ISO[body.language_code ?? ""] ?? body.language_code ?? "";
+  const turns: SpeakerTurn[] = [];
+  for (const word of (body.words ?? []).slice(0, 1500)) {
+    if (word.type !== "word" || typeof word.text !== "string" || !Number.isFinite(word.start) || !Number.isFinite(word.end) || word.start < 0 || word.end < word.start || word.end > seconds + 0.5) continue;
+    const speaker = typeof word.speaker_id === "string" && /^[\w-]{1,80}$/.test(word.speaker_id) ? word.speaker_id : "unknown";
+    const last = turns.at(-1);
+    if (last && last.speaker === speaker && word.start - last.end < 0.8 && last.text.length + word.text.length < 400) {
+      last.text += ` ${word.text}`; last.end = Math.max(last.end, word.end);
+      last.words.push({ text: word.text, start: word.start, end: word.end });
+    } else turns.push({ start: word.start, end: word.end, text: word.text, speaker, profile: "unknown", words: [{ text: word.text, start: word.start, end: word.end }] });
+  }
+  const pcm = Buffer.alloc(wav.samples.length * 2);
+  wav.samples.forEach((sample, i) => pcm.writeInt16LE(Math.round(Math.max(-1, Math.min(1, sample)) * 32767), i * 2));
+  for (const turn of turns) {
+    turn.text = reliableText(turn.text, Math.max(1, turn.end - turn.start));
+    turn.profile = voiceProfile(pcm.subarray(Math.floor(turn.start * 16000) * 2, Math.ceil(turn.end * 16000) * 2));
+  }
+  return { language, seconds, turns: turns.filter(turn => turn.text !== "") };
+}

package/src/speech.ts

@@ -40,6 +40,8 @@ export const SECONDS_PER_MINUTE = 300;
 /** How many may wait for the one CPU. Past this, the honest answer is "later". */
 export const QUEUE_LIMIT = 8;
 export const DEFAULT_MODEL = "onnx-community/whisper-base";
+/** Stored live lines from older recognition settings must be heard again. */
+export const NATIVE_REVISION = "native-v2";
 
 export class SpeechError extends Error {
   constructor(message: string, readonly status: number) {
@@ -215,6 +217,29 @@ export function tidy(text: string): string {
   return text.replace(/\[[A-Z_ ]+\]|\([A-Za-z ]+\)/g, " ").replace(/\s+/g, " ").trim();
 }
 
+/** Reject decoding loops, without removing normal emphasis like “no, no, no”. */
+export function reliableText(text: string, seconds: number): string {
+  const clean = text.replace(/\s+/g, " ").trim();
+  if (clean.length > Math.max(120, seconds * 55)) return "";
+  const words = clean.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? [];
+  for (let size = 1; size <= Math.min(20, Math.floor(words.length / 4)); size++) {
+    for (let start = 0; start + size * 4 <= words.length; start++) {
+      let repeated = true;
+      for (let i = size; i < size * 4; i++) {
+        if (words[start + i] !== words[start + i % size]) { repeated = false; break; }
+      }
+      if (repeated) return "";
+    }
+  }
+  return clean;
+}
+
+export function quietSamples(pcm: Float32Array): boolean {
+  let energy = 0;
+  for (const sample of pcm) energy += sample * sample;
+  return pcm.length === 0 || Math.sqrt(energy / pcm.length) < 0.004;
+}
+
 /** A two-letter language code, or nothing: Whisper guesses when not told. */
 export function languageOf(value: unknown): string | undefined {
   if (typeof value !== "string") return undefined;
@@ -229,7 +254,7 @@ interface AsrOutput {
 }
 
 /** The pipeline, and the parts under it that language detection needs. */
-interface AsrPipeline {
+export interface AsrPipeline {
   (audio: Float32Array, options: Record<string, unknown>): Promise<AsrOutput | AsrOutput[]>;
   model: {
     (inputs: Record<string, unknown>): Promise<{ logits: { data: Float32Array | number[] } }>;
@@ -239,7 +264,7 @@ interface AsrPipeline {
 }
 
 /** The module, as much of it as this file touches. Typed here so the import can be by name. */
-interface Transformers {
+export interface Transformers {
   env: { cacheDir?: string; allowLocalModels?: boolean };
   Tensor: new (type: string, data: BigInt64Array, dims: number[]) => unknown;
   pipeline(task: "automatic-speech-recognition", model: string, options: { dtype: string }): Promise<AsrPipeline>;
@@ -296,13 +321,25 @@ async function loadWhisper(model: string, cacheDir: string): Promise<Recognizer>
   const transformers = await loadTransformers<Transformers>();
   transformers.env.cacheDir = cacheDir;
   const recognize = await transformers.pipeline("automatic-speech-recognition", model, { dtype: "q8" });
+  return whisperRecognizer(transformers, recognize);
+}
+
+/** Detection is from each recording, never from the caption translation selection. */
+export function whisperRecognizer(transformers: Transformers, recognize: AsrPipeline): Recognizer {
   return async (pcm, { language, timestamps }) => {
-    const spoken = language ?? (await detectLanguage(transformers, recognize, pcm));
+    const multilingual = recognize.model.generation_config.is_multilingual !== false;
+    const spoken = language ?? (multilingual ? await detectLanguage(transformers, recognize, pcm) : "en");
+    if (multilingual && !spoken) throw new SpeechError("could not detect the audio language; try another speech segment", 422);
     const heard = await recognize(pcm, {
       // Whisper hears thirty seconds at a time; longer is heard in overlapping pieces.
       chunk_length_s: 30,
       stride_length_s: 5,
-      ...(spoken ? { language: spoken, task: "transcribe" } : {}),
+      ...(multilingual ? { language: spoken, task: "transcribe" } : {}),
+      // A five-second clip used to be allowed hundreds of tokens of hallucinated loops.
+      max_new_tokens: Math.min(384, Math.max(32, Math.ceil(Math.min(30, pcm.length / RATE) * 10) + 16)),
+      do_sample: false,
+      num_beams: 1,
+      no_repeat_ngram_size: 6,
       ...(timestamps ? { return_timestamps: true } : {}),
     });
     const pieces = Array.isArray(heard) ? heard : [heard];
@@ -385,7 +422,7 @@ export class Speech {
    * The words in a WAV. Refuses what is not a WAV, or is too long, or more
    * than the account may have heard this minute, with a status.
    */
-  async transcribe(bytes: Uint8Array, options: { language?: string; timestamps?: boolean; by?: string } = {}): Promise<Heard> {
+  async transcribe(bytes: Uint8Array, options: { language?: string; timestamps?: boolean; by?: string; deadline?: number } = {}): Promise<Heard> {
     if (bytes.length > MAX_BYTES) throw new SpeechError(`that is too much sound: ${MAX_SECONDS} seconds at most`, 413);
     const wav = decodeWav(bytes);
     const seconds = wav.samples.length / wav.rate;
@@ -393,9 +430,11 @@ export class Speech {
     if (options.by) this.allow(options.by, seconds);
     if (wav.samples.length < wav.rate / 10) return { text: "", seconds };
     const pcm = resample(wav.samples, wav.rate, RATE);
+    if (quietSamples(pcm)) return { text: "", seconds };
     if (this.waiting >= QUEUE_LIMIT) throw new SpeechError("too many people are talking at once; try again in a moment", 503);
     this.waiting += 1;
     const turn = this.tail.then(async () => {
+      if (options.deadline && this.now() > options.deadline) throw new SpeechError("this live audio is too old; waiting for the next segment", 408);
       const recognize = await this.ear();
       return recognize(pcm, {
         ...(options.language ? { language: options.language } : {}),
@@ -407,10 +446,10 @@ export class Speech {
     try {
       const heard = await turn;
       const segments = heard.segments
-        ?.map((segment) => ({ start: segment.start, end: segment.end, text: tidy(segment.text) }))
+        ?.map((segment) => ({ start: segment.start, end: segment.end, text: reliableText(tidy(segment.text), segment.end - segment.start) }))
         .filter((segment) => segment.text !== "");
       return {
-        text: tidy(heard.text),
+        text: reliableText(tidy(heard.text), seconds),
         seconds,
         ...(heard.language ? { language: heard.language } : {}),
         ...(segments ? { segments } : {}),

package/src/transcript-client.ts

@@ -53,9 +53,11 @@ export async function fetchTranscript(
   id: string,
   language = "",
   fetcher: typeof fetch = fetch,
+  cachedOnly = false,
 ): Promise<Got<StoredTranscript>> {
   const url = new URL(`${base(signed.site)}/api/v1/transcripts/${encodeURIComponent(id)}`);
   if (language) url.searchParams.set("language", language);
+  if (cachedOnly) url.searchParams.set("cached", "1");
   try {
     const response = await fetcher(url.toString(), { headers: { authorization: `Bearer ${signed.token}` } });
     return await asJson<StoredTranscript>(response);
@@ -107,12 +109,14 @@ export async function translateTexts(
   from: string,
   to: string,
   fetcher: typeof fetch = fetch,
+  signal?: AbortSignal,
 ): Promise<Got<Translated>> {
   try {
     const response = await fetcher(`${base(signed.site)}/api/v1/translate`, {
       method: "POST",
       headers: { authorization: `Bearer ${signed.token}`, "content-type": "application/json" },
       body: JSON.stringify({ texts, from, to }),
+      signal,
     });
     return await asJson<Translated>(response);
   } catch (error) {

package/src/transcripts.ts

@@ -30,6 +30,11 @@ export interface TranscriptLine {
   start: number;
   end: number;
   text: string;
+  /** Live transcripts may change language within one programme. */
+  language?: string;
+  revision?: string;
+  original?: string;
+  voiceProfile?: "lower" | "higher" | "unknown";
 }
 
 export type MediaKind = "file" | "url" | "live";
@@ -159,12 +164,17 @@ export function linesFrom(value: unknown): TranscriptLine[] {
   const lines: TranscriptLine[] = [];
   for (const one of parsed) {
     if (!one || typeof one !== "object") continue;
-    const { start, end, text } = one as Record<string, unknown>;
+    const { start, end, text, language, revision, original, voiceProfile } = one as Record<string, unknown>;
     if (typeof text !== "string" || typeof start !== "number" || !Number.isFinite(start) || start < 0) continue;
     const words = text.replace(/\s+/g, " ").trim().slice(0, MAX_LINE_CHARS);
     if (words === "") continue;
     const until = typeof end === "number" && Number.isFinite(end) && end > start ? end : start;
-    lines.push({ start: round(start), end: round(until), text: words });
+    lines.push({ start: round(start), end: round(until), text: words,
+      ...(typeof language === "string" && /^[a-z]{2,3}$/.test(language) ? { language } : {}),
+      ...(typeof revision === "string" ? { revision: revision.slice(0, 32) } : {}),
+      ...(typeof original === "string" ? { original: original.slice(0, MAX_LINE_CHARS) } : {}),
+      ...(voiceProfile === "lower" || voiceProfile === "higher" || voiceProfile === "unknown" ? { voiceProfile } : {}),
+    });
   }
   return lines;
 }
@@ -371,7 +381,7 @@ export class Transcripts {
     const { rows } = language === ""
       ? await this.db.query(
           `SELECT * FROM transcripts WHERE id = $1 AND translated_from IS NULL
-           ORDER BY complete DESC, updated_at DESC LIMIT 1`,
+           ORDER BY (language = '') DESC, complete DESC, updated_at DESC LIMIT 1`,
           [id],
         )
       : await this.db.query("SELECT * FROM transcripts WHERE id = $1 AND language = $2 LIMIT 1", [id, language]);

package/src/translate-jobs.ts

@@ -60,9 +60,10 @@ export class StoredTranslations {
     const translation = await this.store.get(id, language);
     const missing = StoredTranslations.missing(original, translation);
     if (translation && missing.length === 0) return { status: 200, transcript: translation };
-    if (original.language === "") return { status: 409, error: "the language this was heard in is not known, so it cannot be translated" };
+    const sources = new Set(missing.map((line) => line.language || original.language));
+    if (sources.has("")) return { status: 409, error: "the language this was heard in is not known, so it cannot be translated" };
     if (!this.translator) return { status: 503, error: "this nixamp cannot translate: no model here. nixamp.com can." };
-    if (!this.translator.can(original.language, language)) {
+    if ([...sources].some((source) => !this.translator!.can(source, language))) {
       return { status: 409, error: `there is no model here from ${original.language} to ${language}` };
     }
     const key = `${id}|${language}`;
@@ -107,19 +108,24 @@ export class StoredTranslations {
     let model = "";
     for (let at = 0; at < missing.length; at += BATCH) {
       const batch = missing.slice(at, at + BATCH);
-      const done = await translator.translate(batch.map((line) => line.text), original.language, language);
-      model = done.model;
-      const lines = batch.map((line, i) => ({ start: line.start, end: line.end, text: done.texts[i] ?? "" })).filter((line) => line.text !== "");
+      const lines: TranscriptLine[] = [];
+      for (const source of new Set(batch.map((line) => line.language || original.language))) {
+        const group = batch.filter((line) => (line.language || original.language) === source);
+        const done = await translator.translate(group.map((line) => line.text), source, language);
+        model = done.model;
+        lines.push(...group.map((line, i) => ({ ...line, language, original: line.text, text: done.texts[i] ?? "" })).filter((line) => line.text !== ""));
+      }
+      lines.sort((a, b) => a.start - b.start);
       made.push(...lines);
       await this.store.save({
-        media: original.media, language, translatedFrom: original.language, model, title: original.title, by, lines,
+        media: original.media, language, translatedFrom: original.language || "mul", model, title: original.title, by, lines,
       });
       job.progress.done += batch.length;
     }
     if (original.complete) {
       // Whole, like the original: the pieces are replaced by the lot, and nothing partial touches it again.
       await this.store.save({
-        media: original.media, language, translatedFrom: original.language, model, title: original.title, by, lines: made, complete: true,
+        media: original.media, language, translatedFrom: original.language || "mul", model, title: original.title, by, lines: made, complete: true,
       });
     }
     if (missing.length > 0) this.onEvent(`  translated ${missing.length} lines of ${original.title || original.media} to ${language}`);

package/src/translate.ts

@@ -164,6 +164,7 @@ export class Translator {
   private readonly keep: number;
   private readonly pairs: ReadonlySet<string>;
   private readonly loaded = new Map<string, { pair: Promise<Pair>; usedAt: number }>();
+  private readonly downloadCooldown = new Map<string, number>();
   private tail: Promise<unknown> = Promise.resolve();
   private waiting = 0;
   private readonly asked = new Map<string, { minute: number; chars: number }>();
@@ -189,6 +190,7 @@ export class Translator {
 
   private pair(from: string, to: string): Promise<Pair> {
     const model = modelFor(from, to);
+    if ((this.downloadCooldown.get(model) ?? 0) > this.now()) return Promise.reject(new SpeechError("the translation model download was rate limited; retry in a minute", 503));
     const held = this.loaded.get(model);
     if (held) {
       held.usedAt = this.now();
@@ -197,6 +199,7 @@ export class Translator {
     const pair = this.load(model, this.cacheDir).catch((error: unknown) => {
       // A failed load is tried again next time, not remembered forever.
       this.loaded.delete(model);
+      if (/\b429\b|rate.?limit/i.test(String(error))) this.downloadCooldown.set(model, this.now() + 60_000);
       throw error;
     });
     this.loaded.set(model, { pair, usedAt: this.now() });
@@ -255,7 +258,7 @@ export class Translator {
    * one account, or too many waiting, each with a status. Empty texts come
    * back empty and cost nothing.
    */
-  async translate(texts: string[], from: string, to: string, options: { by?: string } = {}): Promise<Translated> {
+  async translate(texts: string[], from: string, to: string, options: { by?: string; deadline?: number } = {}): Promise<Translated> {
     const hops = route(from, to, this.pairs);
     if (hops === null) {
       const known = Object.keys(LANGUAGES).includes(from) && Object.keys(LANGUAGES).includes(to);
@@ -268,12 +271,15 @@ export class Translator {
     if (this.waiting >= QUEUE_LIMIT) throw new SpeechError("too much is being translated at once; try again in a moment", 503);
     this.waiting += 1;
     const turn = this.tail.then(async () => {
+      const fresh = (): void => { if (options.deadline && this.now() > options.deadline) throw new SpeechError("live translation expired while waiting; try the next line", 503); };
+      fresh();
       // Only the lines with something in them go to the model; the rest keep their place.
       const spoken = texts.map((text) => text.trim());
       const which = spoken.map((text, i) => (text === "" ? -1 : i)).filter((i) => i >= 0);
       let current = which.map((i) => spoken[i] as string);
       for (const [a, b] of hops) {
         const pair = await this.pair(a, b);
+        fresh();
         current = await pair.translate(current);
       }
       const out = [...spoken];

package/src/voice-profile.ts

@@ -0,0 +1,46 @@
+/** Acoustic matching only: pitch does not establish a speaker's gender or identity. */
+export type VoiceProfile = "lower" | "higher" | "unknown";
+
+/** Median fundamental frequency from periodic voiced frames, sampled at 8 kHz.
+ * Ambiguous pitch, overlapping voices and unvoiced sound use the selected default.
+ * This is deliberately inexpensive and does not load another model beside ASR.
+ */
+export function voiceProfile(pcm: Buffer): VoiceProfile {
+  const pitches: number[] = [];
+  const frame = 320;
+  const samples = Math.floor(pcm.length / 4);
+  for (let start = 0; start + frame < samples; start += 1600) {
+    const values = new Float32Array(frame);
+    let mean = 0;
+    for (let i = 0; i < frame; i++) mean += values[i] = pcm.readInt16LE((start + i) * 4) / 32768;
+    mean /= frame;
+    let energy = 0;
+    for (let i = 0; i < frame; i++) { values[i] = (values[i] as number) - mean; energy += (values[i] as number) ** 2; }
+    if (Math.sqrt(energy / frame) < 0.01) continue;
+    let best = 0;
+    let period = 0;
+    // Prefer the first strong peak; a later multiple of the period has the same correlation.
+    const correlations: number[] = [];
+    for (let lag = 20; lag <= 114; lag++) {
+      let cross = 0, left = 0, right = 0;
+      for (let i = 0; i < frame - lag; i++) {
+        const a = values[i] as number, b = values[i + lag] as number;
+        cross += a * b; left += a * a; right += b * b;
+      }
+      correlations[lag] = cross / Math.sqrt(left * right || 1);
+    }
+    for (let lag = 21; lag < 114; lag++) {
+      const score = correlations[lag] as number;
+      if (score > 0.8 && score > (correlations[lag - 1] as number) && score >= (correlations[lag + 1] as number) && score > best + 0.03) {
+        best = score; period = lag;
+      }
+    }
+    if (period) pitches.push(8000 / period);
+  }
+  if (pitches.length < 3) return "unknown";
+  pitches.sort((a, b) => a - b);
+  const median = pitches[Math.floor(pitches.length / 2)] as number;
+  const lower = pitches.filter((pitch) => pitch < 155).length / pitches.length;
+  const higher = pitches.filter((pitch) => pitch > 185).length / pitches.length;
+  return median < 155 && lower >= 0.75 ? "lower" : median > 185 && higher >= 0.75 ? "higher" : "unknown";
+}

package/src/warm.ts

@@ -7,15 +7,15 @@
  * ask after each one, and the first person to speak waited for it. The
  * models land in NIXAMP_STT_CACHE and the image keeps them.
  *
- * NIXAMP_MT_WARM names the pairs (en-de,en-sv,de-en,sv-en by default):
- * German and Swedish both ways, which are the two that were asked for.
+ * NIXAMP_MT_WARM names the pairs: German and Swedish both ways with
+ * English, and Spanish both ways with English and German by default.
  * Exits non-zero when anything could not be fetched, so a build does not
  * quietly ship without its ear.
  */
 import { Speech } from "./speech.ts";
 import { Translator } from "./translate.ts";
 
-const DEFAULT_PAIRS = "en-de,en-sv,de-en,sv-en";
+const DEFAULT_PAIRS = "en-de,en-sv,de-en,sv-en,es-en,en-es,es-de,de-es";
 
 async function main(): Promise<number> {
   const speech = new Speech();

package/web/dist/assets/{hls-3VKVEQE3-B4ltbKDh.js → hls-3VKVEQE3-vgax_tk1.js}

@@ -1 +1 @@
-import{t as e}from"./index-d7TvpeFZ.js";var t=3;async function n(n){let{media:r,src:i,isTv:a}=n,{default:o}=await e(async()=>{let{default:e}=await import(`./hls-n74Cnh8A.js`);return{default:e}},[]);if(!o.isSupported())return n.onError(`This browser cannot play HLS streams.`),{destroy:()=>void 0,levels:()=>[]};let s=new o({...a?{maxBufferLength:60,maxMaxBufferLength:120,backBufferLength:30,liveSyncDurationCount:4}:{backBufferLength:90},enableWorker:!0}),c=0,l=!1;s.on(o.Events.ERROR,(e,r)=>{if(!l&&r.fatal){if(c>=t){n.onError(`This stream kept failing and has been stopped.`),s.destroy();return}switch(c+=1,r.type){case o.ErrorTypes.NETWORK_ERROR:n.onNotice(`Reconnecting…`),s.startLoad();break;case o.ErrorTypes.MEDIA_ERROR:n.onNotice(`Recovering…`),s.recoverMediaError();break;default:n.onError(`This stream could not be played.`),s.destroy()}}}),s.on(o.Events.MANIFEST_PARSED,()=>{l||(n.onNotice(null),n.onReady?.({live:s.levels.length>0&&!Number.isFinite(r.duration),levels:u()}))}),s.on(o.Events.LEVEL_LOADED,(e,t)=>{l||n.onReady?.({live:t.details.live,levels:u()})}),s.on(o.Events.FRAG_BUFFERED,()=>{l||n.onNotice(null)});function u(){return s.levels.map((e,t)=>({index:t,height:e.height||null,bitrate:e.bitrate||null,label:e.height?`${String(e.height)}p`:`${String(Math.round((e.bitrate||0)/1e3))}k`}))}return s.loadSource(i),s.attachMedia(r),{destroy(){l=!0,s.destroy()},levels:u,setLevel(e){s.currentLevel=e},currentLevel:()=>s.autoLevelEnabled?-1:s.currentLevel}}export{n as createHlsEngine};
+import{t as e}from"./index-BzjrTOLf.js";var t=3;async function n(n){let{media:r,src:i,isTv:a}=n,{default:o}=await e(async()=>{let{default:e}=await import(`./hls-n74Cnh8A.js`);return{default:e}},[]);if(!o.isSupported())return n.onError(`This browser cannot play HLS streams.`),{destroy:()=>void 0,levels:()=>[]};let s=new o({...a?{maxBufferLength:60,maxMaxBufferLength:120,backBufferLength:30,liveSyncDurationCount:4}:{backBufferLength:90},enableWorker:!0}),c=0,l=!1;s.on(o.Events.ERROR,(e,r)=>{if(!l&&r.fatal){if(c>=t){n.onError(`This stream kept failing and has been stopped.`),s.destroy();return}switch(c+=1,r.type){case o.ErrorTypes.NETWORK_ERROR:n.onNotice(`Reconnecting…`),s.startLoad();break;case o.ErrorTypes.MEDIA_ERROR:n.onNotice(`Recovering…`),s.recoverMediaError();break;default:n.onError(`This stream could not be played.`),s.destroy()}}}),s.on(o.Events.MANIFEST_PARSED,()=>{l||(n.onNotice(null),n.onReady?.({live:s.levels.length>0&&!Number.isFinite(r.duration),levels:u()}))}),s.on(o.Events.LEVEL_LOADED,(e,t)=>{l||n.onReady?.({live:t.details.live,levels:u()})}),s.on(o.Events.FRAG_BUFFERED,()=>{l||n.onNotice(null)});function u(){return s.levels.map((e,t)=>({index:t,height:e.height||null,bitrate:e.bitrate||null,label:e.height?`${String(e.height)}p`:`${String(Math.round((e.bitrate||0)/1e3))}k`}))}return s.loadSource(i),s.attachMedia(r),{destroy(){l=!0,s.destroy()},levels:u,setLevel(e){s.currentLevel=e},currentLevel:()=>s.autoLevelEnabled?-1:s.currentLevel}}export{n as createHlsEngine};