nixamp 0.25.1 → 0.26.0

package/dist/live-voice.js

@@ -0,0 +1,299 @@
+/** ElevenLabs Flash for live captions. The key stays on the account server. */
+import { createHash, randomBytes } from "node:crypto";
+import { SpeechError, decodeWav, encodeWav, quietSamples } from "./speech.js";
+import { speakerTurns } from "./speaker-turns.js";
+import { Guard } from "./guard.js";
+export const LIVE_VOICE_MODEL = "eleven_flash_v2_5";
+export const LIVE_VOICE_RATE = 16_000;
+export const LIVE_VOICE_LANGUAGES = new Set("en ja zh de hi fr ko pt it es id nl tr fil pl sv bg ro ar cs el fi hr ms sk da ta uk ru hu no vi".split(" "));
+const HEADERS = { "content-type": "audio/pcm", "cache-control": "no-store", "x-audio-sample-rate": String(LIVE_VOICE_RATE) };
+const budget = (value, fallback) => Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
+export class LiveVoice {
+    key;
+    fetcher;
+    now;
+    catalog = null;
+    catalogUntil = 0;
+    cache = new Map();
+    pending = new Map();
+    usage = new Map();
+    charsPerMinute;
+    requests;
+    grants = new Map();
+    activeBy = new Map();
+    db;
+    schema = null;
+    dailyChars;
+    cleanupAt = 0;
+    hearing = new Set();
+    dailyAudioSeconds;
+    userDailyChars;
+    userDailyAudioSeconds;
+    constructor(options = {}) {
+        this.key = options.apiKey ?? process.env["ELEVENLABS_API_KEY"] ?? "";
+        this.fetcher = options.fetcher ?? fetch;
+        this.now = options.now ?? Date.now;
+        this.charsPerMinute = budget(options.charsPerMinute, 3000);
+        this.dailyChars = budget(options.dailyChars, 200_000);
+        this.dailyAudioSeconds = budget(options.dailyAudioSeconds, 86_400);
+        this.userDailyChars = budget(options.userDailyChars, 120_000);
+        this.userDailyAudioSeconds = budget(options.userDailyAudioSeconds, 43_200);
+        this.requests = new Guard(this.now);
+        this.db = options.db;
+    }
+    available() { return this.key !== ""; }
+    /** Optional diarization, billed only while a signed-in listener requests it.
+     * Rolling audio is bounded to 15 seconds, including overlap. Every second
+     * submitted (also repeated context) consumes the persistent provider budget. */
+    async hear(bytes, by, signal) {
+        if (!this.available())
+            throw new SpeechError("speaker voices are unavailable", 503);
+        const wav = decodeWav(bytes);
+        const seconds = wav.samples.length / wav.rate;
+        if (wav.rate !== 16000 || wav.channels !== 1 || seconds < 0.2 || seconds > 15.1)
+            throw new SpeechError("send up to 15 seconds of mono 16 kHz WAV", 400);
+        if (wav.samples.some(sample => !Number.isFinite(sample)))
+            throw new SpeechError("invalid audio samples", 400);
+        if (!this.requests.check(`hear:${by}`, { allowed: 20, windowMs: 60_000 }).ok)
+            throw new SpeechError("too many speaker transcription requests", 429);
+        if (quietSamples(wav.samples))
+            return { language: "", seconds, turns: [] };
+        if (this.hearing.has(by) || this.hearing.size >= 4)
+            throw new SpeechError("speaker transcription is busy", 429);
+        this.hearing.add(by);
+        try {
+            const billed = Math.ceil(seconds);
+            await this.reserve(`scribe:user:${by}`, billed, 300, 60_000);
+            await this.reserve("scribe:server", billed, 600, 60_000);
+            await this.reserve(`scribe:user:${by}`, billed, this.userDailyAudioSeconds, 86_400_000);
+            await this.reserve("scribe:server", billed, this.dailyAudioSeconds, 86_400_000);
+            signal?.throwIfAborted();
+            const form = new FormData();
+            // Canonical PCM prevents a crafted container from billing more audio
+            // than the duration we validated, and removes uploaded metadata.
+            form.set("file", new Blob([new Uint8Array(encodeWav(wav.samples))], { type: "audio/wav" }), "listening.wav");
+            form.set("model_id", "scribe_v2");
+            form.set("diarize", "true");
+            form.set("tag_audio_events", "false");
+            form.set("timestamps_granularity", "word");
+            // No language_code: preserve the source language, including Spanish.
+            const answer = await this.fetcher("https://api.elevenlabs.io/v1/speech-to-text", {
+                method: "POST", headers: { "xi-api-key": this.key }, body: form,
+                signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
+            });
+            if (!answer.ok)
+                throw new SpeechError("speaker transcription could not run; check provider quota and permissions", answer.status === 429 ? 429 : 502);
+            return speakerTurns(await answer.json(), wav);
+        }
+        finally {
+            this.hearing.delete(by);
+        }
+    }
+    async voices() {
+        if (!this.available())
+            throw new SpeechError("translated audio needs ELEVENLABS_API_KEY on the account server", 503);
+        if (!this.catalog || this.now() >= this.catalogUntil) {
+            this.catalogUntil = this.now() + 3600_000;
+            this.catalog = this.fetcher("https://api.elevenlabs.io/v2/voices?page_size=100&voice_type=default", {
+                headers: { "xi-api-key": this.key }, signal: AbortSignal.timeout(8000),
+            }).then(async (response) => {
+                if (!response.ok)
+                    throw new SpeechError("ElevenLabs could not list voices; check the server key and its voice permissions", 503);
+                const body = await response.json();
+                return body.voices.filter(voice => /^[a-zA-Z0-9_-]+$/.test(voice.voice_id)).map(voice => ({
+                    id: voice.voice_id, name: voice.name, gender: voice.labels?.["gender"] ?? "neutral", language: voice.labels?.["language"] ?? "",
+                }));
+            }).catch(error => { this.catalog = null; throw error; });
+        }
+        return this.catalog;
+    }
+    /** A 90-second capability for one channel, never the viewer's account credential. */
+    async grant(by, channel) {
+        if (!/^[\w-]{1,80}$/.test(channel))
+            throw new SpeechError("choose a playback session or live channel", 400);
+        if (!this.requests.check(`grant:${by}`, { allowed: 10, windowMs: 60_000 }).ok)
+            throw new SpeechError("too many audio authorization requests", 429);
+        for (const [token, grant] of this.grants)
+            if (grant.expires <= this.now())
+                this.grants.delete(token);
+        if (this.grants.size >= 5000)
+            throw new SpeechError("audio authorization is busy", 503);
+        const token = `nxd_${randomBytes(32).toString("base64url")}`;
+        const expires = this.now() + 90_000;
+        if (this.db) {
+            await this.ensure();
+            await this.db.query("INSERT INTO live_voice_grants (token_hash, by_account, channel, expires_at, remaining) VALUES ($1, $2, $3, $4, 2000)", [createHash("sha256").update(token).digest("hex"), by, channel, new Date(expires)]);
+        }
+        else
+            this.grants.set(token, { by, channel, expires, remaining: 2000 });
+        return { token, expires };
+    }
+    async authorize(token, channel, chars) {
+        if (!/^nxd_[A-Za-z0-9_-]{43}$/.test(token))
+            throw new SpeechError("sign in to enable translated audio", 401);
+        if (!Number.isFinite(chars) || chars < 1 || chars > 600)
+            throw new SpeechError("invalid caption length", 400);
+        if (this.db) {
+            await this.ensure();
+            const result = await this.db.query(`UPDATE live_voice_grants SET remaining = remaining - $3
+        WHERE token_hash = $1 AND channel = $2 AND expires_at > $4 AND remaining >= $3 RETURNING by_account`, [createHash("sha256").update(token).digest("hex"), channel, chars, new Date(this.now())]);
+            if (!result.rows.length)
+                throw new SpeechError("audio authorization expired or reached its limit; enable translated audio again", 401);
+            return String(result.rows[0]?.["by_account"]);
+        }
+        const grant = this.grants.get(token);
+        if (!grant || grant.expires <= this.now() || grant.channel !== channel)
+            throw new SpeechError("sign in to renew translated audio", 401);
+        if (grant.remaining < chars)
+            throw new SpeechError("this audio authorization reached its character limit", 429);
+        grant.remaining -= chars;
+        return grant.by;
+    }
+    async ensure() {
+        if (!this.db)
+            return;
+        this.schema ??= (async () => {
+            await this.db.query("CREATE TABLE IF NOT EXISTS live_voice_usage (bucket TEXT PRIMARY KEY, chars BIGINT NOT NULL, expires_at TIMESTAMPTZ NOT NULL)");
+            await this.db.query("CREATE TABLE IF NOT EXISTS live_voice_grants (token_hash TEXT PRIMARY KEY, by_account TEXT NOT NULL, channel TEXT NOT NULL, expires_at TIMESTAMPTZ NOT NULL, remaining INTEGER NOT NULL)");
+        })().catch(error => { this.schema = null; throw error; });
+        await this.schema;
+        if (this.now() >= this.cleanupAt) {
+            this.cleanupAt = this.now() + 60_000;
+            // Keep only live windows; accounting survives restarts and is shared by replicas.
+            await this.db.query("DELETE FROM live_voice_usage WHERE expires_at <= $1", [new Date(this.now())]);
+            await this.db.query("DELETE FROM live_voice_grants WHERE expires_at <= $1", [new Date(this.now())]);
+        }
+    }
+    async reserve(bucket, chars, limit, windowMs) {
+        if (chars > limit)
+            throw new SpeechError("translated audio reached its character budget; try again later", 429);
+        const window = Math.floor(this.now() / windowMs);
+        const key = `${bucket}:${windowMs}:${window}`;
+        if (this.db) {
+            await this.ensure();
+            const result = await this.db.query(`INSERT INTO live_voice_usage (bucket, chars, expires_at) VALUES ($1, $2, $4)
+        ON CONFLICT (bucket) DO UPDATE SET chars = live_voice_usage.chars + EXCLUDED.chars
+        WHERE live_voice_usage.chars + EXCLUDED.chars <= $3 RETURNING chars`, [key, chars, limit, new Date((window + 1) * windowMs)]);
+            if (!result.rows.length)
+                throw new SpeechError("translated audio reached its character budget; try again later", 429);
+            return;
+        }
+        // Local/single-process installations without a database. Account servers pass their pool.
+        const record = this.usage.get(key) ?? { minute: (window + 1) * windowMs, chars: 0 };
+        if (record.chars + chars > limit)
+            throw new SpeechError("translated audio reached its character budget; try again later", 429);
+        record.chars += chars;
+        this.usage.set(key, record);
+    }
+    async charge(by, channel, chars) {
+        for (const [key, record] of this.usage)
+            if (record.minute <= this.now())
+                this.usage.delete(key);
+        await this.reserve(`user:${by}`, chars, this.charsPerMinute, 60_000);
+        await this.reserve(`channel:${channel}`, chars, 6000, 60_000);
+        await this.reserve("server", chars, 12_000, 60_000);
+        await this.reserve(`user:${by}`, chars, this.userDailyChars, 86400_000);
+        await this.reserve("server", chars, this.dailyChars, 86400_000);
+    }
+    checkRequest(by) {
+        if (!this.requests.check(`audio:${by}`, { allowed: 60, windowMs: 60_000 }).ok)
+            throw new SpeechError("too many translated audio requests", 429);
+    }
+    /** Stream the first request immediately; concurrent listeners share its cached result. */
+    async stream(ask, by, signal) {
+        this.checkRequest(by);
+        const text = typeof ask.text === "string" ? ask.text.trim() : "";
+        if (!text || text.length > 600)
+            throw new SpeechError("translated audio needs a caption of 1–600 characters", 400);
+        if (!LIVE_VOICE_LANGUAGES.has(ask.language))
+            throw new SpeechError("this language is not supported by Flash voices", 400);
+        const voices = await this.voices();
+        signal?.throwIfAborted();
+        const gender = ask.profile === "lower" ? "male" : ask.profile === "higher" ? "female" : "neutral";
+        const voice = ask.voice && ask.voice !== "auto"
+            ? voices.find(voice => voice.id === ask.voice)
+            : voices.find(voice => voice.gender === gender) ?? voices[0];
+        if (!voice)
+            throw new SpeechError("choose an available voice", 400);
+        const id = createHash("sha256").update(JSON.stringify([LIVE_VOICE_MODEL, voice.id, ask.language, text])).digest("hex");
+        for (const [key, item] of this.cache)
+            if (item.until < this.now())
+                this.cache.delete(key);
+        const cached = this.cache.get(id);
+        if (cached)
+            return new Response(new Uint8Array(cached.bytes), { headers: HEADERS });
+        const pending = this.pending.get(id);
+        if (pending)
+            return new Response(new Uint8Array(await pending), { headers: HEADERS });
+        if (this.pending.size >= 4)
+            throw new SpeechError("translated audio is busy; waiting for the next caption", 429);
+        if ((this.activeBy.get(by) ?? 0) >= 2)
+            throw new SpeechError("two voice requests are already active for this account", 429);
+        // Register before the first provider await, so identical requests cannot both bill.
+        let done;
+        let fail;
+        const finished = new Promise((resolve, reject) => { done = resolve; fail = reject; });
+        this.pending.set(id, finished);
+        this.activeBy.set(by, (this.activeBy.get(by) ?? 0) + 1);
+        const release = () => { this.pending.delete(id); this.activeBy.set(by, Math.max(0, (this.activeBy.get(by) ?? 1) - 1)); if (!this.activeBy.get(by))
+            this.activeBy.delete(by); };
+        void finished.catch(() => undefined);
+        try {
+            await this.charge(by, ask.channel ?? "direct", text.length);
+            signal?.throwIfAborted();
+            const response = await this.fetcher(`https://api.elevenlabs.io/v1/text-to-speech/${voice.id}/stream?output_format=pcm_16000`, {
+                method: "POST",
+                headers: { "xi-api-key": this.key, "content-type": "application/json" },
+                body: JSON.stringify({ text, model_id: LIVE_VOICE_MODEL, language_code: ask.language }),
+                signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
+            });
+            if (!response.ok || !response.body)
+                throw new SpeechError(response.status === 429 ? "ElevenLabs audio quota is temporarily exhausted" : "ElevenLabs could not generate audio; check the server key and quota", response.status === 429 ? 429 : 502);
+            const [play, keep] = response.body.tee();
+            void (async () => {
+                const reader = keep.getReader();
+                const chunks = [];
+                let size = 0;
+                try {
+                    while (true) {
+                        const { done: ended, value } = await reader.read();
+                        if (ended)
+                            break;
+                        size += value.length;
+                        if (size > 2 * 1024 * 1024) {
+                            await reader.cancel();
+                            throw new SpeechError("voice response was too long", 502);
+                        }
+                        chunks.push(value);
+                    }
+                    if (!size || size % 2)
+                        throw new SpeechError("voice response contained incomplete audio", 502);
+                    const bytes = new Uint8Array(size);
+                    let at = 0;
+                    for (const chunk of chunks) {
+                        bytes.set(chunk, at);
+                        at += chunk.length;
+                    }
+                    this.cache.set(id, { until: this.now() + 60_000, bytes });
+                    // At most a few MB for recent lines, never a recording archive.
+                    while (this.cache.size > 32)
+                        this.cache.delete(this.cache.keys().next().value);
+                    done(bytes);
+                }
+                catch (error) {
+                    fail(error);
+                }
+                finally {
+                    reader.releaseLock();
+                    release();
+                }
+            })();
+            return new Response(play, { headers: HEADERS });
+        }
+        catch (error) {
+            fail(error);
+            release();
+            throw error;
+        }
+    }
+}

package/dist/server.d.ts

@@ -1,4 +1,5 @@
 import { type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import { LiveVoice } from "./live-voice.ts";
 import { Connections } from "./connections.ts";
 import { Broadcaster, type Destination, type EncoderSettings } from "./broadcast.ts";
 import { Ingest } from "./ingest.ts";
@@ -703,6 +704,7 @@ export interface HandlerOptions {
     trollbox?: Trollbox;
     /** Speech to text: a line said out loud, heard here. Needs the optional model. */
     speech?: Speech;
+    liveVoice?: LiveVoice;
     /** Translation: texts in another language, by a model here. The same optional library. */
     translator?: Translator;
     /** The transcript store: what was heard, kept under the media's identity. Where the accounts are. */

package/dist/server.js

@@ -16,6 +16,9 @@ import { randomBytes } from "node:crypto";
 import { hostname } from "node:os";
 import { spawn, spawnSync } from "node:child_process";
 import { readFileSync } from "node:fs";
+import { Readable } from "node:stream";
+import { pipeline as pipeStream } from "node:stream/promises";
+import { LiveVoice, LIVE_VOICE_LANGUAGES, LIVE_VOICE_MODEL } from "./live-voice.js";
 import { Connections } from "./connections.js";
 import { Broadcaster, DEFAULT_ENCODER, PRESETS, redact, } from "./broadcast.js";
 import { Ingest, normaliseFormat } from "./ingest.js";
@@ -65,7 +68,7 @@ import { Tickets, needsTicket, ticketFrom, ticketsFromEnv } from "./tickets.js";
 import { Layouts } from "./layouts.js";
 import { Rooms } from "./rooms.js";
 import { Trollbox, TrollboxError, fallbackHandle, roomFor } from "./trollbox.js";
-import { MAX_BYTES as SPEECH_BYTES, Speech, SpeechError, isWav, languageOf } from "./speech.js";
+import { MAX_BYTES as SPEECH_BYTES, NATIVE_REVISION, Speech, SpeechError, isWav, languageOf } from "./speech.js";
 import { Captions } from "./captions.js";
 import { LANGUAGES, Translator } from "./translate.js";
 import { StoredTranslations } from "./translate-jobs.js";
@@ -1106,7 +1109,7 @@ const CORS = {
     // paths and takes six commands; binding to 127.0.0.1 is what keeps it shut.
     "access-control-allow-origin": "*",
     "access-control-allow-methods": "GET, POST, PUT, DELETE, OPTIONS",
-    "access-control-allow-headers": "content-type",
+    "access-control-allow-headers": "content-type, authorization",
     "access-control-max-age": "86400",
 };
 /**
@@ -1278,6 +1281,8 @@ export function joinDocument(shell, subject, site) {
         .replace("</head>", `${card}<meta name="nixamp-shell-title" content="${htmlText(shellTitle)}" />\n  </head>`);
 }
 function json(response, code, body) {
+    if (code === 429)
+        response.setHeader("retry-after", "60");
     const text = JSON.stringify(body);
     response.writeHead(code, {
         ...CORS,
@@ -1287,6 +1292,18 @@ function json(response, code, body) {
     });
     response.end(text);
 }
+/** Stream PCM without waiting for a complete utterance, respecting browser backpressure. */
+async function voiceResponse(response, upstream) {
+    response.writeHead(upstream.status, {
+        ...CORS, "content-type": upstream.headers.get("content-type") ?? "application/json",
+        "cache-control": "no-store", "x-accel-buffering": "no",
+    });
+    if (!upstream.body) {
+        response.end();
+        return;
+    }
+    await pipeStream(Readable.fromWeb(upstream.body), response);
+}
 async function readBody(request, limit = 64 * 1024) {
     const chunks = [];
     let size = 0;
@@ -2314,6 +2331,80 @@ export function createHandler(engine, options) {
          * room, the words go straight into that room's trollbox as a line by
          * whoever spoke them. The page, the CLI and the MCP tools all come here.
          */
+        if (["/api/v1/speech/voices", "/api/v1/speech/grant", "/api/v1/speech/synthesize", "/api/v1/speech/speakers"].includes(path) && options.accounts) {
+            const controller = new AbortController();
+            response.once("close", () => controller.abort());
+            try {
+                const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
+                if (!guard.check(`voice-ip:${caller}`, { allowed: 600, windowMs: 60_000 }).ok) {
+                    json(response, 429, { error: "too many voice requests" });
+                    return;
+                }
+                const service = options.liveVoice;
+                if (!service?.available()) {
+                    json(response, 503, { error: "translated audio needs ELEVENLABS_API_KEY on the account server" });
+                    return;
+                }
+                if (path.endsWith("/synthesize") && request.method === "POST") {
+                    let body;
+                    try {
+                        body = JSON.parse(await readBody(request, 4096));
+                    }
+                    catch {
+                        json(response, 400, { error: "send a caption as JSON" });
+                        return;
+                    }
+                    if (!body || typeof body !== "object") {
+                        json(response, 400, { error: "send a caption as JSON" });
+                        return;
+                    }
+                    const by = await service.authorize(tokenFrom(request.headers), body.channel ?? "", typeof body.text === "string" ? body.text.length : 0);
+                    await voiceResponse(response, await service.stream(body, by, controller.signal));
+                }
+                else {
+                    const who = await options.accounts.whoIs(tokenFrom(request.headers));
+                    if (!who) {
+                        json(response, 401, { error: "sign in to use translated audio" });
+                        return;
+                    }
+                    if (path.endsWith("/voices") && request.method === "GET") {
+                        json(response, 200, { voices: await service.voices(), model: LIVE_VOICE_MODEL, languages: [...LIVE_VOICE_LANGUAGES] });
+                    }
+                    else if (path.endsWith("/speakers") && request.method === "POST") {
+                        if (!request.headers["content-type"]?.startsWith("audio/wav")) {
+                            json(response, 415, { error: "send a mono 16 kHz WAV" });
+                            return;
+                        }
+                        const bytes = await readBytes(request, 484_000);
+                        json(response, 200, await service.hear(bytes, who.id, controller.signal));
+                    }
+                    else if (path.endsWith("/grant") && request.method === "POST") {
+                        if (!request.headers["content-type"]?.startsWith("application/json")) {
+                            json(response, 415, { error: "send JSON" });
+                            return;
+                        }
+                        let body;
+                        try {
+                            body = JSON.parse(await readBody(request, 1024));
+                        }
+                        catch {
+                            json(response, 400, { error: "send a channel as JSON" });
+                            return;
+                        }
+                        json(response, 200, await service.grant(who.id, typeof body?.channel === "string" ? body.channel : ""));
+                    }
+                    else
+                        json(response, 405, { error: "GET voices or POST an audio request" });
+                }
+            }
+            catch (error) {
+                if (response.headersSent)
+                    response.destroy();
+                else
+                    json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "translated audio is unavailable" });
+            }
+            return;
+        }
         if (path === "/api/v1/speech/transcribe" && options.speech && options.accounts) {
             const speech = options.speech;
             try {
@@ -2343,6 +2434,7 @@ export function createHandler(engine, options) {
                     // Pieces with their timing, for a whole file being written down.
                     timestamps: url.searchParams.get("timestamps") === "1",
                     by: who.id,
+                    ...(url.searchParams.get("live") === "1" ? { deadline: Date.now() + 10_000 } : {}),
                 });
                 // The language travels back: as told, or as the ear guessed it, so
                 // a captioner can say it next time and a transcript can be kept as it.
@@ -2563,7 +2655,7 @@ export function createHandler(engine, options) {
                 return;
             }
             try {
-                json(response, 200, await translator.translate(texts, from, to, { by: who.id }));
+                json(response, 200, await translator.translate(texts, from, to, { by: who.id, ...(url.searchParams.get("live") === "1" ? { deadline: Date.now() + 10_000 } : {}) }));
             }
             catch (error) {
                 if (error instanceof SpeechError)
@@ -2620,6 +2712,12 @@ export function createHandler(engine, options) {
                         json(response, 400, { error: "format is json, srt, vtt or txt" });
                         return;
                     }
+                    // Joining a live must not start translating an entire old transcript.
+                    if (url.searchParams.get("cached") === "1") {
+                        const saved = await store.get(id, language);
+                        json(response, saved ? 200 : 404, saved ? wire(saved) : { error: "no cached transcript" });
+                        return;
+                    }
                     const answer = await options.translations.get(id, language, who.id);
                     if (answer.status !== 200 && answer.status !== 202) {
                         json(response, answer.status, { error: answer.error });
@@ -3930,6 +4028,44 @@ export function createHandler(engine, options) {
              * started by the first person asking. `captions` is the live stream
              * of lines; `transcript` is the recent ones as JSON, for a poll.
              */
+            if ((action === "voice" || action === "voice-options") && request.method === "GET") {
+                const controller = new AbortController();
+                response.once("close", () => controller.abort());
+                const timeout = setTimeout(() => controller.abort(), 12_000);
+                try {
+                    const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
+                    if (!guard.check(`channel-voice-ip:${caller}`, { allowed: 60, windowMs: 60_000 }).ok || !guard.check(`channel-voice:${id}`, { allowed: 240, windowMs: 60_000 }).ok) {
+                        json(response, 429, { error: "too many translated audio requests" });
+                        return;
+                    }
+                    if (!channels.has(id)) {
+                        json(response, 404, { error: "nothing is playing on that channel" });
+                        return;
+                    }
+                    if (!options.captions) {
+                        json(response, 503, { error: "this server cannot caption" });
+                        return;
+                    }
+                    const at = action === "voice-options" ? null : Number(url.searchParams.get("at"));
+                    const language = languageCode(url.searchParams.get("language"));
+                    if ((at !== null && (!url.searchParams.has("at") || !Number.isFinite(at))) || language === null) {
+                        json(response, 400, { error: "audio needs a caption time and language" });
+                        return;
+                    }
+                    const upstream = await options.captions.voiceRequest(id, at, language, url.searchParams.get("voice") ?? "auto", controller.signal, tokenFrom(request.headers));
+                    await voiceResponse(response, upstream);
+                }
+                catch (error) {
+                    if (response.headersSent)
+                        response.destroy();
+                    else
+                        json(response, error instanceof SpeechError ? error.status : 502, { error: error instanceof SpeechError ? error.message : "translated audio is unavailable" });
+                }
+                finally {
+                    clearTimeout(timeout);
+                }
+                return;
+            }
             if ((action === "captions" || action === "transcript") && request.method === "GET") {
                 const captions = options.captions;
                 if (!captions) {
@@ -3940,6 +4076,11 @@ export function createHandler(engine, options) {
                     json(response, 503, { error: "this server is not signed in to nixamp.com, so it cannot caption; run `nixamp login` on it" });
                     return;
                 }
+                const caller = callerOf(request.headers, request.socket.remoteAddress, options.behindProxy ?? false);
+                if (!guard.check(`captions:${caller}`, { allowed: 120, windowMs: 60_000 }).ok || !captions.capacity(id)) {
+                    json(response, 429, { error: "captioning is at capacity; try again in a moment" });
+                    return;
+                }
                 if (!channels.has(id)) {
                     json(response, 404, { error: "nothing is playing on that channel" });
                     return;
@@ -3958,7 +4099,7 @@ export function createHandler(engine, options) {
                     // Asking keeps the captioner up: it stops a minute after the last ask.
                     captions.subscribe(id, () => undefined, wanted)?.();
                     json(response, 200, {
-                        channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted),
+                        channel: id, recognitionRevision: NATIVE_REVISION, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted),
                     });
                     return;
                 }
@@ -3979,7 +4120,7 @@ export function createHandler(engine, options) {
                 }
                 // How far behind the live edge a newcomer's playback starts, so the
                 // page can hold each line until its own sound gets there.
-                write("hello", { channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted) });
+                write("hello", { channel: id, recognitionRevision: NATIVE_REVISION, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted) });
                 const beat = setInterval(() => response.write(": beat\n\n"), 20_000);
                 beat.unref?.();
                 const done = () => {
@@ -5276,6 +5417,13 @@ export async function serve(argv, version = "0.1.0") {
     // the first person to speak does not wait for the model to arrive; a box
     // without the optional model never says it is ready, and answers 503.
     const speech = accounts && process.env["NIXAMP_STT"] !== "off" ? new Speech() : undefined;
+    const liveVoice = accounts && process.env["NIXAMP_DUBBING"] !== "off" ? new LiveVoice({
+        ...(pool ? { db: pool } : {}),
+        dailyChars: Number(process.env["NIXAMP_DUB_DAILY_CHARS"] ?? 200_000),
+        dailyAudioSeconds: Number(process.env["NIXAMP_DUB_DAILY_AUDIO_SECONDS"] ?? 86_400),
+        userDailyChars: Number(process.env["NIXAMP_DUB_USER_DAILY_CHARS"] ?? 120_000),
+        userDailyAudioSeconds: Number(process.env["NIXAMP_DUB_USER_DAILY_AUDIO_SECONDS"] ?? 43_200),
+    }) : undefined;
     if (speech) {
         void speech.warm().then((ready) => {
             console.error(ready ? `nixamp: hearing with ${speech.model}` : `nixamp: not hearing: ${speech.lastFailure}`);
@@ -5759,6 +5907,7 @@ export async function serve(argv, version = "0.1.0") {
         ...(rooms ? { rooms } : {}),
         ...(trollbox ? { trollbox } : {}),
         ...(speech ? { speech } : {}),
+        ...(liveVoice ? { liveVoice } : {}),
         ...(translator ? { translator } : {}),
         ...(transcripts ? { transcripts } : {}),
         ...(translations ? { translations } : {}),

package/dist/speaker-turns.d.ts

@@ -0,0 +1,32 @@
+/** Provider speaker IDs belong to a clip. Clients reconcile the overlapping
+ * word timestamps before assigning a voice; these are not people's identities. */
+import { type Wav } from "./speech.ts";
+import { type VoiceProfile } from "./voice-profile.ts";
+export interface SpeakerTurn {
+    start: number;
+    end: number;
+    text: string;
+    speaker: string;
+    profile: VoiceProfile;
+    words: {
+        text: string;
+        start: number;
+        end: number;
+    }[];
+}
+export interface SpeakerTranscript {
+    language: string;
+    seconds: number;
+    turns: SpeakerTurn[];
+}
+export interface ScribeResult {
+    language_code?: string;
+    words?: {
+        text: string;
+        start: number;
+        end: number;
+        type: string;
+        speaker_id?: string | null;
+    }[];
+}
+export declare function speakerTurns(body: ScribeResult, wav: Wav): SpeakerTranscript;

package/dist/speaker-turns.js

@@ -0,0 +1,30 @@
+/** Provider speaker IDs belong to a clip. Clients reconcile the overlapping
+ * word timestamps before assigning a voice; these are not people's identities. */
+import { reliableText } from "./speech.js";
+import { voiceProfile } from "./voice-profile.js";
+const ISO = Object.fromEntries("eng:en spa:es deu:de ger:de fra:fr fre:fr por:pt ita:it nld:nl dut:nl swe:sv dan:da fin:fi rus:ru ukr:uk ces:cs cze:cs hun:hu cmn:zh zho:zh ara:ar hin:hi vie:vi ind:id jpn:ja kor:ko pol:pl tur:tr ron:ro bul:bg ell:el nor:no nob:no nno:no".split(" ").map(pair => pair.split(":")));
+export function speakerTurns(body, wav) {
+    const seconds = wav.samples.length / wav.rate;
+    const language = ISO[body.language_code ?? ""] ?? body.language_code ?? "";
+    const turns = [];
+    for (const word of (body.words ?? []).slice(0, 1500)) {
+        if (word.type !== "word" || typeof word.text !== "string" || !Number.isFinite(word.start) || !Number.isFinite(word.end) || word.start < 0 || word.end < word.start || word.end > seconds + 0.5)
+            continue;
+        const speaker = typeof word.speaker_id === "string" && /^[\w-]{1,80}$/.test(word.speaker_id) ? word.speaker_id : "unknown";
+        const last = turns.at(-1);
+        if (last && last.speaker === speaker && word.start - last.end < 0.8 && last.text.length + word.text.length < 400) {
+            last.text += ` ${word.text}`;
+            last.end = Math.max(last.end, word.end);
+            last.words.push({ text: word.text, start: word.start, end: word.end });
+        }
+        else
+            turns.push({ start: word.start, end: word.end, text: word.text, speaker, profile: "unknown", words: [{ text: word.text, start: word.start, end: word.end }] });
+    }
+    const pcm = Buffer.alloc(wav.samples.length * 2);
+    wav.samples.forEach((sample, i) => pcm.writeInt16LE(Math.round(Math.max(-1, Math.min(1, sample)) * 32767), i * 2));
+    for (const turn of turns) {
+        turn.text = reliableText(turn.text, Math.max(1, turn.end - turn.start));
+        turn.profile = voiceProfile(pcm.subarray(Math.floor(turn.start * 16000) * 2, Math.ceil(turn.end * 16000) * 2));
+    }
+    return { language, seconds, turns: turns.filter(turn => turn.text !== "") };
+}