nixamp 0.21.5 → 0.22.0

package/README.md

@@ -293,6 +293,81 @@ something else.
 Entries expire a few minutes after a stream stops renewing, so the list is
 always what is actually live.
 
+## The trollbox, and saying a line out loud
+
+Every live room has a trollbox: the chat for whoever has joined that stream,
+kept at nixamp.com and keyed by the server and the channel, so everybody
+watching one stream is in the same box whichever page they came from.
+Reading it needs nobody. A line needs a nixamp.com sign-in, and is signed
+with the account's public handle, never its address.
+
+A line can be said rather than typed. The microphone button beside the box
+is tap, talk, tap: the page records, brings the sound to 16 kHz mono itself,
+and sends nixamp.com a small WAV; the words come back into the box, and
+**Send** is still yours, so a misheard word is fixed before the room sees it.
+The ear is [Whisper](https://github.com/openai/whisper) run through
+[Transformers.js](https://github.com/huggingface/transformers.js), an
+Apache-2.0 library carrying MIT-licensed models, on nixamp.com's own CPU.
+Nothing is sent to a speech vendor and nothing is billed. It works in the
+PWA, the desktop app and on a phone, wherever the browser can record; the
+button only appears where a line can be sent from, which is signed in on
+nixamp.com.
+
+The same ear is one route, for anything else that has a recording:
+
+```
+POST /api/v1/speech/transcribe            a WAV in (16-bit PCM; 16 kHz mono is ideal), {text} out
+POST /api/v1/speech/transcribe?server=URL&channel=ID   and the words posted to that room
+```
+
+Signed in only, up to a minute at a time, twelve asks a minute per account,
+`?language=de` when Whisper should not guess. The CLI and the MCP server
+front the same route:
+
+```
+nixamp transcribe clip.m4a                        the words in a recording
+nixamp transcribe clip.m4a --say https://server1.chovy.nixamp.com:4321
+nixamp transcribe clip.m4a --say URL --channel cat-1
+```
+
+Anything ffmpeg can read is converted here first; a WAV needs no ffmpeg.
+`nixamp mcp` offers `transcribe_audio` (with the same optional room),
+`trollbox_say` and `trollbox_read`.
+
+### Subtitles: what a live is saying
+
+Every live channel can be captioned. The server carrying it listens to its
+own stream, turns the sound into five-second windows with its ffmpeg, and
+has nixamp.com's ear turn each window into a line stamped with the moment
+its sound was at the live edge. The lines go out as Server-Sent Events:
+
+```
+GET /api/channels/ID/captions      an event stream: `hello` with the recent lines, then a `line` each
+GET /api/channels/ID/transcript    the recent lines as JSON (?after=MS for only the new ones)
+```
+
+Both are read with the same key as the sound. The page opens the stream as
+soon as you join a live and shows a **Transcript** panel, on by default:
+each line is held until your own playback has reached the sound it came
+from (the backlog you were handed, plus a little buffering) and then shown,
+on the picture when there is one and in the panel always. Close to the
+voice, not on it: a line is a window, not a word. The switch in the panel
+turns captions off for that device; the Panels list hides the panel.
+
+A captioner runs only while somebody is asking, and stops a minute after
+the last one leaves; silence between songs is never sent. The server needs
+an ffmpeg and a sign-in (`nixamp login`) for the ear to answer it. In the
+terminal, `nixamp transcript --channel ID --follow` prints the lines as
+they come; an agent reads them with the `transcript_read` tool.
+
+The model is an optional dependency, because it is hundreds of megabytes
+with the ONNX runtime under it and the CLI tarball is pure JavaScript. A
+`nixamp serve` on a laptop answers 503 to this route and every client asks
+nixamp.com instead. `NIXAMP_STT_MODEL` picks another Whisper
+(`onnx-community/whisper-base` by default; `whisper-small` hears better and
+takes twice as long), `NIXAMP_STT_CACHE` says where its files are kept, and
+`NIXAMP_STT=off` leaves the ear out of a deployment altogether.
+
 ## Several streams at once
 
 A channel is one publisher and everybody listening to them. Two or three devices

package/dist/captions.d.ts

@@ -0,0 +1,75 @@
+import type { Listener } from "./channels.ts";
+export interface CaptionLine {
+    /** The channel's id. */
+    channel: string;
+    /** When the sound this line is from began and ended, wall clock, ms. */
+    at: number;
+    until: number;
+    text: string;
+}
+/** What turns a channel's bytes into 16 kHz mono 16-bit PCM. ffmpeg, or a test's stand-in. */
+export interface Decoder {
+    write(chunk: Buffer): boolean;
+    end(): void;
+}
+export interface CaptionsOptions {
+    /** A listener on a channel, or null when there is no such channel. */
+    listen: (id: string, listener: Listener) => (() => void) | null;
+    ffmpeg: string[];
+    /** Whose ear to use: this server's own sign-in, read when a captioner starts. Null means no captions. */
+    session: () => {
+        site: string;
+        token: string;
+    } | null;
+    fetcher?: typeof fetch;
+    /** How bytes become PCM. The default spawns ffmpeg; the tests hand in something quieter. */
+    decoder?: (onPcm: (pcm: Buffer) => void, onEnd: () => void) => Decoder;
+    now?: () => number;
+    onEvent?: (message: string) => void;
+    windowMs?: number;
+    idleMs?: number;
+}
+export declare const WINDOW_MS = 5000;
+/** Lines kept per channel for whoever arrives late. */
+export declare const KEEP = 200;
+export declare const IDLE_MS = 60000;
+/** Below this RMS (about -48 dBFS) a window is silence, and never sent. */
+export declare const QUIET = 0.004;
+/** Windows waiting on the ear at once. Past this the sound is dropped, not queued: late words are worse than none. */
+export declare const IN_FLIGHT = 2;
+/**
+ * The ffmpeg arguments: whatever arrives on stdin, as PCM on stdout, with
+ * a short probe so the first line is not long in coming. Never `-fflags
+ * nobuffer`: it drops the packets it probed, which took the first 1.7
+ * seconds out of every captioner and began every transcript mid-sentence.
+ */
+export declare function decoderArgs(): string[];
+/** Whether a window of 16-bit PCM has anything in it worth hearing. */
+export declare function isQuiet(pcm: Buffer, threshold?: number): boolean;
+/** A WAV around 16-bit mono PCM, without copying it through floats. */
+export declare function wavAround(pcm: Buffer, rate?: number): Buffer;
+type Subscriber = (line: CaptionLine) => void;
+export declare class Captions {
+    private readonly options;
+    private readonly running;
+    constructor(options: CaptionsOptions);
+    /** Whether this server can caption at all: it has to be signed in for the ear to answer it. */
+    available(): boolean;
+    /**
+     * Lines for a channel as they are heard, starting the captioner if it is
+     * not running. Null when there is no such channel. The returned function
+     * is how to stop listening; the captioner itself stops a minute after the
+     * last listener does.
+     */
+    subscribe(id: string, subscriber: Subscriber): (() => void) | null;
+    /** The recent lines of a channel, oldest first, after a moment when given. Empty when nobody has asked for them. */
+    recent(id: string, after?: number): CaptionLine[];
+    /** Whether a channel is being captioned, and what last went wrong if the lines are not coming. */
+    status(id: string): {
+        on: boolean;
+        lines: number;
+        error: string;
+    };
+    stopAll(): void;
+}
+export {};

package/dist/captions.js

@@ -0,0 +1,308 @@
+/**
+ * Captions: what a live channel is saying, written down as it says it.
+ *
+ * One captioner per channel, started when the first person asks for the
+ * transcript and stopped a minute after the last one leaves, because it
+ * costs a CPU somewhere for as long as it runs. It listens to the channel
+ * exactly as a browser does -- the same bytes, from the same backlog --
+ * hands them to an ffmpeg that turns them into 16 kHz mono PCM, cuts that
+ * into five-second windows, and sends each window to nixamp.com's ear
+ * (see speech.ts) signed in as this server. The words come back as a
+ * line, stamped with the wall-clock moment the sound was heard, so a page
+ * can hold each line until its own playback gets there and the subtitles
+ * land close to the voice.
+ *
+ * Five seconds is the trade. Shorter windows hear less context and cost
+ * more asks; longer ones make the words later than the sound. A line is a
+ * window: no word-level timing, because the ear takes twice as long when
+ * asked for it and the page is only ever close to the voice, not on it.
+ *
+ * A quiet window -- the gap between songs, a picture with no talking -- is
+ * never sent. Most of a music channel is that, and hearing it costs the
+ * same as hearing speech.
+ */
+import { spawn } from "node:child_process";
+import { RATE } from "./speech.js";
+export const WINDOW_MS = 5000;
+/** Lines kept per channel for whoever arrives late. */
+export const KEEP = 200;
+export const IDLE_MS = 60_000;
+/** Below this RMS (about -48 dBFS) a window is silence, and never sent. */
+export const QUIET = 0.004;
+/** Windows waiting on the ear at once. Past this the sound is dropped, not queued: late words are worse than none. */
+export const IN_FLIGHT = 2;
+/**
+ * The ffmpeg arguments: whatever arrives on stdin, as PCM on stdout, with
+ * a short probe so the first line is not long in coming. Never `-fflags
+ * nobuffer`: it drops the packets it probed, which took the first 1.7
+ * seconds out of every captioner and began every transcript mid-sentence.
+ */
+export function decoderArgs() {
+    return [
+        "-v", "error", "-nostats",
+        "-flags", "low_delay",
+        "-analyzeduration", "500000", "-probesize", "262144",
+        "-i", "pipe:0",
+        "-vn", "-ac", "1", "-ar", String(RATE), "-f", "s16le", "pipe:1",
+    ];
+}
+function ffmpegDecoder(ffmpeg, onPcm, onEnd) {
+    const [command, ...prefix] = ffmpeg;
+    const child = spawn(command, [...prefix, ...decoderArgs()], { stdio: ["pipe", "pipe", "ignore"] });
+    let ended = false;
+    const end = () => {
+        if (ended)
+            return;
+        ended = true;
+        onEnd();
+    };
+    child.stdin.on("error", () => undefined);
+    child.stdout.on("data", (chunk) => onPcm(chunk));
+    child.on("error", end);
+    child.on("close", end);
+    return {
+        write: (chunk) => {
+            if (ended || child.stdin.destroyed)
+                return false;
+            try {
+                child.stdin.write(chunk);
+            }
+            catch {
+                return false;
+            }
+            return true;
+        },
+        end: () => {
+            try {
+                child.stdin.end();
+            }
+            catch {
+                // Already gone.
+            }
+            if (!ended) {
+                const kill = setTimeout(() => child.kill("SIGKILL"), 2000);
+                kill.unref?.();
+            }
+        },
+    };
+}
+/** Whether a window of 16-bit PCM has anything in it worth hearing. */
+export function isQuiet(pcm, threshold = QUIET) {
+    const samples = Math.floor(pcm.length / 2);
+    if (samples === 0)
+        return true;
+    let sum = 0;
+    for (let i = 0; i < samples; i++) {
+        const value = pcm.readInt16LE(i * 2) / 32768;
+        sum += value * value;
+    }
+    return Math.sqrt(sum / samples) < threshold;
+}
+/** A WAV around 16-bit mono PCM, without copying it through floats. */
+export function wavAround(pcm, rate = RATE) {
+    const header = Buffer.alloc(44);
+    header.write("RIFF", 0, "ascii");
+    header.writeUInt32LE(36 + pcm.length, 4);
+    header.write("WAVE", 8, "ascii");
+    header.write("fmt ", 12, "ascii");
+    header.writeUInt32LE(16, 16);
+    header.writeUInt16LE(1, 20);
+    header.writeUInt16LE(1, 22);
+    header.writeUInt32LE(rate, 24);
+    header.writeUInt32LE(rate * 2, 28);
+    header.writeUInt16LE(2, 32);
+    header.writeUInt16LE(16, 34);
+    header.write("data", 36, "ascii");
+    header.writeUInt32LE(pcm.length, 40);
+    return Buffer.concat([header, pcm]);
+}
+class Captioner {
+    id;
+    options;
+    onStop;
+    lines = [];
+    subscribers = new Set();
+    /** The last thing that went wrong, for whoever asks why there are no lines. */
+    error = "";
+    decoder = null;
+    detach = null;
+    idle = null;
+    pending = [];
+    pendingBytes = 0;
+    inFlight = 0;
+    stopped = false;
+    complainedAt = 0;
+    constructor(id, options, onStop) {
+        this.id = id;
+        this.options = options;
+        this.onStop = onStop;
+    }
+    get windowBytes() {
+        return Math.round(((this.options.windowMs ?? WINDOW_MS) / 1000) * RATE) * 2;
+    }
+    start() {
+        const make = this.options.decoder ?? ((onPcm, onEnd) => ffmpegDecoder(this.options.ffmpeg, onPcm, onEnd));
+        this.decoder = make((pcm) => this.onPcm(pcm), () => this.stop());
+        this.detach = this.options.listen(this.id, {
+            write: (chunk) => this.decoder?.write(chunk) ?? false,
+            end: () => this.stop(),
+        });
+        if (this.detach === null) {
+            this.stop();
+            return false;
+        }
+        return true;
+    }
+    onPcm(pcm) {
+        if (this.stopped)
+            return;
+        this.pending.push(pcm);
+        this.pendingBytes += pcm.length;
+        const size = this.windowBytes;
+        while (this.pendingBytes >= size) {
+            const all = Buffer.concat(this.pending);
+            const window = all.subarray(0, size);
+            const rest = all.subarray(size);
+            this.pending = rest.length > 0 ? [Buffer.from(rest)] : [];
+            this.pendingBytes = rest.length;
+            const until = (this.options.now ?? Date.now)();
+            void this.hear(Buffer.from(window), until - (this.options.windowMs ?? WINDOW_MS), until);
+        }
+    }
+    async hear(pcm, at, until) {
+        if (isQuiet(pcm))
+            return;
+        if (this.inFlight >= IN_FLIGHT)
+            return;
+        const session = this.options.session();
+        if (session === null) {
+            this.complain("this server is not signed in, so it cannot caption; `nixamp login` on it");
+            return;
+        }
+        this.inFlight += 1;
+        try {
+            const wav = wavAround(pcm);
+            const response = await (this.options.fetcher ?? fetch)(`${session.site.replace(/\/+$/, "")}/api/v1/speech/transcribe`, {
+                method: "POST",
+                headers: { authorization: `Bearer ${session.token}`, "content-type": "audio/wav" },
+                body: new Blob([wav.buffer.slice(wav.byteOffset, wav.byteOffset + wav.byteLength)]),
+            });
+            const body = (await response.json().catch(() => ({})));
+            if (!response.ok) {
+                this.complain(body.error ?? `nixamp.com answered ${response.status}`);
+                return;
+            }
+            const text = (body.text ?? "").trim();
+            if (text === "" || this.stopped)
+                return;
+            this.error = "";
+            const line = { channel: this.id, at, until, text };
+            this.lines.push(line);
+            while (this.lines.length > KEEP)
+                this.lines.shift();
+            for (const subscriber of this.subscribers) {
+                try {
+                    subscriber(line);
+                }
+                catch {
+                    // A listener that throws is not this channel's problem.
+                }
+            }
+        }
+        catch (error) {
+            this.complain(`could not reach the ear: ${error.message}`);
+        }
+        finally {
+            this.inFlight -= 1;
+        }
+    }
+    /** Said once a minute at most: a broken ear would otherwise say so twelve times a minute. */
+    complain(message) {
+        this.error = message;
+        const now = (this.options.now ?? Date.now)();
+        if (now - this.complainedAt < 60_000)
+            return;
+        this.complainedAt = now;
+        this.options.onEvent?.(`captions for "${this.id}": ${message}`);
+    }
+    subscribe(subscriber) {
+        this.subscribers.add(subscriber);
+        if (this.idle)
+            clearTimeout(this.idle);
+        this.idle = null;
+        return () => {
+            this.subscribers.delete(subscriber);
+            if (this.subscribers.size === 0 && !this.stopped) {
+                this.idle = setTimeout(() => {
+                    if (this.subscribers.size === 0)
+                        this.stop();
+                }, this.options.idleMs ?? IDLE_MS);
+                this.idle.unref?.();
+            }
+        };
+    }
+    stop() {
+        if (this.stopped)
+            return;
+        this.stopped = true;
+        if (this.idle)
+            clearTimeout(this.idle);
+        this.idle = null;
+        this.detach?.();
+        this.detach = null;
+        this.decoder?.end();
+        this.decoder = null;
+        this.pending = [];
+        this.pendingBytes = 0;
+        this.subscribers.clear();
+        this.onStop();
+    }
+}
+export class Captions {
+    options;
+    running = new Map();
+    constructor(options) {
+        this.options = options;
+    }
+    /** Whether this server can caption at all: it has to be signed in for the ear to answer it. */
+    available() {
+        return this.options.session() !== null;
+    }
+    /**
+     * Lines for a channel as they are heard, starting the captioner if it is
+     * not running. Null when there is no such channel. The returned function
+     * is how to stop listening; the captioner itself stops a minute after the
+     * last listener does.
+     */
+    subscribe(id, subscriber) {
+        let captioner = this.running.get(id);
+        if (!captioner) {
+            const made = new Captioner(id, this.options, () => {
+                if (this.running.get(id) === made)
+                    this.running.delete(id);
+            });
+            this.running.set(id, made);
+            if (!made.start())
+                return null;
+            this.options.onEvent?.(`captions for "${id}": started`);
+            captioner = made;
+        }
+        return captioner.subscribe(subscriber);
+    }
+    /** The recent lines of a channel, oldest first, after a moment when given. Empty when nobody has asked for them. */
+    recent(id, after = 0) {
+        const captioner = this.running.get(id);
+        if (!captioner)
+            return [];
+        return after > 0 ? captioner.lines.filter((line) => line.at > after) : [...captioner.lines];
+    }
+    /** Whether a channel is being captioned, and what last went wrong if the lines are not coming. */
+    status(id) {
+        const captioner = this.running.get(id);
+        return captioner ? { on: true, lines: captioner.lines.length, error: captioner.error } : { on: false, lines: 0, error: "" };
+    }
+    stopAll() {
+        for (const captioner of [...this.running.values()])
+            captioner.stop();
+    }
+}

package/dist/main.js

@@ -59,6 +59,8 @@ const HELP = `nixamp — it really whips the terminal's ass.
   nixamp server list|add|remove  the machines you run, kept against your account
   nixamp party list|join|host   watch parties, here and on the sites nixamp is connected to
   nixamp mcp                     speak Model Context Protocol on stdin, for an agent
+  nixamp transcribe FILE [--say SERVER]  the words in a recording, and into a trollbox
+  nixamp transcript --channel ID [--follow]  what a channel is saying, as it says it
   nixamp opendir list|add|remove  folders found on the web, published for everyone
   nixamp update [version]        re-run the installer, keeping your choices
   nixamp uninstall [--yes]       remove everything the installer created
@@ -236,10 +238,43 @@ takes it away again.
   nixamp mcp    speak Model Context Protocol on stdin and stdout
 
 It offers the watch party tools: list them, read one, put one on the air,
-say where playback is, end it. It acts as whoever this machine is signed in
-as, so \`nixamp login\` (or NIXAMP_TOKEN) comes first.
+say where playback is, end it. And the room tools: transcribe a recording
+(transcribe_audio, which can post the words straight into a trollbox), say a
+line in a room (trollbox_say), read a room (trollbox_read), read what a
+channel is saying (transcript_read). It acts as
+whoever this machine is signed in as, so \`nixamp login\` (or NIXAMP_TOKEN)
+comes first.
 
 Point an MCP client at it as a stdio server running \`nixamp mcp\`.
+`,
+    transcribe: `nixamp transcribe — say it, and have it written down.
+
+  nixamp transcribe FILE                  the words in a recording
+  nixamp transcribe FILE --say SERVER     and post them to that server's trollbox
+  nixamp transcribe FILE --say SERVER --channel ID   to one channel's room (default: live)
+  nixamp transcribe FILE --language de    when Whisper should not guess
+  nixamp transcribe FILE --json           the answer as JSON
+
+FILE is any recording ffmpeg can read; a WAV needs no ffmpeg at all. The
+hearing is done by nixamp.com with an open-source model (Whisper, through
+Transformers.js) on its own CPU: nothing goes to a speech vendor. It needs a
+sign-in (\`nixamp login\`) and nothing else. Up to a minute at a time.
+
+The same ear is behind the microphone button in every nixamp.com trollbox,
+and behind the transcribe_audio tool of \`nixamp mcp\`.
+`,
+    transcript: `nixamp transcript — what a channel is saying, written down.
+
+  nixamp transcript --channel ID                 the recent lines from this machine's daemon
+  nixamp transcript --url URL --key K --channel ID   from another server, with its share link
+  nixamp transcript ... --follow                 and keep printing as it speaks
+  nixamp transcript ... --json                   the lines as JSON
+
+A server captions a channel while somebody is asking for its transcript: its
+own ffmpeg turns the sound into five-second windows, nixamp.com's ear turns
+those into lines, each stamped with when its sound was heard. The page shows
+them as subtitles, held until its own sound gets there; this prints them.
+The server needs an ffmpeg and a sign-in (\`nixamp login\`).
 `,
     attach: `nixamp attach — the player, in front of the running daemon.
 
@@ -437,6 +472,16 @@ export async function main() {
         process.exitCode = await party(rest);
         return;
     }
+    if (first === "transcript" || first === "captions") {
+        const { transcript } = await import("./transcript.js");
+        process.exitCode = await transcript(rest);
+        return;
+    }
+    if (first === "transcribe" || first === "dictate") {
+        const { transcribe } = await import("./transcribe.js");
+        process.exitCode = await transcribe(rest);
+        return;
+    }
     if (first === "mcp") {
         const { mcp } = await import("./mcp.js");
         process.exitCode = await mcp();

package/dist/mcp.d.ts

@@ -1,3 +1,4 @@
+import { wavOf } from "./transcribe.ts";
 export declare const PROTOCOL_VERSION = "2025-06-18";
 interface Request {
     jsonrpc: "2.0";
@@ -18,6 +19,8 @@ export interface McpOptions {
         site: string;
         token: string;
     } | null;
+    /** How a recording becomes a WAV; the tests hand in a fake. */
+    wavOf?: typeof wavOf;
     say?: (line: string) => void;
 }
 /** A tool answer, in the shape MCP wants: content blocks, and a flag for failure. */