nixamp 0.21.4 → 0.22.0

package/dist/mcp.js

@@ -8,7 +8,9 @@
  *
  * What it offers is the watch party, because that is the part of nixamp an
  * agent can usefully do something with: find the party, say where it is, put
- * one on the air, move everybody to the same second. It signs in as whoever
+ * one on the air, move everybody to the same second. And the room: hear a
+ * recording (nixamp.com's own ear, see speech.ts), say a line in a trollbox,
+ * read one back. It signs in as whoever
  * this machine is signed in as -- the session on disk, or NIXAMP_TOKEN --
  * because an agent holding its own credential is a credential nobody revokes.
  *
@@ -18,6 +20,8 @@
 import { createInterface } from "node:readline";
 import { clock } from "./party.js";
 import { readSession } from "./session.js";
+import { askToHear, wavOf } from "./transcribe.js";
+import { readTranscript } from "./transcript.js";
 export const PROTOCOL_VERSION = "2025-06-18";
 const STRING = { type: "string" };
 export const TOOLS = [
@@ -74,7 +78,64 @@ export const TOOLS = [
         description: "End a watch party. Only its host may.",
         inputSchema: { type: "object", properties: { code: STRING }, required: ["code"] },
     },
+    {
+        name: "transcribe_audio",
+        description: "The words in a recording on this machine, heard by nixamp.com's own open-source ear (Whisper). Any format ffmpeg reads; up to a minute. Given a server, the words are also posted to that server's trollbox as this account.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                path: { ...STRING, description: "The recording's path on this machine." },
+                language: { ...STRING, description: "A two-letter language code, when Whisper should not guess." },
+                server: { ...STRING, description: "Post the words to this nixamp's trollbox: its address, as in its share link." },
+                channel: { ...STRING, description: "Which of that server's channels; its own stream (live) by default." },
+            },
+            required: ["path"],
+        },
+    },
+    {
+        name: "trollbox_say",
+        description: "Say a line in a live room's trollbox, as this account and under its public handle. A room is a nixamp server's address and one of its channels (or `live`, the server's own stream).",
+        inputSchema: {
+            type: "object",
+            properties: {
+                server: { ...STRING, description: "The nixamp server's address, as in its share link." },
+                channel: { ...STRING, description: "The channel's id, or live (default)." },
+                text: { ...STRING, description: "The line. At most 500 characters." },
+            },
+            required: ["server", "text"],
+        },
+    },
+    {
+        name: "transcript_read",
+        description: "What a live channel is saying: the recent lines of its transcript, oldest first, each with when its sound was heard. The server carrying the channel captions it while somebody asks. Pass the server's address and share key, and the channel's id.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                url: { ...STRING, description: "The nixamp server's address, e.g. https://server1.chovy.nixamp.com:4321." },
+                key: { ...STRING, description: "The share key from its link, when it has one." },
+                channel: { ...STRING, description: "The channel's id on that server (default: main)." },
+                after: { type: "number", description: "Only lines heard after this moment (ms since the epoch)." },
+            },
+            required: ["url"],
+        },
+    },
+    {
+        name: "trollbox_read",
+        description: "The recent lines in a live room's trollbox, oldest first: who said what, and when.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                server: { ...STRING, description: "The nixamp server's address, as in its share link." },
+                channel: { ...STRING, description: "The channel's id, or live (default)." },
+                after: { ...STRING, description: "Only lines after this moment (an ISO timestamp)." },
+            },
+            required: ["server"],
+        },
+    },
 ];
+function said(line) {
+    return `${line.createdAt}  ${line.handle}: ${line.body}`;
+}
 function text(value) {
     return { content: [{ type: "text", text: value }] };
 }
@@ -172,6 +233,77 @@ export async function callTool(name, args, options = {}) {
                 return failed(await answerOf(response));
             return text(`Ended ${code}.`);
         }
+        const server = typeof args["server"] === "string" ? args["server"].trim() : "";
+        const channel = typeof args["channel"] === "string" && args["channel"].trim() ? args["channel"].trim() : "live";
+        if (name === "transcribe_audio") {
+            const path = typeof args["path"] === "string" ? args["path"] : "";
+            if (!path)
+                return failed("Which recording? Pass its path.");
+            let wav;
+            try {
+                wav = (options.wavOf ?? wavOf)(path);
+            }
+            catch (error) {
+                return failed(error.message);
+            }
+            const answer = await askToHear(session, {
+                wav,
+                ...(typeof args["language"] === "string" ? { language: args["language"] } : {}),
+                ...(server ? { server, channel } : {}),
+            }, send, site);
+            if (!answer.ok)
+                return failed(answer.error);
+            if (answer.heard.text === "")
+                return text("Heard nothing in that recording.");
+            return text(answer.heard.message
+                ? `${answer.heard.text}\n\nSaid in the room for ${channel} at ${server} as ${answer.heard.message.handle}.`
+                : answer.heard.text);
+        }
+        if (name === "trollbox_say") {
+            if (!server)
+                return failed("Which room? Pass the server's address.");
+            const line = typeof args["text"] === "string" ? args["text"] : "";
+            if (!line.trim())
+                return failed("Say what? Pass the text.");
+            const response = await send(`${site}/api/v1/trollbox`, {
+                method: "POST",
+                headers,
+                body: JSON.stringify({ server, channel, body: line }),
+            });
+            if (!response.ok)
+                return failed(await answerOf(response));
+            const body = (await response.json());
+            return text(body.message ? `Said, as ${body.message.handle}: ${body.message.body}` : "Said.");
+        }
+        if (name === "transcript_read") {
+            const url = typeof args["url"] === "string" ? args["url"].trim() : "";
+            if (!url)
+                return failed("Which server? Pass its address.");
+            const got = await readTranscript({ url, key: typeof args["key"] === "string" && args["key"] ? args["key"] : null }, channel === "live" ? "main" : channel, typeof args["after"] === "number" ? args["after"] : 0, send);
+            if (!got.ok)
+                return failed(got.error);
+            if (got.answer.recent.length === 0) {
+                return text(got.answer.error
+                    ? `Nothing yet: ${got.answer.error}`
+                    : "Nothing said yet. The server has just started listening; ask again in a few seconds.");
+            }
+            return text(got.answer.recent.map((line) => `${new Date(line.at).toISOString()}  ${line.text}`).join("\n"));
+        }
+        if (name === "trollbox_read") {
+            if (!server)
+                return failed("Which room? Pass the server's address.");
+            const url = new URL(`${site}/api/v1/trollbox`);
+            url.searchParams.set("server", server);
+            url.searchParams.set("channel", channel);
+            if (typeof args["after"] === "string" && args["after"])
+                url.searchParams.set("after", args["after"]);
+            const response = await send(url.toString(), { headers });
+            if (!response.ok)
+                return failed(await answerOf(response));
+            const body = (await response.json());
+            const lines = body.messages ?? [];
+            return text(lines.length === 0 ? `Nobody has said anything in the room for ${channel} at ${server}.` : lines.map(said).join("\n"));
+        }
     }
     catch (error) {
         return failed(`Could not reach ${site}: ${error.message}`);
@@ -186,7 +318,7 @@ export async function handleMessage(message, options = {}) {
         return reply({
             protocolVersion: PROTOCOL_VERSION,
             capabilities: { tools: { listChanged: false } },
-            serverInfo: { name: "nixamp", title: "nixamp watch parties", version: "1" },
+            serverInfo: { name: "nixamp", title: "nixamp: watch parties and rooms", version: "1" },
             instructions: "Watch parties on nixamp. A party lives on the site hosting the film and is bridged here as a room every nixamp client can join. Codes are the ones that site shows; positions are seconds into the film.",
         });
     }

package/dist/server.d.ts

@@ -28,6 +28,8 @@ import { Tickets } from "./tickets.ts";
 import { Layouts } from "./layouts.ts";
 import { Rooms } from "./rooms.ts";
 import { Trollbox } from "./trollbox.ts";
+import { Speech } from "./speech.ts";
+import { Captions } from "./captions.ts";
 import { type Codecs } from "./audio.ts";
 import { type Tools, type Track } from "./audio.ts";
 import { type Command, type RemoteTrack, type Snapshot } from "./protocol.ts";
@@ -511,6 +513,8 @@ export interface HandlerOptions {
     ytdlp?: string[] | null;
     /** Channels as HLS, for Safari on a phone, which plays a live stream no other way. */
     hls?: HlsPackagers;
+    /** Captions for a channel: its sound, as lines, as they are heard. Needs ffmpeg and a sign-in. */
+    captions?: Captions;
     /** Lossless relay compression, its policies, diagnostics and static representations. */
     compression?: CompressionService;
     /** What a name is -- a film, a channel, a fixture -- asked of nichedb.dev and remembered. */
@@ -658,6 +662,8 @@ export interface HandlerOptions {
     rooms?: Rooms;
     /** The trollbox: a chat per live room, kept at nixamp.com. */
     trollbox?: Trollbox;
+    /** Speech to text: a line said out loud, heard here. Needs the optional model. */
+    speech?: Speech;
     /** Tickets: a paid pass to one event's room. Absent means every show is free. */
     tickets?: Tickets;
     /**

package/dist/server.js

@@ -19,7 +19,7 @@ import { readFileSync } from "node:fs";
 import { Connections } from "./connections.js";
 import { Broadcaster, DEFAULT_ENCODER, PRESETS, redact, } from "./broadcast.js";
 import { Ingest, normaliseFormat } from "./ingest.js";
-import { Channels, cleanId, generatedId, rememberChannels, rememberedChannels, rememberedNow, } from "./channels.js";
+import { BACKLOG_SECONDS, Channels, cleanId, generatedId, rememberChannels, rememberedChannels, rememberedNow, } from "./channels.js";
 import { RtmpListeners } from "./rtmp-in.js";
 import { Accounts, clearedCookie, sessionCookie, tokenFrom } from "./accounts.js";
 import { anonymousHandle, Handles } from "./handles.js";
@@ -63,6 +63,8 @@ import { Tickets, needsTicket, ticketFrom, ticketsFromEnv } from "./tickets.js";
 import { Layouts } from "./layouts.js";
 import { Rooms } from "./rooms.js";
 import { Trollbox, TrollboxError, fallbackHandle, roomFor } from "./trollbox.js";
+import { MAX_BYTES as SPEECH_BYTES, Speech, SpeechError, isWav, languageOf } from "./speech.js";
+import { Captions } from "./captions.js";
 import { confirm, DEFAULT_DIRECTORY, Publisher } from "./publish.js";
 import { applyRemoteConfig, createPaywall, FREE_LISTENERS, paywallFromEnv, } from "./paywall.js";
 import { isRemote, isTransportStream, playsInBrowser, sourceLabel } from "./sources.js";
@@ -1182,6 +1184,19 @@ async function readBody(request, limit = 64 * 1024) {
     }
     return Buffer.concat(chunks).toString("utf8");
 }
+/** The body as bytes: sound, not text. */
+async function readBytes(request, limit) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of request) {
+        const buffer = chunk;
+        size += buffer.length;
+        if (size > limit)
+            throw new SpeechError("that is too much sound", 413);
+        chunks.push(buffer);
+    }
+    return new Uint8Array(Buffer.concat(chunks));
+}
 /**
  * The whole HTTP surface, as a plain function of a request — so a test can
  * drive it with a real socket and no ffmpeg in sight.
@@ -2127,6 +2142,56 @@ export function createHandler(engine, options) {
             }
             return;
         }
+        /*
+         * Speech to text: a WAV in, the words out. Signed in only -- the ear
+         * costs CPU and a trollbox line needs a name anyway -- and, given a
+         * room, the words go straight into that room's trollbox as a line by
+         * whoever spoke them. The page, the CLI and the MCP tools all come here.
+         */
+        if (path === "/api/v1/speech/transcribe" && options.speech && options.accounts) {
+            const speech = options.speech;
+            try {
+                if (request.method !== "POST") {
+                    json(response, 405, { error: "POST a WAV" });
+                    return;
+                }
+                const who = await options.accounts.whoIs(tokenFrom(request.headers));
+                if (who === null) {
+                    json(response, 401, { error: "sign in to nixamp.com to dictate" });
+                    return;
+                }
+                const bytes = await readBytes(request, SPEECH_BYTES);
+                if (!isWav(bytes)) {
+                    json(response, 415, { error: "send a WAV: 16-bit PCM, mono, 16 kHz is ideal" });
+                    return;
+                }
+                // A room named is a line posted, by the same rules as typing it.
+                const roomAsked = url.searchParams.has("server") || url.searchParams.has("channel");
+                const where = roomAsked ? roomFor(url.searchParams.get("server"), url.searchParams.get("channel")) : null;
+                if (roomAsked && !where) {
+                    json(response, 400, { error: "a room is a server address and a channel" });
+                    return;
+                }
+                const heard = await speech.transcribe(bytes, { language: languageOf(url.searchParams.get("language")), by: who.id });
+                if (where && options.trollbox && heard.text !== "") {
+                    const handle = (await options.handles?.of(who.id)) || fallbackHandle(who.id);
+                    const line = await options.trollbox.post(where, who.id, handle, heard.text);
+                    json(response, 201, {
+                        text: heard.text, seconds: heard.seconds, model: speech.model,
+                        message: { id: line.id, handle: line.handle, body: line.body, createdAt: line.createdAt, mine: true },
+                    });
+                    return;
+                }
+                json(response, 200, { text: heard.text, seconds: heard.seconds, model: speech.model });
+            }
+            catch (error) {
+                if (error instanceof SpeechError || error instanceof TrollboxError)
+                    json(response, error.status, { error: error.message });
+                else
+                    throw error;
+            }
+            return;
+        }
         if (path === "/api/v1/me/handle" && options.handles && options.accounts) {
             const handles = options.handles;
             const who = await options.accounts.whoIs(tokenFrom(request.headers));
@@ -3182,6 +3247,67 @@ export function createHandler(engine, options) {
                 createReadStream(segment).pipe(response);
                 return;
             }
+            /*
+             * The channel's captions: what it is saying, as lines, as they are
+             * heard by nixamp.com's ear on this server's behalf. Read the way the
+             * sound is read -- the key gate above has already passed -- and
+             * started by the first person asking. `captions` is the live stream
+             * of lines; `transcript` is the recent ones as JSON, for a poll.
+             */
+            if ((action === "captions" || action === "transcript") && request.method === "GET") {
+                const captions = options.captions;
+                if (!captions) {
+                    json(response, 503, { error: "this server cannot caption: it has no ffmpeg" });
+                    return;
+                }
+                if (!captions.available()) {
+                    json(response, 503, { error: "this server is not signed in to nixamp.com, so it cannot caption; run `nixamp login` on it" });
+                    return;
+                }
+                if (!channels.has(id)) {
+                    json(response, 404, { error: "nothing is playing on that channel" });
+                    return;
+                }
+                // Lines after a moment: a poll's ?after=, or the last line a
+                // reconnecting EventSource saw, which the browser sends by itself.
+                const lastSeen = request.headers["last-event-id"];
+                const after = Number(url.searchParams.get("after") ?? (Array.isArray(lastSeen) ? lastSeen[0] : lastSeen) ?? 0) || 0;
+                if (action === "transcript") {
+                    // Asking keeps the captioner up: it stops a minute after the last ask.
+                    captions.subscribe(id, () => undefined)?.();
+                    json(response, 200, {
+                        channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), ...captions.status(id), lines: captions.recent(id, after),
+                    });
+                    return;
+                }
+                response.writeHead(200, {
+                    ...CORS,
+                    "content-type": "text/event-stream; charset=utf-8",
+                    "cache-control": "no-store",
+                    connection: "keep-alive",
+                    "x-accel-buffering": "no",
+                });
+                const write = (event, data, eventId) => {
+                    response.write(`${eventId === undefined ? "" : `id: ${eventId}\n`}event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
+                };
+                const off = captions.subscribe(id, (line) => write("line", line, line.at));
+                if (off === null) {
+                    response.end();
+                    return;
+                }
+                // How far behind the live edge a newcomer's playback starts, so the
+                // page can hold each line until its own sound gets there.
+                write("hello", { channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), ...captions.status(id), lines: captions.recent(id, after) });
+                const beat = setInterval(() => response.write(": beat\n\n"), 20_000);
+                beat.unref?.();
+                const done = () => {
+                    clearInterval(beat);
+                    off();
+                };
+                request.on("close", done);
+                response.on("close", done);
+                return;
+            }
             if (action === undefined && request.method === "GET") {
                 // Listening. The response is the fan-out target: whatever ffmpeg
                 // produces for this channel is written to it until one end goes away.
@@ -4450,6 +4576,16 @@ export async function serve(argv, version = "0.1.0") {
             secret: process.env["NIXAMP_JWT_SECRET"] ?? "",
         })
         : undefined;
+    // The ear lives where the accounts live. Loaded now, in the background, so
+    // the first person to speak does not wait for the model to arrive; a box
+    // without the optional model never says it is ready, and answers 503.
+    const speech = accounts && process.env["NIXAMP_STT"] !== "off" ? new Speech() : undefined;
+    if (speech) {
+        void speech.warm().then((ready) => {
+            if (ready)
+                console.error(`nixamp: hearing with ${speech.model}`);
+        });
+    }
     // nixamp as an OAuth 2.1 authorization server, and the watch parties a
     // client site bridges through it. Both need the same three things -- a
     // database, accounts, and a site to be the issuer of -- so both appear or
@@ -4627,6 +4763,18 @@ export async function serve(argv, version = "0.1.0") {
         void durable.sweep(Date.now() - ENDED_TTL_MS, new Date(Date.now() - 30 * 24 * 60 * 60 * 1000));
     }
     const session = readSession();
+    // Captions for a channel: this server's ffmpeg turns the sound into PCM
+    // and nixamp.com's ear, asked as this server, turns that into lines. A
+    // server without ffmpeg cannot; a server nobody signed in on is refused
+    // by the ear, and says so to whoever asks.
+    const captions = tools.carries
+        ? new Captions({
+            ffmpeg: tools.ffmpeg,
+            listen: (id, listener) => channels.listen(id, listener),
+            session: () => readSession(),
+            onEvent: (message) => console.log(`  ${message}`),
+        })
+        : undefined;
     const owner = new Owner({
         ownerId: options.owner || (session?.token ? await ownerIdOf(session) : ""),
         site: session?.site ?? DEFAULT_DIRECTORY,
@@ -4810,6 +4958,7 @@ export async function serve(argv, version = "0.1.0") {
         carries: tools.carries !== false,
         cookies: cookiesFile(),
         hls,
+        ...(captions ? { captions } : {}),
         compression,
         enricher,
         ...(tls ? { tls } : {}),
@@ -4824,6 +4973,7 @@ export async function serve(argv, version = "0.1.0") {
         ...(layouts ? { layouts } : {}),
         ...(rooms ? { rooms } : {}),
         ...(trollbox ? { trollbox } : {}),
+        ...(speech ? { speech } : {}),
         ...(tickets ? { tickets } : {}),
         ...(authServer ? { authServer } : {}),
         ...(parties ? { parties } : {}),

package/dist/speech.d.ts

@@ -0,0 +1,96 @@
+/** What Whisper listens at. Everything is brought to this before it is heard. */
+export declare const RATE = 16000;
+/** A trollbox line, said out loud, is seconds long; a minute is the ceiling. */
+export declare const MAX_SECONDS = 60;
+/** A minute of 16-bit mono at 16 kHz is under 2 MB; 48 kHz stereo is under 12. */
+export declare const MAX_BYTES: number;
+/**
+ * How much one account may have heard in a minute, in seconds of sound.
+ * Counted in sound rather than asks because a live channel being captioned
+ * asks twelve times a minute for five seconds each, and a person dictating
+ * asks twice for thirty: the cost is the sound, not the call. Five minutes
+ * of sound a minute is four channels captioned, or a conversation, and
+ * not a firehose.
+ */
+export declare const SECONDS_PER_MINUTE = 300;
+/** How many may wait for the one CPU. Past this, the honest answer is "later". */
+export declare const QUEUE_LIMIT = 8;
+export declare const DEFAULT_MODEL = "onnx-community/whisper-base";
+export declare class SpeechError extends Error {
+    readonly status: number;
+    constructor(message: string, status: number);
+}
+export interface Heard {
+    text: string;
+    /** How long the audio was, in seconds. */
+    seconds: number;
+}
+/** Mono samples in [-1, 1] at RATE -> words. What the loaded model is, to this file. */
+export type Recognizer = (pcm: Float32Array, options: {
+    language?: string;
+}) => Promise<{
+    text: string;
+}>;
+export interface SpeechOptions {
+    /** A Hugging Face model id; NIXAMP_STT_MODEL otherwise; whisper-base by default. */
+    model?: string;
+    /** Where the model's files are kept between runs. */
+    cacheDir?: string;
+    /** How the model is loaded. The tests hand in a fake; nothing else does. */
+    load?: (model: string, cacheDir: string) => Promise<Recognizer>;
+    now?: () => number;
+}
+export interface Wav {
+    rate: number;
+    channels: number;
+    /** Mixed down to one channel, in [-1, 1]. */
+    samples: Float32Array;
+}
+/** Whether these bytes are a RIFF/WAVE file, whatever the request called them. */
+export declare function isWav(bytes: Uint8Array): boolean;
+/**
+ * A WAV file's samples, mixed to mono. PCM of 8, 16, 24 or 32 bits, or
+ * 32-bit float; the WAVE_FORMAT_EXTENSIBLE wrapper around either. That is
+ * what every encoder that matters writes, including the one in the page.
+ */
+export declare function decodeWav(bytes: Uint8Array): Wav;
+/**
+ * Samples at one rate, at another. Down is an average over each output
+ * sample's span, which is a crude low-pass and enough for speech; up is a
+ * straight line between neighbours. Whisper resamples nothing itself.
+ */
+export declare function resample(samples: Float32Array, from: number, to: number): Float32Array;
+/** 16-bit mono PCM WAV bytes from samples: what the CLI's tests, and anybody, can send. */
+export declare function encodeWav(samples: Float32Array, rate?: number): Uint8Array;
+/** Whisper's own spacing and blank-audio tokens, tidied into a line. */
+export declare function tidy(text: string): string;
+/** A two-letter language code, or nothing: Whisper guesses when not told. */
+export declare function languageOf(value: unknown): string | undefined;
+export declare class Speech {
+    readonly model: string;
+    private readonly cacheDir;
+    private readonly load;
+    private readonly now;
+    private recognizer;
+    /** One at a time: the model is CPU-bound, and two at once is slower than two in turn. */
+    private tail;
+    private waiting;
+    private readonly asked;
+    constructor(options?: SpeechOptions);
+    private ear;
+    /**
+     * Load the model now, so the first person to speak is not the one who
+     * waits for the download. Says whether it could; never throws.
+     */
+    warm(): Promise<boolean>;
+    /** Whether this account may have this much heard now, and the bookkeeping if so. */
+    allow(accountId: string, seconds: number): void;
+    /**
+     * The words in a WAV. Refuses what is not a WAV, or is too long, or more
+     * than the account may have heard this minute, with a status.
+     */
+    transcribe(bytes: Uint8Array, options?: {
+        language?: string;
+        by?: string;
+    }): Promise<Heard>;
+}