nixamp 0.23.7 → 0.24.0

package/dist/server.js

@@ -28,7 +28,7 @@ import { Servers } from "./servers.js";
 import { DeviceGrants } from "./device.js";
 import { BAD_KEY_LIMIT, callerOf, Guard, SIGN_IN_LIMIT } from "./guard.js";
 import { deviceDonePage, devicePage, exchangeCode, providersFrom, signInFailedPage, SignIn, } from "./oauth.js";
-import { AuthorizationServer, clientsFrom } from "./oauth-server.js";
+import { AuthorizationServer, SCOPE_NAMES, clientsFrom } from "./oauth-server.js";
 import { handleOAuthApi, oauthApiPath } from "./oauth-api.js";
 import { WatchParties } from "./watch-party.js";
 import { needsAdmin, needsMember, Owner } from "./owner.js";
@@ -67,6 +67,10 @@ import { Rooms } from "./rooms.js";
 import { Trollbox, TrollboxError, fallbackHandle, roomFor } from "./trollbox.js";
 import { MAX_BYTES as SPEECH_BYTES, Speech, SpeechError, isWav, languageOf } from "./speech.js";
 import { Captions } from "./captions.js";
+import { LANGUAGES, Translator } from "./translate.js";
+import { StoredTranslations } from "./translate-jobs.js";
+import { Transcripts, fileFingerprint, formatOf, idFrom, languageCode, linesFrom, mediaOfLive, mediaOfUrl, toSrt, toText, toVtt, transcriptIdOf, wire, } from "./transcripts.js";
+import { handleMessage as mcpMessage } from "./mcp.js";
 import { Outro } from "./outro.js";
 import { Profiles, Voices, spokenLine, spokenVoiceFor } from "./voices.js";
 import { confirm, DEFAULT_DIRECTORY, Publisher } from "./publish.js";
@@ -2350,18 +2354,30 @@ export function createHandler(engine, options) {
                     json(response, 400, { error: "a room is a server address and a channel" });
                     return;
                 }
-                const heard = await speech.transcribe(bytes, { language: languageOf(url.searchParams.get("language")), by: who.id });
+                const heard = await speech.transcribe(bytes, {
+                    language: languageOf(url.searchParams.get("language")),
+                    // Pieces with their timing, for a whole file being written down.
+                    timestamps: url.searchParams.get("timestamps") === "1",
+                    by: who.id,
+                });
+                // The language travels back: as told, or as the ear guessed it, so
+                // a captioner can say it next time and a transcript can be kept as it.
+                const said = {
+                    text: heard.text, seconds: heard.seconds, model: speech.model,
+                    ...(heard.language ? { language: heard.language } : {}),
+                    ...(heard.segments ? { segments: heard.segments } : {}),
+                };
                 if (where && options.trollbox && heard.text !== "") {
                     const handle = (await options.handles?.of(who.id)) || fallbackHandle(who.id);
                     const line = await options.trollbox.post(where, who.id, handle, heard.text);
                     readOnThePhone(where, who.id, line.handle, line.body);
                     json(response, 201, {
-                        text: heard.text, seconds: heard.seconds, model: speech.model,
+                        ...said,
                         message: { id: line.id, handle: line.handle, body: line.body, createdAt: line.createdAt, mine: true },
                     });
                     return;
                 }
-                json(response, 200, { text: heard.text, seconds: heard.seconds, model: speech.model });
+                json(response, 200, said);
             }
             catch (error) {
                 if (error instanceof SpeechError || error instanceof TrollboxError)
@@ -2371,6 +2387,282 @@ export function createHandler(engine, options) {
             }
             return;
         }
+        /*
+         * Translation: texts in one language, said in another, by a model on
+         * this CPU (see translate.ts). Signed in only, like the ear. GET says
+         * which languages, and which each can be turned into here.
+         */
+        if (path === "/api/v1/translate" && options.accounts) {
+            const translator = options.translator;
+            if (request.method === "GET") {
+                json(response, 200, {
+                    available: translator !== undefined,
+                    languages: Object.entries(LANGUAGES).map(([code, names]) => ({ code, ...names, targets: translator ? translator.targets(code) : [] })),
+                });
+                return;
+            }
+            if (request.method !== "POST") {
+                json(response, 405, { error: "POST texts, or GET the languages" });
+                return;
+            }
+            if (!translator) {
+                json(response, 503, { error: "this nixamp cannot translate: the models are not here. nixamp.com can." });
+                return;
+            }
+            const who = await options.accounts.whoIs(tokenFrom(request.headers));
+            if (who === null) {
+                json(response, 401, { error: "sign in to nixamp.com to translate" });
+                return;
+            }
+            let body = {};
+            try {
+                body = JSON.parse(await readBody(request, 256 * 1024));
+            }
+            catch {
+                json(response, 400, { error: "bad JSON" });
+                return;
+            }
+            const texts = Array.isArray(body.texts)
+                ? body.texts.filter((one) => typeof one === "string")
+                : typeof body.text === "string" ? [body.text] : [];
+            if (texts.length === 0) {
+                json(response, 400, { error: "say what: texts, a list of strings" });
+                return;
+            }
+            if (texts.length > 200) {
+                json(response, 413, { error: "200 texts at a time at most" });
+                return;
+            }
+            const from = languageCode(body.from);
+            const to = languageCode(body.to);
+            if (from === null || to === null || to === "") {
+                json(response, 400, { error: "from and to are two-letter language codes, such as en and sv" });
+                return;
+            }
+            if (from === "") {
+                json(response, 400, { error: "say which language the texts are in: from" });
+                return;
+            }
+            try {
+                json(response, 200, await translator.translate(texts, from, to, { by: who.id }));
+            }
+            catch (error) {
+                if (error instanceof SpeechError)
+                    json(response, error.status, { error: error.message });
+                else
+                    throw error;
+            }
+            return;
+        }
+        /*
+         * The transcript store: what a file, a link or a live said, kept once
+         * under the media's identity (see transcripts.ts). Signed in to read
+         * and to keep, the way the ear is: a captioning server, the CLI, the
+         * MCP tools. A language asks for a translation, made before answering
+         * when it is short and as a job when it is not; a format asks for it
+         * as subtitles.
+         */
+        if ((path === "/api/v1/transcripts" || path.startsWith("/api/v1/transcripts/")) && options.transcripts && options.translations && options.accounts) {
+            const store = options.transcripts;
+            const parts = path.slice("/api/v1/transcripts".length).split("/").filter(Boolean);
+            const who = await options.accounts.whoIs(tokenFrom(request.headers));
+            if (who === null) {
+                json(response, 401, { error: "sign in to nixamp.com to read or keep transcripts" });
+                return;
+            }
+            try {
+                if (parts.length === 0) {
+                    if (request.method !== "GET") {
+                        json(response, 405, { error: "GET: what you have had written down" });
+                        return;
+                    }
+                    json(response, 200, { transcripts: await store.list(who.id, Number(url.searchParams.get("limit") ?? 50) || 50) });
+                    return;
+                }
+                let named = "";
+                try {
+                    named = decodeURIComponent(parts[0]);
+                }
+                catch {
+                    json(response, 400, { error: "bad id" });
+                    return;
+                }
+                // By id, or by the media identity itself: both name the same row.
+                const id = idFrom(named);
+                const action = parts[1];
+                if (action === undefined && request.method === "GET") {
+                    const language = languageCode(url.searchParams.get("language"));
+                    const format = formatOf(url.searchParams.get("format"));
+                    if (language === null) {
+                        json(response, 400, { error: "language is a two-letter code, or original" });
+                        return;
+                    }
+                    if (format === null) {
+                        json(response, 400, { error: "format is json, srt, vtt or txt" });
+                        return;
+                    }
+                    const answer = await options.translations.get(id, language, who.id);
+                    if (answer.status !== 200 && answer.status !== 202) {
+                        json(response, answer.status, { error: answer.error });
+                        return;
+                    }
+                    if (answer.status === 202) {
+                        json(response, 202, {
+                            ...(answer.transcript ? wire(answer.transcript, await store.languages(id)) : { id, language, lines: [], languages: await store.languages(id) }),
+                            translating: answer.translating,
+                        });
+                        return;
+                    }
+                    const transcript = answer.transcript;
+                    if (format === "json") {
+                        json(response, 200, wire(transcript, await store.languages(id)));
+                        return;
+                    }
+                    const text = format === "srt" ? toSrt(transcript.lines) : format === "vtt" ? toVtt(transcript.lines) : toText(transcript.lines);
+                    const name = (transcript.title || id).replace(/[^\w.-]+/g, "_").slice(0, 60) || "transcript";
+                    response.writeHead(200, {
+                        ...CORS,
+                        "content-type": `${format === "vtt" ? "text/vtt" : "text/plain"}; charset=utf-8`,
+                        "content-length": Buffer.byteLength(text),
+                        "cache-control": "no-store",
+                        "content-disposition": `inline; filename="${name}${transcript.language ? `.${transcript.language}` : ""}.${format}"`,
+                    });
+                    response.end(text);
+                    return;
+                }
+                if (action === "lines" && request.method === "POST") {
+                    let body = {};
+                    try {
+                        body = JSON.parse(await readBody(request, 4 * 1024 * 1024));
+                    }
+                    catch {
+                        json(response, 400, { error: "bad JSON, or more than 4 MB of it" });
+                        return;
+                    }
+                    const media = typeof body.media === "string" ? body.media.trim() : "";
+                    if (media === "" || media.length > 2048 || !/^(file|url|live):/.test(media)) {
+                        json(response, 400, { error: "media is the identity: file:v1:<hash>, url:<address> or live:<server>/<channel>@<started>" });
+                        return;
+                    }
+                    if (transcriptIdOf(media) !== id) {
+                        json(response, 400, { error: "that media is not this transcript's id" });
+                        return;
+                    }
+                    const language = languageCode(body.language);
+                    const translatedFrom = body.translatedFrom === undefined || body.translatedFrom === null ? "" : languageCode(body.translatedFrom);
+                    if (language === null || translatedFrom === null) {
+                        json(response, 400, { error: "language and translatedFrom are two-letter codes" });
+                        return;
+                    }
+                    const lines = linesFrom(body.lines);
+                    if (lines.length === 0 && body.complete !== true) {
+                        json(response, 400, { error: "lines: a list of {start, end, text}, seconds into the media" });
+                        return;
+                    }
+                    const saved = await store.save({
+                        media,
+                        language,
+                        translatedFrom: translatedFrom === "" ? null : translatedFrom,
+                        model: typeof body.model === "string" ? body.model.slice(0, 100) : "",
+                        title: typeof body.title === "string" ? body.title : "",
+                        complete: body.complete === true,
+                        by: who.id,
+                        lines,
+                    });
+                    json(response, 200, { id, language: saved?.language ?? language, saved: lines.length, seconds: saved?.seconds ?? 0, complete: saved?.complete ?? false });
+                    return;
+                }
+                if (action === undefined && request.method === "DELETE") {
+                    const gone = await store.forget(id, who.id);
+                    json(response, gone ? 200 : 404, gone ? { ok: true } : { error: "nothing of yours under that id" });
+                    return;
+                }
+                json(response, 405, { error: "GET a transcript, POST its lines, or DELETE it" });
+            }
+            catch (error) {
+                if (error instanceof SpeechError)
+                    json(response, error.status, { error: error.message });
+                else
+                    throw error;
+            }
+            return;
+        }
+        /*
+         * MCP over HTTP: the tools `nixamp mcp` offers on a pipe, at
+         * nixamp.com/mcp for an agent with no nixamp installed. A JSON-RPC
+         * message, or a batch of them, in a POST; the answers as JSON. The
+         * bearer token says whose tools they are, as on every other route, and
+         * the tools call nixamp.com as that account. No stream is offered on
+         * GET: nothing here has to be pushed, so a client is told so and gets
+         * on with POST.
+         */
+        if (path === "/mcp" && options.accounts && options.site) {
+            if (request.method === "GET") {
+                response.writeHead(405, { ...CORS, allow: "POST, DELETE" });
+                response.end();
+                return;
+            }
+            if (request.method === "DELETE") {
+                response.writeHead(204, CORS);
+                response.end();
+                return;
+            }
+            if (request.method !== "POST") {
+                json(response, 405, { error: "POST a JSON-RPC message" });
+                return;
+            }
+            const token = tokenFrom(request.headers);
+            const who = await options.accounts.whoIs(token);
+            if (who === null) {
+                const error = JSON.stringify({ error: "sign in: a nixamp token as the bearer (`nixamp token create`), or OAuth" });
+                response.writeHead(401, {
+                    ...CORS,
+                    "content-type": "application/json; charset=utf-8",
+                    "content-length": Buffer.byteLength(error),
+                    "www-authenticate": `Bearer resource_metadata="${options.site}/.well-known/oauth-protected-resource"`,
+                });
+                response.end(error);
+                return;
+            }
+            let parsed;
+            try {
+                parsed = JSON.parse(await readBody(request, 1024 * 1024));
+            }
+            catch {
+                json(response, 400, { jsonrpc: "2.0", id: null, error: { code: -32700, message: "parse error" } });
+                return;
+            }
+            const messages = Array.isArray(parsed) ? parsed : [parsed];
+            const session = { site: options.site, token };
+            const answers = [];
+            for (const message of messages) {
+                if (!message || typeof message !== "object" || typeof message.method !== "string") {
+                    answers.push({ jsonrpc: "2.0", id: null, error: { code: -32600, message: "invalid request" } });
+                    continue;
+                }
+                const answer = await mcpMessage(message, { session, fetcher: fetch });
+                if (answer)
+                    answers.push(answer);
+            }
+            if (answers.length === 0) {
+                response.writeHead(202, CORS);
+                response.end();
+                return;
+            }
+            json(response, 200, Array.isArray(parsed) ? answers : answers[0]);
+            return;
+        }
+        /* Where a client of /mcp finds the authorization server (RFC 9728). */
+        if (path === "/.well-known/oauth-protected-resource" && options.site && options.authServer) {
+            json(response, 200, {
+                resource: options.site,
+                authorization_servers: [options.site],
+                bearer_methods_supported: ["header"],
+                scopes_supported: SCOPE_NAMES,
+                resource_name: "nixamp",
+            });
+            return;
+        }
         if (path === "/api/v1/me/handle" && options.handles && options.accounts) {
             const handles = options.handles;
             const who = await options.accounts.whoIs(tokenFrom(request.headers));
@@ -3537,11 +3829,17 @@ export function createHandler(engine, options) {
                 // reconnecting EventSource saw, which the browser sends by itself.
                 const lastSeen = request.headers["last-event-id"];
                 const after = Number(url.searchParams.get("after") ?? (Array.isArray(lastSeen) ? lastSeen[0] : lastSeen) ?? 0) || 0;
+                // The lines in another language, translated as they are heard; "" is what was said.
+                const wanted = languageCode(url.searchParams.get("language"));
+                if (wanted === null) {
+                    json(response, 400, { error: "language is a two-letter code, such as de or sv" });
+                    return;
+                }
                 if (action === "transcript") {
                     // Asking keeps the captioner up: it stops a minute after the last ask.
-                    captions.subscribe(id, () => undefined)?.();
+                    captions.subscribe(id, () => undefined, wanted)?.();
                     json(response, 200, {
-                        channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), ...captions.status(id), lines: captions.recent(id, after),
+                        channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted),
                     });
                     return;
                 }
@@ -3555,14 +3853,14 @@ export function createHandler(engine, options) {
                 const write = (event, data, eventId) => {
                     response.write(`${eventId === undefined ? "" : `id: ${eventId}\n`}event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
                 };
-                const off = captions.subscribe(id, (line) => write("line", line, line.at));
+                const off = captions.subscribe(id, (line) => write("line", line, line.at), wanted);
                 if (off === null) {
                     response.end();
                     return;
                 }
                 // How far behind the live edge a newcomer's playback starts, so the
                 // page can hold each line until its own sound gets there.
-                write("hello", { channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), ...captions.status(id), lines: captions.recent(id, after) });
+                write("hello", { channel: id, backlog: BACKLOG_SECONDS, now: Date.now(), wanted, ...captions.status(id), lines: captions.recent(id, after, wanted) });
                 const beat = setInterval(() => response.write(": beat\n\n"), 20_000);
                 beat.unref?.();
                 const done = () => {
@@ -4864,6 +5162,22 @@ export async function serve(argv, version = "0.1.0") {
             console.error(ready ? `nixamp: hearing with ${speech.model}` : `nixamp: not hearing: ${speech.lastFailure}`);
         });
     }
+    // Translation lives beside the ear: the same optional library, a model
+    // per pair of languages, loaded when first asked for. NIXAMP_MT_WARM
+    // names pairs to load at boot (en-de,en-sv) so the first line does not
+    // wait; NIXAMP_MT=off turns it off.
+    const translator = accounts && process.env["NIXAMP_MT"] !== "off" ? new Translator() : undefined;
+    if (translator) {
+        const warm = (process.env["NIXAMP_MT_WARM"] ?? "").split(",").map((one) => one.trim()).filter(Boolean);
+        if (warm.length > 0) {
+            void translator.warm(warm).then((ready) => {
+                console.error(ready ? `nixamp: translating (${warm.join(", ")} loaded)` : `nixamp: not translating everything: ${translator.lastFailure}`);
+            });
+        }
+    }
+    // What was heard, kept: where the accounts are, under the media's identity.
+    const transcripts = pool ? new Transcripts(pool, (message) => console.log(message)) : undefined;
+    const translations = transcripts ? new StoredTranslations(transcripts, translator, (message) => console.log(message)) : undefined;
     // nixamp as an OAuth 2.1 authorization server, and the watch parties a
     // client site bridges through it. Both need the same three things -- a
     // database, accounts, and a site to be the issuer of -- so both appear or
@@ -5051,6 +5365,33 @@ export async function serve(argv, version = "0.1.0") {
             listen: (id, listener) => channels.listen(id, listener),
             session: () => readSession(),
             onEvent: (message) => console.log(`  ${message}`),
+            // What a channel is playing, for the transcript store: a film by
+            // its bytes and where it has got to, a link by its address, and
+            // anything live as the broadcast it is.
+            mediaOf: (id) => {
+                const info = channels.info(id);
+                if (!info)
+                    return null;
+                const base = { title: info.name, startedAt: info.startedAt, backlog: info.codecs?.video ? BACKLOG_SECONDS : 3 };
+                if (info.via === "pull" && !info.live && typeof info.position === "number" && info.source) {
+                    if (/^https?:\/\//.test(info.source)) {
+                        // The pasted address while it is still known: it outlives a media URL with a token in it.
+                        let pasted = info.source;
+                        for (const [link] of links)
+                            if (linkChannelId(link) === id)
+                                pasted = link;
+                        return { ...base, media: mediaOfUrl(pasted), position: info.position };
+                    }
+                    try {
+                        return { ...base, media: fileFingerprint(info.source), position: info.position };
+                    }
+                    catch {
+                        return null;
+                    }
+                }
+                const self = publishable_?.url || options.publicUrl || `${options.host}:${options.port}`;
+                return { ...base, media: mediaOfLive(self, id, info.startedAt) };
+            },
         })
         : undefined;
     const owner = new Owner({
@@ -5254,6 +5595,10 @@ export async function serve(argv, version = "0.1.0") {
         ...(rooms ? { rooms } : {}),
         ...(trollbox ? { trollbox } : {}),
         ...(speech ? { speech } : {}),
+        ...(translator ? { translator } : {}),
+        ...(transcripts ? { transcripts } : {}),
+        ...(translations ? { translations } : {}),
+        site: nixampSite,
         // Other people's OpenProfiles, read for the voice a line is spoken in,
         // and the voices to speak in: ElevenLabs when the Telnyx account holds
         // the key for it (an integration secret named "elevenlabs"), Kokoro

package/dist/speech.d.ts

@@ -20,16 +20,31 @@ export declare class SpeechError extends Error {
     readonly status: number;
     constructor(message: string, status: number);
 }
+/** A stretch of the clip and what was said in it, seconds from the clip's start. */
+export interface Segment {
+    start: number;
+    end: number;
+    text: string;
+}
 export interface Heard {
     text: string;
     /** How long the audio was, in seconds. */
     seconds: number;
+    /** The language heard, as told or as guessed; absent when neither was possible. */
+    language?: string;
+    /** The clip in pieces with their timing, when asked for. */
+    segments?: Segment[];
 }
-/** Mono samples in [-1, 1] at RATE -> words. What the loaded model is, to this file. */
-export type Recognizer = (pcm: Float32Array, options: {
+export interface RecognizeOptions {
     language?: string;
-}) => Promise<{
+    /** Also say when each piece was said. Slower: the ear takes about twice as long. */
+    timestamps?: boolean;
+}
+/** Mono samples in [-1, 1] at RATE -> words. What the loaded model is, to this file. */
+export type Recognizer = (pcm: Float32Array, options: RecognizeOptions) => Promise<{
     text: string;
+    language?: string;
+    segments?: Segment[];
 }>;
 export interface SpeechOptions {
     /** A Hugging Face model id; NIXAMP_STT_MODEL otherwise; whisper-base by default. */
@@ -66,6 +81,8 @@ export declare function encodeWav(samples: Float32Array, rate?: number): Uint8Ar
 export declare function tidy(text: string): string;
 /** A two-letter language code, or nothing: Whisper guesses when not told. */
 export declare function languageOf(value: unknown): string | undefined;
+/** Load the library, or say why not with a status. */
+export declare function loadTransformers<T>(): Promise<T>;
 export declare class Speech {
     readonly model: string;
     private readonly cacheDir;
@@ -93,6 +110,7 @@ export declare class Speech {
      */
     transcribe(bytes: Uint8Array, options?: {
         language?: string;
+        timestamps?: boolean;
         by?: string;
     }): Promise<Heard>;
 }

package/dist/speech.js

@@ -196,13 +196,13 @@ export function languageOf(value) {
     const code = value.trim().toLowerCase();
     return /^[a-z]{2}$/.test(code) ? code : undefined;
 }
-async function loadWhisper(model, cacheDir) {
+/** Load the library, or say why not with a status. */
+export async function loadTransformers() {
     // By name held in a variable: an optional dependency that is not on disk
     // must fail here, at the first ask, and not when the file is imported.
     const name = "@huggingface/transformers";
-    let transformers;
     try {
-        transformers = (await import(__rewriteRelativeImportExtension(name)));
+        return (await import(__rewriteRelativeImportExtension(name)));
     }
     catch (error) {
         // The reason travels with the refusal: "not installed" and "installed
@@ -211,16 +211,62 @@ async function loadWhisper(model, cacheDir) {
         const why = String(error.message ?? error).split("\n")[0] ?? "";
         throw new SpeechError(`this nixamp cannot hear: @huggingface/transformers did not load here (${why}). nixamp.com can.`, 503);
     }
+}
+/**
+ * Which language a clip is in, the way whisper.cpp asks: the encoder once
+ * over the first thirty seconds, one decoder step from the start token,
+ * and the loudest of the language tokens wins. Whisper told nothing
+ * assumes English, and a Swedish sentence heard as English comes back as
+ * the same three English words repeated to the end of the clip; guessing
+ * first costs an encoder pass and is what makes a caption in Swedish say
+ * something. Undefined for a model that knows one language.
+ */
+async function detectLanguage(transformers, asr, pcm) {
+    const config = asr.model.generation_config;
+    const languages = config.lang_to_id;
+    if (!languages || config.is_multilingual === false)
+        return undefined;
+    const inputs = await asr.processor(pcm.subarray(0, 30 * RATE));
+    const start = new transformers.Tensor("int64", BigInt64Array.from([BigInt(config.decoder_start_token_id)]), [1, 1]);
+    const out = await asr.model({ input_features: inputs.input_features, decoder_input_ids: start });
+    const logits = out.logits.data;
+    let best;
+    let loudest = -Infinity;
+    for (const [token, id] of Object.entries(languages)) {
+        const score = logits[id];
+        if (typeof score === "number" && score > loudest) {
+            loudest = score;
+            best = token;
+        }
+    }
+    // The token is written <|sv|>.
+    const code = best?.replace(/^<\|/, "").replace(/\|>$/, "");
+    return code && /^[a-z]{2,3}$/.test(code) ? code : undefined;
+}
+async function loadWhisper(model, cacheDir) {
+    const transformers = await loadTransformers();
     transformers.env.cacheDir = cacheDir;
     const recognize = await transformers.pipeline("automatic-speech-recognition", model, { dtype: "q8" });
-    return async (pcm, { language }) => {
+    return async (pcm, { language, timestamps }) => {
+        const spoken = language ?? (await detectLanguage(transformers, recognize, pcm));
         const heard = await recognize(pcm, {
             // Whisper hears thirty seconds at a time; longer is heard in overlapping pieces.
             chunk_length_s: 30,
             stride_length_s: 5,
-            ...(language ? { language, task: "transcribe" } : {}),
+            ...(spoken ? { language: spoken, task: "transcribe" } : {}),
+            ...(timestamps ? { return_timestamps: true } : {}),
         });
-        return { text: Array.isArray(heard) ? heard.map((piece) => piece.text).join(" ") : heard.text };
+        const pieces = Array.isArray(heard) ? heard : [heard];
+        const text = pieces.map((piece) => piece.text).join(" ");
+        const seconds = pcm.length / RATE;
+        const segments = timestamps
+            ? pieces.flatMap((piece) => piece.chunks ?? []).map((chunk) => ({
+                start: Math.max(0, chunk.timestamp[0] ?? 0),
+                end: Math.min(seconds, chunk.timestamp[1] ?? seconds),
+                text: chunk.text,
+            }))
+            : undefined;
+        return { text, ...(spoken ? { language: spoken } : {}), ...(segments ? { segments } : {}) };
     };
 }
 export class Speech {
@@ -303,13 +349,24 @@ export class Speech {
         this.waiting += 1;
         const turn = this.tail.then(async () => {
             const recognize = await this.ear();
-            return recognize(pcm, options.language ? { language: options.language } : {});
+            return recognize(pcm, {
+                ...(options.language ? { language: options.language } : {}),
+                ...(options.timestamps ? { timestamps: true } : {}),
+            });
         });
         // The queue moves on whether or not this one was heard.
         this.tail = turn.catch(() => undefined);
         try {
             const heard = await turn;
-            return { text: tidy(heard.text), seconds };
+            const segments = heard.segments
+                ?.map((segment) => ({ start: segment.start, end: segment.end, text: tidy(segment.text) }))
+                .filter((segment) => segment.text !== "");
+            return {
+                text: tidy(heard.text),
+                seconds,
+                ...(heard.language ? { language: heard.language } : {}),
+                ...(segments ? { segments } : {}),
+            };
         }
         finally {
             this.waiting -= 1;