nixamp 0.25.1 → 0.26.0

package/dist/speech.d.ts

@@ -16,6 +16,8 @@ export declare const SECONDS_PER_MINUTE = 300;
 /** How many may wait for the one CPU. Past this, the honest answer is "later". */
 export declare const QUEUE_LIMIT = 8;
 export declare const DEFAULT_MODEL = "onnx-community/whisper-base";
+/** Stored live lines from older recognition settings must be heard again. */
+export declare const NATIVE_REVISION = "native-v2";
 export declare class SpeechError extends Error {
     readonly status: number;
     constructor(message: string, status: number);
@@ -79,10 +81,53 @@ export declare function resample(samples: Float32Array, from: number, to: number
 export declare function encodeWav(samples: Float32Array, rate?: number): Uint8Array;
 /** Whisper's own spacing and blank-audio tokens, tidied into a line. */
 export declare function tidy(text: string): string;
+/** Reject decoding loops, without removing normal emphasis like “no, no, no”. */
+export declare function reliableText(text: string, seconds: number): string;
+export declare function quietSamples(pcm: Float32Array): boolean;
 /** A two-letter language code, or nothing: Whisper guesses when not told. */
 export declare function languageOf(value: unknown): string | undefined;
+/** What the pipeline answers: the words, and the pieces with their timing when asked. */
+interface AsrOutput {
+    text: string;
+    chunks?: {
+        timestamp: [number, number | null];
+        text: string;
+    }[];
+}
+/** The pipeline, and the parts under it that language detection needs. */
+export interface AsrPipeline {
+    (audio: Float32Array, options: Record<string, unknown>): Promise<AsrOutput | AsrOutput[]>;
+    model: {
+        (inputs: Record<string, unknown>): Promise<{
+            logits: {
+                data: Float32Array | number[];
+            };
+        }>;
+        generation_config: {
+            decoder_start_token_id: number;
+            is_multilingual?: boolean;
+            lang_to_id?: Record<string, number>;
+        };
+    };
+    processor: (audio: Float32Array) => Promise<{
+        input_features: unknown;
+    }>;
+}
+/** The module, as much of it as this file touches. Typed here so the import can be by name. */
+export interface Transformers {
+    env: {
+        cacheDir?: string;
+        allowLocalModels?: boolean;
+    };
+    Tensor: new (type: string, data: BigInt64Array, dims: number[]) => unknown;
+    pipeline(task: "automatic-speech-recognition", model: string, options: {
+        dtype: string;
+    }): Promise<AsrPipeline>;
+}
 /** Load the library, or say why not with a status. */
 export declare function loadTransformers<T>(): Promise<T>;
+/** Detection is from each recording, never from the caption translation selection. */
+export declare function whisperRecognizer(transformers: Transformers, recognize: AsrPipeline): Recognizer;
 export declare class Speech {
     readonly model: string;
     private readonly cacheDir;
@@ -112,5 +157,7 @@ export declare class Speech {
         language?: string;
         timestamps?: boolean;
         by?: string;
+        deadline?: number;
     }): Promise<Heard>;
 }
+export {};

package/dist/speech.js

@@ -47,6 +47,8 @@ export const SECONDS_PER_MINUTE = 300;
 /** How many may wait for the one CPU. Past this, the honest answer is "later". */
 export const QUEUE_LIMIT = 8;
 export const DEFAULT_MODEL = "onnx-community/whisper-base";
+/** Stored live lines from older recognition settings must be heard again. */
+export const NATIVE_REVISION = "native-v2";
 export class SpeechError extends Error {
     status;
     constructor(message, status) {
@@ -189,6 +191,33 @@ export function encodeWav(samples, rate = RATE) {
 export function tidy(text) {
     return text.replace(/\[[A-Z_ ]+\]|\([A-Za-z ]+\)/g, " ").replace(/\s+/g, " ").trim();
 }
+/** Reject decoding loops, without removing normal emphasis like “no, no, no”. */
+export function reliableText(text, seconds) {
+    const clean = text.replace(/\s+/g, " ").trim();
+    if (clean.length > Math.max(120, seconds * 55))
+        return "";
+    const words = clean.toLowerCase().match(/[\p{L}\p{N}']+/gu) ?? [];
+    for (let size = 1; size <= Math.min(20, Math.floor(words.length / 4)); size++) {
+        for (let start = 0; start + size * 4 <= words.length; start++) {
+            let repeated = true;
+            for (let i = size; i < size * 4; i++) {
+                if (words[start + i] !== words[start + i % size]) {
+                    repeated = false;
+                    break;
+                }
+            }
+            if (repeated)
+                return "";
+        }
+    }
+    return clean;
+}
+export function quietSamples(pcm) {
+    let energy = 0;
+    for (const sample of pcm)
+        energy += sample * sample;
+    return pcm.length === 0 || Math.sqrt(energy / pcm.length) < 0.004;
+}
 /** A two-letter language code, or nothing: Whisper guesses when not told. */
 export function languageOf(value) {
     if (typeof value !== "string")
@@ -247,13 +276,25 @@ async function loadWhisper(model, cacheDir) {
     const transformers = await loadTransformers();
     transformers.env.cacheDir = cacheDir;
     const recognize = await transformers.pipeline("automatic-speech-recognition", model, { dtype: "q8" });
+    return whisperRecognizer(transformers, recognize);
+}
+/** Detection is from each recording, never from the caption translation selection. */
+export function whisperRecognizer(transformers, recognize) {
     return async (pcm, { language, timestamps }) => {
-        const spoken = language ?? (await detectLanguage(transformers, recognize, pcm));
+        const multilingual = recognize.model.generation_config.is_multilingual !== false;
+        const spoken = language ?? (multilingual ? await detectLanguage(transformers, recognize, pcm) : "en");
+        if (multilingual && !spoken)
+            throw new SpeechError("could not detect the audio language; try another speech segment", 422);
         const heard = await recognize(pcm, {
             // Whisper hears thirty seconds at a time; longer is heard in overlapping pieces.
             chunk_length_s: 30,
             stride_length_s: 5,
-            ...(spoken ? { language: spoken, task: "transcribe" } : {}),
+            ...(multilingual ? { language: spoken, task: "transcribe" } : {}),
+            // A five-second clip used to be allowed hundreds of tokens of hallucinated loops.
+            max_new_tokens: Math.min(384, Math.max(32, Math.ceil(Math.min(30, pcm.length / RATE) * 10) + 16)),
+            do_sample: false,
+            num_beams: 1,
+            no_repeat_ngram_size: 6,
             ...(timestamps ? { return_timestamps: true } : {}),
         });
         const pieces = Array.isArray(heard) ? heard : [heard];
@@ -344,10 +385,14 @@ export class Speech {
         if (wav.samples.length < wav.rate / 10)
             return { text: "", seconds };
         const pcm = resample(wav.samples, wav.rate, RATE);
+        if (quietSamples(pcm))
+            return { text: "", seconds };
         if (this.waiting >= QUEUE_LIMIT)
             throw new SpeechError("too many people are talking at once; try again in a moment", 503);
         this.waiting += 1;
         const turn = this.tail.then(async () => {
+            if (options.deadline && this.now() > options.deadline)
+                throw new SpeechError("this live audio is too old; waiting for the next segment", 408);
             const recognize = await this.ear();
             return recognize(pcm, {
                 ...(options.language ? { language: options.language } : {}),
@@ -359,10 +404,10 @@ export class Speech {
         try {
             const heard = await turn;
             const segments = heard.segments
-                ?.map((segment) => ({ start: segment.start, end: segment.end, text: tidy(segment.text) }))
+                ?.map((segment) => ({ start: segment.start, end: segment.end, text: reliableText(tidy(segment.text), segment.end - segment.start) }))
                 .filter((segment) => segment.text !== "");
             return {
-                text: tidy(heard.text),
+                text: reliableText(tidy(heard.text), seconds),
                 seconds,
                 ...(heard.language ? { language: heard.language } : {}),
                 ...(segments ? { segments } : {}),

package/dist/transcript-client.d.ts

@@ -45,7 +45,7 @@ export type Got<T> = {
     error: string;
 };
 /** The transcript of some media in a language ("" for the original), if the store has one. */
-export declare function fetchTranscript(signed: Signed, id: string, language?: string, fetcher?: typeof fetch): Promise<Got<StoredTranscript>>;
+export declare function fetchTranscript(signed: Signed, id: string, language?: string, fetcher?: typeof fetch, cachedOnly?: boolean): Promise<Got<StoredTranscript>>;
 export interface LinesToKeep {
     media: string;
     language: string;
@@ -68,7 +68,7 @@ export interface Translated {
     model: string;
 }
 /** Texts in another language. `from` may be "" when nixamp.com should tell. */
-export declare function translateTexts(signed: Signed, texts: string[], from: string, to: string, fetcher?: typeof fetch): Promise<Got<Translated>>;
+export declare function translateTexts(signed: Signed, texts: string[], from: string, to: string, fetcher?: typeof fetch, signal?: AbortSignal): Promise<Got<Translated>>;
 /** What a machine tells nixamp.com about a file it has, for the record at /hash/<id>. */
 export interface MediaToKeep {
     name?: string;

package/dist/transcript-client.js

@@ -11,10 +11,12 @@ function unreachable(site, error) {
     return { ok: false, status: 0, error: `could not reach ${site}: ${error.message}` };
 }
 /** The transcript of some media in a language ("" for the original), if the store has one. */
-export async function fetchTranscript(signed, id, language = "", fetcher = fetch) {
+export async function fetchTranscript(signed, id, language = "", fetcher = fetch, cachedOnly = false) {
     const url = new URL(`${base(signed.site)}/api/v1/transcripts/${encodeURIComponent(id)}`);
     if (language)
         url.searchParams.set("language", language);
+    if (cachedOnly)
+        url.searchParams.set("cached", "1");
     try {
         const response = await fetcher(url.toString(), { headers: { authorization: `Bearer ${signed.token}` } });
         return await asJson(response);
@@ -38,12 +40,13 @@ export async function keepLines(signed, id, ask, fetcher = fetch) {
     }
 }
 /** Texts in another language. `from` may be "" when nixamp.com should tell. */
-export async function translateTexts(signed, texts, from, to, fetcher = fetch) {
+export async function translateTexts(signed, texts, from, to, fetcher = fetch, signal) {
     try {
         const response = await fetcher(`${base(signed.site)}/api/v1/translate`, {
             method: "POST",
             headers: { authorization: `Bearer ${signed.token}`, "content-type": "application/json" },
             body: JSON.stringify({ texts, from, to }),
+            signal,
         });
         return await asJson(response);
     }

package/dist/transcripts.d.ts

@@ -4,6 +4,11 @@ export interface TranscriptLine {
     start: number;
     end: number;
     text: string;
+    /** Live transcripts may change language within one programme. */
+    language?: string;
+    revision?: string;
+    original?: string;
+    voiceProfile?: "lower" | "higher" | "unknown";
 }
 export type MediaKind = "file" | "url" | "live";
 export interface Transcript {

package/dist/transcripts.js

@@ -120,14 +120,19 @@ export function linesFrom(value) {
     for (const one of parsed) {
         if (!one || typeof one !== "object")
             continue;
-        const { start, end, text } = one;
+        const { start, end, text, language, revision, original, voiceProfile } = one;
         if (typeof text !== "string" || typeof start !== "number" || !Number.isFinite(start) || start < 0)
             continue;
         const words = text.replace(/\s+/g, " ").trim().slice(0, MAX_LINE_CHARS);
         if (words === "")
             continue;
         const until = typeof end === "number" && Number.isFinite(end) && end > start ? end : start;
-        lines.push({ start: round(start), end: round(until), text: words });
+        lines.push({ start: round(start), end: round(until), text: words,
+            ...(typeof language === "string" && /^[a-z]{2,3}$/.test(language) ? { language } : {}),
+            ...(typeof revision === "string" ? { revision: revision.slice(0, 32) } : {}),
+            ...(typeof original === "string" ? { original: original.slice(0, MAX_LINE_CHARS) } : {}),
+            ...(voiceProfile === "lower" || voiceProfile === "higher" || voiceProfile === "unknown" ? { voiceProfile } : {}),
+        });
     }
     return lines;
 }
@@ -306,7 +311,7 @@ export class Transcripts {
         await this.ensure();
         const { rows } = language === ""
             ? await this.db.query(`SELECT * FROM transcripts WHERE id = $1 AND translated_from IS NULL
-           ORDER BY complete DESC, updated_at DESC LIMIT 1`, [id])
+           ORDER BY (language = '') DESC, complete DESC, updated_at DESC LIMIT 1`, [id])
             : await this.db.query("SELECT * FROM transcripts WHERE id = $1 AND language = $2 LIMIT 1", [id, language]);
         const row = rows[0];
         return row ? rowToTranscript(row) : null;

package/dist/translate-jobs.js

@@ -46,11 +46,12 @@ export class StoredTranslations {
         const missing = StoredTranslations.missing(original, translation);
         if (translation && missing.length === 0)
             return { status: 200, transcript: translation };
-        if (original.language === "")
+        const sources = new Set(missing.map((line) => line.language || original.language));
+        if (sources.has(""))
             return { status: 409, error: "the language this was heard in is not known, so it cannot be translated" };
         if (!this.translator)
             return { status: 503, error: "this nixamp cannot translate: no model here. nixamp.com can." };
-        if (!this.translator.can(original.language, language)) {
+        if ([...sources].some((source) => !this.translator.can(source, language))) {
             return { status: 409, error: `there is no model here from ${original.language} to ${language}` };
         }
         const key = `${id}|${language}`;
@@ -95,19 +96,24 @@ export class StoredTranslations {
         let model = "";
         for (let at = 0; at < missing.length; at += BATCH) {
             const batch = missing.slice(at, at + BATCH);
-            const done = await translator.translate(batch.map((line) => line.text), original.language, language);
-            model = done.model;
-            const lines = batch.map((line, i) => ({ start: line.start, end: line.end, text: done.texts[i] ?? "" })).filter((line) => line.text !== "");
+            const lines = [];
+            for (const source of new Set(batch.map((line) => line.language || original.language))) {
+                const group = batch.filter((line) => (line.language || original.language) === source);
+                const done = await translator.translate(group.map((line) => line.text), source, language);
+                model = done.model;
+                lines.push(...group.map((line, i) => ({ ...line, language, original: line.text, text: done.texts[i] ?? "" })).filter((line) => line.text !== ""));
+            }
+            lines.sort((a, b) => a.start - b.start);
             made.push(...lines);
             await this.store.save({
-                media: original.media, language, translatedFrom: original.language, model, title: original.title, by, lines,
+                media: original.media, language, translatedFrom: original.language || "mul", model, title: original.title, by, lines,
             });
             job.progress.done += batch.length;
         }
         if (original.complete) {
             // Whole, like the original: the pieces are replaced by the lot, and nothing partial touches it again.
             await this.store.save({
-                media: original.media, language, translatedFrom: original.language, model, title: original.title, by, lines: made, complete: true,
+                media: original.media, language, translatedFrom: original.language || "mul", model, title: original.title, by, lines: made, complete: true,
             });
         }
         if (missing.length > 0)

package/dist/translate.d.ts

@@ -59,6 +59,7 @@ export declare class Translator {
     private readonly keep;
     private readonly pairs;
     private readonly loaded;
+    private readonly downloadCooldown;
     private tail;
     private waiting;
     private readonly asked;
@@ -82,5 +83,6 @@ export declare class Translator {
      */
     translate(texts: string[], from: string, to: string, options?: {
         by?: string;
+        deadline?: number;
     }): Promise<Translated>;
 }

package/dist/translate.js

@@ -126,6 +126,7 @@ export class Translator {
     keep;
     pairs;
     loaded = new Map();
+    downloadCooldown = new Map();
     tail = Promise.resolve();
     waiting = 0;
     asked = new Map();
@@ -147,6 +148,8 @@ export class Translator {
     }
     pair(from, to) {
         const model = modelFor(from, to);
+        if ((this.downloadCooldown.get(model) ?? 0) > this.now())
+            return Promise.reject(new SpeechError("the translation model download was rate limited; retry in a minute", 503));
         const held = this.loaded.get(model);
         if (held) {
             held.usedAt = this.now();
@@ -155,6 +158,8 @@ export class Translator {
         const pair = this.load(model, this.cacheDir).catch((error) => {
             // A failed load is tried again next time, not remembered forever.
             this.loaded.delete(model);
+            if (/\b429\b|rate.?limit/i.test(String(error)))
+                this.downloadCooldown.set(model, this.now() + 60_000);
             throw error;
         });
         this.loaded.set(model, { pair, usedAt: this.now() });
@@ -231,12 +236,16 @@ export class Translator {
             throw new SpeechError("too much is being translated at once; try again in a moment", 503);
         this.waiting += 1;
         const turn = this.tail.then(async () => {
+            const fresh = () => { if (options.deadline && this.now() > options.deadline)
+                throw new SpeechError("live translation expired while waiting; try the next line", 503); };
+            fresh();
             // Only the lines with something in them go to the model; the rest keep their place.
             const spoken = texts.map((text) => text.trim());
             const which = spoken.map((text, i) => (text === "" ? -1 : i)).filter((i) => i >= 0);
             let current = which.map((i) => spoken[i]);
             for (const [a, b] of hops) {
                 const pair = await this.pair(a, b);
+                fresh();
                 current = await pair.translate(current);
             }
             const out = [...spoken];

package/dist/voice-profile.d.ts

@@ -0,0 +1,7 @@
+/** Acoustic matching only: pitch does not establish a speaker's gender or identity. */
+export type VoiceProfile = "lower" | "higher" | "unknown";
+/** Median fundamental frequency from periodic voiced frames, sampled at 8 kHz.
+ * Ambiguous pitch, overlapping voices and unvoiced sound use the selected default.
+ * This is deliberately inexpensive and does not load another model beside ASR.
+ */
+export declare function voiceProfile(pcm: Buffer): VoiceProfile;

package/dist/voice-profile.js

@@ -0,0 +1,53 @@
+/** Median fundamental frequency from periodic voiced frames, sampled at 8 kHz.
+ * Ambiguous pitch, overlapping voices and unvoiced sound use the selected default.
+ * This is deliberately inexpensive and does not load another model beside ASR.
+ */
+export function voiceProfile(pcm) {
+    const pitches = [];
+    const frame = 320;
+    const samples = Math.floor(pcm.length / 4);
+    for (let start = 0; start + frame < samples; start += 1600) {
+        const values = new Float32Array(frame);
+        let mean = 0;
+        for (let i = 0; i < frame; i++)
+            mean += values[i] = pcm.readInt16LE((start + i) * 4) / 32768;
+        mean /= frame;
+        let energy = 0;
+        for (let i = 0; i < frame; i++) {
+            values[i] = values[i] - mean;
+            energy += values[i] ** 2;
+        }
+        if (Math.sqrt(energy / frame) < 0.01)
+            continue;
+        let best = 0;
+        let period = 0;
+        // Prefer the first strong peak; a later multiple of the period has the same correlation.
+        const correlations = [];
+        for (let lag = 20; lag <= 114; lag++) {
+            let cross = 0, left = 0, right = 0;
+            for (let i = 0; i < frame - lag; i++) {
+                const a = values[i], b = values[i + lag];
+                cross += a * b;
+                left += a * a;
+                right += b * b;
+            }
+            correlations[lag] = cross / Math.sqrt(left * right || 1);
+        }
+        for (let lag = 21; lag < 114; lag++) {
+            const score = correlations[lag];
+            if (score > 0.8 && score > correlations[lag - 1] && score >= correlations[lag + 1] && score > best + 0.03) {
+                best = score;
+                period = lag;
+            }
+        }
+        if (period)
+            pitches.push(8000 / period);
+    }
+    if (pitches.length < 3)
+        return "unknown";
+    pitches.sort((a, b) => a - b);
+    const median = pitches[Math.floor(pitches.length / 2)];
+    const lower = pitches.filter((pitch) => pitch < 155).length / pitches.length;
+    const higher = pitches.filter((pitch) => pitch > 185).length / pitches.length;
+    return median < 155 && lower >= 0.75 ? "lower" : median > 185 && higher >= 0.75 ? "higher" : "unknown";
+}

package/dist/warm.js

@@ -7,14 +7,14 @@
  * ask after each one, and the first person to speak waited for it. The
  * models land in NIXAMP_STT_CACHE and the image keeps them.
  *
- * NIXAMP_MT_WARM names the pairs (en-de,en-sv,de-en,sv-en by default):
- * German and Swedish both ways, which are the two that were asked for.
+ * NIXAMP_MT_WARM names the pairs: German and Swedish both ways with
+ * English, and Spanish both ways with English and German by default.
  * Exits non-zero when anything could not be fetched, so a build does not
  * quietly ship without its ear.
  */
 import { Speech } from "./speech.js";
 import { Translator } from "./translate.js";
-const DEFAULT_PAIRS = "en-de,en-sv,de-en,sv-en";
+const DEFAULT_PAIRS = "en-de,en-sv,de-en,sv-en,es-en,en-es,es-de,de-es";
 async function main() {
     const speech = new Speech();
     console.error(`warming ${speech.model}...`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nixamp",
-  "version": "0.25.1",
+  "version": "0.26.0",
   "description": "It really whips the terminal's ass. A Winamp-shaped audio player for your terminal.",
   "license": "MIT",
   "type": "module",