create-pmos 0.74.0 → 0.76.0

package/workspace/lib/notes-whisper.js

@@ -3,10 +3,11 @@
  *
  *   detect(runtimeDir, { env }) → { ready, bin, model, missing, setup, model_dir }
  *   setupCommand(runtimeDir)     → the shell commands the PM runs once: install whisper.cpp and download its model
- *   createTranscriber({ runtimeDir, onText, run?, detect? }) → { push, flush, pending, stopAll }
- *   wav(pcm, rate) · levels(pcm, rate) · cleanText(text)
+ *   createTranscriber({ runtimeDir, onText, run?, detect?, vocab?, prompt? }) → { push, flush, drop, pending, idle }
+ *   wav(pcm, rate) · levels(pcm, rate) · cleanText(text, { voicedRatio }) · promptFor(vocab, previous)
  *   getLanguage(runtimeDir) · setLanguage(runtimeDir, code)   the spoken language Whisper writes in (LANGUAGES)
  *   getModel(runtimeDir) · setModel(runtimeDir, id)             the model: accurate (1.6 GB) or faster (about 550 MB) (MODELS)
+ *   getAccuracy(runtimeDir) · setAccuracy(runtimeDir, id)       the decoding: fast (greedy, default) or accurate (beam search) (ACCURACIES)
  *
  * WHAT IT IS. The page captures the microphone and, when the PM picks it, a meeting tab's audio. It sends each source as
  * 16 kHz mono 16-bit samples to the server on the loopback address. This module keeps a buffer per note and source, cuts it
@@ -20,7 +21,17 @@
  * CUTTING AT PAUSES. A 100 ms frame counts as speech when its loudness is well above the quiet level measured so far. A cut
  * happens when at least MIN_CUT_MS is buffered and the last PAUSE_MS were quiet, or when MAX_CUT_MS is buffered, or when the
  * page stops. A cut with less than MIN_VOICED_MS of speech is dropped without running the program, which also keeps
- * Whisper from inventing words for silence.
+ * Whisper from inventing words for silence. A pause of LONG_PAUSE_MS also cuts a short phrase, so it is not held back.
+ * The cuts are long (a pause of 1 s, at least 4 s) so that a cut holds a whole sentence more often than a fragment;
+ * lib/notes.js joins a cut that still ends mid-sentence to the one before it.
+ *
+ * VOCABULARY AND CONTEXT. Each run gets `--prompt` with the names and terms of the repository (the `vocab` callback, or the
+ * `prompt` string) and the last line heard from the same source, at most PROMPT_MAX characters. Whisper then spells the
+ * repository's people and product terms as the repository does, and reads the cut as the continuation of the line before.
+ *
+ * INVENTED LINES. whisper.cpp's no-speech threshold (`-nth`) drops segments it thinks are silence. A cut that is mostly
+ * quiet and comes back as only "Thank you." or "you" is a known invention and is dropped (cleanText). A program too old
+ * for `-nth` is run again without it, and is remembered.
  */
 const fs = require('fs');
 const os = require('os');
@@ -30,9 +41,13 @@ const { execFile } = require('child_process');
 
 const RATE = 16000;
 const FRAME_MS = 100;
-const MIN_CUT_MS = 2000;
+const MIN_CUT_MS = 4000;
 const MAX_CUT_MS = 12000;
-const PAUSE_MS = 600;
+const PAUSE_MS = 1000;
+const LONG_PAUSE_MS = 2000;          // a pause this long cuts whatever was said, however short
+const PROMPT_MAX = 200;              // characters of --prompt: the vocabulary first, then the end of the line before
+const NO_SPEECH_THOLD = '0.6';
+const LOW_VOICED_RATIO = 0.4;        // a cut with less speech than this, as a share of its length, is mostly quiet
 const MIN_VOICED_MS = 300;
 const LEAD_MS = 300;                 // speech that starts a cut keeps this much of the quiet before it
 const MIN_RMS = 180;                 // in 16-bit units; below this a frame is quiet whatever the room
@@ -126,13 +141,33 @@ function levels(pcm, rate = RATE) {
 // Whisper marks what is not speech in brackets, and on near-silence it is known to produce stock closing lines
 const NOT_SPEECH = /\[[^\]]*\]|\([^)]*\)|[♪♫]+|>>/g;
 const STOCK = /^(?:thanks? (?:you )?(?:so much )?for watching[.!]?|please subscribe[.!]?|subtitles? by .*|transcribed by .*)$/i;
-/** The words of a run, without the non-speech marks. Empty when nothing was said. */
-function cleanText(text) {
+// what Whisper writes for a mostly quiet cut: a lone thanks, "you", "bye", "okay"; dropped only when the cut was mostly quiet
+const QUIET_STOCK = /^(?:(?:thank you|thanks)(?: (?:so much|very much|everyone|all))?|you|bye(?: bye)?|okay|ok|oh|uh|um|hmm|so)(?:[.!?,…]+(?:\s+(?:(?:thank you|thanks)(?: (?:so much|very much))?|you|bye|okay))?)*[.!?…]*$/i;
+/** The words of a run, without the non-speech marks. Empty when nothing was said, or when a mostly quiet cut
+ * (`voicedRatio` under LOW_VOICED_RATIO) came back as only a stock line such as "Thank you." */
+function cleanText(text, { voicedRatio = 1 } = {}) {
   const t = String(text || '').split(/\r?\n/).map((l) => l.replace(/^\s*\[\d[\d:.]*\s*-->\s*[\d:.]*\]\s*/, '')).join(' ')
     .replace(NOT_SPEECH, ' ').replace(/\s+/g, ' ').trim();
   if (!/[\p{L}\p{N}]/u.test(t) || STOCK.test(t)) return '';
+  if (voicedRatio < LOW_VOICED_RATIO && QUIET_STOCK.test(t)) return '';
   return t;
 }
+/** The --prompt for one run: the vocabulary, then the end of the previous line, at most PROMPT_MAX characters. '' when
+ * there is neither. The vocabulary keeps at most two thirds of the room when there is a previous line. */
+function promptFor(vocab, previous) {
+  const v = String(vocab || '').replace(/\s+/g, ' ').trim().replace(/[.,;:\s]+$/, '');
+  const p = String(previous || '').replace(/\s+/g, ' ').trim();
+  const cutAt = (s, n, fromEnd) => {
+    if (s.length <= n) return s;
+    if (fromEnd) { const x = s.slice(s.length - n); const sp = x.indexOf(' '); return (sp > 0 && sp < 30 ? x.slice(sp + 1) : x).trim(); }
+    const x = s.slice(0, n); const c = Math.max(x.lastIndexOf(','), x.lastIndexOf(' ')); return (c > n / 2 ? x.slice(0, c) : x).replace(/[.,;:\s]+$/, '');
+  };
+  if (!p) return cutAt(v, PROMPT_MAX, false);
+  if (!v) return cutAt(p, PROMPT_MAX, true);
+  const vv = cutAt(v, Math.min(v.length, Math.floor(PROMPT_MAX * 2 / 3)), false);
+  const room = PROMPT_MAX - vv.length - 2;
+  return room > 10 ? vv + '. ' + cutAt(p, room, true) : vv;
+}
 
 /* THE SPOKEN LANGUAGE. Whisper writes in the language it is told. With 'auto' it guesses the language of each cut, and on a
  * few seconds of accented speech it can guess wrong and write the line in that language, which reads as a translation. So
@@ -160,6 +195,19 @@ function setLanguage(runtimeDir, code) {
   writeSettings(runtimeDir, { language: code });
   return { ok: true, language: code };
 }
+/* THE DECODING. Greedy (fast, the default) or a five-way beam search (accurate): slower on every line, better on hard words. */
+const ACCURACIES = Object.freeze([
+  Object.freeze({ id: 'fast', label: 'Faster (greedy decoding)', args: ['-bs', '1', '-bo', '1'] }),
+  Object.freeze({ id: 'accurate', label: 'More accurate (beam search, slower)', args: ['-bs', '5', '-bo', '5'] }),
+]);
+const DEFAULT_ACCURACY = 'fast';
+function accuracyOf(runtimeDir) { const id = readSettings(runtimeDir).accuracy; return ACCURACIES.find((a) => a.id === id) || ACCURACIES.find((a) => a.id === DEFAULT_ACCURACY); }
+function getAccuracy(runtimeDir) { return accuracyOf(runtimeDir).id; }
+function setAccuracy(runtimeDir, id) {
+  if (!ACCURACIES.some((a) => a.id === id)) return { ok: false, error: 'the accuracy must be one of ' + ACCURACIES.map((a) => a.id).join(', ') };
+  writeSettings(runtimeDir, { accuracy: id });
+  return { ok: true, accuracy: id };
+}
 function modelOf(runtimeDir) { const id = readSettings(runtimeDir).model; return MODELS.find((m) => m.id === id) || MODELS.find((m) => m.id === DEFAULT_MODEL); }
 function getModel(runtimeDir) { return modelOf(runtimeDir).id; }
 function setModel(runtimeDir, id) {
@@ -168,18 +216,32 @@ function setModel(runtimeDir, id) {
   return { ok: true, model: id };
 }
 
-/** Run the program on one WAV file. Resolves { ok, text } or { ok: false, error }. */
-function runWhisper({ bin, model, file, lang = DEFAULT_LANGUAGE }) {
+const noNth = new Set();   // programs too old for -nth, found when they refused it
+/** Run the program on one WAV file. Resolves { ok, text } or { ok: false, error }.
+ * `accuracy` is an ACCURACIES id; `prompt` is the --prompt text, left out when empty. */
+function runWhisper({ bin, model, file, lang = DEFAULT_LANGUAGE, accuracy = DEFAULT_ACCURACY, prompt = '' }) {
   // The GPU does most of the work on Apple silicon, so four threads are enough; more only compete with the browser.
   // Greedy decoding (-bs 1 -bo 1) instead of the program's default five-way beam search makes each line several times
-  // faster for a small loss on hard words.
+  // faster for a small loss on hard words; the accurate setting uses the beam search.
   const threads = String(Math.max(2, Math.min(4, (os.cpus() || []).length || 4)));
-  return new Promise((resolve) => {
-    execFile(bin, ['-m', model, '-f', file, '-nt', '-np', '-l', lang, '-t', threads, '-bs', '1', '-bo', '1'], { timeout: RUN_TIMEOUT_MS, maxBuffer: 1 << 20 }, (e, stdout, stderr) => {
-      if (e) resolve({ ok: false, error: e.killed ? 'the transcription took too long' : String((stderr || e.message || 'the program failed')).trim().split('\n').pop().slice(0, 200) });
-      else resolve({ ok: true, text: String(stdout || '') });
-    });
+  const dec = (ACCURACIES.find((a) => a.id === accuracy) || ACCURACIES[0]).args;
+  const base = ['-m', model, '-f', file, '-nt', '-np', '-l', lang, '-t', threads, ...dec];
+  const extra = prompt ? ['--prompt', String(prompt)] : [];
+  const once = (args) => new Promise((resolve) => {
+    execFile(bin, args, { timeout: RUN_TIMEOUT_MS, maxBuffer: 1 << 20 }, (e, stdout, stderr) => resolve({ e, stdout: String(stdout || ''), stderr: String(stderr || '') }));
   });
+  const done = ({ e, stdout, stderr }) => (e
+    ? { ok: false, error: e.killed ? 'the transcription took too long' : String((stderr || e.message || 'the program failed')).trim().split('\n').pop().slice(0, 200) }
+    : { ok: true, text: stdout });
+  const refused = (r) => /unknown argument|unrecognized option|invalid option/i.test(r.stderr) && /-nth|no-speech/i.test(r.stderr);
+  return (async () => {
+    if (!noNth.has(bin)) {
+      const r = await once([...base, '-nth', NO_SPEECH_THOLD, ...extra]);
+      if (!refused(r)) return done(r);
+      noNth.add(bin);
+    }
+    return done(await once([...base, ...extra]));
+  })();
 }
 
 /**
@@ -191,8 +253,10 @@ function runWhisper({ bin, model, file, lang = DEFAULT_LANGUAGE }) {
  * @param {Function} [a.run]     runWhisper, replaceable in the suites
  * @param {Function} [a.detect]  detect, replaceable in the suites
  */
-function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWhisper, detect: det = () => detect(runtimeDir) }) {
+function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWhisper, detect: det = () => detect(runtimeDir), vocab = null, prompt = '' }) {
   const bufs = new Map();     // `${id}|${source}` → { id, source, chunks, ms, voicedMs, quietMs, t, noise }
+  const lastText = new Map(); // `${id}|${source}` → the last words heard there, the context of the next cut
+  const vocabOf = (job) => { if (typeof vocab !== 'function') return prompt || ''; try { return String(vocab({ id: job.id, source: job.source }) || ''); } catch { return prompt || ''; } };
   const noiseOf = new Map();  // source → the quiet level last measured, so a new cut starts from it
   let queue = Promise.resolve();
   let waiting = 0;
@@ -206,6 +270,7 @@ function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWh
     if (waiting >= MAX_QUEUE) { onError({ id: b.id, source: b.source, error: 'transcription is falling behind, so a few seconds were skipped' }); return false; }
     const pcm = Buffer.concat(b.chunks);
     const job = { id: b.id, source: b.source, t: b.t, ms: b.ms };
+    const voicedRatio = b.ms > 0 ? b.voicedMs / b.ms : 1;
     waiting++;
     queue = queue.then(async () => {
       const d = det();
@@ -214,10 +279,11 @@ function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWh
       const file = path.join(tmpDir, 'cut-' + crypto.randomBytes(6).toString('hex') + '.wav');
       try {
         fs.writeFileSync(file, wav(pcm), { mode: 0o600 });
-        const r = await run({ bin: d.bin, model: d.model, file, lang: getLanguage(runtimeDir) });
+        const key = job.id + '|' + job.source;
+        const r = await run({ bin: d.bin, model: d.model, file, lang: getLanguage(runtimeDir), accuracy: getAccuracy(runtimeDir), prompt: promptFor(vocabOf(job), lastText.get(key)) });
         if (!r.ok) { onError({ ...job, error: r.error }); return; }
-        const text = cleanText(r.text);
-        if (text) await onText({ ...job, text });
+        const text = cleanText(r.text, { voicedRatio });
+        if (text) { lastText.set(key, text); await onText({ ...job, text }); }
       } catch (e) { onError({ ...job, error: String((e && e.message) || e) }); }
       finally { try { fs.unlinkSync(file); } catch { /* already gone */ } }
     }).catch(() => {}).finally(() => { waiting--; });
@@ -249,14 +315,20 @@ function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWh
       b.t += b.ms - LEAD_MS; b.ms = LEAD_MS;
     }
     let did = false;
-    if (final || b.ms >= MAX_CUT_MS || (b.ms >= MIN_CUT_MS && b.voicedMs >= MIN_VOICED_MS && b.quietMs >= PAUSE_MS)) did = cut(b);
+    const spoke = b.voicedMs >= MIN_VOICED_MS;
+    if (final || b.ms >= MAX_CUT_MS || (spoke && b.ms >= MIN_CUT_MS && b.quietMs >= PAUSE_MS) || (spoke && b.quietMs >= LONG_PAUSE_MS)) did = cut(b);
     return { ok: true, cut: did };
   }
   /** Cut whatever is buffered for a note (the page stopped). */
   function flush(id) { for (const b of Array.from(bufs.values())) if (b.id === id) cut(b); }
   /** Drop a note's buffers without transcribing them (the note was marked private or deleted). */
-  function drop(id) { for (const [k, b] of Array.from(bufs.entries())) if (b.id === id) bufs.delete(k); }
+  function drop(id) {
+    for (const [k, b] of Array.from(bufs.entries())) if (b.id === id) bufs.delete(k);
+    for (const k of Array.from(lastText.keys())) if (k.startsWith(id + '|')) lastText.delete(k);
+  }
   return { push, flush, drop, pending: () => waiting, idle: () => queue };
 }
 
-module.exports = { detect, setupCommand, getLanguage, setLanguage, LANGUAGES, DEFAULT_LANGUAGE, getModel, setModel, MODELS, DEFAULT_MODEL, createTranscriber, runWhisper, wav, levels, cleanText, modelDir, RATE, MODEL_FILE, MODEL_URL, MIN_CUT_MS, MAX_CUT_MS, PAUSE_MS };
+module.exports = { detect, setupCommand, getLanguage, setLanguage, LANGUAGES, DEFAULT_LANGUAGE, getModel, setModel, MODELS, DEFAULT_MODEL,
+  getAccuracy, setAccuracy, ACCURACIES, DEFAULT_ACCURACY, createTranscriber, runWhisper, wav, levels, cleanText, promptFor, modelDir, RATE, MODEL_FILE, MODEL_URL,
+  MIN_CUT_MS, MAX_CUT_MS, PAUSE_MS, LONG_PAUSE_MS, PROMPT_MAX, LOW_VOICED_RATIO, NO_SPEECH_THOLD };