create-pmos 0.62.0 → 0.63.1

package/workspace/lib/notes-whisper.js

@@ -0,0 +1,262 @@
+'use strict';
+/* notes-whisper.js — local speech-to-text for the Notes view: whisper.cpp on the PM's machine (meeting-notes Run 2, D91).
+ *
+ *   detect(runtimeDir, { env }) → { ready, bin, model, missing, setup, model_dir }
+ *   setupCommand(runtimeDir)     → the shell commands the PM runs once: install whisper.cpp and download its model
+ *   createTranscriber({ runtimeDir, onText, run?, detect? }) → { push, flush, pending, stopAll }
+ *   wav(pcm, rate) · levels(pcm, rate) · cleanText(text)
+ *   getLanguage(runtimeDir) · setLanguage(runtimeDir, code)   the spoken language Whisper writes in (LANGUAGES)
+ *   getModel(runtimeDir) · setModel(runtimeDir, id)             the model: accurate (1.6 GB) or faster (about 550 MB) (MODELS)
+ *
+ * WHAT IT IS. The page captures the microphone and, when the PM picks it, a meeting tab's audio. It sends each source as
+ * 16 kHz mono 16-bit samples to the server on the loopback address. This module keeps a buffer per note and source, cuts it
+ * at a pause, writes the cut to a temporary WAV file, runs `whisper-cli` on it, deletes the file and hands the words to
+ * `onText`. One run at a time, in the order the cuts were made, so the transcript keeps the order things were said.
+ *
+ * WHAT IT NEVER DOES. It keeps no audio: a buffer is dropped once it is cut, and the WAV file is deleted when the program
+ * ends, whatever the outcome. It makes no request: the PM installs whisper.cpp and downloads the model in their own terminal
+ * (setupCommand), and this module only looks for the program and the model file. It opens no network module.
+ *
+ * CUTTING AT PAUSES. A 100 ms frame counts as speech when its loudness is well above the quiet level measured so far. A cut
+ * happens when at least MIN_CUT_MS is buffered and the last PAUSE_MS were quiet, or when MAX_CUT_MS is buffered, or when the
+ * page stops. A cut with less than MIN_VOICED_MS of speech is dropped without running the program, which also keeps
+ * Whisper from inventing words for silence.
+ */
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const crypto = require('crypto');
+const { execFile } = require('child_process');
+
+const RATE = 16000;
+const FRAME_MS = 100;
+const MIN_CUT_MS = 2000;
+const MAX_CUT_MS = 12000;
+const PAUSE_MS = 600;
+const MIN_VOICED_MS = 300;
+const LEAD_MS = 300;                 // speech that starts a cut keeps this much of the quiet before it
+const MIN_RMS = 180;                 // in 16-bit units; below this a frame is quiet whatever the room
+const MAX_QUEUE = 24;
+const RUN_TIMEOUT_MS = 120000;
+const MODEL_BASE = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/';
+// The faster model is the same one quantized to 5 bits: a third of the size, so it loads and runs faster on an M1, and it
+// makes a few more mistakes. Every cut starts the program again, so the time to load the model is paid on every line.
+const MODELS = Object.freeze([
+  Object.freeze({ id: 'accurate', file: 'ggml-large-v3-turbo.bin', size: '1.6 GB', label: 'Most accurate (1.6 GB)' }),
+  Object.freeze({ id: 'fast', file: 'ggml-large-v3-turbo-q5_0.bin', size: 'about 550 MB', label: 'Faster (about 550 MB)' }),
+]);
+const DEFAULT_MODEL = 'accurate';
+const MODEL_FILE = MODELS[0].file;
+const MODEL_URL = MODEL_BASE + MODEL_FILE;
+const BIN_NAMES = ['whisper-cli', 'whisper-cpp'];
+const BIN_DIRS = ['/opt/homebrew/bin', '/usr/local/bin'];
+
+const modelDir = (runtimeDir) => path.join(runtimeDir, 'notes', 'whisper');
+const isFile = (p) => { try { return fs.statSync(p).isFile(); } catch { return false; } };
+const isExec = (p) => { try { fs.accessSync(p, fs.constants.X_OK); return isFile(p); } catch { return false; } };
+const shq = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'";
+
+function findBin(env) {
+  if (env.PMOS_WHISPER_BIN) return isExec(env.PMOS_WHISPER_BIN) ? env.PMOS_WHISPER_BIN : null;
+  const dirs = String(env.PATH || '').split(path.delimiter).filter(Boolean).concat(BIN_DIRS);
+  for (const name of BIN_NAMES) for (const d of dirs) { const p = path.join(d, name); if (isExec(p)) return p; }
+  return null;
+}
+function findModel(runtimeDir, env) {
+  if (env.PMOS_WHISPER_MODEL) return isFile(env.PMOS_WHISPER_MODEL) ? env.PMOS_WHISPER_MODEL : null;
+  const dir = modelDir(runtimeDir);
+  const chosen = modelOf(runtimeDir);
+  if (isFile(path.join(dir, chosen.file))) return path.join(dir, chosen.file);
+  // a model the PM chose and has not downloaded yet is missing, so the setup downloads it; with no choice made, any
+  // whole model in the folder will do
+  if (readSettings(runtimeDir).model) return null;
+  let names = [];
+  try { names = fs.readdirSync(dir).filter((f) => /^ggml-.+\.bin$/.test(f)).sort(); } catch { /* none */ }
+  return names.length ? path.join(dir, names[names.length - 1]) : null;
+}
+
+/** The commands the PM runs once, in their own terminal. The server never runs them.
+ *
+ * Homebrew's own note after the install says that no model is downloaded; the next command is that download. It goes to a
+ * `.part` file that is renamed only when it is complete, so a stopped download never reads as ready, and `-C -` continues
+ * a stopped download when the PM runs the commands again. A program already installed is not installed again. */
+function setupCommand(runtimeDir) {
+  const dir = modelDir(runtimeDir);
+  const m = modelOf(runtimeDir);
+  const file = path.join(dir, m.file);
+  return '{ command -v whisper-cli >/dev/null 2>&1 || brew install whisper-cpp; }'
+    + ' && mkdir -p ' + shq(dir)
+    + ' && echo ' + shq('Downloading the Whisper model (' + m.size + '). Keep this terminal open.')
+    + ' && curl -L --fail -C - -# -o ' + shq(file + '.part') + ' ' + MODEL_BASE + m.file
+    + ' && mv ' + shq(file + '.part') + ' ' + shq(file)
+    + ' && echo ' + shq('Whisper is set up. Open the Notes settings again.');
+}
+
+/** Is local transcription ready? `missing` names what is not there: 'program', 'model'. */
+function detect(runtimeDir, { env = process.env } = {}) {
+  const bin = findBin(env);
+  const model = findModel(runtimeDir, env);
+  const missing = [];
+  if (!bin) missing.push('program');
+  if (!model) missing.push('model');
+  return { ready: missing.length === 0, bin, model, missing, setup: setupCommand(runtimeDir), model_dir: modelDir(runtimeDir), model_size: modelOf(runtimeDir).size };
+}
+
+/** A mono 16-bit PCM buffer as a WAV file. */
+function wav(pcm, rate = RATE) {
+  const h = Buffer.alloc(44);
+  h.write('RIFF', 0); h.writeUInt32LE(36 + pcm.length, 4); h.write('WAVE', 8);
+  h.write('fmt ', 12); h.writeUInt32LE(16, 16); h.writeUInt16LE(1, 20); h.writeUInt16LE(1, 22);
+  h.writeUInt32LE(rate, 24); h.writeUInt32LE(rate * 2, 28); h.writeUInt16LE(2, 32); h.writeUInt16LE(16, 34);
+  h.write('data', 36); h.writeUInt32LE(pcm.length, 40);
+  return Buffer.concat([h, pcm]);
+}
+/** The loudness (RMS, 16-bit units) of each 100 ms frame. */
+function levels(pcm, rate = RATE) {
+  const per = Math.round(rate * FRAME_MS / 1000);
+  const n = Math.floor(pcm.length / 2);
+  const out = [];
+  for (let i = 0; i < n; i += per) {
+    const end = Math.min(n, i + per); let sum = 0;
+    for (let k = i; k < end; k++) { const v = pcm.readInt16LE(k * 2); sum += v * v; }
+    out.push(Math.sqrt(sum / Math.max(1, end - i)));
+  }
+  return out;
+}
+// Whisper marks what is not speech in brackets, and on near-silence it is known to produce stock closing lines
+const NOT_SPEECH = /\[[^\]]*\]|\([^)]*\)|[♪♫]+|>>/g;
+const STOCK = /^(?:thanks? (?:you )?(?:so much )?for watching[.!]?|please subscribe[.!]?|subtitles? by .*|transcribed by .*)$/i;
+/** The words of a run, without the non-speech marks. Empty when nothing was said. */
+function cleanText(text) {
+  const t = String(text || '').split(/\r?\n/).map((l) => l.replace(/^\s*\[\d[\d:.]*\s*-->\s*[\d:.]*\]\s*/, '')).join(' ')
+    .replace(NOT_SPEECH, ' ').replace(/\s+/g, ' ').trim();
+  if (!/[\p{L}\p{N}]/u.test(t) || STOCK.test(t)) return '';
+  return t;
+}
+
+/* THE SPOKEN LANGUAGE. Whisper writes in the language it is told. With 'auto' it guesses the language of each cut, and on a
+ * few seconds of accented speech it can guess wrong and write the line in that language, which reads as a translation. So
+ * the PM picks the language of the meeting once (English by default) and every cut uses it. PMOS_WHISPER_LANG wins. */
+const LANGUAGES = Object.freeze([['en', 'English'], ['id', 'Indonesian'], ['auto', 'Detect for each line']]);
+const DEFAULT_LANGUAGE = 'en';
+const settingsFile = (runtimeDir) => path.join(runtimeDir, 'notes', 'whisper-settings.json');
+function readSettings(runtimeDir) {
+  try { const s = JSON.parse(fs.readFileSync(settingsFile(runtimeDir), 'utf8')); return s && typeof s === 'object' ? s : {}; } catch { return {}; }
+}
+function writeSettings(runtimeDir, patch) {
+  const f = settingsFile(runtimeDir);
+  fs.mkdirSync(path.dirname(f), { recursive: true });
+  const tmp = f + '.' + process.pid + '.tmp';
+  fs.writeFileSync(tmp, JSON.stringify({ ...readSettings(runtimeDir), ...patch }), { mode: 0o600 });
+  fs.renameSync(tmp, f);
+}
+function getLanguage(runtimeDir, { env = process.env } = {}) {
+  if (env.PMOS_WHISPER_LANG) return env.PMOS_WHISPER_LANG;
+  const c = readSettings(runtimeDir).language;
+  return LANGUAGES.some(([k]) => k === c) ? c : DEFAULT_LANGUAGE;
+}
+function setLanguage(runtimeDir, code) {
+  if (!LANGUAGES.some(([k]) => k === code)) return { ok: false, error: 'the language must be one of ' + LANGUAGES.map(([k]) => k).join(', ') };
+  writeSettings(runtimeDir, { language: code });
+  return { ok: true, language: code };
+}
+function modelOf(runtimeDir) { const id = readSettings(runtimeDir).model; return MODELS.find((m) => m.id === id) || MODELS.find((m) => m.id === DEFAULT_MODEL); }
+function getModel(runtimeDir) { return modelOf(runtimeDir).id; }
+function setModel(runtimeDir, id) {
+  if (!MODELS.some((m) => m.id === id)) return { ok: false, error: 'the model must be one of ' + MODELS.map((m) => m.id).join(', ') };
+  writeSettings(runtimeDir, { model: id });
+  return { ok: true, model: id };
+}
+
+/** Run the program on one WAV file. Resolves { ok, text } or { ok: false, error }. */
+function runWhisper({ bin, model, file, lang = DEFAULT_LANGUAGE }) {
+  // The GPU does most of the work on Apple silicon, so four threads are enough; more only compete with the browser.
+  // Greedy decoding (-bs 1 -bo 1) instead of the program's default five-way beam search makes each line several times
+  // faster for a small loss on hard words.
+  const threads = String(Math.max(2, Math.min(4, (os.cpus() || []).length || 4)));
+  return new Promise((resolve) => {
+    execFile(bin, ['-m', model, '-f', file, '-nt', '-np', '-l', lang, '-t', threads, '-bs', '1', '-bo', '1'], { timeout: RUN_TIMEOUT_MS, maxBuffer: 1 << 20 }, (e, stdout, stderr) => {
+      if (e) resolve({ ok: false, error: e.killed ? 'the transcription took too long' : String((stderr || e.message || 'the program failed')).trim().split('\n').pop().slice(0, 200) });
+      else resolve({ ok: true, text: String(stdout || '') });
+    });
+  });
+}
+
+/**
+ * The buffers and the one queue.
+ * @param {object} a
+ * @param {string} a.runtimeDir
+ * @param {(cut:{id,source,t,text,ms}) => void} a.onText   called with the words of each cut, in order
+ * @param {(cut:{id,source,error}) => void} [a.onError]
+ * @param {Function} [a.run]     runWhisper, replaceable in the suites
+ * @param {Function} [a.detect]  detect, replaceable in the suites
+ */
+function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWhisper, detect: det = () => detect(runtimeDir) }) {
+  const bufs = new Map();     // `${id}|${source}` → { id, source, chunks, ms, voicedMs, quietMs, t, noise }
+  const noiseOf = new Map();  // source → the quiet level last measured, so a new cut starts from it
+  let queue = Promise.resolve();
+  let waiting = 0;
+  const tmpDir = path.join(runtimeDir, 'notes', 'audio');
+  // a cut left by a process that ended while the program ran is deleted when the next transcriber starts
+  try { for (const f of fs.readdirSync(tmpDir)) if (/^cut-[0-9a-f]+\.wav$/.test(f)) fs.unlinkSync(path.join(tmpDir, f)); } catch { /* no folder yet */ }
+
+  function cut(b) {
+    bufs.delete(b.id + '|' + b.source);
+    if (b.voicedMs < MIN_VOICED_MS || !b.chunks.length) return false;
+    if (waiting >= MAX_QUEUE) { onError({ id: b.id, source: b.source, error: 'transcription is falling behind, so a few seconds were skipped' }); return false; }
+    const pcm = Buffer.concat(b.chunks);
+    const job = { id: b.id, source: b.source, t: b.t, ms: b.ms };
+    waiting++;
+    queue = queue.then(async () => {
+      const d = det();
+      if (!d.ready) { onError({ ...job, error: 'local transcription is not set up' }); return; }
+      fs.mkdirSync(tmpDir, { recursive: true });
+      const file = path.join(tmpDir, 'cut-' + crypto.randomBytes(6).toString('hex') + '.wav');
+      try {
+        fs.writeFileSync(file, wav(pcm), { mode: 0o600 });
+        const r = await run({ bin: d.bin, model: d.model, file, lang: getLanguage(runtimeDir) });
+        if (!r.ok) { onError({ ...job, error: r.error }); return; }
+        const text = cleanText(r.text);
+        if (text) await onText({ ...job, text });
+      } catch (e) { onError({ ...job, error: String((e && e.message) || e) }); }
+      finally { try { fs.unlinkSync(file); } catch { /* already gone */ } }
+    }).catch(() => {}).finally(() => { waiting--; });
+    return true;
+  }
+
+  /**
+   * One chunk from the page. `t` is the chunk's start, in ms from the note's recording start.
+   * @returns {{ ok:true, cut:boolean }}
+   */
+  function push({ id, source, pcm, t = 0, final = false }) {
+    const k = id + '|' + source;
+    let b = bufs.get(k);
+    const lv = pcm && pcm.length ? levels(pcm) : [];
+    if (!b) { b = { id, source, chunks: [], ms: 0, voicedMs: 0, quietMs: 0, t, noise: noiseOf.has(source) ? noiseOf.get(source) : null }; bufs.set(k, b); }
+    for (const v of lv) {
+      // the quiet level is learnt from quiet frames only, so speech at the very start is not taken for the room's noise
+      const voiced = v > Math.max(MIN_RMS, (b.noise == null ? 0 : b.noise) * 3);
+      if (voiced) { b.voicedMs += FRAME_MS; b.quietMs = 0; }
+      else { b.quietMs += FRAME_MS; b.noise = b.noise == null ? v : b.noise * 0.9 + v * 0.1; }
+    }
+    if (pcm && pcm.length) { b.chunks.push(pcm); b.ms += (pcm.length / 2 / RATE) * 1000; }
+    if (b.noise != null) noiseOf.set(source, b.noise);
+    // quiet before anyone speaks is not kept: only the last LEAD_MS of it stays in front of the speech
+    if (!b.voicedMs && b.ms > LEAD_MS + 1000) {
+      const keepBytes = Math.round(RATE * LEAD_MS / 1000) * 2;
+      const all = Buffer.concat(b.chunks);
+      b.chunks = [all.subarray(Math.max(0, all.length - keepBytes))];
+      b.t += b.ms - LEAD_MS; b.ms = LEAD_MS;
+    }
+    let did = false;
+    if (final || b.ms >= MAX_CUT_MS || (b.ms >= MIN_CUT_MS && b.voicedMs >= MIN_VOICED_MS && b.quietMs >= PAUSE_MS)) did = cut(b);
+    return { ok: true, cut: did };
+  }
+  /** Cut whatever is buffered for a note (the page stopped). */
+  function flush(id) { for (const b of Array.from(bufs.values())) if (b.id === id) cut(b); }
+  /** Drop a note's buffers without transcribing them (the note was marked private or deleted). */
+  function drop(id) { for (const [k, b] of Array.from(bufs.entries())) if (b.id === id) bufs.delete(k); }
+  return { push, flush, drop, pending: () => waiting, idle: () => queue };
+}
+
+module.exports = { detect, setupCommand, getLanguage, setLanguage, LANGUAGES, DEFAULT_LANGUAGE, getModel, setModel, MODELS, DEFAULT_MODEL, createTranscriber, runWhisper, wav, levels, cleanText, modelDir, RATE, MODEL_FILE, MODEL_URL, MIN_CUT_MS, MAX_CUT_MS, PAUSE_MS };