create-pmos 0.76.0 → 0.78.0

package/workspace/lib/notes-final.js

@@ -0,0 +1,293 @@
+'use strict';
+/* notes-final.js — the final pass after Stop: the whole meeting transcribed again, with who said what (meeting-notes Run 9).
+ *
+ *   detect(runtimeDir, { env }) → { ready, transcriber, parakeet, whisper, vad, diarize, missing, setup }
+ *   setupCommand(runtimeDir)    → the shell commands the PM runs once for the better engine, speech detection and speakers
+ *   createFinal({ runtimeDir, onDone, onState, run?, detect? }) → { queue(id, opts), cancel(id), state(id), idle() }
+ *   parseWhisper(json) · parseParakeet(json) · assignSpeakers(segs, diar) · dropEcho(mic, meeting) · mergeSources(parts)
+ *
+ * WHY. The live transcript is cut every few seconds so that lines appear while people talk. A cut that short gives Whisper
+ * little context, so names come out wrong and sentences break in two, and nothing in it says who spoke. Meeting tools that
+ * write good notes (Wispr Flow, Granola, Notion) treat the live text as a draft and write the notes from one pass over the
+ * whole recording. This module is that pass, on this machine.
+ *
+ * WHAT IT DOES, for each source the note heard (lib/notes-audio.js keeps the microphone and the meeting apart):
+ *   1. The source's samples become one temporary WAV file.
+ *   2. The best engine set up transcribes the whole file: Parakeet (parakeet-mlx, English only, more accurate and faster on
+ *      Apple silicon) when it is set up and the meeting is in English, else whisper.cpp with a five-way beam search, the
+ *      repository's names and terms as the prompt, and speech detection (`--vad`) when its model is there. Whisper reads the
+ *      whole file in 30-second windows, each with the text before it, which is what the live cuts lack.
+ *   3. When speaker separation is set up (lib/notes-diarize.py, sherpa-onnx, local), the meeting audio is split into
+ *      speakers: Speaker 1, Speaker 2 and so on. The microphone is the PM. In a room with one microphone and no meeting
+ *      audio, the microphone is split instead.
+ *   4. The sources are merged in time order. A microphone line that repeats a meeting line said at the same moment is the
+ *      meeting heard through the PM's speakers, and is dropped.
+ * The result goes to `onDone(id, final)`; lib/routes-notes.js stores it on the note and then asks the PM's agent session for
+ * the notes. One job runs at a time, so the model and the speaker separation never share the memory of an 8 GB machine.
+ *
+ * WHAT IT NEVER DOES. It makes no request. The setup commands download the models in the PM's own terminal; at run time
+ * Parakeet is started with HF_HUB_OFFLINE=1. The temporary files are deleted when the job ends, whatever the outcome.
+ */
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const crypto = require('crypto');
+const { execFile } = require('child_process');
+const whisper = require('./notes-whisper.js');
+const notesAudio = require('./notes-audio.js');
+
+const RUN_TIMEOUT_MS = 45 * 60 * 1000;
+const PARAKEET_MODEL = 'mlx-community/parakeet-tdt-0.6b-v2';
+const VAD_FILE = 'ggml-silero-v5.1.2.bin';
+const VAD_URL = 'https://huggingface.co/ggml-org/whisper-vad/resolve/main/' + VAD_FILE;
+const SEG_URL = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2';
+const EMB_URL = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/nemo_en_titanet_small.onnx';
+const DIARIZE_THRESHOLD = '0.9';      // tried on a four-speaker sample: 0.9 found four, 0.5 to 0.7 split them into more
+const ECHO_OVERLAP = 0.5;
+const ECHO_WORDS = 0.5;
+
+const finalDir = (runtimeDir) => path.join(runtimeDir, 'notes', 'final');
+const venvBin = (runtimeDir, name) => path.join(finalDir(runtimeDir), 'venv', 'bin', name);
+const diarDir = (runtimeDir) => path.join(finalDir(runtimeDir), 'speakers');
+const SCRIPT = path.join(__dirname, 'notes-diarize.py');
+const isFile = (p) => { try { return fs.statSync(p).isFile(); } catch { return false; } };
+const isExec = (p) => { try { fs.accessSync(p, fs.constants.X_OK); return isFile(p); } catch { return false; } };
+const shq = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'";
+
+/** What the final pass can use on this machine. `ready` is true when any engine can transcribe: Whisper alone is enough. */
+function detect(runtimeDir, { env = process.env } = {}) {
+  const w = whisper.detect(runtimeDir, { env });
+  const pk = env.PMOS_NOTES_PARAKEET_BIN || venvBin(runtimeDir, 'parakeet-mlx');
+  const parakeet = isExec(pk) ? pk : null;
+  const lang = whisper.getLanguage(runtimeDir, { env });
+  const transcriber = parakeet && lang === 'en' ? 'parakeet' : w.ready ? 'whisper' : null;
+  const vadFile = env.PMOS_NOTES_VAD_MODEL || path.join(whisper.modelDir(runtimeDir), VAD_FILE);
+  const python = env.PMOS_NOTES_DIARIZE_PYTHON || venvBin(runtimeDir, 'python3');
+  const seg = path.join(diarDir(runtimeDir), 'sherpa-onnx-pyannote-segmentation-3-0', 'model.onnx');
+  const emb = path.join(diarDir(runtimeDir), 'nemo_en_titanet_small.onnx');
+  const diarize = isExec(python) && isFile(seg) && isFile(emb) && isFile(SCRIPT) ? { python, script: SCRIPT, seg, emb } : null;
+  const missing = [];
+  if (!parakeet) missing.push('parakeet');
+  if (!isFile(vadFile)) missing.push('vad');
+  if (!diarize) missing.push('speakers');
+  return { ready: !!transcriber, transcriber, parakeet, whisper: w.ready ? { bin: w.bin, model: w.model } : null, vad: isFile(vadFile) ? vadFile : null,
+    diarize, missing, lang, setup: setupCommand(runtimeDir) };
+}
+
+/** The commands the PM runs once, in their own terminal, for the better final pass. The server never runs them.
+ * Python 3.12 from Homebrew (the system Python is too old for MLX), a virtual environment in the workspace's own folder
+ * with parakeet-mlx and sherpa-onnx, the Parakeet model fetched once so it runs offline, ffmpeg (parakeet-mlx reads audio
+ * with it), Whisper's speech detection model, and the two speaker models. Each download goes to a `.part` file first. */
+function setupCommand(runtimeDir) {
+  const dir = finalDir(runtimeDir);
+  const sp = diarDir(runtimeDir);
+  const wdir = whisper.modelDir(runtimeDir);
+  const py = venvBin(runtimeDir, 'python3');
+  const dl = (url, file) => 'curl -L --fail -C - -# -o ' + shq(file + '.part') + ' ' + url + ' && mv ' + shq(file + '.part') + ' ' + shq(file);
+  return '{ command -v ffmpeg >/dev/null 2>&1 || brew install ffmpeg; }'
+    + ' && { command -v python3.12 >/dev/null 2>&1 || brew install python@3.12; }'
+    + ' && mkdir -p ' + shq(dir) + ' ' + shq(sp) + ' ' + shq(wdir)
+    + ' && { [ -x ' + shq(py) + ' ] || python3.12 -m venv ' + shq(path.join(dir, 'venv')) + '; }'
+    + ' && ' + shq(py) + ' -m pip install --upgrade parakeet-mlx sherpa-onnx numpy'
+    + ' && echo ' + shq('Downloading the Parakeet model (about 2.5 GB). Keep this terminal open.')
+    + ' && ' + shq(py) + ' -c ' + shq("from parakeet_mlx import from_pretrained; from_pretrained('" + PARAKEET_MODEL + "')")
+    + ' && ' + dl(VAD_URL, path.join(wdir, VAD_FILE))
+    + ' && ' + dl(SEG_URL, path.join(sp, 'segmentation.tar.bz2')) + ' && tar -xjf ' + shq(path.join(sp, 'segmentation.tar.bz2')) + ' -C ' + shq(sp)
+    + ' && ' + dl(EMB_URL, path.join(sp, 'nemo_en_titanet_small.onnx'))
+    + ' && echo ' + shq('The final pass is set up. Open the Notes settings again.');
+}
+
+/* ── parsing what the programs write ──────────────────────────────────────────────────────────── */
+const clean = (t) => whisper.cleanText(String(t || ''));
+/** whisper-cli -oj: { transcription: [{ offsets: { from, to }, text }] } in ms. → [{ start, end, text }] in ms. */
+function parseWhisper(json) {
+  const rows = json && Array.isArray(json.transcription) ? json.transcription : [];
+  return rows.map((r) => ({ start: Number(r.offsets && r.offsets.from) || 0, end: Number(r.offsets && r.offsets.to) || 0, text: clean(r.text) })).filter((s) => s.text);
+}
+/** parakeet-mlx --output-format json: { sentences: [{ start, end, text }] } in seconds (or segments). → ms. */
+function parseParakeet(json) {
+  const rows = json && (Array.isArray(json.sentences) ? json.sentences : Array.isArray(json.segments) ? json.segments : []);
+  const out = (rows || []).map((r) => ({ start: Math.round((Number(r.start) || 0) * 1000), end: Math.round((Number(r.end) || 0) * 1000), text: clean(r.text) })).filter((s) => s.text);
+  if (!out.length && json && typeof json.text === 'string' && clean(json.text)) out.push({ start: 0, end: 0, text: clean(json.text) });
+  return out;
+}
+/** Each segment gets the speaker who talks most during it. `diar`: [{ start, end, speaker }] in seconds. Speakers are
+ * numbered in the order they first speak. → segments with `spk` (0, 1, …) and the number of speakers. */
+function assignSpeakers(segs, diar) {
+  const d = (diar || []).map((x) => ({ s: x.start * 1000, e: x.end * 1000, k: x.speaker }));
+  const order = new Map();
+  for (const x of [...d].sort((a, b) => a.s - b.s)) if (!order.has(x.k)) order.set(x.k, order.size);
+  const out = segs.map((g) => {
+    const talk = new Map();
+    for (const x of d) { const o = Math.min(g.end || g.start + 1, x.e) - Math.max(g.start, x.s); if (o > 0) talk.set(x.k, (talk.get(x.k) || 0) + o); }
+    let best = null; let most = 0;
+    for (const [k, v] of talk) if (v > most) { most = v; best = k; }
+    if (best === null && d.length) {
+      // a segment between two turns goes to the nearest turn
+      let near = Infinity;
+      for (const x of d) { const gap = Math.min(Math.abs(g.start - x.e), Math.abs(x.s - (g.end || g.start))); if (gap < near) { near = gap; best = x.k; } }
+    }
+    return { ...g, spk: best === null ? null : order.get(best) };
+  });
+  return { segments: out, speakers: order.size };
+}
+const wordsOf = (t) => new Set(String(t || '').toLowerCase().match(/[\p{L}\p{N}']{3,}/gu) || []);
+function sameWords(a, b) {
+  const x = wordsOf(a); const y = wordsOf(b);
+  if (!x.size || !y.size) return 0;
+  let both = 0; for (const w of x) if (y.has(w)) both++;
+  return both / Math.min(x.size, y.size);
+}
+/** The microphone lines that are not the meeting heard through the PM's speakers. */
+function dropEcho(mic, meeting) {
+  return mic.filter((m) => {
+    const len = Math.max(1, (m.end || m.start) - m.start);
+    return !meeting.some((g) => {
+      const o = Math.min(m.end || m.start, g.end || g.start) - Math.max(m.start, g.start);
+      return o / len >= ECHO_OVERLAP && sameWords(m.text, g.text) >= ECHO_WORDS;
+    });
+  });
+}
+/** parts: [{ source, segments, speakers }] → one transcript in time order: [{ t, end, text, source, spk }], where `spk` is
+ * the speaker key: the source alone ('mic', 'meeting') or the source and its speaker ('meeting-1'). */
+function mergeSources(parts) {
+  const mic = parts.find((p) => p.source === 'mic');
+  const meet = parts.find((p) => p.source === 'meeting');
+  if (mic && meet) mic.segments = dropEcho(mic.segments, meet.segments);
+  const out = [];
+  for (const p of parts) {
+    for (const g of p.segments) {
+      const spk = p.speakers > 1 && g.spk !== null && g.spk !== undefined ? p.source + '-' + (g.spk + 1) : p.source;
+      out.push({ t: g.start, end: g.end, text: g.text, source: p.source, spk });
+    }
+  }
+  return out.sort((a, b) => a.t - b.t || (a.source < b.source ? -1 : 1));
+}
+
+/* ── running the programs ─────────────────────────────────────────────────────────────────────── */
+function runProgram(bin, args, { env, timeout = RUN_TIMEOUT_MS } = {}) {
+  return new Promise((resolve) => {
+    execFile(bin, args, { timeout, maxBuffer: 16 << 20, env: env || process.env }, (e, stdout, stderr) => {
+      const out = String(stdout || ''); const err = String(stderr || '');
+      resolve(e ? { ok: false, stdout: out, stderr: err, error: e.killed ? 'it took too long' : (err || e.message || 'the program failed').trim().split('\n').pop().slice(0, 300) } : { ok: true, stdout: out, stderr: err });
+    });
+  });
+}
+const refusedFlag = (r, flag) => !r.ok && /unknown argument|unrecognized option|invalid option|error: unknown/i.test(r.stderr + ' ' + r.error) && (r.stderr + r.error).includes(flag);
+
+function writeWav(pcmFile, wavFile) {
+  const size = fs.statSync(pcmFile).size;
+  fs.writeFileSync(wavFile, notesAudio.wavHeader(size), { mode: 0o600 });
+  fs.appendFileSync(wavFile, fs.readFileSync(pcmFile));
+}
+async function transcribeWhisper(run, d, wav, tmp, { lang, prompt }) {
+  const threads = String(Math.max(2, Math.min(4, (os.cpus() || []).length || 4)));
+  const base = path.join(tmp, 'whisper-' + crypto.randomBytes(4).toString('hex'));
+  let vad = d.vad ? ['--vad', '-vm', d.vad] : [];
+  let nth = ['-nth', whisper.NO_SPEECH_THOLD];
+  const pr = prompt ? ['--prompt', String(prompt)] : [];
+  for (let i = 0; i < 3; i++) {
+    const r = await run(d.whisper.bin, ['-m', d.whisper.model, '-f', wav, '-l', lang, '-t', threads, '-bs', '5', '-bo', '5', '-np', '-oj', '-of', base, ...nth, ...vad, ...pr]);
+    if (r.ok) {
+      let json = null;
+      try { json = JSON.parse(fs.readFileSync(base + '.json', 'utf8')); } catch { return { ok: false, error: 'Whisper wrote no transcript' }; }
+      return { ok: true, segments: parseWhisper(json), engine: 'whisper', vad: vad.length > 0 };
+    }
+    if (vad.length && refusedFlag(r, '--vad')) { vad = []; continue; }        // a whisper.cpp older than its speech detection
+    if (nth.length && refusedFlag(r, '-nth')) { nth = []; continue; }
+    return { ok: false, error: 'Whisper: ' + r.error };
+  }
+  return { ok: false, error: 'Whisper refused its options' };
+}
+async function transcribeParakeet(run, d, wav, tmp) {
+  const out = path.join(tmp, 'parakeet');
+  fs.mkdirSync(out, { recursive: true });
+  const r = await run(d.parakeet, [wav, '--model', PARAKEET_MODEL, '--output-dir', out, '--output-format', 'json', '--max-duration', '20'], { env: { ...process.env, HF_HUB_OFFLINE: '1' } });
+  if (!r.ok) return { ok: false, error: 'Parakeet: ' + r.error };
+  const f = fs.readdirSync(out).find((x) => x.endsWith('.json'));
+  if (!f) return { ok: false, error: 'Parakeet wrote no transcript' };
+  try { return { ok: true, segments: parseParakeet(JSON.parse(fs.readFileSync(path.join(out, f), 'utf8'))), engine: 'parakeet', vad: false }; } catch { return { ok: false, error: 'Parakeet wrote a transcript that does not read' }; }
+}
+async function diarizeFile(run, d, wav) {
+  const r = await run(d.diarize.python, [d.diarize.script, '--segmentation', d.diarize.seg, '--embedding', d.diarize.emb, '--wav', wav, '--threshold', DIARIZE_THRESHOLD]);
+  let j = null;
+  try { j = JSON.parse(r.stdout.trim().split('\n').pop()); } catch { /* below */ }
+  if (!j || !j.ok) return { ok: false, error: 'speaker separation: ' + ((j && j.error) || r.error || 'it gave no answer') };
+  return { ok: true, segments: j.segments || [] };
+}
+
+/**
+ * The queue of final passes.
+ * @param {object} a
+ * @param {string} a.runtimeDir
+ * @param {(id:string, final:object) => void|Promise<void>} a.onDone   the merged transcript of a finished pass
+ * @param {(id:string, state:{state,step?,reason?}) => void} [a.onState]
+ * @param {Function} [a.run]     runProgram, replaceable in the suites
+ * @param {Function} [a.detect]  detect, replaceable in the suites
+ */
+function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram, detect: det = () => detect(runtimeDir) } = {}) {
+  let queue = Promise.resolve();
+  const gen = new Map();          // id → the generation of its latest job; a cancelled or newer job makes an older one moot
+  const states = new Map();
+  const set = (id, s) => { states.set(id, s); try { onState(id, s); } catch { /* the caller's */ } };
+
+  async function job(id, g, { prompt = '' }) {
+    const live = () => gen.get(id) === g;
+    if (!live()) return;
+    const d = det();
+    if (!d.ready) { set(id, { state: 'failed', reason: 'no engine is set up for the final pass' }); return; }
+    const srcs = notesAudio.sources(runtimeDir, id);
+    if (!srcs.length) { set(id, { state: 'failed', reason: 'no audio was kept for this note' }); return; }
+    const tmp = path.join(finalDir(runtimeDir), 'tmp-' + crypto.randomBytes(6).toString('hex'));
+    const began = Date.now();
+    try {
+      fs.mkdirSync(tmp, { recursive: true, mode: 0o700 });
+      const parts = []; let engine = d.transcriber; let vad = false; let diarized = false;
+      const splitMic = !srcs.some((s) => s.source === 'meeting');
+      for (const s of srcs) {
+        if (!live()) return;
+        const wav = path.join(tmp, s.source + '.wav');
+        writeWav(s.file, wav);
+        set(id, { state: 'running', step: 'transcribing ' + (s.source === 'mic' ? 'your microphone' : 'the meeting') + ' again' });
+        let r = d.transcriber === 'parakeet' ? await transcribeParakeet(run, d, wav, tmp) : await transcribeWhisper(run, d, wav, tmp, { lang: d.lang, prompt });
+        if (!r.ok && d.transcriber === 'parakeet' && d.whisper) r = await transcribeWhisper(run, d, wav, tmp, { lang: d.lang, prompt });   // Parakeet failed: Whisper still can
+        if (!r.ok) { set(id, { state: 'failed', reason: r.error }); return; }
+        engine = r.engine; vad = vad || r.vad;
+        let part = { source: s.source, segments: r.segments, speakers: 1 };
+        if (d.diarize && (s.source === 'meeting' || splitMic) && r.segments.length) {
+          if (!live()) return;
+          set(id, { state: 'running', step: 'telling the speakers apart' });
+          const dz = await diarizeFile(run, d, wav);
+          if (dz.ok) { const a = assignSpeakers(r.segments, dz.segments); part = { source: s.source, segments: a.segments, speakers: a.speakers }; diarized = true; }
+        }
+        parts.push(part);
+        try { fs.unlinkSync(wav); } catch { /* gone */ }
+      }
+      if (!live()) return;
+      const final = { at: new Date().toISOString(), engine, vad, diarized, took_ms: Date.now() - began, segments: mergeSources(parts) };
+      await onDone(id, final);
+      set(id, { state: 'done' });
+    } catch (e) {
+      set(id, { state: 'failed', reason: String((e && e.message) || e).slice(0, 300) });
+    } finally {
+      try { fs.rmSync(tmp, { recursive: true, force: true }); } catch { /* gone */ }
+    }
+  }
+  return {
+    /** Queue the final pass of a note; a pass already queued for it is replaced. */
+    queue(id, opts = {}) {
+      const g = (gen.get(id) || 0) + 1;
+      gen.set(id, g);
+      set(id, { state: 'queued' });
+      queue = queue.then(() => job(id, g, opts)).catch(() => {});
+      return { ok: true };
+    },
+    /** Listening started again, the note was deleted or made private: its pass is dropped. */
+    cancel(id) { gen.set(id, (gen.get(id) || 0) + 1); states.delete(id); },
+    state(id) { return states.get(id) || null; },
+    idle() { return queue; },
+  };
+}
+
+module.exports = { detect, setupCommand, createFinal, runProgram, parseWhisper, parseParakeet, assignSpeakers, dropEcho, mergeSources, finalDir,
+  PARAKEET_MODEL, VAD_FILE, DIARIZE_THRESHOLD };