create-pmos 0.78.0 → 0.80.0

package/workspace/lib/notes-final.js

@@ -1,10 +1,11 @@
 'use strict';
 /* notes-final.js — the final pass after Stop: the whole meeting transcribed again, with who said what (meeting-notes Run 9).
  *
- *   detect(runtimeDir, { env }) → { ready, transcriber, parakeet, whisper, vad, diarize, missing, setup }
+ *   detect(runtimeDir, { env }) → { ready, transcriber, qwen, parakeet, whisper, vad, diarize, missing, setup }
  *   setupCommand(runtimeDir)    → the shell commands the PM runs once for the better engine, speech detection and speakers
- *   createFinal({ runtimeDir, onDone, onState, run?, detect? }) → { queue(id, opts), cancel(id), state(id), idle() }
- *   parseWhisper(json) · parseParakeet(json) · assignSpeakers(segs, diar) · dropEcho(mic, meeting) · mergeSources(parts)
+ *   createFinal({ runtimeDir, onDone, onState, run?, detect?, qwen? }) → { queue(id, opts), cancel(id), state(id), idle() }
+ *   parseWhisper(json) · parseParakeet(json) · splitAtPauses(pcm) · assignSpeakers(segs, diar) · dropEcho(mic, meeting)
+ *   mergeSources(parts)
  *
  * WHY. The live transcript is cut every few seconds so that lines appear while people talk. A cut that short gives Whisper
  * little context, so names come out wrong and sentences break in two, and nothing in it says who spoke. Meeting tools that
@@ -13,8 +14,11 @@
  *
  * WHAT IT DOES, for each source the note heard (lib/notes-audio.js keeps the microphone and the meeting apart):
  *   1. The source's samples become one temporary WAV file.
- *   2. The best engine set up transcribes the whole file: Parakeet (parakeet-mlx, English only, more accurate and faster on
- *      Apple silicon) when it is set up and the meeting is in English, else whisper.cpp with a five-way beam search, the
+ *   2. The best engine set up transcribes the whole file. Qwen3-ASR first (lib/notes-qwen.js, the 1.7B model; English,
+ *      Chinese, Indonesian and speech that mixes them): it gives no times for its words, so the source is cut at its pauses
+ *      into pieces of at most about 25 seconds, and each piece is one line with the piece's start and end. Else Parakeet
+ *      (parakeet-mlx, English only, more accurate and faster on Apple silicon) when it is set up and the meeting is in
+ *      English, else whisper.cpp with a five-way beam search, the
  *      repository's names and terms as the prompt, and speech detection (`--vad`) when its model is there. Whisper reads the
  *      whole file in 30-second windows, each with the text before it, which is what the live cuts lack.
  *   3. When speaker separation is set up (lib/notes-diarize.py, sherpa-onnx, local), the meeting audio is split into
@@ -26,7 +30,7 @@
  * the notes. One job runs at a time, so the model and the speaker separation never share the memory of an 8 GB machine.
  *
  * WHAT IT NEVER DOES. It makes no request. The setup commands download the models in the PM's own terminal; at run time
- * Parakeet is started with HF_HUB_OFFLINE=1. The temporary files are deleted when the job ends, whatever the outcome.
+ * Qwen3-ASR and Parakeet are started with HF_HUB_OFFLINE=1. The temporary files are deleted when the job ends, whatever the outcome.
  */
 const fs = require('fs');
 const os = require('os');
@@ -35,6 +39,7 @@ const crypto = require('crypto');
 const { execFile } = require('child_process');
 const whisper = require('./notes-whisper.js');
 const notesAudio = require('./notes-audio.js');
+const notesQwen = require('./notes-qwen.js');
 
 const RUN_TIMEOUT_MS = 45 * 60 * 1000;
 const PARAKEET_MODEL = 'mlx-community/parakeet-tdt-0.6b-v2';
@@ -45,6 +50,12 @@ const EMB_URL = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker
 const DIARIZE_THRESHOLD = '0.9';      // tried on a four-speaker sample: 0.9 found four, 0.5 to 0.7 split them into more
 const ECHO_OVERLAP = 0.5;
 const ECHO_WORDS = 0.5;
+// Qwen3-ASR pieces: cut at a pause of PIECE_PAUSE_MS once a piece is PIECE_MIN_MS long; by PIECE_MAX_MS cut at the quietest frame
+const PIECE_MIN_MS = 8000;
+const PIECE_MAX_MS = 25000;
+const PIECE_PAUSE_MS = 500;
+const PIECE_MIN_VOICED_MS = 300;
+const PIECE_TIMEOUT_MS = 5 * 60 * 1000;
 
 const finalDir = (runtimeDir) => path.join(runtimeDir, 'notes', 'final');
 const venvBin = (runtimeDir, name) => path.join(finalDir(runtimeDir), 'venv', 'bin', name);
@@ -60,23 +71,26 @@ function detect(runtimeDir, { env = process.env } = {}) {
   const pk = env.PMOS_NOTES_PARAKEET_BIN || venvBin(runtimeDir, 'parakeet-mlx');
   const parakeet = isExec(pk) ? pk : null;
   const lang = whisper.getLanguage(runtimeDir, { env });
-  const transcriber = parakeet && lang === 'en' ? 'parakeet' : w.ready ? 'whisper' : null;
+  const q = notesQwen.detect(runtimeDir, { env });
+  const qwen = q.python && q.final ? { model: q.final } : null;
+  const transcriber = qwen ? 'qwen' : parakeet && lang === 'en' ? 'parakeet' : w.ready ? 'whisper' : null;
   const vadFile = env.PMOS_NOTES_VAD_MODEL || path.join(whisper.modelDir(runtimeDir), VAD_FILE);
   const python = env.PMOS_NOTES_DIARIZE_PYTHON || venvBin(runtimeDir, 'python3');
   const seg = path.join(diarDir(runtimeDir), 'sherpa-onnx-pyannote-segmentation-3-0', 'model.onnx');
   const emb = path.join(diarDir(runtimeDir), 'nemo_en_titanet_small.onnx');
   const diarize = isExec(python) && isFile(seg) && isFile(emb) && isFile(SCRIPT) ? { python, script: SCRIPT, seg, emb } : null;
   const missing = [];
+  if (q.missing.length) missing.push('qwen');
   if (!parakeet) missing.push('parakeet');
   if (!isFile(vadFile)) missing.push('vad');
   if (!diarize) missing.push('speakers');
-  return { ready: !!transcriber, transcriber, parakeet, whisper: w.ready ? { bin: w.bin, model: w.model } : null, vad: isFile(vadFile) ? vadFile : null,
+  return { ready: !!transcriber, transcriber, qwen, parakeet, whisper: w.ready ? { bin: w.bin, model: w.model } : null, vad: isFile(vadFile) ? vadFile : null,
     diarize, missing, lang, setup: setupCommand(runtimeDir) };
 }
 
 /** The commands the PM runs once, in their own terminal, for the better final pass. The server never runs them.
  * Python 3.12 from Homebrew (the system Python is too old for MLX), a virtual environment in the workspace's own folder
- * with parakeet-mlx and sherpa-onnx, the Parakeet model fetched once so it runs offline, ffmpeg (parakeet-mlx reads audio
+ * with parakeet-mlx and sherpa-onnx, the Parakeet model fetched once so it runs offline, mlx-qwen3-asr and its two models, ffmpeg (parakeet-mlx reads audio
  * with it), Whisper's speech detection model, and the two speaker models. Each download goes to a `.part` file first. */
 function setupCommand(runtimeDir) {
   const dir = finalDir(runtimeDir);
@@ -91,6 +105,7 @@ function setupCommand(runtimeDir) {
     + ' && ' + shq(py) + ' -m pip install --upgrade parakeet-mlx sherpa-onnx numpy'
     + ' && echo ' + shq('Downloading the Parakeet model (about 2.5 GB). Keep this terminal open.')
     + ' && ' + shq(py) + ' -c ' + shq("from parakeet_mlx import from_pretrained; from_pretrained('" + PARAKEET_MODEL + "')")
+    + notesQwen.setupLines(runtimeDir).map((l) => ' && ' + l).join('')
     + ' && ' + dl(VAD_URL, path.join(wdir, VAD_FILE))
     + ' && ' + dl(SEG_URL, path.join(sp, 'segmentation.tar.bz2')) + ' && tar -xjf ' + shq(path.join(sp, 'segmentation.tar.bz2')) + ' -C ' + shq(sp)
     + ' && ' + dl(EMB_URL, path.join(sp, 'nemo_en_titanet_small.onnx'))
@@ -111,6 +126,42 @@ function parseParakeet(json) {
   if (!out.length && json && typeof json.text === 'string' && clean(json.text)) out.push({ start: 0, end: 0, text: clean(json.text) });
   return out;
 }
+/** The pieces Qwen3-ASR transcribes one at a time: [{ start, end }] in ms. A piece ends at a pause of PIECE_PAUSE_MS once it
+ * is PIECE_MIN_MS long, or at its quietest frame between PIECE_MIN_MS and PIECE_MAX_MS. Quiet at either end is left out,
+ * and a piece with less than PIECE_MIN_VOICED_MS of speech is dropped. */
+function splitAtPauses(pcm, rate = whisper.RATE) {
+  const lv = whisper.levels(pcm, rate);
+  const F = 100;
+  // a frame is speech above three times the room's quiet (the 10th percentile), but never above a third of the loud frames
+  // (the 90th), so a recording that is speech from end to end still has its pauses
+  const sorted = lv.filter((v) => v > 0).sort((x, y) => x - y);
+  const at = (q) => (sorted.length ? sorted[Math.floor((sorted.length - 1) * q)] : 0);
+  const floor = Math.max(180, Math.min(at(0.1) * 3, at(0.9) * 0.3));
+  const voiced = lv.map((v) => v > floor);
+  const out = [];
+  let i = 0;
+  while (i < lv.length) {
+    while (i < lv.length && !voiced[i]) i++;
+    if (i >= lv.length) break;
+    const s = i; let end = -1; let run = 0;
+    for (let k = s; k < lv.length; k++) {
+      run = voiced[k] ? 0 : run + 1;
+      const len = (k - s + 1) * F;
+      if (len >= PIECE_MIN_MS && run * F >= PIECE_PAUSE_MS) { end = k - run + 1; break; }
+      if (len >= PIECE_MAX_MS) {
+        let at = k; let low = Infinity;
+        for (let j = s + PIECE_MIN_MS / F; j <= k; j++) if (lv[j] < low) { low = lv[j]; at = j; }
+        end = at; break;
+      }
+    }
+    if (end < 0) { end = lv.length; while (end > s && !voiced[end - 1]) end--; }
+    let v = 0; for (let k = s; k < end; k++) if (voiced[k]) v++;
+    if (v * F >= PIECE_MIN_VOICED_MS) out.push({ start: s * F, end: end * F });
+    i = Math.max(end, s + 1);
+  }
+  return out;
+}
+
 /** Each segment gets the speaker who talks most during it. `diar`: [{ start, end, speaker }] in seconds. Speakers are
  * numbered in the order they first speak. → segments with `spk` (0, 1, …) and the number of speakers. */
 function assignSpeakers(segs, diar) {
@@ -208,6 +259,23 @@ async function transcribeParakeet(run, d, wav, tmp) {
   if (!f) return { ok: false, error: 'Parakeet wrote no transcript' };
   try { return { ok: true, segments: parseParakeet(JSON.parse(fs.readFileSync(path.join(out, f), 'utf8'))), engine: 'parakeet', vad: false }; } catch { return { ok: false, error: 'Parakeet wrote a transcript that does not read' }; }
 }
+/** Qwen3-ASR on each piece of a source. `qwen` is lib/notes-qwen.js's runQwen with the worker, or a stand-in. */
+async function transcribeQwen(qwen, d, pcmFile, tmp, { lang, prompt, live = () => true }) {
+  const pcm = fs.readFileSync(pcmFile);
+  const segments = [];
+  for (const p of splitAtPauses(pcm)) {
+    if (!live()) break;
+    const f = path.join(tmp, 'piece-' + crypto.randomBytes(4).toString('hex') + '.wav');
+    fs.writeFileSync(f, whisper.wav(pcm.subarray(Math.round(p.start * whisper.RATE / 1000) * 2, Math.round(p.end * whisper.RATE / 1000) * 2)), { mode: 0o600 });
+    let r;
+    try { r = await qwen({ file: f, model: d.qwen.model, lang, prompt, timeoutMs: PIECE_TIMEOUT_MS }); }
+    finally { try { fs.unlinkSync(f); } catch { /* gone */ } }
+    if (!r || !r.ok) return { ok: false, error: 'Qwen3-ASR: ' + ((r && r.error) || 'it gave no answer') };
+    const text = clean(r.text);
+    if (text) segments.push({ start: p.start, end: p.end, text });
+  }
+  return { ok: true, segments, engine: 'qwen', vad: true };
+}
 async function diarizeFile(run, d, wav) {
   const r = await run(d.diarize.python, [d.diarize.script, '--segmentation', d.diarize.seg, '--embedding', d.diarize.emb, '--wav', wav, '--threshold', DIARIZE_THRESHOLD]);
   let j = null;
@@ -224,8 +292,10 @@ async function diarizeFile(run, d, wav) {
  * @param {(id:string, state:{state,step?,reason?}) => void} [a.onState]
  * @param {Function} [a.run]     runProgram, replaceable in the suites
  * @param {Function} [a.detect]  detect, replaceable in the suites
+ * @param {Function} [a.qwen]    ({ file, model, lang, prompt, timeoutMs }) → { ok, text }: Qwen3-ASR, replaceable in the suites
  */
-function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram, detect: det = () => detect(runtimeDir) } = {}) {
+function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram, detect: det = () => detect(runtimeDir),
+  qwen = (args) => notesQwen.runQwen(notesQwen.workerFor(runtimeDir), args) } = {}) {
   let queue = Promise.resolve();
   const gen = new Map();          // id → the generation of its latest job; a cancelled or newer job makes an older one moot
   const states = new Map();
@@ -249,8 +319,18 @@ function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram,
         const wav = path.join(tmp, s.source + '.wav');
         writeWav(s.file, wav);
         set(id, { state: 'running', step: 'transcribing ' + (s.source === 'mic' ? 'your microphone' : 'the meeting') + ' again' });
-        let r = d.transcriber === 'parakeet' ? await transcribeParakeet(run, d, wav, tmp) : await transcribeWhisper(run, d, wav, tmp, { lang: d.lang, prompt });
-        if (!r.ok && d.transcriber === 'parakeet' && d.whisper) r = await transcribeWhisper(run, d, wav, tmp, { lang: d.lang, prompt });   // Parakeet failed: Whisper still can
+        // the best engine first; when it fails, the next one set up still can: Qwen3-ASR, Parakeet (English), Whisper
+        const order = [];
+        if (d.qwen) order.push('qwen');
+        if (d.parakeet && d.lang === 'en') order.push('parakeet');
+        if (d.whisper) order.push('whisper');
+        let r = { ok: false, error: 'no engine is set up for the final pass' };
+        for (const e of order) {
+          if (!live()) return;
+          r = e === 'qwen' ? await transcribeQwen(qwen, d, s.file, tmp, { lang: d.lang, prompt, live })
+            : e === 'parakeet' ? await transcribeParakeet(run, d, wav, tmp) : await transcribeWhisper(run, d, wav, tmp, { lang: d.lang, prompt });
+          if (r.ok) break;
+        }
         if (!r.ok) { set(id, { state: 'failed', reason: r.error }); return; }
         engine = r.engine; vad = vad || r.vad;
         let part = { source: s.source, segments: r.segments, speakers: 1 };
@@ -289,5 +369,5 @@ function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram,
   };
 }
 
-module.exports = { detect, setupCommand, createFinal, runProgram, parseWhisper, parseParakeet, assignSpeakers, dropEcho, mergeSources, finalDir,
+module.exports = { detect, setupCommand, createFinal, runProgram, parseWhisper, parseParakeet, splitAtPauses, assignSpeakers, dropEcho, mergeSources, finalDir,
   PARAKEET_MODEL, VAD_FILE, DIARIZE_THRESHOLD };

package/workspace/lib/notes-qwen.js

@@ -0,0 +1,182 @@
+'use strict';
+/* notes-qwen.js — Qwen3-ASR on the PM's machine, kept loaded, for the live transcript and the final pass (meeting-notes Run 10).
+ *
+ *   detect(runtimeDir, { env }) → { ready, python, script, live, final, cache, missing }
+ *   setupLines(runtimeDir)      → the shell commands that install mlx-qwen3-asr and download its two models (into the final
+ *                                 pass's setup, lib/notes-final.js)
+ *   createWorker({ cmd, args, env, idleMs, timeoutMs }) → { run({ file, model, language, context, timeoutMs }), stop(), alive() }
+ *   workerFor(runtimeDir, { env }) → the one worker of this process
+ *   runQwen(worker, { file, model, lang, prompt }) → { ok, text } | { ok: false, error }   the shape lib/notes-whisper.js runs
+ *   languageOf(code) → the Qwen3-ASR language name of a Notes language code, or null to detect it
+ *
+ * WHY. Qwen3-ASR (Alibaba's Qwen team, open weights) is more accurate than Whisper large-v3 in English and in Chinese,
+ * and it also understands Indonesian and speech that mixes them. Its Mac port (mlx-qwen3-asr) runs in Python, and
+ * loading a model takes seconds. The live transcript sends a cut every few seconds, so starting the program for each cut,
+ * as whisper.cpp is started, would fall behind. This module starts lib/notes-qwen.py once, keeps the model loaded, and
+ * sends it one cut at a time. It stops the program after IDLE_MS without work, which frees the memory.
+ *
+ * THE MODELS. The live transcript uses the 0.6B model in 8 bits (about 800 MB, the same text as the full model). The final
+ * pass after Stop uses the 1.7B model in 8 bits (about 2 GB, the same text as the full model), which is more accurate; the
+ * two together use about 3 GB, which a 16 GB machine holds beside the browser. Both are downloaded once, in the PM's own
+ * terminal, into the workspace's own folder.
+ *
+ * WHAT IT NEVER DOES. It makes no request: the program is started with HF_HUB_OFFLINE=1 and reads the models from the
+ * workspace's folder. It writes no file; the caller writes each WAV file and deletes it.
+ */
+const fs = require('fs');
+const path = require('path');
+const { spawn } = require('child_process');
+
+const LIVE_MODEL = 'moona3k/mlx-qwen3-asr-0.6b-8bit';
+const FINAL_MODEL = 'moona3k/mlx-qwen3-asr-1.7b-8bit';
+const IDLE_MS = 10 * 60 * 1000;
+const TIMEOUT_MS = 120000;           // one live cut; the final pass passes a longer time
+const START_TIMEOUT_MS = 120000;     // importing MLX the first time can be slow
+const SCRIPT = path.join(__dirname, 'notes-qwen.py');
+// The Notes languages (lib/notes-whisper.js LANGUAGES) as Qwen3-ASR names them; 'auto' lets it detect the language
+const LANGUAGE_NAMES = Object.freeze({ en: 'English', id: 'Indonesian', zh: 'Chinese' });
+
+const finalDir = (runtimeDir) => path.join(runtimeDir, 'notes', 'final');
+const cacheDir = (runtimeDir) => path.join(finalDir(runtimeDir), 'hf');
+const venvPython = (runtimeDir) => path.join(finalDir(runtimeDir), 'venv', 'bin', 'python3');
+const isFile = (p) => { try { return fs.statSync(p).isFile(); } catch { return false; } };
+const isExec = (p) => { try { fs.accessSync(p, fs.constants.X_OK); return isFile(p); } catch { return false; } };
+const shq = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'";
+const languageOf = (code) => LANGUAGE_NAMES[code] || null;
+
+/** Whether the package is in the virtual environment: lib/python3.*\/site-packages/mlx_qwen3_asr. */
+function hasPackage(python) {
+  const lib = path.join(path.dirname(path.dirname(python)), 'lib');
+  try {
+    return fs.readdirSync(lib).some((d) => /^python3/.test(d) && fs.existsSync(path.join(lib, d, 'site-packages', 'mlx_qwen3_asr', '__init__.py')));
+  } catch { return false; }
+}
+/** Whether a model is in the cache: models--<org>--<name>/snapshots/<rev>/ with its config and weights. */
+function hasModel(cache, repo) {
+  const snaps = path.join(cache, 'models--' + repo.replace('/', '--'), 'snapshots');
+  try {
+    return fs.readdirSync(snaps).some((rev) => {
+      const files = fs.readdirSync(path.join(snaps, rev));
+      return files.includes('config.json') && files.some((f) => /\.safetensors$/.test(f));
+    });
+  } catch { return false; }
+}
+
+/** What Qwen3-ASR can do on this machine. PMOS_NOTES_QWEN 'off' turns it off; PMOS_NOTES_QWEN_PYTHON names the Python. */
+function detect(runtimeDir, { env = process.env } = {}) {
+  const cache = cacheDir(runtimeDir);
+  if (String(env.PMOS_NOTES_QWEN || '') === 'off') return { ready: false, python: null, script: SCRIPT, live: null, final: null, cache, missing: ['qwen'] };
+  const python = env.PMOS_NOTES_QWEN_PYTHON || venvPython(runtimeDir);
+  const pkg = isExec(python) && (env.PMOS_NOTES_QWEN_PYTHON ? true : hasPackage(python));
+  const live = pkg && hasModel(cache, LIVE_MODEL) ? LIVE_MODEL : null;
+  const fin = pkg && hasModel(cache, FINAL_MODEL) ? FINAL_MODEL : null;
+  return { ready: !!live, python: pkg ? python : null, script: SCRIPT, live, final: fin || live, cache, missing: live && fin ? [] : ['qwen'] };
+}
+
+/** The setup lines, run after the virtual environment exists. Each model is downloaded once into the workspace's folder. */
+function setupLines(runtimeDir) {
+  const py = venvPython(runtimeDir);
+  const get = (repo) => 'from huggingface_hub import snapshot_download; snapshot_download(' + JSON.stringify(repo) + ')';
+  return [
+    shq(py) + ' -m pip install --upgrade mlx-qwen3-asr',
+    'echo ' + shq('Downloading the Qwen3-ASR models (about 3 GB). Keep this terminal open.'),
+    'HF_HUB_CACHE=' + shq(cacheDir(runtimeDir)) + ' ' + shq(py) + ' -c ' + shq(get(LIVE_MODEL) + '; ' + get(FINAL_MODEL)),
+  ];
+}
+
+/**
+ * The program, started on the first request and kept until it is idle.
+ * @param {object} a
+ * @param {string} a.cmd          the Python (or, in the suites, a stand-in)
+ * @param {string[]} a.args       [script]
+ * @param {object} [a.env]
+ * @param {number} [a.idleMs]
+ */
+function createWorker({ cmd, args = [], env = {}, idleMs = IDLE_MS, timeoutMs = TIMEOUT_MS }) {
+  let child = null; let ready = null; let buf = '';
+  let seq = 0; let idleTimer = null;
+  const pending = new Map();   // id → { resolve, timer }
+
+  const failAll = (error) => { for (const [k, p] of pending) { clearTimeout(p.timer); p.resolve({ ok: false, error }); pending.delete(k); } };
+  function kill(error) {
+    const c = child; child = null; ready = null; buf = '';
+    if (c) { try { c.stdin.end(); } catch { /* closed */ } try { c.kill(); } catch { /* gone */ } }
+    failAll(error);
+  }
+  function armIdle() {
+    clearTimeout(idleTimer);
+    idleTimer = setTimeout(() => { if (!pending.size) kill('the speech program was stopped'); }, idleMs);
+    if (idleTimer.unref) idleTimer.unref();
+  }
+  function start() {
+    if (ready) return ready;
+    ready = new Promise((resolve) => {
+      let c;
+      try { c = spawn(cmd, args, { env: { ...process.env, ...env, HF_HUB_OFFLINE: '1', PYTHONUNBUFFERED: '1' }, stdio: ['pipe', 'pipe', 'pipe'] }); }
+      catch (e) { resolve({ ok: false, error: 'the speech program did not start: ' + ((e && e.message) || e) }); return; }
+      child = c;
+      let started = false; let errText = '';
+      const t = setTimeout(() => { if (!started) { resolve({ ok: false, error: 'the speech program took too long to start' }); kill('the speech program took too long to start'); } }, START_TIMEOUT_MS);
+      c.stderr.on('data', (d) => { errText = (errText + d).slice(-2000); });
+      c.on('error', (e) => { clearTimeout(t); if (!started) resolve({ ok: false, error: 'the speech program did not start: ' + e.message }); if (child === c) kill('the speech program stopped'); });
+      c.on('exit', () => {
+        clearTimeout(t);
+        const why = String(errText).trim().split('\n').pop() || 'the speech program stopped';
+        if (!started) resolve({ ok: false, error: why.slice(0, 200) });
+        if (child === c) kill(why.slice(0, 200));
+      });
+      c.stdout.on('data', (d) => {
+        buf += d;
+        let nl;
+        while ((nl = buf.indexOf('\n')) >= 0) {
+          const line = buf.slice(0, nl); buf = buf.slice(nl + 1);
+          let m; try { m = JSON.parse(line); } catch { continue; }
+          if (!started && m && 'ready' in m) {
+            started = true; clearTimeout(t);
+            resolve(m.ready ? { ok: true } : { ok: false, error: String(m.error || 'the speech program is not set up').slice(0, 200) });
+            continue;
+          }
+          const p = m && pending.get(m.id);
+          if (!p) continue;
+          clearTimeout(p.timer); pending.delete(m.id);
+          p.resolve(m.ok ? { ok: true, text: String(m.text || ''), language: String(m.language || '') } : { ok: false, error: String(m.error || 'the transcription failed').slice(0, 200) });
+          if (!pending.size) armIdle();
+        }
+      });
+    });
+    return ready;
+  }
+  /** One file. Resolves { ok, text, language } or { ok: false, error }; never rejects. */
+  async function run({ file, model, language = null, context = '', timeoutMs: tm = timeoutMs }) {
+    clearTimeout(idleTimer);
+    const s = await start();
+    if (!s.ok) { kill(s.error); return s; }
+    const id = ++seq;
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => { pending.delete(id); resolve({ ok: false, error: 'the transcription took too long' }); kill('the transcription took too long'); }, tm);
+      pending.set(id, { resolve, timer });
+      try { child.stdin.write(JSON.stringify({ id, file, model, language, context: String(context || '') }) + '\n'); }
+      catch (e) { clearTimeout(timer); pending.delete(id); resolve({ ok: false, error: 'the speech program stopped' }); }
+    });
+  }
+  return { run, stop: () => { clearTimeout(idleTimer); kill('the speech program was stopped'); }, alive: () => !!child };
+}
+
+let shared = null; let sharedKey = null;
+/** The one worker of this process, for the live transcript and the final pass alike, so the models load once. */
+function workerFor(runtimeDir, { env = process.env } = {}) {
+  const d = detect(runtimeDir, { env });
+  const key = runtimeDir + '|' + d.python;
+  if (shared && sharedKey === key) return shared;
+  if (shared) shared.stop();
+  sharedKey = key;
+  shared = createWorker({ cmd: d.python || 'python3', args: [SCRIPT], env: { HF_HUB_CACHE: d.cache } });
+  return shared;
+}
+
+/** The run function lib/notes-whisper.js's transcriber calls, for Qwen3-ASR: the same arguments, the same answer. */
+function runQwen(worker, { file, model, lang, prompt = '', timeoutMs }) {
+  return worker.run({ file, model, language: languageOf(lang), context: prompt, ...(timeoutMs ? { timeoutMs } : {}) });
+}
+
+module.exports = { detect, setupLines, createWorker, workerFor, runQwen, languageOf, hasModel, cacheDir, LIVE_MODEL, FINAL_MODEL, SCRIPT, IDLE_MS };

package/workspace/lib/notes-qwen.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""notes-qwen.py: Qwen3-ASR speech to text that stays loaded, on this machine (meeting-notes Run 10).
+
+    python3 notes-qwen.py
+
+It reads one JSON request per line on stdin and writes one JSON answer per line on stdout:
+    {"id": 1, "file": "/path/cut.wav", "model": "moona3k/mlx-qwen3-asr-0.6b-8bit", "language": "English", "context": "names"}
+ -> {"id": 1, "ok": true, "text": "...", "language": "English"}
+ or {"id": 1, "ok": false, "error": "..."}
+When it has started it writes {"ready": true}. It stops when stdin closes.
+
+The WAV is 16 kHz mono 16-bit. `language` is a Qwen3-ASR language name, or null to detect it. `context` is the meeting's
+names and terms, which Qwen3-ASR reads as a hint. The model is loaded on the first request for it and kept, so a live
+line costs the transcription only; at most two models are kept (the live one and the final one). lib/notes-qwen.js starts
+it, sends one request at a time, and stops it when it has been idle. It reads no network (the server sets
+HF_HUB_OFFLINE=1, so the models must already be in the cache) and writes no file.
+"""
+import json
+import sys
+
+MAX_SESSIONS = 2
+
+
+def say(obj):
+    sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
+    sys.stdout.flush()
+
+
+def main():
+    try:
+        from mlx_qwen3_asr import Session
+    except Exception as e:  # noqa: BLE001
+        say({"ready": False, "error": "mlx-qwen3-asr is not installed: " + str(e)})
+        return 1
+    sessions = {}  # model id -> Session, oldest first
+    say({"ready": True})
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        rid = None
+        try:
+            req = json.loads(line)
+            rid = req.get("id")
+            model = str(req.get("model") or "")
+            if not model:
+                raise ValueError("no model was named")
+            s = sessions.get(model)
+            if s is None:
+                while len(sessions) >= MAX_SESSIONS:
+                    sessions.pop(next(iter(sessions)))
+                s = Session(model=model)
+                sessions[model] = s
+            r = s.transcribe(str(req["file"]), language=req.get("language") or None, context=str(req.get("context") or ""))
+            say({"id": rid, "ok": True, "text": r.text or "", "language": r.language or ""})
+        except Exception as e:  # noqa: BLE001
+            say({"id": rid, "ok": False, "error": (str(e) or e.__class__.__name__)[:300]})
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/workspace/lib/notes-whisper.js

@@ -172,7 +172,7 @@ function promptFor(vocab, previous) {
 /* THE SPOKEN LANGUAGE. Whisper writes in the language it is told. With 'auto' it guesses the language of each cut, and on a
  * few seconds of accented speech it can guess wrong and write the line in that language, which reads as a translation. So
  * the PM picks the language of the meeting once (English by default) and every cut uses it. PMOS_WHISPER_LANG wins. */
-const LANGUAGES = Object.freeze([['en', 'English'], ['id', 'Indonesian'], ['auto', 'Detect for each line']]);
+const LANGUAGES = Object.freeze([['en', 'English'], ['id', 'Indonesian'], ['zh', 'Chinese'], ['auto', 'Detect for each line']]);
 const DEFAULT_LANGUAGE = 'en';
 const settingsFile = (runtimeDir) => path.join(runtimeDir, 'notes', 'whisper-settings.json');
 function readSettings(runtimeDir) {
@@ -280,7 +280,8 @@ function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWh
       try {
         fs.writeFileSync(file, wav(pcm), { mode: 0o600 });
         const key = job.id + '|' + job.source;
-        const r = await run({ bin: d.bin, model: d.model, file, lang: getLanguage(runtimeDir), accuracy: getAccuracy(runtimeDir), prompt: promptFor(vocabOf(job), lastText.get(key)) });
+        // `engine` is 'qwen' when the caller's detect chose Qwen3-ASR (lib/notes-qwen.js); `run` then sends the cut there
+        const r = await run({ engine: d.engine || 'whisper', bin: d.bin, model: d.model, file, lang: getLanguage(runtimeDir), accuracy: getAccuracy(runtimeDir), prompt: promptFor(vocabOf(job), lastText.get(key)) });
         if (!r.ok) { onError({ ...job, error: r.error }); return; }
         const text = cleanText(r.text, { voicedRatio });
         if (text) { lastText.set(key, text); await onText({ ...job, text }); }

package/workspace/lib/routes-notes.js

@@ -61,6 +61,7 @@ const whisper = require('./notes-whisper.js');
 const notesRepo = require('./notes-repo.js');
 const notesAudio = require('./notes-audio.js');
 const notesFinal = require('./notes-final.js');
+const notesQwen = require('./notes-qwen.js');
 const notesLinks = require('./notes-links.js');
 const jevNotes = require('./jev-notes.js');
 const tsSettings = require('./typesafe-settings.js');
@@ -283,7 +284,20 @@ function vocabFor(ctx, id) {
   return text;
 }
 
-/* ── local Whisper: one transcriber for the process ───────────────────────────────────────────── */
+/* ── local transcription: one transcriber for the process ─────────────────────────────────────── */
+/* THE LIVE ENGINE (Run 10). Qwen3-ASR when it is set up (lib/notes-qwen.js: more accurate, Chinese and Indonesian too, kept
+ * loaded so each cut is quick), else whisper.cpp. ctx.notesQwenDetect and ctx.notesQwenRun replace them in the suites. */
+function qwenDetect(ctx) { return ctx.notesQwenDetect ? ctx.notesQwenDetect() : notesQwen.detect(ctx.runtimeDir); }
+function liveDetect(ctx) {
+  const q = qwenDetect(ctx);
+  if (q.ready) return { ready: true, engine: 'qwen', model: q.live, bin: null };
+  return { ...whisper.detect(ctx.runtimeDir), engine: 'whisper' };
+}
+function liveRun(ctx, args) {
+  if (args.engine !== 'qwen') return whisper.runWhisper(args);
+  if (ctx.notesQwenRun) return ctx.notesQwenRun(args);
+  return notesQwen.runQwen(notesQwen.workerFor(ctx.runtimeDir), args);
+}
 let transcriber = null; let transcriberDir = null;
 const audioErrors = new Map();   // note id → the last thing that went wrong, for the page
 function transcriberFor(ctx) {
@@ -291,6 +305,8 @@ function transcriberFor(ctx) {
   transcriberDir = ctx.runtimeDir;
   transcriber = whisper.createTranscriber({
     runtimeDir: ctx.runtimeDir,
+    detect: () => liveDetect(ctx),
+    run: (args) => liveRun(ctx, args),
     vocab: ({ id }) => vocabFor(ctx, id),
     onText: async (c) => {
       const r = notes.appendTranscript(ctx.runtimeDir, { id: c.id, text: c.text, source: c.source, t: c.t, ms: c.ms });
@@ -302,7 +318,8 @@ function transcriberFor(ctx) {
 }
 function engineInfo(ctx, w) {
   const d = whisper.detect(ctx.runtimeDir);
-  const out = { ready: d.ready, missing: d.missing, setup: d.setup, model_dir: d.model_dir, model_size: d.model_size,
+  const live = liveDetect(ctx);
+  const out = { ready: live.ready, live: live.ready ? live.engine : null, missing: d.missing, setup: d.setup, model_dir: d.model_dir, model_size: d.model_size,
     language: whisper.getLanguage(ctx.runtimeDir), languages: whisper.LANGUAGES.map(([value, label]) => ({ value, label })),
     model: whisper.getModel(ctx.runtimeDir), models: whisper.MODELS.map((m) => ({ value: m.id, label: m.label })),
     accuracy: whisper.getAccuracy(ctx.runtimeDir), accuracies: whisper.ACCURACIES.map((a) => ({ value: a.id, label: a.label })), keep_audio: notesAudio.getKeep(ctx.runtimeDir),
@@ -336,6 +353,7 @@ function finalFor(ctx) {
     runtimeDir: ctx.runtimeDir,
     ...(ctx.notesFinalRun ? { run: ctx.notesFinalRun } : {}),
     ...(ctx.notesFinalDetect ? { detect: ctx.notesFinalDetect } : {}),
+    ...(ctx.notesQwenRun ? { qwen: ctx.notesQwenRun } : {}),
     onState: (id, st) => { if (st.state !== 'done') notes.setFinalState(ctx.runtimeDir, { id, state: st }); },
     onDone: (id, final) => {
       const dir = ctx.runtimeDir;
@@ -564,7 +582,7 @@ async function handle(req, res, url, ctx) {
     if (b64.length > Math.ceil(MAX_PCM_BYTES / 3) * 4) return done({ ok: false, status: 413, error: 'that audio chunk is too long' });
     const pcm = Buffer.from(b64, 'base64');
     if (pcm.length % 2) return done({ ok: false, status: 400, error: 'the audio must be 16-bit samples' });
-    if (!whisper.detect(dir).ready) return done({ ok: false, status: 409, error: 'local transcription is not set up' });
+    if (!liveDetect(ctx).ready) return done({ ok: false, status: 409, error: 'local transcription is not set up' });
     const ms = (pcm.length / 2 / whisper.RATE) * 1000;
     const t = Math.max(0, notes.meetingMs(note.note) - ms);
     // the audio is kept for replay unless the PM switched that off (D92); a private note never reaches this line

package/workspace/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "pmos-workspace",
-  "version": "0.78.0",
+  "version": "0.80.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "pmos-workspace",
-      "version": "0.37.0",
+      "version": "0.80.0",
       "license": "MIT",
       "dependencies": {
         "@xterm/addon-fit": "^0.11.0",

package/workspace/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pmos-workspace",
-  "version": "0.78.0",
+  "version": "0.80.0",
   "private": true,
   "description": "PMOS Terminal Workspace — local companion hosting real PTYs beside the library and gates (D62)",
   "license": "MIT",

package/workspace/public/pm/view-notes.js

@@ -397,7 +397,7 @@ async function finalSetup(ctx) {
   const ws = wsOf(ctx);
   const r = await pmPost('/api/notes/final-setup', ws ? { ws } : {});
   banner = r && r.ok && r.started
-    ? { tone: 'ok', text: 'A terminal opened with the setup commands typed in. Read them and press Enter there. The downloads are about 3 GB. Then open Settings again.' }
+    ? { tone: 'ok', text: 'A terminal opened with the setup commands typed in. Read them and press Enter there. The downloads are about 6 GB. Then open Settings again.' }
     : { tone: 'bad', text: 'No terminal opened' + (r && r.reason ? ': ' + r.reason : '') + '. Run this in a terminal: ' + ((r && r.command) || '') };
   ctx.setState({});
 }
@@ -783,9 +783,11 @@ function openSettings(ctx, n) {
     const parts = [];
     parts.push(el('div', { class: 'nt-set', 'data-nt-whisper': w ? (w.ready ? 'ready' : 'missing') : 'unknown' },
       el('div', { class: 'nt-set__head' }, 'Local transcription'),
-      el('p', { class: 'nt-hint' }, !w ? 'Reading …' : w.ready ? 'Whisper is set up. The microphone and the meeting are transcribed on this machine.'
+      el('p', { class: 'nt-hint', 'data-nt-live-engine': w ? (w.live || '') : null }, !w ? 'Reading …'
+        : w.live === 'qwen' ? 'Qwen3-ASR is set up. The microphone and the meeting are transcribed on this machine, in English, Chinese or Indonesian.'
+        : w.ready ? 'Whisper is set up. The microphone and the meeting are transcribed on this machine.'
         : 'Whisper is not set up (' + w.missing.map((m) => (m === 'program' ? 'the program' : 'the model, ' + w.model_size)).join(' and ') + ' missing). The setup installs whisper.cpp with Homebrew and downloads the model.'),
-      w && w.models ? field({ label: 'Model', kind: 'select', value: w.model, options: w.models,
+      w && w.models && w.live !== 'qwen' ? field({ label: 'Model', kind: 'select', value: w.model, options: w.models,
         onCommit: (v) => setSwitch(ctx, { model: v }).then(() => { h.close(); openSettings(ctx, n); }), attrs: { 'data-nt-model': '' } }) : null,
       w && !w.ready ? button({ label: 'Set up in a terminal', onClick: () => { h.close(); whisperSetup(ctx); }, attrs: { 'data-nt-whisper-setup': '' } }) : null,
       w && w.ready && w.languages ? field({ label: 'Language spoken in the meeting', kind: 'select', value: w.language, options: w.languages,
@@ -797,15 +799,15 @@ function openSettings(ctx, n) {
         : 'No audio is kept: each clip is deleted once it is transcribed, and a meeting cannot be replayed.') : null));
     const fp = w && w.final;
     if (fp) {
-      const have = [fp.transcriber === 'parakeet' ? 'Parakeet' : fp.transcriber === 'whisper' ? 'Whisper with beam search' : null,
+      const have = [fp.transcriber === 'qwen' ? 'Qwen3-ASR' : fp.transcriber === 'parakeet' ? 'Parakeet' : fp.transcriber === 'whisper' ? 'Whisper with beam search' : null,
         fp.vad ? 'speech detection' : null, fp.speakers ? 'speaker separation' : null].filter(Boolean);
       parts.push(el('div', { class: 'nt-set', 'data-nt-final-set': fp.ready ? 'ready' : 'missing' },
         el('div', { class: 'nt-set__head' }, 'Final notes after Stop'),
         el('p', { class: 'nt-hint' }, fp.ready
           ? 'After Stop, the whole meeting is transcribed again on this machine (' + have.join(', ') + '), and your agent session writes the notes from it.'
           : 'Set up local transcription first. After Stop, the whole meeting is then transcribed again on this machine, and your agent session writes the notes from it.'),
-        fp.missing.length ? el('p', { class: 'nt-hint', 'data-nt-final-missing': '' }, 'For better notes, set up ' + fp.missing.map((m) => (m === 'parakeet' ? 'Parakeet (more accurate in English)' : m === 'vad' ? 'speech detection' : 'speaker separation')).join(', ')
-          + '. The setup installs them into the workspace\'s own folder; the downloads are about 3 GB.') : null,
+        fp.missing.length ? el('p', { class: 'nt-hint', 'data-nt-final-missing': '' }, 'For better notes, set up ' + fp.missing.map((m) => (m === 'qwen' ? 'Qwen3-ASR (English, Chinese and Indonesian, more accurate)' : m === 'parakeet' ? 'Parakeet (more accurate in English)' : m === 'vad' ? 'speech detection' : 'speaker separation')).join(', ')
+          + '. The setup installs them into the workspace\'s own folder; the downloads are about 6 GB.') : null,
         fp.missing.length ? button({ label: 'Set up in a terminal', onClick: () => { h.close(); finalSetup(ctx); }, attrs: { 'data-nt-final-setup': '' } }) : null,
         field({ label: 'Fetch Wispr Flow notes after the meeting', kind: 'check', value: w.wispr !== false,
           onCommit: (v) => setSwitch(ctx, { wispr: v }).then(() => { h.close(); openSettings(ctx, n); }), attrs: { 'data-nt-wispr-switch': '' } }),
@@ -1263,7 +1265,7 @@ function finalTranscript(ctx, n, opts) {
   });
   const box = el('div', { class: 'nt-lines', 'data-nt-transcript': 'final' }, lines);
   const where = opts.inline ? 'inline' : 'full';
-  const how = [n.final.engine === 'parakeet' ? 'Parakeet' : 'Whisper', n.final.diarized ? 'speakers told apart' : null].filter(Boolean).join(', ');
+  const how = [n.final.engine === 'qwen' ? 'Qwen3-ASR' : n.final.engine === 'parakeet' ? 'Parakeet' : 'Whisper', n.final.diarized ? 'speakers told apart' : null].filter(Boolean).join(', ');
   const d = el('details', { class: 'nt-trans', 'data-nt-part': 'transcript', open: transcriptOpen[where] },
     el('summary', { class: 'nt-sec__h nt-fold' }, 'Transcript', el('span', { class: 'nt-src' }, ' · from the whole recording (' + how + '), ' + rows.length + (rows.length === 1 ? ' line' : ' lines'))),
     button({ label: 'Show the live transcript', variant: 'ghost', size: 'sm', onClick: () => { liveTranscript = true; ctx.setState({}); }, attrs: { 'data-nt-live-transcript': '' } }), box);