create-pmos 0.79.0 → 0.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/payload/KIT-VERSION +1 -1
- package/payload/planning/evals/eval-calibration.md +1 -1
- package/payload/planning/okrs/health-budgets.md +1 -1
- package/payload/state/discoveries.md +1 -1
- package/workspace/guide/notes.md +9 -5
- package/workspace/lib/notes-final.js +93 -13
- package/workspace/lib/notes-qwen.js +182 -0
- package/workspace/lib/notes-qwen.py +62 -0
- package/workspace/lib/notes-whisper.js +3 -2
- package/workspace/lib/routes-notes.js +21 -3
- package/workspace/package-lock.json +2 -2
- package/workspace/package.json +1 -1
- package/workspace/public/pm/view-notes.js +9 -7
- package/workspace/test-lists/meeting-notes.json +5 -3
- package/workspace/test-timings.json +1 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "create-pmos",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.80.0",
|
|
4
4
|
"description": "Scaffold the PMOS portable kit — run the PMOS product-management workflow (initiative → lane-fit spec → contract → build → three gates, with work-type lanes for features, research, maintenance, incidents, and content) entirely inside your own repo, with all state as git-versioned files. Includes `workflow-sweep`, which audits the state every board derives from and separates real defects from work you simply have not done. `create-pmos workspace` also installs + launches the local Terminal Workspace GUI. No database, no network, no telemetry.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"bin": {
|
package/payload/KIT-VERSION
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
type: Reference
|
|
3
3
|
title: "Eval calibration log"
|
|
4
4
|
description: "Append-only record of every PM override of an evaluator verdict."
|
|
5
|
-
generated: {by: process:create-pmos, at: "2026-10-
|
|
5
|
+
generated: {by: process:create-pmos, at: "2026-10-07T13:06:30Z"}
|
|
6
6
|
status: stable
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
type: Reference
|
|
3
3
|
title: "Health budgets"
|
|
4
4
|
description: "Closed registry of health-budget anchors (D59), edited only by the PM."
|
|
5
|
-
generated: {by: process:create-pmos, at: "2026-10-
|
|
5
|
+
generated: {by: process:create-pmos, at: "2026-10-07T13:06:30Z"}
|
|
6
6
|
status: stable
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
type: Reference
|
|
3
3
|
title: "Discoveries and friction log"
|
|
4
4
|
description: "Append-only record of what runs discovered and where the process rubbed."
|
|
5
|
-
generated: {by: process:create-pmos, at: "2026-10-
|
|
5
|
+
generated: {by: process:create-pmos, at: "2026-10-07T13:06:30Z"}
|
|
6
6
|
status: stable
|
|
7
7
|
---
|
|
8
8
|
|
package/workspace/guide/notes.md
CHANGED
|
@@ -42,7 +42,7 @@ After **Stop**, the final notes replace this live draft. See [Final notes after
|
|
|
42
42
|
|
|
43
43
|
The notes taken live are a draft. After **Stop**, the workspace transcribes the whole recording again on this machine, with more time for each sentence, and then your agent session writes the final notes from that transcript and your typed lines.
|
|
44
44
|
|
|
45
|
-
1. The workspace transcribes your microphone and the meeting apart. It uses **Parakeet** when it is set up and the meeting is in English, else Whisper with beam search, speech detection and the names of this meeting.
|
|
45
|
+
1. The workspace transcribes your microphone and the meeting apart. It uses **Qwen3-ASR** when it is set up, in any of the languages. Else it uses **Parakeet** when it is set up and the meeting is in English, else Whisper with beam search, speech detection and the names of this meeting. When one engine fails, the next one set up is used.
|
|
46
46
|
2. It tells the speakers apart. Your microphone is you. The meeting's voices are **Speaker 1**, **Speaker 2** and so on. When no meeting audio was captured, the voices on your microphone are told apart instead.
|
|
47
47
|
3. Your agent session writes the purpose, the overview, the topics, the decisions, the next steps with an owner, and the open questions. Each item shows its meeting time and **Said:** with the words that support it. An item with no supporting words in the transcript is left out, and the note says how many were left out.
|
|
48
48
|
|
|
@@ -50,9 +50,9 @@ For a 30-minute meeting on an M1, this takes about 3 to 10 minutes. Until it is
|
|
|
50
50
|
|
|
51
51
|
- To name a speaker, press the name in the transcript, type the name and press **Save**. The name is kept when the meeting is transcribed again.
|
|
52
52
|
- **Show the live transcript** shows what was heard while listening. **Show the final transcript** goes back.
|
|
53
|
-
- **Transcribe again** in the settings runs the final pass again, for example after you set up
|
|
53
|
+
- **Transcribe again** in the settings runs the final pass again, for example after you set up Qwen3-ASR.
|
|
54
54
|
|
|
55
|
-
To set up Parakeet, speech detection and speakers, open the settings and press **Set up in a terminal** under **Final notes after Stop**. A terminal opens with the commands typed in. They install ffmpeg and Python with Homebrew, then Parakeet
|
|
55
|
+
To set up Qwen3-ASR, Parakeet, speech detection and speakers, open the settings and press **Set up in a terminal** under **Final notes after Stop**. A terminal opens with the commands typed in. They install ffmpeg and Python with Homebrew, then Parakeet, sherpa-onnx and mlx-qwen3-asr in the workspace's own folder, and download the models (about 6 GB in all, of which Qwen3-ASR is about 3 GB). Read the commands and press Enter. Without them, the final pass uses Whisper and does not tell speakers apart.
|
|
56
56
|
|
|
57
57
|
The kept audio holds your microphone and the meeting apart until the final pass is done. When **Keep the audio for replay** is off, both are deleted after the final pass.
|
|
58
58
|
|
|
@@ -77,7 +77,11 @@ Local transcription uses whisper.cpp, a free program that runs on your machine.
|
|
|
77
77
|
|
|
78
78
|
If lines come slowly, choose **Faster (about 550 MB)** under **Model** in the settings, press **Set up in a terminal** and press Enter to download it. It is the same model in a smaller form: it starts and runs faster on an M1 and makes a few more mistakes. **Most accurate (1.6 GB)** is the default.
|
|
79
79
|
|
|
80
|
-
|
|
80
|
+
The transcript is written in the language you choose in the settings under **Language spoken in the meeting**: English unless you change it, Indonesian or Chinese. **Detect for each line** lets the engine find the language of each few seconds of speech. Whisper can guess wrong and write a line in another language, so with Whisper choose the language when you know it. Qwen3-ASR detects the language better, so with it **Detect for each line** is the choice for a meeting that mixes languages.
|
|
81
|
+
|
|
82
|
+
### Qwen3-ASR for the live transcript
|
|
83
|
+
|
|
84
|
+
When Qwen3-ASR is set up (see [Final notes after Stop](#final-notes-after-stop)), the live transcript uses it instead of Whisper. It is more accurate in English and Chinese, and it understands Indonesian and speech that mixes these languages. The workspace starts it once, keeps the model loaded while you listen, and stops it after 10 minutes with nothing to transcribe, which frees the memory. The live transcript uses the smaller model (about 800 MB) and the final pass the larger one (about 2 GB). The settings say which engine the live transcript uses. To go back to Whisper, start the workspace with `PMOS_NOTES_QWEN=off`.
|
|
81
85
|
|
|
82
86
|
If the download stops, press **Set up in a terminal** again and press Enter. The download continues where it stopped, and a part-downloaded model is never used.
|
|
83
87
|
|
|
@@ -148,6 +152,6 @@ To take notes for a calendar event, open the event's detail on the Agenda and pr
|
|
|
148
152
|
- Audio never leaves this machine. The page sends it to the workspace on this machine, which gives it to whisper.cpp in a temporary file and deletes that file at once. For replay, the workspace keeps one recording of each note in `notes/recordings/` in the state folder, until you delete the audio, the note, or mark the note private. The browser's own recognition is used only on the device, never in the cloud.
|
|
149
153
|
- Notes are files in the workspace's own folder on this machine (`notes/` in the state folder). They are never written into a repository.
|
|
150
154
|
- Only Jev, with the switches on, and your agent session (the final notes, **Summary by agent**, and the Wispr Flow search) receive any text from a note. The Wispr Flow search receives the meeting's time and title only.
|
|
151
|
-
- The final pass runs Parakeet, Whisper and the speaker models on this machine, from temporary files that it deletes when it is done.
|
|
155
|
+
- The final pass runs Qwen3-ASR, Parakeet, Whisper and the speaker models on this machine, from temporary files that it deletes when it is done.
|
|
152
156
|
|
|
153
157
|
Live notes is a different tool: it maps each talking point to an initiative. See [Live notes](live-notes.md).
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
/* notes-final.js — the final pass after Stop: the whole meeting transcribed again, with who said what (meeting-notes Run 9).
|
|
3
3
|
*
|
|
4
|
-
* detect(runtimeDir, { env }) → { ready, transcriber, parakeet, whisper, vad, diarize, missing, setup }
|
|
4
|
+
* detect(runtimeDir, { env }) → { ready, transcriber, qwen, parakeet, whisper, vad, diarize, missing, setup }
|
|
5
5
|
* setupCommand(runtimeDir) → the shell commands the PM runs once for the better engine, speech detection and speakers
|
|
6
|
-
* createFinal({ runtimeDir, onDone, onState, run?, detect? }) → { queue(id, opts), cancel(id), state(id), idle() }
|
|
7
|
-
* parseWhisper(json) · parseParakeet(json) · assignSpeakers(segs, diar) · dropEcho(mic, meeting)
|
|
6
|
+
* createFinal({ runtimeDir, onDone, onState, run?, detect?, qwen? }) → { queue(id, opts), cancel(id), state(id), idle() }
|
|
7
|
+
* parseWhisper(json) · parseParakeet(json) · splitAtPauses(pcm) · assignSpeakers(segs, diar) · dropEcho(mic, meeting)
|
|
8
|
+
* mergeSources(parts)
|
|
8
9
|
*
|
|
9
10
|
* WHY. The live transcript is cut every few seconds so that lines appear while people talk. A cut that short gives Whisper
|
|
10
11
|
* little context, so names come out wrong and sentences break in two, and nothing in it says who spoke. Meeting tools that
|
|
@@ -13,8 +14,11 @@
|
|
|
13
14
|
*
|
|
14
15
|
* WHAT IT DOES, for each source the note heard (lib/notes-audio.js keeps the microphone and the meeting apart):
|
|
15
16
|
* 1. The source's samples become one temporary WAV file.
|
|
16
|
-
* 2. The best engine set up transcribes the whole file
|
|
17
|
-
*
|
|
17
|
+
* 2. The best engine set up transcribes the whole file. Qwen3-ASR first (lib/notes-qwen.js, the 1.7B model; English,
|
|
18
|
+
* Chinese, Indonesian and speech that mixes them): it gives no times for its words, so the source is cut at its pauses
|
|
19
|
+
* into pieces of at most about 25 seconds, and each piece is one line with the piece's start and end. Else Parakeet
|
|
20
|
+
* (parakeet-mlx, English only, more accurate and faster on Apple silicon) when it is set up and the meeting is in
|
|
21
|
+
* English, else whisper.cpp with a five-way beam search, the
|
|
18
22
|
* repository's names and terms as the prompt, and speech detection (`--vad`) when its model is there. Whisper reads the
|
|
19
23
|
* whole file in 30-second windows, each with the text before it, which is what the live cuts lack.
|
|
20
24
|
* 3. When speaker separation is set up (lib/notes-diarize.py, sherpa-onnx, local), the meeting audio is split into
|
|
@@ -26,7 +30,7 @@
|
|
|
26
30
|
* the notes. One job runs at a time, so the model and the speaker separation never share the memory of an 8 GB machine.
|
|
27
31
|
*
|
|
28
32
|
* WHAT IT NEVER DOES. It makes no request. The setup commands download the models in the PM's own terminal; at run time
|
|
29
|
-
* Parakeet
|
|
33
|
+
* Qwen3-ASR and Parakeet are started with HF_HUB_OFFLINE=1. The temporary files are deleted when the job ends, whatever the outcome.
|
|
30
34
|
*/
|
|
31
35
|
const fs = require('fs');
|
|
32
36
|
const os = require('os');
|
|
@@ -35,6 +39,7 @@ const crypto = require('crypto');
|
|
|
35
39
|
const { execFile } = require('child_process');
|
|
36
40
|
const whisper = require('./notes-whisper.js');
|
|
37
41
|
const notesAudio = require('./notes-audio.js');
|
|
42
|
+
const notesQwen = require('./notes-qwen.js');
|
|
38
43
|
|
|
39
44
|
const RUN_TIMEOUT_MS = 45 * 60 * 1000;
|
|
40
45
|
const PARAKEET_MODEL = 'mlx-community/parakeet-tdt-0.6b-v2';
|
|
@@ -45,6 +50,12 @@ const EMB_URL = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker
|
|
|
45
50
|
const DIARIZE_THRESHOLD = '0.9'; // tried on a four-speaker sample: 0.9 found four, 0.5 to 0.7 split them into more
|
|
46
51
|
const ECHO_OVERLAP = 0.5;
|
|
47
52
|
const ECHO_WORDS = 0.5;
|
|
53
|
+
// Qwen3-ASR pieces: cut at a pause of PIECE_PAUSE_MS once a piece is PIECE_MIN_MS long; by PIECE_MAX_MS cut at the quietest frame
|
|
54
|
+
const PIECE_MIN_MS = 8000;
|
|
55
|
+
const PIECE_MAX_MS = 25000;
|
|
56
|
+
const PIECE_PAUSE_MS = 500;
|
|
57
|
+
const PIECE_MIN_VOICED_MS = 300;
|
|
58
|
+
const PIECE_TIMEOUT_MS = 5 * 60 * 1000;
|
|
48
59
|
|
|
49
60
|
const finalDir = (runtimeDir) => path.join(runtimeDir, 'notes', 'final');
|
|
50
61
|
const venvBin = (runtimeDir, name) => path.join(finalDir(runtimeDir), 'venv', 'bin', name);
|
|
@@ -60,23 +71,26 @@ function detect(runtimeDir, { env = process.env } = {}) {
|
|
|
60
71
|
const pk = env.PMOS_NOTES_PARAKEET_BIN || venvBin(runtimeDir, 'parakeet-mlx');
|
|
61
72
|
const parakeet = isExec(pk) ? pk : null;
|
|
62
73
|
const lang = whisper.getLanguage(runtimeDir, { env });
|
|
63
|
-
const
|
|
74
|
+
const q = notesQwen.detect(runtimeDir, { env });
|
|
75
|
+
const qwen = q.python && q.final ? { model: q.final } : null;
|
|
76
|
+
const transcriber = qwen ? 'qwen' : parakeet && lang === 'en' ? 'parakeet' : w.ready ? 'whisper' : null;
|
|
64
77
|
const vadFile = env.PMOS_NOTES_VAD_MODEL || path.join(whisper.modelDir(runtimeDir), VAD_FILE);
|
|
65
78
|
const python = env.PMOS_NOTES_DIARIZE_PYTHON || venvBin(runtimeDir, 'python3');
|
|
66
79
|
const seg = path.join(diarDir(runtimeDir), 'sherpa-onnx-pyannote-segmentation-3-0', 'model.onnx');
|
|
67
80
|
const emb = path.join(diarDir(runtimeDir), 'nemo_en_titanet_small.onnx');
|
|
68
81
|
const diarize = isExec(python) && isFile(seg) && isFile(emb) && isFile(SCRIPT) ? { python, script: SCRIPT, seg, emb } : null;
|
|
69
82
|
const missing = [];
|
|
83
|
+
if (q.missing.length) missing.push('qwen');
|
|
70
84
|
if (!parakeet) missing.push('parakeet');
|
|
71
85
|
if (!isFile(vadFile)) missing.push('vad');
|
|
72
86
|
if (!diarize) missing.push('speakers');
|
|
73
|
-
return { ready: !!transcriber, transcriber, parakeet, whisper: w.ready ? { bin: w.bin, model: w.model } : null, vad: isFile(vadFile) ? vadFile : null,
|
|
87
|
+
return { ready: !!transcriber, transcriber, qwen, parakeet, whisper: w.ready ? { bin: w.bin, model: w.model } : null, vad: isFile(vadFile) ? vadFile : null,
|
|
74
88
|
diarize, missing, lang, setup: setupCommand(runtimeDir) };
|
|
75
89
|
}
|
|
76
90
|
|
|
77
91
|
/** The commands the PM runs once, in their own terminal, for the better final pass. The server never runs them.
|
|
78
92
|
* Python 3.12 from Homebrew (the system Python is too old for MLX), a virtual environment in the workspace's own folder
|
|
79
|
-
* with parakeet-mlx and sherpa-onnx, the Parakeet model fetched once so it runs offline, ffmpeg (parakeet-mlx reads audio
|
|
93
|
+
* with parakeet-mlx and sherpa-onnx, the Parakeet model fetched once so it runs offline, mlx-qwen3-asr and its two models, ffmpeg (parakeet-mlx reads audio
|
|
80
94
|
* with it), Whisper's speech detection model, and the two speaker models. Each download goes to a `.part` file first. */
|
|
81
95
|
function setupCommand(runtimeDir) {
|
|
82
96
|
const dir = finalDir(runtimeDir);
|
|
@@ -91,6 +105,7 @@ function setupCommand(runtimeDir) {
|
|
|
91
105
|
+ ' && ' + shq(py) + ' -m pip install --upgrade parakeet-mlx sherpa-onnx numpy'
|
|
92
106
|
+ ' && echo ' + shq('Downloading the Parakeet model (about 2.5 GB). Keep this terminal open.')
|
|
93
107
|
+ ' && ' + shq(py) + ' -c ' + shq("from parakeet_mlx import from_pretrained; from_pretrained('" + PARAKEET_MODEL + "')")
|
|
108
|
+
+ notesQwen.setupLines(runtimeDir).map((l) => ' && ' + l).join('')
|
|
94
109
|
+ ' && ' + dl(VAD_URL, path.join(wdir, VAD_FILE))
|
|
95
110
|
+ ' && ' + dl(SEG_URL, path.join(sp, 'segmentation.tar.bz2')) + ' && tar -xjf ' + shq(path.join(sp, 'segmentation.tar.bz2')) + ' -C ' + shq(sp)
|
|
96
111
|
+ ' && ' + dl(EMB_URL, path.join(sp, 'nemo_en_titanet_small.onnx'))
|
|
@@ -111,6 +126,42 @@ function parseParakeet(json) {
|
|
|
111
126
|
if (!out.length && json && typeof json.text === 'string' && clean(json.text)) out.push({ start: 0, end: 0, text: clean(json.text) });
|
|
112
127
|
return out;
|
|
113
128
|
}
|
|
129
|
+
/** The pieces Qwen3-ASR transcribes one at a time: [{ start, end }] in ms. A piece ends at a pause of PIECE_PAUSE_MS once it
|
|
130
|
+
* is PIECE_MIN_MS long, or at its quietest frame between PIECE_MIN_MS and PIECE_MAX_MS. Quiet at either end is left out,
|
|
131
|
+
* and a piece with less than PIECE_MIN_VOICED_MS of speech is dropped. */
|
|
132
|
+
function splitAtPauses(pcm, rate = whisper.RATE) {
|
|
133
|
+
const lv = whisper.levels(pcm, rate);
|
|
134
|
+
const F = 100;
|
|
135
|
+
// a frame is speech above three times the room's quiet (the 10th percentile), but never above a third of the loud frames
|
|
136
|
+
// (the 90th), so a recording that is speech from end to end still has its pauses
|
|
137
|
+
const sorted = lv.filter((v) => v > 0).sort((x, y) => x - y);
|
|
138
|
+
const at = (q) => (sorted.length ? sorted[Math.floor((sorted.length - 1) * q)] : 0);
|
|
139
|
+
const floor = Math.max(180, Math.min(at(0.1) * 3, at(0.9) * 0.3));
|
|
140
|
+
const voiced = lv.map((v) => v > floor);
|
|
141
|
+
const out = [];
|
|
142
|
+
let i = 0;
|
|
143
|
+
while (i < lv.length) {
|
|
144
|
+
while (i < lv.length && !voiced[i]) i++;
|
|
145
|
+
if (i >= lv.length) break;
|
|
146
|
+
const s = i; let end = -1; let run = 0;
|
|
147
|
+
for (let k = s; k < lv.length; k++) {
|
|
148
|
+
run = voiced[k] ? 0 : run + 1;
|
|
149
|
+
const len = (k - s + 1) * F;
|
|
150
|
+
if (len >= PIECE_MIN_MS && run * F >= PIECE_PAUSE_MS) { end = k - run + 1; break; }
|
|
151
|
+
if (len >= PIECE_MAX_MS) {
|
|
152
|
+
let at = k; let low = Infinity;
|
|
153
|
+
for (let j = s + PIECE_MIN_MS / F; j <= k; j++) if (lv[j] < low) { low = lv[j]; at = j; }
|
|
154
|
+
end = at; break;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (end < 0) { end = lv.length; while (end > s && !voiced[end - 1]) end--; }
|
|
158
|
+
let v = 0; for (let k = s; k < end; k++) if (voiced[k]) v++;
|
|
159
|
+
if (v * F >= PIECE_MIN_VOICED_MS) out.push({ start: s * F, end: end * F });
|
|
160
|
+
i = Math.max(end, s + 1);
|
|
161
|
+
}
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
|
|
114
165
|
/** Each segment gets the speaker who talks most during it. `diar`: [{ start, end, speaker }] in seconds. Speakers are
|
|
115
166
|
* numbered in the order they first speak. → segments with `spk` (0, 1, …) and the number of speakers. */
|
|
116
167
|
function assignSpeakers(segs, diar) {
|
|
@@ -208,6 +259,23 @@ async function transcribeParakeet(run, d, wav, tmp) {
|
|
|
208
259
|
if (!f) return { ok: false, error: 'Parakeet wrote no transcript' };
|
|
209
260
|
try { return { ok: true, segments: parseParakeet(JSON.parse(fs.readFileSync(path.join(out, f), 'utf8'))), engine: 'parakeet', vad: false }; } catch { return { ok: false, error: 'Parakeet wrote a transcript that does not read' }; }
|
|
210
261
|
}
|
|
262
|
+
/** Qwen3-ASR on each piece of a source. `qwen` is lib/notes-qwen.js's runQwen with the worker, or a stand-in. */
|
|
263
|
+
async function transcribeQwen(qwen, d, pcmFile, tmp, { lang, prompt, live = () => true }) {
|
|
264
|
+
const pcm = fs.readFileSync(pcmFile);
|
|
265
|
+
const segments = [];
|
|
266
|
+
for (const p of splitAtPauses(pcm)) {
|
|
267
|
+
if (!live()) break;
|
|
268
|
+
const f = path.join(tmp, 'piece-' + crypto.randomBytes(4).toString('hex') + '.wav');
|
|
269
|
+
fs.writeFileSync(f, whisper.wav(pcm.subarray(Math.round(p.start * whisper.RATE / 1000) * 2, Math.round(p.end * whisper.RATE / 1000) * 2)), { mode: 0o600 });
|
|
270
|
+
let r;
|
|
271
|
+
try { r = await qwen({ file: f, model: d.qwen.model, lang, prompt, timeoutMs: PIECE_TIMEOUT_MS }); }
|
|
272
|
+
finally { try { fs.unlinkSync(f); } catch { /* gone */ } }
|
|
273
|
+
if (!r || !r.ok) return { ok: false, error: 'Qwen3-ASR: ' + ((r && r.error) || 'it gave no answer') };
|
|
274
|
+
const text = clean(r.text);
|
|
275
|
+
if (text) segments.push({ start: p.start, end: p.end, text });
|
|
276
|
+
}
|
|
277
|
+
return { ok: true, segments, engine: 'qwen', vad: true };
|
|
278
|
+
}
|
|
211
279
|
async function diarizeFile(run, d, wav) {
|
|
212
280
|
const r = await run(d.diarize.python, [d.diarize.script, '--segmentation', d.diarize.seg, '--embedding', d.diarize.emb, '--wav', wav, '--threshold', DIARIZE_THRESHOLD]);
|
|
213
281
|
let j = null;
|
|
@@ -224,8 +292,10 @@ async function diarizeFile(run, d, wav) {
|
|
|
224
292
|
* @param {(id:string, state:{state,step?,reason?}) => void} [a.onState]
|
|
225
293
|
* @param {Function} [a.run] runProgram, replaceable in the suites
|
|
226
294
|
* @param {Function} [a.detect] detect, replaceable in the suites
|
|
295
|
+
* @param {Function} [a.qwen] ({ file, model, lang, prompt, timeoutMs }) → { ok, text }: Qwen3-ASR, replaceable in the suites
|
|
227
296
|
*/
|
|
228
|
-
function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram, detect: det = () => detect(runtimeDir)
|
|
297
|
+
function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram, detect: det = () => detect(runtimeDir),
|
|
298
|
+
qwen = (args) => notesQwen.runQwen(notesQwen.workerFor(runtimeDir), args) } = {}) {
|
|
229
299
|
let queue = Promise.resolve();
|
|
230
300
|
const gen = new Map(); // id → the generation of its latest job; a cancelled or newer job makes an older one moot
|
|
231
301
|
const states = new Map();
|
|
@@ -249,8 +319,18 @@ function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram,
|
|
|
249
319
|
const wav = path.join(tmp, s.source + '.wav');
|
|
250
320
|
writeWav(s.file, wav);
|
|
251
321
|
set(id, { state: 'running', step: 'transcribing ' + (s.source === 'mic' ? 'your microphone' : 'the meeting') + ' again' });
|
|
252
|
-
|
|
253
|
-
|
|
322
|
+
// the best engine first; when it fails, the next one set up still can: Qwen3-ASR, Parakeet (English), Whisper
|
|
323
|
+
const order = [];
|
|
324
|
+
if (d.qwen) order.push('qwen');
|
|
325
|
+
if (d.parakeet && d.lang === 'en') order.push('parakeet');
|
|
326
|
+
if (d.whisper) order.push('whisper');
|
|
327
|
+
let r = { ok: false, error: 'no engine is set up for the final pass' };
|
|
328
|
+
for (const e of order) {
|
|
329
|
+
if (!live()) return;
|
|
330
|
+
r = e === 'qwen' ? await transcribeQwen(qwen, d, s.file, tmp, { lang: d.lang, prompt, live })
|
|
331
|
+
: e === 'parakeet' ? await transcribeParakeet(run, d, wav, tmp) : await transcribeWhisper(run, d, wav, tmp, { lang: d.lang, prompt });
|
|
332
|
+
if (r.ok) break;
|
|
333
|
+
}
|
|
254
334
|
if (!r.ok) { set(id, { state: 'failed', reason: r.error }); return; }
|
|
255
335
|
engine = r.engine; vad = vad || r.vad;
|
|
256
336
|
let part = { source: s.source, segments: r.segments, speakers: 1 };
|
|
@@ -289,5 +369,5 @@ function createFinal({ runtimeDir, onDone, onState = () => {}, run = runProgram,
|
|
|
289
369
|
};
|
|
290
370
|
}
|
|
291
371
|
|
|
292
|
-
module.exports = { detect, setupCommand, createFinal, runProgram, parseWhisper, parseParakeet, assignSpeakers, dropEcho, mergeSources, finalDir,
|
|
372
|
+
module.exports = { detect, setupCommand, createFinal, runProgram, parseWhisper, parseParakeet, splitAtPauses, assignSpeakers, dropEcho, mergeSources, finalDir,
|
|
293
373
|
PARAKEET_MODEL, VAD_FILE, DIARIZE_THRESHOLD };
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/* notes-qwen.js — Qwen3-ASR on the PM's machine, kept loaded, for the live transcript and the final pass (meeting-notes Run 10).
|
|
3
|
+
*
|
|
4
|
+
* detect(runtimeDir, { env }) → { ready, python, script, live, final, cache, missing }
|
|
5
|
+
* setupLines(runtimeDir) → the shell commands that install mlx-qwen3-asr and download its two models (into the final
|
|
6
|
+
* pass's setup, lib/notes-final.js)
|
|
7
|
+
* createWorker({ cmd, args, env, idleMs, timeoutMs }) → { run({ file, model, language, context, timeoutMs }), stop(), alive() }
|
|
8
|
+
* workerFor(runtimeDir, { env }) → the one worker of this process
|
|
9
|
+
* runQwen(worker, { file, model, lang, prompt }) → { ok, text } | { ok: false, error } the shape lib/notes-whisper.js runs
|
|
10
|
+
* languageOf(code) → the Qwen3-ASR language name of a Notes language code, or null to detect it
|
|
11
|
+
*
|
|
12
|
+
* WHY. Qwen3-ASR (Alibaba's Qwen team, open weights) is more accurate than Whisper large-v3 in English and in Chinese,
|
|
13
|
+
* and it also understands Indonesian and speech that mixes them. Its Mac port (mlx-qwen3-asr) runs in Python, and
|
|
14
|
+
* loading a model takes seconds. The live transcript sends a cut every few seconds, so starting the program for each cut,
|
|
15
|
+
* as whisper.cpp is started, would fall behind. This module starts lib/notes-qwen.py once, keeps the model loaded, and
|
|
16
|
+
* sends it one cut at a time. It stops the program after IDLE_MS without work, which frees the memory.
|
|
17
|
+
*
|
|
18
|
+
* THE MODELS. The live transcript uses the 0.6B model in 8 bits (about 800 MB, the same text as the full model). The final
|
|
19
|
+
* pass after Stop uses the 1.7B model in 8 bits (about 2 GB, the same text as the full model), which is more accurate; the
|
|
20
|
+
* two together use about 3 GB, which a 16 GB machine holds beside the browser. Both are downloaded once, in the PM's own
|
|
21
|
+
* terminal, into the workspace's own folder.
|
|
22
|
+
*
|
|
23
|
+
* WHAT IT NEVER DOES. It makes no request: the program is started with HF_HUB_OFFLINE=1 and reads the models from the
|
|
24
|
+
* workspace's folder. It writes no file; the caller writes each WAV file and deletes it.
|
|
25
|
+
*/
|
|
26
|
+
const fs = require('fs');
|
|
27
|
+
const path = require('path');
|
|
28
|
+
const { spawn } = require('child_process');
|
|
29
|
+
|
|
30
|
+
const LIVE_MODEL = 'moona3k/mlx-qwen3-asr-0.6b-8bit';
|
|
31
|
+
const FINAL_MODEL = 'moona3k/mlx-qwen3-asr-1.7b-8bit';
|
|
32
|
+
const IDLE_MS = 10 * 60 * 1000;
|
|
33
|
+
const TIMEOUT_MS = 120000; // one live cut; the final pass passes a longer time
|
|
34
|
+
const START_TIMEOUT_MS = 120000; // importing MLX the first time can be slow
|
|
35
|
+
const SCRIPT = path.join(__dirname, 'notes-qwen.py');
|
|
36
|
+
// The Notes languages (lib/notes-whisper.js LANGUAGES) as Qwen3-ASR names them; 'auto' lets it detect the language
|
|
37
|
+
const LANGUAGE_NAMES = Object.freeze({ en: 'English', id: 'Indonesian', zh: 'Chinese' });
|
|
38
|
+
|
|
39
|
+
const finalDir = (runtimeDir) => path.join(runtimeDir, 'notes', 'final');
|
|
40
|
+
const cacheDir = (runtimeDir) => path.join(finalDir(runtimeDir), 'hf');
|
|
41
|
+
const venvPython = (runtimeDir) => path.join(finalDir(runtimeDir), 'venv', 'bin', 'python3');
|
|
42
|
+
const isFile = (p) => { try { return fs.statSync(p).isFile(); } catch { return false; } };
|
|
43
|
+
const isExec = (p) => { try { fs.accessSync(p, fs.constants.X_OK); return isFile(p); } catch { return false; } };
|
|
44
|
+
const shq = (s) => "'" + String(s).replace(/'/g, "'\\''") + "'";
|
|
45
|
+
const languageOf = (code) => LANGUAGE_NAMES[code] || null;
|
|
46
|
+
|
|
47
|
+
/** Whether the package is in the virtual environment: lib/python3.*\/site-packages/mlx_qwen3_asr. */
|
|
48
|
+
function hasPackage(python) {
|
|
49
|
+
const lib = path.join(path.dirname(path.dirname(python)), 'lib');
|
|
50
|
+
try {
|
|
51
|
+
return fs.readdirSync(lib).some((d) => /^python3/.test(d) && fs.existsSync(path.join(lib, d, 'site-packages', 'mlx_qwen3_asr', '__init__.py')));
|
|
52
|
+
} catch { return false; }
|
|
53
|
+
}
|
|
54
|
+
/** Whether a model is in the cache: models--<org>--<name>/snapshots/<rev>/ with its config and weights. */
|
|
55
|
+
function hasModel(cache, repo) {
|
|
56
|
+
const snaps = path.join(cache, 'models--' + repo.replace('/', '--'), 'snapshots');
|
|
57
|
+
try {
|
|
58
|
+
return fs.readdirSync(snaps).some((rev) => {
|
|
59
|
+
const files = fs.readdirSync(path.join(snaps, rev));
|
|
60
|
+
return files.includes('config.json') && files.some((f) => /\.safetensors$/.test(f));
|
|
61
|
+
});
|
|
62
|
+
} catch { return false; }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** What Qwen3-ASR can do on this machine. PMOS_NOTES_QWEN 'off' turns it off; PMOS_NOTES_QWEN_PYTHON names the Python. */
|
|
66
|
+
function detect(runtimeDir, { env = process.env } = {}) {
|
|
67
|
+
const cache = cacheDir(runtimeDir);
|
|
68
|
+
if (String(env.PMOS_NOTES_QWEN || '') === 'off') return { ready: false, python: null, script: SCRIPT, live: null, final: null, cache, missing: ['qwen'] };
|
|
69
|
+
const python = env.PMOS_NOTES_QWEN_PYTHON || venvPython(runtimeDir);
|
|
70
|
+
const pkg = isExec(python) && (env.PMOS_NOTES_QWEN_PYTHON ? true : hasPackage(python));
|
|
71
|
+
const live = pkg && hasModel(cache, LIVE_MODEL) ? LIVE_MODEL : null;
|
|
72
|
+
const fin = pkg && hasModel(cache, FINAL_MODEL) ? FINAL_MODEL : null;
|
|
73
|
+
return { ready: !!live, python: pkg ? python : null, script: SCRIPT, live, final: fin || live, cache, missing: live && fin ? [] : ['qwen'] };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The setup lines, run after the virtual environment exists. Each model is downloaded once into the workspace's folder. */
|
|
77
|
+
function setupLines(runtimeDir) {
|
|
78
|
+
const py = venvPython(runtimeDir);
|
|
79
|
+
const get = (repo) => 'from huggingface_hub import snapshot_download; snapshot_download(' + JSON.stringify(repo) + ')';
|
|
80
|
+
return [
|
|
81
|
+
shq(py) + ' -m pip install --upgrade mlx-qwen3-asr',
|
|
82
|
+
'echo ' + shq('Downloading the Qwen3-ASR models (about 3 GB). Keep this terminal open.'),
|
|
83
|
+
'HF_HUB_CACHE=' + shq(cacheDir(runtimeDir)) + ' ' + shq(py) + ' -c ' + shq(get(LIVE_MODEL) + '; ' + get(FINAL_MODEL)),
|
|
84
|
+
];
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The program, started on the first request and kept until it is idle.
|
|
89
|
+
* @param {object} a
|
|
90
|
+
* @param {string} a.cmd the Python (or, in the suites, a stand-in)
|
|
91
|
+
* @param {string[]} a.args [script]
|
|
92
|
+
* @param {object} [a.env]
|
|
93
|
+
* @param {number} [a.idleMs]
|
|
94
|
+
*/
|
|
95
|
+
function createWorker({ cmd, args = [], env = {}, idleMs = IDLE_MS, timeoutMs = TIMEOUT_MS }) {
|
|
96
|
+
let child = null; let ready = null; let buf = '';
|
|
97
|
+
let seq = 0; let idleTimer = null;
|
|
98
|
+
const pending = new Map(); // id → { resolve, timer }
|
|
99
|
+
|
|
100
|
+
const failAll = (error) => { for (const [k, p] of pending) { clearTimeout(p.timer); p.resolve({ ok: false, error }); pending.delete(k); } };
|
|
101
|
+
function kill(error) {
|
|
102
|
+
const c = child; child = null; ready = null; buf = '';
|
|
103
|
+
if (c) { try { c.stdin.end(); } catch { /* closed */ } try { c.kill(); } catch { /* gone */ } }
|
|
104
|
+
failAll(error);
|
|
105
|
+
}
|
|
106
|
+
function armIdle() {
|
|
107
|
+
clearTimeout(idleTimer);
|
|
108
|
+
idleTimer = setTimeout(() => { if (!pending.size) kill('the speech program was stopped'); }, idleMs);
|
|
109
|
+
if (idleTimer.unref) idleTimer.unref();
|
|
110
|
+
}
|
|
111
|
+
function start() {
|
|
112
|
+
if (ready) return ready;
|
|
113
|
+
ready = new Promise((resolve) => {
|
|
114
|
+
let c;
|
|
115
|
+
try { c = spawn(cmd, args, { env: { ...process.env, ...env, HF_HUB_OFFLINE: '1', PYTHONUNBUFFERED: '1' }, stdio: ['pipe', 'pipe', 'pipe'] }); }
|
|
116
|
+
catch (e) { resolve({ ok: false, error: 'the speech program did not start: ' + ((e && e.message) || e) }); return; }
|
|
117
|
+
child = c;
|
|
118
|
+
let started = false; let errText = '';
|
|
119
|
+
const t = setTimeout(() => { if (!started) { resolve({ ok: false, error: 'the speech program took too long to start' }); kill('the speech program took too long to start'); } }, START_TIMEOUT_MS);
|
|
120
|
+
c.stderr.on('data', (d) => { errText = (errText + d).slice(-2000); });
|
|
121
|
+
c.on('error', (e) => { clearTimeout(t); if (!started) resolve({ ok: false, error: 'the speech program did not start: ' + e.message }); if (child === c) kill('the speech program stopped'); });
|
|
122
|
+
c.on('exit', () => {
|
|
123
|
+
clearTimeout(t);
|
|
124
|
+
const why = String(errText).trim().split('\n').pop() || 'the speech program stopped';
|
|
125
|
+
if (!started) resolve({ ok: false, error: why.slice(0, 200) });
|
|
126
|
+
if (child === c) kill(why.slice(0, 200));
|
|
127
|
+
});
|
|
128
|
+
c.stdout.on('data', (d) => {
|
|
129
|
+
buf += d;
|
|
130
|
+
let nl;
|
|
131
|
+
while ((nl = buf.indexOf('\n')) >= 0) {
|
|
132
|
+
const line = buf.slice(0, nl); buf = buf.slice(nl + 1);
|
|
133
|
+
let m; try { m = JSON.parse(line); } catch { continue; }
|
|
134
|
+
if (!started && m && 'ready' in m) {
|
|
135
|
+
started = true; clearTimeout(t);
|
|
136
|
+
resolve(m.ready ? { ok: true } : { ok: false, error: String(m.error || 'the speech program is not set up').slice(0, 200) });
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
const p = m && pending.get(m.id);
|
|
140
|
+
if (!p) continue;
|
|
141
|
+
clearTimeout(p.timer); pending.delete(m.id);
|
|
142
|
+
p.resolve(m.ok ? { ok: true, text: String(m.text || ''), language: String(m.language || '') } : { ok: false, error: String(m.error || 'the transcription failed').slice(0, 200) });
|
|
143
|
+
if (!pending.size) armIdle();
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
return ready;
|
|
148
|
+
}
|
|
149
|
+
/** One file. Resolves { ok, text, language } or { ok: false, error }; never rejects. */
|
|
150
|
+
async function run({ file, model, language = null, context = '', timeoutMs: tm = timeoutMs }) {
|
|
151
|
+
clearTimeout(idleTimer);
|
|
152
|
+
const s = await start();
|
|
153
|
+
if (!s.ok) { kill(s.error); return s; }
|
|
154
|
+
const id = ++seq;
|
|
155
|
+
return new Promise((resolve) => {
|
|
156
|
+
const timer = setTimeout(() => { pending.delete(id); resolve({ ok: false, error: 'the transcription took too long' }); kill('the transcription took too long'); }, tm);
|
|
157
|
+
pending.set(id, { resolve, timer });
|
|
158
|
+
try { child.stdin.write(JSON.stringify({ id, file, model, language, context: String(context || '') }) + '\n'); }
|
|
159
|
+
catch (e) { clearTimeout(timer); pending.delete(id); resolve({ ok: false, error: 'the speech program stopped' }); }
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
return { run, stop: () => { clearTimeout(idleTimer); kill('the speech program was stopped'); }, alive: () => !!child };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
let shared = null; let sharedKey = null;
|
|
166
|
+
/** The one worker of this process, for the live transcript and the final pass alike, so the models load once. */
|
|
167
|
+
function workerFor(runtimeDir, { env = process.env } = {}) {
|
|
168
|
+
const d = detect(runtimeDir, { env });
|
|
169
|
+
const key = runtimeDir + '|' + d.python;
|
|
170
|
+
if (shared && sharedKey === key) return shared;
|
|
171
|
+
if (shared) shared.stop();
|
|
172
|
+
sharedKey = key;
|
|
173
|
+
shared = createWorker({ cmd: d.python || 'python3', args: [SCRIPT], env: { HF_HUB_CACHE: d.cache } });
|
|
174
|
+
return shared;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The run function lib/notes-whisper.js's transcriber calls, for Qwen3-ASR: the same arguments, the same answer. */
|
|
178
|
+
function runQwen(worker, { file, model, lang, prompt = '', timeoutMs }) {
|
|
179
|
+
return worker.run({ file, model, language: languageOf(lang), context: prompt, ...(timeoutMs ? { timeoutMs } : {}) });
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
module.exports = { detect, setupLines, createWorker, workerFor, runQwen, languageOf, hasModel, cacheDir, LIVE_MODEL, FINAL_MODEL, SCRIPT, IDLE_MS };
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""notes-qwen.py: Qwen3-ASR speech to text that stays loaded, on this machine (meeting-notes Run 10).
|
|
3
|
+
|
|
4
|
+
python3 notes-qwen.py
|
|
5
|
+
|
|
6
|
+
It reads one JSON request per line on stdin and writes one JSON answer per line on stdout:
|
|
7
|
+
{"id": 1, "file": "/path/cut.wav", "model": "moona3k/mlx-qwen3-asr-0.6b-8bit", "language": "English", "context": "names"}
|
|
8
|
+
-> {"id": 1, "ok": true, "text": "...", "language": "English"}
|
|
9
|
+
or {"id": 1, "ok": false, "error": "..."}
|
|
10
|
+
When it has started it writes {"ready": true}. It stops when stdin closes.
|
|
11
|
+
|
|
12
|
+
The WAV is 16 kHz mono 16-bit. `language` is a Qwen3-ASR language name, or null to detect it. `context` is the meeting's
|
|
13
|
+
names and terms, which Qwen3-ASR reads as a hint. The model is loaded on the first request for it and kept, so a live
|
|
14
|
+
line costs the transcription only; at most two models are kept (the live one and the final one). lib/notes-qwen.js starts
|
|
15
|
+
it, sends one request at a time, and stops it when it has been idle. It reads no network (the server sets
|
|
16
|
+
HF_HUB_OFFLINE=1, so the models must already be in the cache) and writes no file.
|
|
17
|
+
"""
|
|
18
|
+
import json
|
|
19
|
+
import sys
|
|
20
|
+
|
|
21
|
+
MAX_SESSIONS = 2
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def say(obj):
|
|
25
|
+
sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
|
|
26
|
+
sys.stdout.flush()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main():
|
|
30
|
+
try:
|
|
31
|
+
from mlx_qwen3_asr import Session
|
|
32
|
+
except Exception as e: # noqa: BLE001
|
|
33
|
+
say({"ready": False, "error": "mlx-qwen3-asr is not installed: " + str(e)})
|
|
34
|
+
return 1
|
|
35
|
+
sessions = {} # model id -> Session, oldest first
|
|
36
|
+
say({"ready": True})
|
|
37
|
+
for line in sys.stdin:
|
|
38
|
+
line = line.strip()
|
|
39
|
+
if not line:
|
|
40
|
+
continue
|
|
41
|
+
rid = None
|
|
42
|
+
try:
|
|
43
|
+
req = json.loads(line)
|
|
44
|
+
rid = req.get("id")
|
|
45
|
+
model = str(req.get("model") or "")
|
|
46
|
+
if not model:
|
|
47
|
+
raise ValueError("no model was named")
|
|
48
|
+
s = sessions.get(model)
|
|
49
|
+
if s is None:
|
|
50
|
+
while len(sessions) >= MAX_SESSIONS:
|
|
51
|
+
sessions.pop(next(iter(sessions)))
|
|
52
|
+
s = Session(model=model)
|
|
53
|
+
sessions[model] = s
|
|
54
|
+
r = s.transcribe(str(req["file"]), language=req.get("language") or None, context=str(req.get("context") or ""))
|
|
55
|
+
say({"id": rid, "ok": True, "text": r.text or "", "language": r.language or ""})
|
|
56
|
+
except Exception as e: # noqa: BLE001
|
|
57
|
+
say({"id": rid, "ok": False, "error": (str(e) or e.__class__.__name__)[:300]})
|
|
58
|
+
return 0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
if __name__ == "__main__":
|
|
62
|
+
sys.exit(main())
|
|
@@ -172,7 +172,7 @@ function promptFor(vocab, previous) {
|
|
|
172
172
|
/* THE SPOKEN LANGUAGE. Whisper writes in the language it is told. With 'auto' it guesses the language of each cut, and on a
|
|
173
173
|
* few seconds of accented speech it can guess wrong and write the line in that language, which reads as a translation. So
|
|
174
174
|
* the PM picks the language of the meeting once (English by default) and every cut uses it. PMOS_WHISPER_LANG wins. */
|
|
175
|
-
const LANGUAGES = Object.freeze([['en', 'English'], ['id', 'Indonesian'], ['auto', 'Detect for each line']]);
|
|
175
|
+
const LANGUAGES = Object.freeze([['en', 'English'], ['id', 'Indonesian'], ['zh', 'Chinese'], ['auto', 'Detect for each line']]);
|
|
176
176
|
const DEFAULT_LANGUAGE = 'en';
|
|
177
177
|
const settingsFile = (runtimeDir) => path.join(runtimeDir, 'notes', 'whisper-settings.json');
|
|
178
178
|
function readSettings(runtimeDir) {
|
|
@@ -280,7 +280,8 @@ function createTranscriber({ runtimeDir, onText, onError = () => {}, run = runWh
|
|
|
280
280
|
try {
|
|
281
281
|
fs.writeFileSync(file, wav(pcm), { mode: 0o600 });
|
|
282
282
|
const key = job.id + '|' + job.source;
|
|
283
|
-
|
|
283
|
+
// `engine` is 'qwen' when the caller's detect chose Qwen3-ASR (lib/notes-qwen.js); `run` then sends the cut there
|
|
284
|
+
const r = await run({ engine: d.engine || 'whisper', bin: d.bin, model: d.model, file, lang: getLanguage(runtimeDir), accuracy: getAccuracy(runtimeDir), prompt: promptFor(vocabOf(job), lastText.get(key)) });
|
|
284
285
|
if (!r.ok) { onError({ ...job, error: r.error }); return; }
|
|
285
286
|
const text = cleanText(r.text, { voicedRatio });
|
|
286
287
|
if (text) { lastText.set(key, text); await onText({ ...job, text }); }
|
|
@@ -61,6 +61,7 @@ const whisper = require('./notes-whisper.js');
|
|
|
61
61
|
const notesRepo = require('./notes-repo.js');
|
|
62
62
|
const notesAudio = require('./notes-audio.js');
|
|
63
63
|
const notesFinal = require('./notes-final.js');
|
|
64
|
+
const notesQwen = require('./notes-qwen.js');
|
|
64
65
|
const notesLinks = require('./notes-links.js');
|
|
65
66
|
const jevNotes = require('./jev-notes.js');
|
|
66
67
|
const tsSettings = require('./typesafe-settings.js');
|
|
@@ -283,7 +284,20 @@ function vocabFor(ctx, id) {
|
|
|
283
284
|
return text;
|
|
284
285
|
}
|
|
285
286
|
|
|
286
|
-
/* ── local
|
|
287
|
+
/* ── local transcription: one transcriber for the process ─────────────────────────────────────── */
|
|
288
|
+
/* THE LIVE ENGINE (Run 10). Qwen3-ASR when it is set up (lib/notes-qwen.js: more accurate, Chinese and Indonesian too, kept
|
|
289
|
+
* loaded so each cut is quick), else whisper.cpp. ctx.notesQwenDetect and ctx.notesQwenRun replace them in the suites. */
|
|
290
|
+
function qwenDetect(ctx) { return ctx.notesQwenDetect ? ctx.notesQwenDetect() : notesQwen.detect(ctx.runtimeDir); }
|
|
291
|
+
function liveDetect(ctx) {
|
|
292
|
+
const q = qwenDetect(ctx);
|
|
293
|
+
if (q.ready) return { ready: true, engine: 'qwen', model: q.live, bin: null };
|
|
294
|
+
return { ...whisper.detect(ctx.runtimeDir), engine: 'whisper' };
|
|
295
|
+
}
|
|
296
|
+
function liveRun(ctx, args) {
|
|
297
|
+
if (args.engine !== 'qwen') return whisper.runWhisper(args);
|
|
298
|
+
if (ctx.notesQwenRun) return ctx.notesQwenRun(args);
|
|
299
|
+
return notesQwen.runQwen(notesQwen.workerFor(ctx.runtimeDir), args);
|
|
300
|
+
}
|
|
287
301
|
let transcriber = null; let transcriberDir = null;
|
|
288
302
|
const audioErrors = new Map(); // note id → the last thing that went wrong, for the page
|
|
289
303
|
function transcriberFor(ctx) {
|
|
@@ -291,6 +305,8 @@ function transcriberFor(ctx) {
|
|
|
291
305
|
transcriberDir = ctx.runtimeDir;
|
|
292
306
|
transcriber = whisper.createTranscriber({
|
|
293
307
|
runtimeDir: ctx.runtimeDir,
|
|
308
|
+
detect: () => liveDetect(ctx),
|
|
309
|
+
run: (args) => liveRun(ctx, args),
|
|
294
310
|
vocab: ({ id }) => vocabFor(ctx, id),
|
|
295
311
|
onText: async (c) => {
|
|
296
312
|
const r = notes.appendTranscript(ctx.runtimeDir, { id: c.id, text: c.text, source: c.source, t: c.t, ms: c.ms });
|
|
@@ -302,7 +318,8 @@ function transcriberFor(ctx) {
|
|
|
302
318
|
}
|
|
303
319
|
function engineInfo(ctx, w) {
|
|
304
320
|
const d = whisper.detect(ctx.runtimeDir);
|
|
305
|
-
const
|
|
321
|
+
const live = liveDetect(ctx);
|
|
322
|
+
const out = { ready: live.ready, live: live.ready ? live.engine : null, missing: d.missing, setup: d.setup, model_dir: d.model_dir, model_size: d.model_size,
|
|
306
323
|
language: whisper.getLanguage(ctx.runtimeDir), languages: whisper.LANGUAGES.map(([value, label]) => ({ value, label })),
|
|
307
324
|
model: whisper.getModel(ctx.runtimeDir), models: whisper.MODELS.map((m) => ({ value: m.id, label: m.label })),
|
|
308
325
|
accuracy: whisper.getAccuracy(ctx.runtimeDir), accuracies: whisper.ACCURACIES.map((a) => ({ value: a.id, label: a.label })), keep_audio: notesAudio.getKeep(ctx.runtimeDir),
|
|
@@ -336,6 +353,7 @@ function finalFor(ctx) {
|
|
|
336
353
|
runtimeDir: ctx.runtimeDir,
|
|
337
354
|
...(ctx.notesFinalRun ? { run: ctx.notesFinalRun } : {}),
|
|
338
355
|
...(ctx.notesFinalDetect ? { detect: ctx.notesFinalDetect } : {}),
|
|
356
|
+
...(ctx.notesQwenRun ? { qwen: ctx.notesQwenRun } : {}),
|
|
339
357
|
onState: (id, st) => { if (st.state !== 'done') notes.setFinalState(ctx.runtimeDir, { id, state: st }); },
|
|
340
358
|
onDone: (id, final) => {
|
|
341
359
|
const dir = ctx.runtimeDir;
|
|
@@ -564,7 +582,7 @@ async function handle(req, res, url, ctx) {
|
|
|
564
582
|
if (b64.length > Math.ceil(MAX_PCM_BYTES / 3) * 4) return done({ ok: false, status: 413, error: 'that audio chunk is too long' });
|
|
565
583
|
const pcm = Buffer.from(b64, 'base64');
|
|
566
584
|
if (pcm.length % 2) return done({ ok: false, status: 400, error: 'the audio must be 16-bit samples' });
|
|
567
|
-
if (!
|
|
585
|
+
if (!liveDetect(ctx).ready) return done({ ok: false, status: 409, error: 'local transcription is not set up' });
|
|
568
586
|
const ms = (pcm.length / 2 / whisper.RATE) * 1000;
|
|
569
587
|
const t = Math.max(0, notes.meetingMs(note.note) - ms);
|
|
570
588
|
// the audio is kept for replay unless the PM switched that off (D92); a private note never reaches this line
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pmos-workspace",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.80.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "pmos-workspace",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.80.0",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@xterm/addon-fit": "^0.11.0",
|
package/workspace/package.json
CHANGED
|
@@ -397,7 +397,7 @@ async function finalSetup(ctx) {
|
|
|
397
397
|
const ws = wsOf(ctx);
|
|
398
398
|
const r = await pmPost('/api/notes/final-setup', ws ? { ws } : {});
|
|
399
399
|
banner = r && r.ok && r.started
|
|
400
|
-
? { tone: 'ok', text: 'A terminal opened with the setup commands typed in. Read them and press Enter there. The downloads are about
|
|
400
|
+
? { tone: 'ok', text: 'A terminal opened with the setup commands typed in. Read them and press Enter there. The downloads are about 6 GB. Then open Settings again.' }
|
|
401
401
|
: { tone: 'bad', text: 'No terminal opened' + (r && r.reason ? ': ' + r.reason : '') + '. Run this in a terminal: ' + ((r && r.command) || '') };
|
|
402
402
|
ctx.setState({});
|
|
403
403
|
}
|
|
@@ -783,9 +783,11 @@ function openSettings(ctx, n) {
|
|
|
783
783
|
const parts = [];
|
|
784
784
|
parts.push(el('div', { class: 'nt-set', 'data-nt-whisper': w ? (w.ready ? 'ready' : 'missing') : 'unknown' },
|
|
785
785
|
el('div', { class: 'nt-set__head' }, 'Local transcription'),
|
|
786
|
-
el('p', { class: 'nt-hint'
|
|
786
|
+
el('p', { class: 'nt-hint', 'data-nt-live-engine': w ? (w.live || '') : null }, !w ? 'Reading …'
|
|
787
|
+
: w.live === 'qwen' ? 'Qwen3-ASR is set up. The microphone and the meeting are transcribed on this machine, in English, Chinese or Indonesian.'
|
|
788
|
+
: w.ready ? 'Whisper is set up. The microphone and the meeting are transcribed on this machine.'
|
|
787
789
|
: 'Whisper is not set up (' + w.missing.map((m) => (m === 'program' ? 'the program' : 'the model, ' + w.model_size)).join(' and ') + ' missing). The setup installs whisper.cpp with Homebrew and downloads the model.'),
|
|
788
|
-
w && w.models ? field({ label: 'Model', kind: 'select', value: w.model, options: w.models,
|
|
790
|
+
w && w.models && w.live !== 'qwen' ? field({ label: 'Model', kind: 'select', value: w.model, options: w.models,
|
|
789
791
|
onCommit: (v) => setSwitch(ctx, { model: v }).then(() => { h.close(); openSettings(ctx, n); }), attrs: { 'data-nt-model': '' } }) : null,
|
|
790
792
|
w && !w.ready ? button({ label: 'Set up in a terminal', onClick: () => { h.close(); whisperSetup(ctx); }, attrs: { 'data-nt-whisper-setup': '' } }) : null,
|
|
791
793
|
w && w.ready && w.languages ? field({ label: 'Language spoken in the meeting', kind: 'select', value: w.language, options: w.languages,
|
|
@@ -797,15 +799,15 @@ function openSettings(ctx, n) {
|
|
|
797
799
|
: 'No audio is kept: each clip is deleted once it is transcribed, and a meeting cannot be replayed.') : null));
|
|
798
800
|
const fp = w && w.final;
|
|
799
801
|
if (fp) {
|
|
800
|
-
const have = [fp.transcriber === 'parakeet' ? 'Parakeet' : fp.transcriber === 'whisper' ? 'Whisper with beam search' : null,
|
|
802
|
+
const have = [fp.transcriber === 'qwen' ? 'Qwen3-ASR' : fp.transcriber === 'parakeet' ? 'Parakeet' : fp.transcriber === 'whisper' ? 'Whisper with beam search' : null,
|
|
801
803
|
fp.vad ? 'speech detection' : null, fp.speakers ? 'speaker separation' : null].filter(Boolean);
|
|
802
804
|
parts.push(el('div', { class: 'nt-set', 'data-nt-final-set': fp.ready ? 'ready' : 'missing' },
|
|
803
805
|
el('div', { class: 'nt-set__head' }, 'Final notes after Stop'),
|
|
804
806
|
el('p', { class: 'nt-hint' }, fp.ready
|
|
805
807
|
? 'After Stop, the whole meeting is transcribed again on this machine (' + have.join(', ') + '), and your agent session writes the notes from it.'
|
|
806
808
|
: 'Set up local transcription first. After Stop, the whole meeting is then transcribed again on this machine, and your agent session writes the notes from it.'),
|
|
807
|
-
fp.missing.length ? el('p', { class: 'nt-hint', 'data-nt-final-missing': '' }, 'For better notes, set up ' + fp.missing.map((m) => (m === 'parakeet' ? 'Parakeet (more accurate in English)' : m === 'vad' ? 'speech detection' : 'speaker separation')).join(', ')
|
|
808
|
-
+ '. The setup installs them into the workspace\'s own folder; the downloads are about
|
|
809
|
+
fp.missing.length ? el('p', { class: 'nt-hint', 'data-nt-final-missing': '' }, 'For better notes, set up ' + fp.missing.map((m) => (m === 'qwen' ? 'Qwen3-ASR (English, Chinese and Indonesian, more accurate)' : m === 'parakeet' ? 'Parakeet (more accurate in English)' : m === 'vad' ? 'speech detection' : 'speaker separation')).join(', ')
|
|
810
|
+
+ '. The setup installs them into the workspace\'s own folder; the downloads are about 6 GB.') : null,
|
|
809
811
|
fp.missing.length ? button({ label: 'Set up in a terminal', onClick: () => { h.close(); finalSetup(ctx); }, attrs: { 'data-nt-final-setup': '' } }) : null,
|
|
810
812
|
field({ label: 'Fetch Wispr Flow notes after the meeting', kind: 'check', value: w.wispr !== false,
|
|
811
813
|
onCommit: (v) => setSwitch(ctx, { wispr: v }).then(() => { h.close(); openSettings(ctx, n); }), attrs: { 'data-nt-wispr-switch': '' } }),
|
|
@@ -1263,7 +1265,7 @@ function finalTranscript(ctx, n, opts) {
|
|
|
1263
1265
|
});
|
|
1264
1266
|
const box = el('div', { class: 'nt-lines', 'data-nt-transcript': 'final' }, lines);
|
|
1265
1267
|
const where = opts.inline ? 'inline' : 'full';
|
|
1266
|
-
const how = [n.final.engine === 'parakeet' ? 'Parakeet' : 'Whisper', n.final.diarized ? 'speakers told apart' : null].filter(Boolean).join(', ');
|
|
1268
|
+
const how = [n.final.engine === 'qwen' ? 'Qwen3-ASR' : n.final.engine === 'parakeet' ? 'Parakeet' : 'Whisper', n.final.diarized ? 'speakers told apart' : null].filter(Boolean).join(', ');
|
|
1267
1269
|
const d = el('details', { class: 'nt-trans', 'data-nt-part': 'transcript', open: transcriptOpen[where] },
|
|
1268
1270
|
el('summary', { class: 'nt-sec__h nt-fold' }, 'Transcript', el('span', { class: 'nt-src' }, ' · from the whole recording (' + how + '), ' + rows.length + (rows.length === 1 ? ' line' : ' lines'))),
|
|
1269
1271
|
button({ label: 'Show the live transcript', variant: 'ghost', size: 'sm', onClick: () => { liveTranscript = true; ctx.setState({}); }, attrs: { 'data-nt-live-transcript': '' } }), box);
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
{
|
|
2
|
-
"_comment": "meeting-notes Runs 1 to
|
|
2
|
+
"_comment": "meeting-notes Runs 1 to 10: the Notes engine and routes (notes.test.js), local Whisper, line labels and repository checks (notes-listen.test.js), the final pass after Stop and the Wispr Flow fetch (notes-final.test.js), Qwen3-ASR kept loaded (notes-qwen.test.js), and the Notes view in a browser (pm-notes-test.js).",
|
|
3
3
|
"modules": [
|
|
4
4
|
"notes.test.js",
|
|
5
5
|
"notes-listen.test.js",
|
|
6
|
-
"notes-final.test.js"
|
|
6
|
+
"notes-final.test.js",
|
|
7
|
+
"notes-qwen.test.js"
|
|
7
8
|
],
|
|
8
9
|
"panels": [
|
|
9
10
|
"pm-notes-test.js"
|
|
@@ -12,6 +13,7 @@
|
|
|
12
13
|
"notes.test.js": 12,
|
|
13
14
|
"pm-notes-test.js": 40,
|
|
14
15
|
"notes-listen.test.js": 10,
|
|
15
|
-
"notes-final.test.js": 6
|
|
16
|
+
"notes-final.test.js": 6,
|
|
17
|
+
"notes-qwen.test.js": 4
|
|
16
18
|
}
|
|
17
19
|
}
|