@marver-design/marver 0.25.1 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +116 -0
- package/README.md +5 -3
- package/dist/{anchor-DVV732zR.mjs → anchor-5H1vnoLu.mjs} +6 -1
- package/dist/{anchor-check-DGZnYviN.mjs → anchor-check-l-ny3UzH.mjs} +1 -1
- package/dist/audio-Dw1lfMdb.mjs +443 -0
- package/dist/{auth-DdUKeVKt.mjs → auth-Dix8wrXV.mjs} +1 -1
- package/dist/{bake-BpxMZfij.mjs → bake-C-awKgLA.mjs} +1 -1
- package/dist/{boards-Csxj2WJn.mjs → boards-DYG22tOe.mjs} +2 -2
- package/dist/{build-DKaGhoVN.mjs → build-BDRiO_mP.mjs} +16 -9
- package/dist/capture-vadKjuCo.mjs +730 -0
- package/dist/cli.mjs +24 -18
- package/dist/{collab-gqNdScUE.mjs → collab-Dxg2trOK.mjs} +4 -3
- package/dist/{comments-Zoch254k.mjs → comments-0BSbb04K.mjs} +14 -3
- package/dist/{comments-pTLlhjh4.mjs → comments-BqXua-XL.mjs} +7 -7
- package/dist/{config-cKgZBRj0.mjs → config-Cf7EjAMf.mjs} +12 -1
- package/dist/{context-D2ugHqwO.mjs → context-HQ4Giw58.mjs} +1 -1
- package/dist/daemon-DL6Fwq6D.mjs +2569 -0
- package/dist/{dev-Diqx1Rt8.mjs → dev-Bw_qYWWq.mjs} +202 -18
- package/dist/dev-lock-DWSpa6nj.mjs +179 -0
- package/dist/{events-DnK_AnrW.mjs → events-CLH4ijKP.mjs} +14 -4
- package/dist/{init-DO4DhUt_.mjs → init-D8DtrK_F.mjs} +2 -2
- package/dist/{ledger-BNXW14Xw.mjs → ledger-D54iROST.mjs} +17 -1
- package/dist/{link-C-qUIuKU.mjs → link-BNzLvdhI.mjs} +2 -2
- package/dist/live-Cgzsicwi.mjs +4 -0
- package/dist/{manifest-DnAte_fB.mjs → manifest-Nl3KSVOk.mjs} +1 -1
- package/dist/{marks-koRsbHfe.mjs → marks-CKgzucHZ.mjs} +2 -2
- package/dist/{marks-DFuintmJ.mjs → marks-DyWMv_Z1.mjs} +23 -5
- package/dist/{marver-id-gate-DGANtVyC.mjs → marver-id-gate-BjF6v4bT.mjs} +4 -4
- package/dist/perf-C75MZ73h.mjs +167 -0
- package/dist/{perf-CCisg-ny.mjs → perf-DPbLGhhO.mjs} +1 -1
- package/dist/perf-jam-BU3TVrZu.mjs +239 -0
- package/dist/{plugin-dTF5reck.mjs → plugin-DkGmZs_r.mjs} +298 -38
- package/dist/{poster-D-zjYpgk.mjs → poster-BdtSKMDm.mjs} +2 -1
- package/dist/{profile-Do0nGkXO.mjs → profile-WU9qcbIo.mjs} +1 -1
- package/dist/{publish-bakes-C_9VZ-Mq.mjs → publish-bakes-9RzRToXV.mjs} +3 -3
- package/dist/queue-29ok5_eZ.mjs +1319 -0
- package/dist/reserve-BURNQ2Y3.mjs +53 -0
- package/dist/{rolldown-runtime-D7D4PA-g.mjs → rolldown-runtime-8H4AJuhK.mjs} +1 -0
- package/dist/{serve-wv2QWdxB.mjs → serve-C8rVheYk.mjs} +12 -12
- package/dist/session-BAaJsK5c.mjs +191 -0
- package/dist/{share-ScamJcks.mjs → share--oFJ1bCI.mjs} +2 -2
- package/dist/{shot-CUlmkqFX.mjs → shot-DQeR_Pfn.mjs} +339 -20
- package/dist/shot-XQV7HC9A.mjs +139 -0
- package/dist/speech-models-DomiLQcY.mjs +299 -0
- package/dist/speech-worker.mjs +354 -0
- package/dist/{summary-CrW88e3q.mjs → summary-Y8MY1Wuc.mjs} +2 -2
- package/dist/{sync-C3035OnJ.mjs → sync-DKk-yT1V.mjs} +9 -8
- package/dist/{work-GE88wb_3.mjs → work-F5u-twpJ.mjs} +28 -5
- package/dist/{work-CpJn1gwY.mjs → work-VvX_JIgl.mjs} +2 -2
- package/docs/jam-session.md +194 -0
- package/docs/live-jam.md +29 -3
- package/package.json +7 -1
- package/src/client/frame-host/bridge.js +20 -2
- package/src/client/frame-host/inspect.d.ts +4 -0
- package/src/client/frame-host/inspect.js +96 -6
- package/src/client/frame-host/keys.d.ts +6 -0
- package/src/client/frame-host/keys.js +59 -0
- package/src/client/frame-host/trail.d.ts +7 -0
- package/src/client/frame-host/trail.js +204 -0
- package/src/client/shell/App.tsx +52 -8
- package/src/client/shell/Comments.tsx +20 -6
- package/src/client/shell/JamSurface.tsx +346 -0
- package/src/client/shell/Play.tsx +32 -9
- package/src/client/shell/Tip.tsx +8 -3
- package/src/client/shell/Toolbar.tsx +5 -4
- package/src/client/shell/canvas/Canvas.tsx +12 -2
- package/src/client/shell/canvas/FrameNode.tsx +14 -4
- package/src/client/shell/comments-store.ts +23 -16
- package/src/client/shell/icons.tsx +17 -0
- package/src/client/shell/jam-capture.ts +1005 -0
- package/src/client/shell/jam-keys.ts +358 -0
- package/src/client/shell/jam-lane.tsx +249 -0
- package/src/client/shell/jam-lines.ts +115 -0
- package/src/client/shell/jam-panel.tsx +556 -0
- package/src/client/shell/jam-queue.ts +126 -0
- package/src/client/shell/jam-session.tsx +389 -0
- package/src/client/shell/jam-state.ts +103 -0
- package/src/client/shell/jam-surface-parts.tsx +106 -0
- package/src/client/shell/jam-ui.tsx +104 -0
- package/src/client/shell/jam-voice.ts +479 -0
- package/src/client/shell/sound.ts +22 -0
- package/src/client/shell/stage-size.ts +11 -0
- package/src/client/shell/store.ts +47 -17
- package/src/client/shell/styles.css +564 -2
- package/src/client/stage/main.tsx +18 -2
- package/src/shared/capture-frames.ts +41 -0
- package/src/shared/events.ts +19 -4
- package/src/shared/speech.ts +16 -0
- package/templates/AGENTS-embedded.md +10 -2
- package/templates/AGENTS-studio.md +10 -2
- package/templates/instructions/iterate.md +8 -6
- package/templates/instructions/jam.md +131 -59
- package/dist/daemon-Cl8KXGW4.mjs +0 -1523
- package/dist/perf-QJIWtTw1.mjs +0 -68
- package/dist/shot-jdO_B6Qw.mjs +0 -86
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
import { createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { spawn } from "node:child_process";
|
|
6
|
+
import { Readable } from "node:stream";
|
|
7
|
+
import { pipeline } from "node:stream/promises";
|
|
8
|
+
//#region src/server/jam/speech-models.ts
|
|
9
|
+
/**
|
|
10
|
+
* The speech model, on this Mac (spec 27, Part 4): Parakeet TDT 0.6B v3 (int8) transcribes, Silero VAD
|
|
11
|
+
* cuts phrases. Downloaded once - only after you say yes - to `~/.marver/models/`, and every file is
|
|
12
|
+
* checked against the hash it was published with before it is used. Nothing here runs a model; the
|
|
13
|
+
* worker does (speech-worker.ts).
|
|
14
|
+
*/
|
|
15
|
+
const RELEASE = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/";
|
|
16
|
+
const ARCHIVE = {
|
|
17
|
+
name: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
|
|
18
|
+
sha256: "5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf",
|
|
19
|
+
bytes: 487170055
|
|
20
|
+
};
|
|
21
|
+
const PARAKEET_DIR = "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8";
|
|
22
|
+
/** The files the recognizer reads, in the archive's folder - each checked before use. */
|
|
23
|
+
const PARAKEET_FILES = {
|
|
24
|
+
encoder: {
|
|
25
|
+
file: "encoder.int8.onnx",
|
|
26
|
+
sha256: "acfc2b4456377e15d04f0243af540b7fe7c992f8d898d751cf134c3a55fd2247"
|
|
27
|
+
},
|
|
28
|
+
decoder: {
|
|
29
|
+
file: "decoder.int8.onnx",
|
|
30
|
+
sha256: "179e50c43d1a9de79c8a24149a2f9bac6eb5981823f2a2ed88d655b24248db4e"
|
|
31
|
+
},
|
|
32
|
+
joiner: {
|
|
33
|
+
file: "joiner.int8.onnx",
|
|
34
|
+
sha256: "3164c13fc2821009440d20fcb5fdc78bff28b4db2f8d0f0b329101719c0948b3"
|
|
35
|
+
},
|
|
36
|
+
tokens: {
|
|
37
|
+
file: "tokens.txt",
|
|
38
|
+
sha256: "d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d"
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
const VAD = {
|
|
42
|
+
name: "silero_vad.onnx",
|
|
43
|
+
sha256: "9e2449e1087496d8d4caba907f23e0bd3f78d91fa552479bb9c23ac09cbb1fd6",
|
|
44
|
+
bytes: 643854
|
|
45
|
+
};
|
|
46
|
+
/** About what the download weighs, for the question ("490 MB, once"). */
|
|
47
|
+
const DOWNLOAD_MB = Math.round((ARCHIVE.bytes + VAD.bytes) / 1e6);
|
|
48
|
+
/** Where the models live: `~/.marver/models/` (MARVER_MODELS points elsewhere - the tests). */
|
|
49
|
+
const modelsDir = () => process.env.MARVER_MODELS || join(homedir(), ".marver", "models");
|
|
50
|
+
const modelPaths = (dir = modelsDir()) => ({
|
|
51
|
+
encoder: join(dir, PARAKEET_DIR, PARAKEET_FILES.encoder.file),
|
|
52
|
+
decoder: join(dir, PARAKEET_DIR, PARAKEET_FILES.decoder.file),
|
|
53
|
+
joiner: join(dir, PARAKEET_DIR, PARAKEET_FILES.joiner.file),
|
|
54
|
+
tokens: join(dir, PARAKEET_DIR, PARAKEET_FILES.tokens.file),
|
|
55
|
+
vad: join(dir, VAD.name)
|
|
56
|
+
});
|
|
57
|
+
const sha256File = async (path) => {
|
|
58
|
+
const h = createHash("sha256");
|
|
59
|
+
await pipeline(createReadStream(path), h);
|
|
60
|
+
return h.digest("hex");
|
|
61
|
+
};
|
|
62
|
+
/** A hash checked once per file per process (by its size and mtime): the 650 MB encoder is hashed once. */
|
|
63
|
+
const checked = /* @__PURE__ */ new Map();
|
|
64
|
+
async function fileOk(path, sha256) {
|
|
65
|
+
let st;
|
|
66
|
+
try {
|
|
67
|
+
st = statSync(path);
|
|
68
|
+
} catch {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
const key = `${path}|${st.size}|${st.mtimeMs}`;
|
|
72
|
+
if (checked.get(key) === sha256) return true;
|
|
73
|
+
const ok = await sha256File(path) === sha256;
|
|
74
|
+
if (ok) checked.set(key, sha256);
|
|
75
|
+
return ok;
|
|
76
|
+
}
|
|
77
|
+
/** Are the model files there, each the one published? */
|
|
78
|
+
async function modelsReady(dir = modelsDir()) {
|
|
79
|
+
const p = modelPaths(dir);
|
|
80
|
+
const all = [...Object.keys(PARAKEET_FILES).map((k) => [p[k], PARAKEET_FILES[k].sha256]), [p.vad, VAD.sha256]];
|
|
81
|
+
for (const [path, sha] of all) if (!await fileOk(path, sha)) return false;
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
let download = {
|
|
85
|
+
state: "idle",
|
|
86
|
+
received: 0,
|
|
87
|
+
total: ARCHIVE.bytes + VAD.bytes
|
|
88
|
+
};
|
|
89
|
+
let running = null;
|
|
90
|
+
const downloadState = () => ({ ...download });
|
|
91
|
+
/** Fetch one file to `dest` through a temporary name, counting bytes; refuse it unless its hash matches. */
|
|
92
|
+
async function fetchChecked(url, dest, sha256, onBytes, fetchImpl) {
|
|
93
|
+
const tmp = `${dest}.part`;
|
|
94
|
+
const res = await fetchImpl(url);
|
|
95
|
+
if (!res.ok || !res.body) throw new Error(`download failed (${res.status}) - ${url}`);
|
|
96
|
+
const h = createHash("sha256");
|
|
97
|
+
const body = Readable.fromWeb(res.body);
|
|
98
|
+
body.on("data", (c) => {
|
|
99
|
+
h.update(c);
|
|
100
|
+
onBytes(c.length);
|
|
101
|
+
});
|
|
102
|
+
await pipeline(body, createWriteStream(tmp));
|
|
103
|
+
if (h.digest("hex") !== sha256) {
|
|
104
|
+
rmSync(tmp, { force: true });
|
|
105
|
+
throw new Error(`${dest.split("/").pop()} did not match its published hash - nothing was kept`);
|
|
106
|
+
}
|
|
107
|
+
renameSync(tmp, dest);
|
|
108
|
+
}
|
|
109
|
+
const untar = (archive, into) => new Promise((resolve, reject) => {
|
|
110
|
+
const p = spawn("tar", [
|
|
111
|
+
"-xjf",
|
|
112
|
+
archive,
|
|
113
|
+
"-C",
|
|
114
|
+
into
|
|
115
|
+
], { stdio: "ignore" });
|
|
116
|
+
p.on("error", reject);
|
|
117
|
+
p.on("exit", (code) => code === 0 ? resolve() : reject(/* @__PURE__ */ new Error(`tar exited ${code}`)));
|
|
118
|
+
});
|
|
119
|
+
const alive = (pid) => {
|
|
120
|
+
try {
|
|
121
|
+
process.kill(pid, 0);
|
|
122
|
+
return true;
|
|
123
|
+
} catch (e) {
|
|
124
|
+
return e.code === "EPERM";
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
128
|
+
/** A lock file unreadable (half written by a process that died writing it) for this long is stale. */
|
|
129
|
+
const LOCK_GRACE = 1e4;
|
|
130
|
+
const ageOf = (f) => {
|
|
131
|
+
try {
|
|
132
|
+
return Date.now() - statSync(f).mtimeMs;
|
|
133
|
+
} catch {
|
|
134
|
+
return 0;
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
/** A lock's text, or null when there is no lock; any other read error (permissions) is thrown - waited on,
|
|
138
|
+
* it would never change. */
|
|
139
|
+
const readRaw = (f) => {
|
|
140
|
+
try {
|
|
141
|
+
return readFileSync(f, "utf8");
|
|
142
|
+
} catch (e) {
|
|
143
|
+
if (e.code === "ENOENT") return null;
|
|
144
|
+
throw e;
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
/** Take the download lock (one installer at a time, across dev servers). Resolves to this installer's
|
|
148
|
+
* token, or to 'done' when another process finished the models while this one waited. A stale lock - its
|
|
149
|
+
* process gone, or unreadable past the grace - is reclaimed by one process at a time, under a guard, and
|
|
150
|
+
* only if it is still the same stale lock. */
|
|
151
|
+
async function takeLock(lock, dir) {
|
|
152
|
+
const token = `${process.pid}-${randomBytes(6).toString("hex")}`;
|
|
153
|
+
for (;;) {
|
|
154
|
+
try {
|
|
155
|
+
writeFileSync(lock, JSON.stringify({
|
|
156
|
+
pid: process.pid,
|
|
157
|
+
token
|
|
158
|
+
}), { flag: "wx" });
|
|
159
|
+
return token;
|
|
160
|
+
} catch (e) {
|
|
161
|
+
if (e.code !== "EEXIST") throw e;
|
|
162
|
+
}
|
|
163
|
+
const raw = readRaw(lock);
|
|
164
|
+
if (raw === null) {
|
|
165
|
+
await sleep(50);
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
let pid = 0;
|
|
169
|
+
try {
|
|
170
|
+
pid = Number(JSON.parse(raw).pid) || 0;
|
|
171
|
+
} catch {
|
|
172
|
+
pid = 0;
|
|
173
|
+
}
|
|
174
|
+
if (pid ? !alive(pid) : ageOf(lock) > LOCK_GRACE) {
|
|
175
|
+
const guard = `${lock}.reclaim`;
|
|
176
|
+
try {
|
|
177
|
+
writeFileSync(guard, String(process.pid), { flag: "wx" });
|
|
178
|
+
} catch (e) {
|
|
179
|
+
if (e.code !== "EEXIST") throw e;
|
|
180
|
+
if (ageOf(guard) > LOCK_GRACE) rmSync(guard, { force: true });
|
|
181
|
+
await sleep(200);
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
try {
|
|
185
|
+
if (readRaw(lock) === raw) rmSync(lock, { force: true });
|
|
186
|
+
} finally {
|
|
187
|
+
rmSync(guard, { force: true });
|
|
188
|
+
}
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
await sleep(1e3);
|
|
192
|
+
if (!existsSync(lock) && await modelsReady(dir)) return "done";
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/** Release the lock - only this installer's own. */
|
|
196
|
+
const dropLock = (lock, token) => {
|
|
197
|
+
try {
|
|
198
|
+
const raw = readRaw(lock);
|
|
199
|
+
if (raw && JSON.parse(raw).token === token) rmSync(lock, { force: true });
|
|
200
|
+
} catch {}
|
|
201
|
+
};
|
|
202
|
+
/** Download the models (once - a second call joins the first; another dev server downloading them is
|
|
203
|
+
* waited for, through a lock file). Everything lands in a staging folder of this process first, is
|
|
204
|
+
* checked there, and only then moves into place; a failure leaves nothing behind. */
|
|
205
|
+
function downloadModels(dir = modelsDir(), fetchImpl = fetch) {
|
|
206
|
+
if (running) return running;
|
|
207
|
+
download = {
|
|
208
|
+
state: "downloading",
|
|
209
|
+
received: 0,
|
|
210
|
+
total: ARCHIVE.bytes + VAD.bytes
|
|
211
|
+
};
|
|
212
|
+
running = (async () => {
|
|
213
|
+
const lock = join(dir, ".download.lock");
|
|
214
|
+
let token;
|
|
215
|
+
try {
|
|
216
|
+
mkdirSync(dir, { recursive: true });
|
|
217
|
+
token = await takeLock(lock, dir);
|
|
218
|
+
} catch (err) {
|
|
219
|
+
download = {
|
|
220
|
+
...download,
|
|
221
|
+
state: "failed",
|
|
222
|
+
error: `the download could not start: ${err.message}`
|
|
223
|
+
};
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
if (token === "done") {
|
|
227
|
+
download = {
|
|
228
|
+
...download,
|
|
229
|
+
state: "done",
|
|
230
|
+
received: download.total
|
|
231
|
+
};
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const stage = join(dir, `.staging-${process.pid}-${randomBytes(4).toString("hex")}`);
|
|
235
|
+
try {
|
|
236
|
+
mkdirSync(stage, { recursive: true });
|
|
237
|
+
const count = (n) => {
|
|
238
|
+
download.received += n;
|
|
239
|
+
};
|
|
240
|
+
const p = modelPaths(dir);
|
|
241
|
+
if (!await fileOk(p.vad, VAD.sha256)) await fetchChecked(RELEASE + VAD.name, join(stage, VAD.name), VAD.sha256, count, fetchImpl);
|
|
242
|
+
else download.received += VAD.bytes;
|
|
243
|
+
const parakeetOk = await (async () => {
|
|
244
|
+
for (const k of Object.keys(PARAKEET_FILES)) if (!await fileOk(p[k], PARAKEET_FILES[k].sha256)) return false;
|
|
245
|
+
return true;
|
|
246
|
+
})();
|
|
247
|
+
if (!parakeetOk) {
|
|
248
|
+
const archive = join(stage, ARCHIVE.name);
|
|
249
|
+
await fetchChecked(RELEASE + ARCHIVE.name, archive, ARCHIVE.sha256, count, fetchImpl);
|
|
250
|
+
download.state = "checking";
|
|
251
|
+
await untar(archive, stage);
|
|
252
|
+
rmSync(archive, { force: true });
|
|
253
|
+
if (!await parakeetIn(stage)) throw new Error("the unpacked model did not match its published hashes");
|
|
254
|
+
} else download.received += ARCHIVE.bytes;
|
|
255
|
+
download.state = "checking";
|
|
256
|
+
if (!parakeetOk) {
|
|
257
|
+
rmSync(join(dir, PARAKEET_DIR), {
|
|
258
|
+
recursive: true,
|
|
259
|
+
force: true
|
|
260
|
+
});
|
|
261
|
+
renameSync(join(stage, PARAKEET_DIR), join(dir, PARAKEET_DIR));
|
|
262
|
+
}
|
|
263
|
+
if (existsSync(join(stage, VAD.name))) renameSync(join(stage, VAD.name), p.vad);
|
|
264
|
+
if (!await modelsReady(dir)) throw new Error("the model did not match its published hashes once in place");
|
|
265
|
+
download.state = "done";
|
|
266
|
+
} catch (err) {
|
|
267
|
+
download = {
|
|
268
|
+
...download,
|
|
269
|
+
state: "failed",
|
|
270
|
+
error: err.message
|
|
271
|
+
};
|
|
272
|
+
} finally {
|
|
273
|
+
try {
|
|
274
|
+
rmSync(stage, {
|
|
275
|
+
recursive: true,
|
|
276
|
+
force: true
|
|
277
|
+
});
|
|
278
|
+
} catch {}
|
|
279
|
+
dropLock(lock, token);
|
|
280
|
+
}
|
|
281
|
+
})().catch((err) => {
|
|
282
|
+
download = {
|
|
283
|
+
...download,
|
|
284
|
+
state: "failed",
|
|
285
|
+
error: err.message
|
|
286
|
+
};
|
|
287
|
+
}).finally(() => {
|
|
288
|
+
running = null;
|
|
289
|
+
});
|
|
290
|
+
return running;
|
|
291
|
+
}
|
|
292
|
+
/** The recognizer's files, checked, in a folder (the staging one: the VAD may already be in place). */
|
|
293
|
+
async function parakeetIn(dir) {
|
|
294
|
+
const p = modelPaths(dir);
|
|
295
|
+
for (const k of Object.keys(PARAKEET_FILES)) if (!await fileOk(p[k], PARAKEET_FILES[k].sha256)) return false;
|
|
296
|
+
return true;
|
|
297
|
+
}
|
|
298
|
+
//#endregion
|
|
299
|
+
export { DOWNLOAD_MB, downloadModels, downloadState, modelPaths, modelsReady };
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
+
//#region src/server/jam/speech-cut.ts
|
|
4
|
+
/**
|
|
5
|
+
* One capture's audio, cut into phrases and transcribed in order (spec 27, Part 4) - the logic, free of
|
|
6
|
+
* the model, so it is tested with a fake VAD and a fake recognizer. Samples are 16 kHz mono, numbered
|
|
7
|
+
* from the capture's start (sample 0 is the mark at `S`): pre-roll can never reach before it.
|
|
8
|
+
*
|
|
9
|
+
* - Silero VAD cuts phrases (0.4 s of silence ends one). Marver also cuts at 15 s: it counts the samples
|
|
10
|
+
* since a phrase began and, at 15 s, calls `vad.flush()` - the next phrase starts at the next sample.
|
|
11
|
+
* - Each phrase keeps 250 ms of the audio before it (the VAD clipped first words), never from before the
|
|
12
|
+
* capture, and never across a forced cut (that audio is already in the phrase before).
|
|
13
|
+
* - At most 30 s of audio waits for transcription; past that the oldest waiting phrase is dropped and
|
|
14
|
+
* the transcript says "[words lost]" where it was.
|
|
15
|
+
* - `end(lastSample)` pads the last partial VAD window with silence (the VAD only takes whole windows),
|
|
16
|
+
* flushes, transcribes what is left, drops any word timed after `lastSample`, and resolves.
|
|
17
|
+
* Raw audio lives only in memory, and only as long as it is needed.
|
|
18
|
+
*/
|
|
19
|
+
const RATE = 16e3;
|
|
20
|
+
const PREROLL = Math.round(.25 * RATE);
|
|
21
|
+
/** How much recent audio is kept for pre-roll - a little over the longest phrase plus its pre-roll. */
|
|
22
|
+
const HISTORY = 272e3;
|
|
23
|
+
/** Tokens to words: a token that starts with a space or `▁` starts a word; the word's time is its first
|
|
24
|
+
* token's. */
|
|
25
|
+
function tokensToWords(r, phraseStart) {
|
|
26
|
+
const out = [];
|
|
27
|
+
r.tokens.forEach((t, i) => {
|
|
28
|
+
const at = phraseStart + Math.round((r.timestamps[i] ?? 0) * RATE);
|
|
29
|
+
const starts = /^[\s▁]/.test(t) || !out.length;
|
|
30
|
+
const text = t.replace(/^[\s▁]+/, "");
|
|
31
|
+
if (starts) {
|
|
32
|
+
if (text) out.push({
|
|
33
|
+
w: text,
|
|
34
|
+
at
|
|
35
|
+
});
|
|
36
|
+
} else if (out.length) out[out.length - 1].w += text;
|
|
37
|
+
});
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
var Segmenter = class {
|
|
41
|
+
pending = /* @__PURE__ */ new Float32Array(0);
|
|
42
|
+
fed = 0;
|
|
43
|
+
history = new Float32Array(HISTORY);
|
|
44
|
+
speech = 0;
|
|
45
|
+
forcedAt = -1;
|
|
46
|
+
skipped = 0;
|
|
47
|
+
gapEnd = 0;
|
|
48
|
+
cancelled = false;
|
|
49
|
+
waiting = [];
|
|
50
|
+
busy = false;
|
|
51
|
+
idle = [];
|
|
52
|
+
last = Infinity;
|
|
53
|
+
ended = false;
|
|
54
|
+
vad;
|
|
55
|
+
recognize;
|
|
56
|
+
onWords;
|
|
57
|
+
onTiming;
|
|
58
|
+
decoding = 0;
|
|
59
|
+
/** the audio waiting for the recognizer, in samples - this capture's alone, or (the worker's) every capture sharing it */
|
|
60
|
+
backlog;
|
|
61
|
+
constructor(vad, recognize, onWords, onTiming, backlog) {
|
|
62
|
+
this.vad = vad;
|
|
63
|
+
this.recognize = recognize;
|
|
64
|
+
this.onWords = onWords;
|
|
65
|
+
this.onTiming = onTiming;
|
|
66
|
+
this.backlog = backlog ?? (() => this.waitingSamples + this.decoding);
|
|
67
|
+
}
|
|
68
|
+
/** The samples from `offset` on (the page numbers them; an overlap is skipped). A short gap is silence
|
|
69
|
+
* given to the VAD; a long one (over a second - a microphone that opened late, a resumed capture) ends
|
|
70
|
+
* the phrase being said and is stepped over whole: every sample keeps its exact number. */
|
|
71
|
+
push(offset, pcm) {
|
|
72
|
+
if (this.ended) return;
|
|
73
|
+
let have = this.fed + this.pending.length;
|
|
74
|
+
if (offset > have && offset - have <= 16e3) {
|
|
75
|
+
this.take(new Float32Array(offset - have));
|
|
76
|
+
have = offset;
|
|
77
|
+
} else if (offset > have) {
|
|
78
|
+
if (this.pending.length) {
|
|
79
|
+
const w = /* @__PURE__ */ new Float32Array(512);
|
|
80
|
+
w.set(this.pending);
|
|
81
|
+
this.pending = /* @__PURE__ */ new Float32Array(0);
|
|
82
|
+
this.window(w);
|
|
83
|
+
}
|
|
84
|
+
this.vad.flush();
|
|
85
|
+
this.drain();
|
|
86
|
+
this.speech = 0;
|
|
87
|
+
if (offset > this.fed) {
|
|
88
|
+
this.skipped += offset - this.fed;
|
|
89
|
+
this.fed = offset;
|
|
90
|
+
}
|
|
91
|
+
this.gapEnd = this.fed;
|
|
92
|
+
have = this.fed;
|
|
93
|
+
}
|
|
94
|
+
const skip = Math.max(0, have - offset);
|
|
95
|
+
if (skip >= pcm.length) return;
|
|
96
|
+
this.take(pcm.subarray(skip));
|
|
97
|
+
}
|
|
98
|
+
take(pcm) {
|
|
99
|
+
let buf = this.pending.length ? concat(this.pending, pcm) : pcm;
|
|
100
|
+
let i = 0;
|
|
101
|
+
for (; i + 512 <= buf.length; i += 512) this.window(buf.subarray(i, i + 512));
|
|
102
|
+
this.pending = buf.slice(i);
|
|
103
|
+
}
|
|
104
|
+
window(w) {
|
|
105
|
+
for (let k = 0; k < w.length; k++) this.history[(this.fed + k) % HISTORY] = w[k];
|
|
106
|
+
this.vad.acceptWaveform(w);
|
|
107
|
+
this.fed += w.length;
|
|
108
|
+
if (this.vad.isDetected()) {
|
|
109
|
+
this.speech += w.length;
|
|
110
|
+
if (this.speech >= 24e4) {
|
|
111
|
+
this.vad.flush();
|
|
112
|
+
this.drain();
|
|
113
|
+
this.forcedAt = this.fed;
|
|
114
|
+
this.speech = 0;
|
|
115
|
+
}
|
|
116
|
+
} else this.speech = 0;
|
|
117
|
+
this.drain();
|
|
118
|
+
}
|
|
119
|
+
/** Phrases the VAD has finished, each with its pre-roll, into the waiting list. */
|
|
120
|
+
drain() {
|
|
121
|
+
while (!this.vad.isEmpty()) {
|
|
122
|
+
const raw = this.vad.front();
|
|
123
|
+
this.vad.pop();
|
|
124
|
+
const seg = {
|
|
125
|
+
start: raw.start + this.skipped,
|
|
126
|
+
samples: raw.samples
|
|
127
|
+
};
|
|
128
|
+
const floor = Math.max(0, this.forcedAt >= 0 && this.forcedAt <= seg.start ? this.forcedAt : 0, this.gapEnd <= seg.start ? this.gapEnd : 0, this.fed - HISTORY);
|
|
129
|
+
const from = Math.max(floor, seg.start - PREROLL);
|
|
130
|
+
const pre = new Float32Array(Math.max(0, seg.start - from));
|
|
131
|
+
for (let k = 0; k < pre.length; k++) pre[k] = this.history[(from + k) % HISTORY];
|
|
132
|
+
this.enqueue({
|
|
133
|
+
start: from,
|
|
134
|
+
samples: pre.length ? concat(pre, seg.samples) : seg.samples
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
enqueue(p) {
|
|
139
|
+
p.queuedAt = performance.now();
|
|
140
|
+
p.backlog = this.backlog();
|
|
141
|
+
this.waiting.push(p);
|
|
142
|
+
while (this.waiting.length > 1 && this.waiting.reduce((n, x) => n + x.samples.length, 0) > 48e4) {
|
|
143
|
+
const gone = this.waiting.shift();
|
|
144
|
+
if (!this.cancelled) this.onWords([{
|
|
145
|
+
w: "[words lost]",
|
|
146
|
+
at: gone.start,
|
|
147
|
+
lost: true
|
|
148
|
+
}]);
|
|
149
|
+
}
|
|
150
|
+
this.run();
|
|
151
|
+
}
|
|
152
|
+
async run() {
|
|
153
|
+
if (this.busy) return;
|
|
154
|
+
this.busy = true;
|
|
155
|
+
try {
|
|
156
|
+
while (this.waiting.length) {
|
|
157
|
+
const p = this.waiting.shift();
|
|
158
|
+
let words = [];
|
|
159
|
+
const t0 = performance.now();
|
|
160
|
+
this.decoding = p.samples.length;
|
|
161
|
+
let decodeMs;
|
|
162
|
+
try {
|
|
163
|
+
const r = await this.recognize(p.samples);
|
|
164
|
+
decodeMs = r.ms;
|
|
165
|
+
words = tokensToWords(r, p.start);
|
|
166
|
+
} catch {
|
|
167
|
+
words = [{
|
|
168
|
+
w: "[words lost]",
|
|
169
|
+
at: p.start,
|
|
170
|
+
lost: true
|
|
171
|
+
}];
|
|
172
|
+
}
|
|
173
|
+
this.decoding = 0;
|
|
174
|
+
const end = performance.now();
|
|
175
|
+
const dec = decodeMs ?? end - t0;
|
|
176
|
+
if (!this.cancelled) this.onTiming?.({
|
|
177
|
+
audioMs: p.samples.length / RATE * 1e3,
|
|
178
|
+
waitMs: Math.max(0, end - dec - (p.queuedAt ?? t0)),
|
|
179
|
+
decodeMs: dec,
|
|
180
|
+
backlogMs: (p.backlog ?? 0) / RATE * 1e3
|
|
181
|
+
});
|
|
182
|
+
const kept = words.filter((x) => x.at <= this.last);
|
|
183
|
+
if (kept.length && !this.cancelled) this.onWords(kept);
|
|
184
|
+
}
|
|
185
|
+
} finally {
|
|
186
|
+
this.busy = false;
|
|
187
|
+
if (!this.waiting.length) for (const f of this.idle.splice(0)) f();
|
|
188
|
+
else this.run();
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/** The capture was dropped: everything it holds goes - nothing waiting is transcribed, no word is given. */
|
|
192
|
+
cancel() {
|
|
193
|
+
this.cancelled = true;
|
|
194
|
+
this.ended = true;
|
|
195
|
+
this.last = -Infinity;
|
|
196
|
+
this.waiting = [];
|
|
197
|
+
this.pending = /* @__PURE__ */ new Float32Array(0);
|
|
198
|
+
if (!this.busy) for (const f of this.idle.splice(0)) f();
|
|
199
|
+
}
|
|
200
|
+
/** The phrase being transcribed now, in samples. */
|
|
201
|
+
get decodingSamples() {
|
|
202
|
+
return this.decoding;
|
|
203
|
+
}
|
|
204
|
+
/** Audio given and not yet turned into words: phrases waiting, in samples (the worker's backlog). */
|
|
205
|
+
get waitingSamples() {
|
|
206
|
+
return this.waiting.reduce((n, x) => n + x.samples.length, 0);
|
|
207
|
+
}
|
|
208
|
+
/** The capture ended at `lastSample`: the partial window padded with silence, the VAD flushed, the
|
|
209
|
+
* rest transcribed; resolves when every word up to `lastSample` has been given. */
|
|
210
|
+
end(lastSample) {
|
|
211
|
+
if (!this.ended) {
|
|
212
|
+
this.last = lastSample;
|
|
213
|
+
const keep = Math.max(0, Math.min(this.pending.length, lastSample - this.fed));
|
|
214
|
+
if (keep > 0) {
|
|
215
|
+
const w = /* @__PURE__ */ new Float32Array(512);
|
|
216
|
+
w.set(this.pending.subarray(0, Math.min(keep, 512)));
|
|
217
|
+
this.pending = /* @__PURE__ */ new Float32Array(0);
|
|
218
|
+
this.window(w);
|
|
219
|
+
}
|
|
220
|
+
this.pending = /* @__PURE__ */ new Float32Array(0);
|
|
221
|
+
this.ended = true;
|
|
222
|
+
this.vad.flush();
|
|
223
|
+
this.drain();
|
|
224
|
+
}
|
|
225
|
+
return this.busy || this.waiting.length ? new Promise((r) => {
|
|
226
|
+
this.idle.push(r);
|
|
227
|
+
this.run();
|
|
228
|
+
}) : Promise.resolve();
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
function concat(a, b) {
|
|
232
|
+
const out = new Float32Array(a.length + b.length);
|
|
233
|
+
out.set(a, 0);
|
|
234
|
+
out.set(b, a.length);
|
|
235
|
+
return out;
|
|
236
|
+
}
|
|
237
|
+
//#endregion
|
|
238
|
+
//#region src/server/jam/speech-worker.ts
|
|
239
|
+
/**
|
|
240
|
+
* The speech worker (spec 27, Part 4): a worker thread, so decoding never blocks the dev server. It
|
|
241
|
+
* loads `sherpa-onnx-node` (an optional dependency) and the checked model once, then runs one Segmenter
|
|
242
|
+
* per capture - Silero VAD for phrases, Parakeet v3 for words.
|
|
243
|
+
*
|
|
244
|
+
* in : { type: 'begin', capture } · { type: 'audio', capture, offset, pcm: Int16Array }
|
|
245
|
+
* { type: 'end', capture, lastSample } · { type: 'drop', capture }
|
|
246
|
+
* out : { type: 'ready' } · { type: 'unavailable', reason } · { type: 'words', capture, words }
|
|
247
|
+
* { type: 'final', capture }
|
|
248
|
+
*/
|
|
249
|
+
const port = parentPort;
|
|
250
|
+
const paths = workerData.paths;
|
|
251
|
+
let sherpa;
|
|
252
|
+
let recognizer;
|
|
253
|
+
const loadStart = performance.now();
|
|
254
|
+
try {
|
|
255
|
+
sherpa = createRequire(import.meta.url)("sherpa-onnx-node");
|
|
256
|
+
recognizer = new sherpa.OfflineRecognizer({
|
|
257
|
+
featConfig: {
|
|
258
|
+
sampleRate: RATE,
|
|
259
|
+
featureDim: 80
|
|
260
|
+
},
|
|
261
|
+
modelConfig: {
|
|
262
|
+
transducer: {
|
|
263
|
+
encoder: paths.encoder,
|
|
264
|
+
decoder: paths.decoder,
|
|
265
|
+
joiner: paths.joiner
|
|
266
|
+
},
|
|
267
|
+
tokens: paths.tokens,
|
|
268
|
+
numThreads: 2,
|
|
269
|
+
provider: "cpu",
|
|
270
|
+
modelType: "nemo_transducer"
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
port.postMessage({
|
|
274
|
+
type: "ready",
|
|
275
|
+
modelLoadMs: performance.now() - loadStart
|
|
276
|
+
});
|
|
277
|
+
} catch (err) {
|
|
278
|
+
port.postMessage({
|
|
279
|
+
type: "unavailable",
|
|
280
|
+
reason: err.message
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
const vadFor = () => new sherpa.Vad({
|
|
284
|
+
sileroVad: {
|
|
285
|
+
model: paths.vad,
|
|
286
|
+
threshold: .5,
|
|
287
|
+
minSpeechDuration: .25,
|
|
288
|
+
minSilenceDuration: .4,
|
|
289
|
+
windowSize: 512
|
|
290
|
+
},
|
|
291
|
+
sampleRate: RATE,
|
|
292
|
+
debug: false,
|
|
293
|
+
numThreads: 1
|
|
294
|
+
}, 30);
|
|
295
|
+
const turn = () => new Promise((r) => setImmediate(r));
|
|
296
|
+
const recognize = async (samples) => {
|
|
297
|
+
await turn();
|
|
298
|
+
const t0 = performance.now();
|
|
299
|
+
const s = recognizer.createStream();
|
|
300
|
+
s.acceptWaveform({
|
|
301
|
+
sampleRate: RATE,
|
|
302
|
+
samples
|
|
303
|
+
});
|
|
304
|
+
recognizer.decode(s);
|
|
305
|
+
const r = recognizer.getResult(s);
|
|
306
|
+
return {
|
|
307
|
+
tokens: r.tokens ?? [],
|
|
308
|
+
timestamps: r.timestamps ?? [],
|
|
309
|
+
ms: performance.now() - t0
|
|
310
|
+
};
|
|
311
|
+
};
|
|
312
|
+
const captures = /* @__PURE__ */ new Map();
|
|
313
|
+
port.on("message", (m) => {
|
|
314
|
+
if (!recognizer || !m || typeof m.capture !== "string") return;
|
|
315
|
+
const id = m.capture;
|
|
316
|
+
if (m.type === "begin") {
|
|
317
|
+
if (!captures.has(id)) captures.set(id, new Segmenter(vadFor(), recognize, (words) => port.postMessage({
|
|
318
|
+
type: "words",
|
|
319
|
+
capture: id,
|
|
320
|
+
words
|
|
321
|
+
}), (t) => port.postMessage({
|
|
322
|
+
type: "timing",
|
|
323
|
+
...t
|
|
324
|
+
}), () => [...captures.values()].reduce((n, x) => n + x.waitingSamples + x.decodingSamples, 0)));
|
|
325
|
+
} else if (m.type === "audio") {
|
|
326
|
+
const seg = captures.get(id);
|
|
327
|
+
if (!seg || !(m.pcm instanceof Int16Array)) return;
|
|
328
|
+
const f = new Float32Array(m.pcm.length);
|
|
329
|
+
for (let i = 0; i < f.length; i++) f[i] = m.pcm[i] / 32768;
|
|
330
|
+
seg.push(m.offset, f);
|
|
331
|
+
} else if (m.type === "end") {
|
|
332
|
+
const seg = captures.get(id);
|
|
333
|
+
if (!seg) {
|
|
334
|
+
port.postMessage({
|
|
335
|
+
type: "final",
|
|
336
|
+
capture: id,
|
|
337
|
+
incomplete: true
|
|
338
|
+
});
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
seg.end(Number(m.lastSample) || 0).then(() => {
|
|
342
|
+
captures.delete(id);
|
|
343
|
+
port.postMessage({
|
|
344
|
+
type: "final",
|
|
345
|
+
capture: id
|
|
346
|
+
});
|
|
347
|
+
});
|
|
348
|
+
} else if (m.type === "drop") {
|
|
349
|
+
captures.get(id)?.cancel();
|
|
350
|
+
captures.delete(id);
|
|
351
|
+
}
|
|
352
|
+
});
|
|
353
|
+
//#endregion
|
|
354
|
+
export {};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { r as replay } from "./events-
|
|
2
|
-
import { x as withLock } from "./auth-
|
|
1
|
+
import { r as replay } from "./events-CLH4ijKP.mjs";
|
|
2
|
+
import { x as withLock } from "./auth-Dix8wrXV.mjs";
|
|
3
3
|
import { closeSync, fsyncSync, openSync, readFileSync, renameSync, writeSync } from "node:fs";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { createHash, createPrivateKey, createPublicKey, createVerify, generateKeyPairSync, randomBytes, sign } from "node:crypto";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { t as __exportAll } from "./rolldown-runtime-
|
|
2
|
-
import { n as diffEvents } from "./events-
|
|
3
|
-
import { a as readLog, i as listBoards, t as appendEvents } from "./comments-
|
|
4
|
-
import { t as CLI_TOKEN_CHARS } from "./auth-
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.mjs";
|
|
2
|
+
import { n as diffEvents } from "./events-CLH4ijKP.mjs";
|
|
3
|
+
import { a as readLog, i as listBoards, t as appendEvents } from "./comments-0BSbb04K.mjs";
|
|
4
|
+
import { t as CLI_TOKEN_CHARS } from "./auth-Dix8wrXV.mjs";
|
|
5
5
|
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, realpathSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
|
6
6
|
import { join } from "node:path";
|
|
7
7
|
import { createHash, randomBytes } from "node:crypto";
|
|
@@ -305,11 +305,12 @@ async function connectToken(root, url, token) {
|
|
|
305
305
|
}
|
|
306
306
|
/** The thread an event belongs to: a reply names its root; everything else names its comment. */
|
|
307
307
|
const threadOf = (e) => e.parentId ?? e.commentId ?? "";
|
|
308
|
-
/** Threads that live on this machine only: started by Marver (an agent's `comments new`)
|
|
309
|
-
* that resolves to one - the thread, and each reply
|
|
310
|
-
* in the set, so `threadOf` of any event in such a
|
|
308
|
+
/** Threads that live on this machine only: started by Marver (an agent's `comments new`) or by a Jam
|
|
309
|
+
* capture (its words, spoken or typed - 0.26). Every id that resolves to one - the thread, and each reply
|
|
310
|
+
* in it (an edit or reaction names the reply) - is in the set, so `threadOf` of any event in such a
|
|
311
|
+
* thread hits it. */
|
|
311
312
|
function localOnlyThreads(events) {
|
|
312
|
-
return threadIds(events, (e) => !!e.agent);
|
|
313
|
+
return threadIds(events, (e) => !!e.agent || !!e.capture);
|
|
313
314
|
}
|
|
314
315
|
/** Every id that resolves to a thread whose root passes `root` - the thread, and each reply in it. */
|
|
315
316
|
function threadIds(events, root) {
|