@openparachute/vault 0.6.4 → 0.6.5-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/__fixtures__/golden-vault.ts +125 -0
- package/core/src/__fixtures__/portable-export-golden.json +10 -0
- package/core/src/do-param-cap.test.ts +161 -0
- package/core/src/links.ts +8 -13
- package/core/src/mcp.ts +306 -314
- package/core/src/notes.ts +54 -62
- package/core/src/onboarding.ts +14 -296
- package/core/src/portable-md-batching.test.ts +67 -0
- package/core/src/portable-md-golden.test.ts +55 -0
- package/core/src/portable-md.ts +579 -435
- package/core/src/schema.ts +21 -43
- package/core/src/seed-packs.test.ts +191 -0
- package/core/src/seed-packs.ts +559 -0
- package/core/src/sql-in.test.ts +58 -0
- package/core/src/sql-in.ts +76 -0
- package/core/src/store.ts +14 -9
- package/core/src/transcription/provider.ts +141 -0
- package/core/src/txn.test.ts +229 -0
- package/core/src/txn.ts +105 -0
- package/core/src/types.ts +9 -0
- package/core/src/vault-projection.ts +1 -1
- package/core/src/wikilinks.ts +10 -4
- package/package.json +1 -1
- package/src/add-pack.test.ts +142 -0
- package/src/cli.ts +778 -7
- package/src/onboarding-seed.test.ts +126 -40
- package/src/onboarding-seed.ts +41 -46
- package/src/routes.ts +25 -7
- package/src/server.ts +108 -31
- package/src/transcription/build.test.ts +224 -0
- package/src/transcription/build.ts +252 -0
- package/src/transcription/capability.test.ts +118 -0
- package/src/transcription/capability.ts +96 -0
- package/src/transcription/install-python.test.ts +366 -0
- package/src/transcription/install-python.ts +471 -0
- package/src/transcription/install.test.ts +167 -0
- package/src/transcription/install.ts +296 -0
- package/src/transcription/providers/onnx-asr.test.ts +229 -0
- package/src/transcription/providers/onnx-asr.ts +239 -0
- package/src/transcription/providers/parakeet-mlx.test.ts +290 -0
- package/src/transcription/providers/parakeet-mlx.ts +242 -0
- package/src/transcription/providers/scribe-http.test.ts +195 -0
- package/src/transcription/providers/scribe-http.ts +144 -0
- package/src/transcription/providers/transcribe-cpp.test.ts +314 -0
- package/src/transcription/providers/transcribe-cpp.ts +293 -0
- package/src/transcription/select.test.ts +259 -0
- package/src/transcription/select.ts +334 -0
- package/src/transcription/tiers.test.ts +197 -0
- package/src/transcription/tiers.ts +184 -0
- package/src/transcription-worker.test.ts +44 -0
- package/src/transcription-worker.ts +57 -122
- package/src/vault-create.test.ts +38 -10
- package/src/vault.test.ts +48 -0
|
@@ -0,0 +1,471 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runnable install routine for the Python-based local transcription providers
|
|
3
|
+
* (parakeet-mlx / onnx-asr) — scribe-fold Phase 2b.
|
|
4
|
+
*
|
|
5
|
+
* Ported from scribe's proven `src/install-backend.ts` (apt system deps +
|
|
6
|
+
* venv + warm-pull + re-verify; idempotent; non-fatal), adapted to vault's
|
|
7
|
+
* conventions:
|
|
8
|
+
*
|
|
9
|
+
* - **Managed venv under PARACHUTE_HOME.** The package installs into
|
|
10
|
+
* `<root>/transcription/venv/` and vault resolves the binary by ABSOLUTE
|
|
11
|
+
* path (see `select.ts`), so there's no "add the venv to your PATH"
|
|
12
|
+
* caveat and the whole install sandboxes under `PARACHUTE_HOME`. (Scribe
|
|
13
|
+
* preferred `uv tool install` with a `~/.venvs` fallback — the venv-only
|
|
14
|
+
* strategy here is the deliberate deviation; an operator's own on-PATH
|
|
15
|
+
* install still wins via the `select.ts` resolution ladder.)
|
|
16
|
+
* - **Total-RAM tiers, not free-RAM.** The ratified table (and Phase 2a's
|
|
17
|
+
* `install.ts`) gates on TOTAL RAM — a 2GB box shouldn't pick a 4GB
|
|
18
|
+
* provider just because it's momentarily idle. The onnx-asr 4GB floor IS
|
|
19
|
+
* the scribe#82 fix (Parakeet peak RAM exceeds 2GB on meeting-length
|
|
20
|
+
* audio; scribe auto-installed it at 2GB and OOMed).
|
|
21
|
+
* - **Honest activation stays in the CLI.** This module reports a runnable
|
|
22
|
+
* binary (or not); the caller flips `TRANSCRIPTION_PROVIDER` only when
|
|
23
|
+
* one exists (PRs #532/#533 precedent).
|
|
24
|
+
*
|
|
25
|
+
* Design contract (unchanged from scribe):
|
|
26
|
+
*
|
|
27
|
+
* - **Non-fatal + idempotent.** Re-running is safe: a present binary is a
|
|
28
|
+
* no-op success; apt/pip of present packages is a no-op; the model
|
|
29
|
+
* warm-pull is best-effort. Every failure mode is a structured step, and
|
|
30
|
+
* `installPythonBackend` never throws.
|
|
31
|
+
* - **Privilege split.** pip runs user-level in the venv; apt is
|
|
32
|
+
* system-level — `sudo` when not root, and when neither is available we
|
|
33
|
+
* instruct rather than fail opaquely.
|
|
34
|
+
* - **Injected runner.** Every subprocess + platform probe goes through the
|
|
35
|
+
* `PythonInstallDeps` seam so tests exercise the tier guards, apt/venv
|
|
36
|
+
* logic, and idempotency WITHOUT installing anything.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { existsSync } from "fs";
|
|
40
|
+
import { join } from "path";
|
|
41
|
+
import {
|
|
42
|
+
DEFAULT_ONNX_ASR_MODEL,
|
|
43
|
+
DEFAULT_PARAKEET_MLX_MODEL,
|
|
44
|
+
pythonVenvDir,
|
|
45
|
+
resolveOnnxAsrBin,
|
|
46
|
+
resolveParakeetMlxBin,
|
|
47
|
+
} from "./select.ts";
|
|
48
|
+
import { detectTotalRamBytes } from "./install.ts";
|
|
49
|
+
import { NOMINAL_SLACK_GB } from "./tiers.ts";
|
|
50
|
+
|
|
51
|
+
const GB = 2 ** 30;
|
|
52
|
+
|
|
53
|
+
export type PythonProviderName = "parakeet-mlx" | "onnx-asr";
|
|
54
|
+
|
|
55
|
+
/** Install spec per python provider — the single source of truth for the
|
|
56
|
+
* pip target, platform gates, and the ratified RAM floors. */
|
|
57
|
+
export interface PythonProviderSpec {
|
|
58
|
+
provider: PythonProviderName;
|
|
59
|
+
/** Binary the package installs (and the venv/PATH resolution target). */
|
|
60
|
+
bin: string;
|
|
61
|
+
/** pip install target, extras included. */
|
|
62
|
+
pipTarget: string;
|
|
63
|
+
/** OS restriction, when the provider only runs on one platform. */
|
|
64
|
+
platform?: "darwin" | "linux";
|
|
65
|
+
/** CPU-arch restriction (parakeet-mlx is MLX → Apple Silicon only). */
|
|
66
|
+
arch?: "arm64";
|
|
67
|
+
/** Ratified total-RAM floor, GB. `--force` bypasses; platform gates don't. */
|
|
68
|
+
minRamGb: number;
|
|
69
|
+
/** Default model id (env/manifest can override; see select.ts). */
|
|
70
|
+
defaultModel: string;
|
|
71
|
+
/** Approx model download footprint, MB (for the confirm prompt). */
|
|
72
|
+
approxDiskMb: number;
|
|
73
|
+
/** Human peak-RAM note (for the confirm prompt). */
|
|
74
|
+
peakRamNote: string;
|
|
75
|
+
/** Why the RAM floor sits where it does — printed on refusal. */
|
|
76
|
+
ramFloorRationale: string;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export const PYTHON_PROVIDERS: Record<PythonProviderName, PythonProviderSpec> = {
|
|
80
|
+
"parakeet-mlx": {
|
|
81
|
+
provider: "parakeet-mlx",
|
|
82
|
+
bin: "parakeet-mlx",
|
|
83
|
+
pipTarget: "parakeet-mlx",
|
|
84
|
+
platform: "darwin",
|
|
85
|
+
arch: "arm64",
|
|
86
|
+
minRamGb: 8,
|
|
87
|
+
defaultModel: DEFAULT_PARAKEET_MLX_MODEL,
|
|
88
|
+
approxDiskMb: 2500,
|
|
89
|
+
peakRamNote: "~2.5–3GB peak while transcribing",
|
|
90
|
+
ramFloorRationale:
|
|
91
|
+
"parakeet-mlx peaks at ~2.5–3GB while transcribing; below 8GB total, use transcribe-cpp instead (`transcription install`).",
|
|
92
|
+
},
|
|
93
|
+
"onnx-asr": {
|
|
94
|
+
provider: "onnx-asr",
|
|
95
|
+
bin: "onnx-asr",
|
|
96
|
+
pipTarget: "onnx-asr[cpu,hub]",
|
|
97
|
+
minRamGb: 4,
|
|
98
|
+
defaultModel: DEFAULT_ONNX_ASR_MODEL,
|
|
99
|
+
approxDiskMb: 670,
|
|
100
|
+
peakRamNote: "~1.2–2GB+ peak while transcribing (meeting-length audio uses the high end)",
|
|
101
|
+
ramFloorRationale:
|
|
102
|
+
"Parakeet's peak RAM exceeds 2GB on meeting-length audio (scribe#82) — below 4GB total, use transcribe-cpp with a small whisper model instead (`transcription install`).",
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/** Outcome of one subprocess step. */
|
|
107
|
+
export type RunResult = { exitCode: number; stdout: string; stderr: string };
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Injectable side-effect seam. Defaults run real subprocesses / read real
|
|
111
|
+
* platform state; tests inject stubs so nothing is actually installed.
|
|
112
|
+
*/
|
|
113
|
+
export interface PythonInstallDeps {
|
|
114
|
+
/** Run a command, capturing exit code + output. Never throws. */
|
|
115
|
+
run: (cmd: string[]) => Promise<RunResult>;
|
|
116
|
+
/** PATH lookup. */
|
|
117
|
+
which: (bin: string) => string | null;
|
|
118
|
+
/** `process.platform`. */
|
|
119
|
+
platform: string;
|
|
120
|
+
/** `process.arch`. */
|
|
121
|
+
arch: string;
|
|
122
|
+
/** Detected total physical RAM in bytes. */
|
|
123
|
+
totalRamBytes: number;
|
|
124
|
+
/** Effective uid; `0` means root. */
|
|
125
|
+
uid: () => number;
|
|
126
|
+
/** Environment (for bin resolution + PARACHUTE_HOME). */
|
|
127
|
+
env: NodeJS.ProcessEnv;
|
|
128
|
+
/** Existence probe. */
|
|
129
|
+
existsImpl: (p: string) => boolean;
|
|
130
|
+
/** Sink for human-readable progress lines. */
|
|
131
|
+
log: (line: string) => void;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Real subprocess runner — captures output, never throws. */
|
|
135
|
+
const defaultRun: PythonInstallDeps["run"] = async (cmd) => {
|
|
136
|
+
try {
|
|
137
|
+
const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe", stdin: "ignore" });
|
|
138
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
139
|
+
new Response(proc.stdout).text(),
|
|
140
|
+
new Response(proc.stderr).text(),
|
|
141
|
+
proc.exited,
|
|
142
|
+
]);
|
|
143
|
+
return { exitCode, stdout, stderr };
|
|
144
|
+
} catch (err) {
|
|
145
|
+
// ENOENT / spawn failure → a reportable step failure, not a throw.
|
|
146
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
147
|
+
return { exitCode: 127, stdout: "", stderr: message };
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
/** Build the default (real-side-effect) deps. */
|
|
152
|
+
export function buildDefaultPythonDeps(): PythonInstallDeps {
|
|
153
|
+
return {
|
|
154
|
+
run: defaultRun,
|
|
155
|
+
which: (bin) => Bun.which(bin),
|
|
156
|
+
platform: process.platform,
|
|
157
|
+
arch: process.arch,
|
|
158
|
+
totalRamBytes: detectTotalRamBytes(),
|
|
159
|
+
uid: () => (typeof process.getuid === "function" ? process.getuid() : 0),
|
|
160
|
+
env: process.env,
|
|
161
|
+
existsImpl: existsSync,
|
|
162
|
+
log: (line) => console.log(line),
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export type PythonInstallStep = {
|
|
167
|
+
name: string;
|
|
168
|
+
status: "ok" | "skipped" | "failed" | "refused";
|
|
169
|
+
detail: string;
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
/** Result of an install run. */
|
|
173
|
+
export interface PythonInstallOutcome {
|
|
174
|
+
/** True when a runnable binary was verified at the end. */
|
|
175
|
+
ok: boolean;
|
|
176
|
+
provider: PythonProviderName;
|
|
177
|
+
/** Ordered step log (machine-readable companion to the human `log`). */
|
|
178
|
+
steps: PythonInstallStep[];
|
|
179
|
+
/** One-line summary for the caller. */
|
|
180
|
+
summary: string;
|
|
181
|
+
/** The resolved runnable binary, when `ok`. The caller activates off this. */
|
|
182
|
+
binPath?: string;
|
|
183
|
+
/** The venv the package landed in ("" when the binary came from PATH/env). */
|
|
184
|
+
venv?: string;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export interface PythonInstallOptions {
|
|
188
|
+
provider: PythonProviderName;
|
|
189
|
+
/** Bypass the RAM floor (operator override). Platform gates are never bypassed. */
|
|
190
|
+
force?: boolean;
|
|
191
|
+
/** Skip the (slow) model warm-pull. */
|
|
192
|
+
skipModel?: boolean;
|
|
193
|
+
/** Model id to warm-pull. Defaults to the spec's defaultModel. */
|
|
194
|
+
model?: string;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Resolve the provider's binary via select.ts's ladder against these deps. */
|
|
198
|
+
function resolveBin(spec: PythonProviderSpec, deps: PythonInstallDeps): string | undefined {
|
|
199
|
+
const resolveDeps = { existsImpl: deps.existsImpl, whichImpl: deps.which };
|
|
200
|
+
return spec.provider === "parakeet-mlx"
|
|
201
|
+
? resolveParakeetMlxBin(deps.env, resolveDeps)
|
|
202
|
+
: resolveOnnxAsrBin(deps.env, resolveDeps);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Install + verify a python transcription provider. Pure orchestration over
|
|
207
|
+
* the injected `deps`; returns a structured outcome and never throws.
|
|
208
|
+
*/
|
|
209
|
+
export async function installPythonBackend(
|
|
210
|
+
deps: PythonInstallDeps,
|
|
211
|
+
opts: PythonInstallOptions,
|
|
212
|
+
): Promise<PythonInstallOutcome> {
|
|
213
|
+
const spec = PYTHON_PROVIDERS[opts.provider];
|
|
214
|
+
const steps: PythonInstallStep[] = [];
|
|
215
|
+
const record = (s: PythonInstallStep): PythonInstallStep => {
|
|
216
|
+
steps.push(s);
|
|
217
|
+
return s;
|
|
218
|
+
};
|
|
219
|
+
const fail = (summary: string): PythonInstallOutcome => ({
|
|
220
|
+
ok: false,
|
|
221
|
+
provider: spec.provider,
|
|
222
|
+
steps,
|
|
223
|
+
summary,
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
// --- Platform / arch gates (never bypassed — the runtime can't exist) -----
|
|
227
|
+
if (spec.platform && spec.platform !== deps.platform) {
|
|
228
|
+
const detail = `${spec.provider} only runs on ${spec.platform}; this host is ${deps.platform}. Run \`transcription install\` (no --provider) for this host's tier default.`;
|
|
229
|
+
deps.log(detail);
|
|
230
|
+
record({ name: "platform-guard", status: "refused", detail });
|
|
231
|
+
return fail(detail);
|
|
232
|
+
}
|
|
233
|
+
if (spec.arch && spec.arch !== deps.arch) {
|
|
234
|
+
const detail = `${spec.provider} needs ${spec.arch} (Apple Silicon); this host is ${deps.platform}/${deps.arch}. Run \`transcription install\` (no --provider) for this host's tier default.`;
|
|
235
|
+
deps.log(detail);
|
|
236
|
+
record({ name: "platform-guard", status: "refused", detail });
|
|
237
|
+
return fail(detail);
|
|
238
|
+
}
|
|
239
|
+
record({ name: "platform-guard", status: "ok", detail: `${deps.platform}/${deps.arch} supported.` });
|
|
240
|
+
|
|
241
|
+
// --- RAM floor (ratified tiers; --force bypasses with a loud warning) -----
|
|
242
|
+
const totalGb = Math.round((deps.totalRamBytes / GB) * 10) / 10;
|
|
243
|
+
if (totalGb < spec.minRamGb - NOMINAL_SLACK_GB) {
|
|
244
|
+
if (opts.force) {
|
|
245
|
+
const detail = `--force: proceeding below the ${spec.minRamGb}GB floor (${totalGb}GB detected). ${spec.ramFloorRationale}`;
|
|
246
|
+
deps.log(`⚠ ${detail}`);
|
|
247
|
+
record({ name: "ram-guard", status: "skipped", detail });
|
|
248
|
+
} else {
|
|
249
|
+
const detail = `only ${totalGb}GB RAM detected — ${spec.provider} needs ~${spec.minRamGb}GB. ${spec.ramFloorRationale} (--force overrides.)`;
|
|
250
|
+
deps.log(detail);
|
|
251
|
+
record({ name: "ram-guard", status: "refused", detail });
|
|
252
|
+
return fail(detail);
|
|
253
|
+
}
|
|
254
|
+
} else {
|
|
255
|
+
record({
|
|
256
|
+
name: "ram-guard",
|
|
257
|
+
status: "ok",
|
|
258
|
+
detail: `${totalGb}GB total RAM ≥ the ${spec.minRamGb}GB ${spec.provider} floor.`,
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
deps.log(`Installing local transcription provider: ${spec.provider} (${spec.pipTarget})`);
|
|
263
|
+
|
|
264
|
+
// --- System deps (python3 + ffmpeg) ---------------------------------------
|
|
265
|
+
if (deps.platform === "linux") {
|
|
266
|
+
const aptStep = await ensureLinuxSystemDeps(deps);
|
|
267
|
+
record(aptStep);
|
|
268
|
+
if (aptStep.status === "failed") {
|
|
269
|
+
return fail(`System dependency install failed: ${aptStep.detail}`);
|
|
270
|
+
}
|
|
271
|
+
} else {
|
|
272
|
+
// macOS: python3 ships with the CLT; ffmpeg is a brew concern we check +
|
|
273
|
+
// instruct rather than drive blindly (scribe's proven split).
|
|
274
|
+
if (deps.which("python3") === null) {
|
|
275
|
+
const detail =
|
|
276
|
+
"python3 not found. Install the Xcode Command Line Tools (`xcode-select --install`) or `brew install python3`, then re-run.";
|
|
277
|
+
record({ name: "system-deps", status: "failed", detail });
|
|
278
|
+
return fail(detail);
|
|
279
|
+
}
|
|
280
|
+
record({
|
|
281
|
+
name: "system-deps",
|
|
282
|
+
status: deps.which("ffmpeg") === null ? "skipped" : "ok",
|
|
283
|
+
detail:
|
|
284
|
+
deps.which("ffmpeg") === null
|
|
285
|
+
? "python3 present. ffmpeg is missing (needed to decode non-WAV capture audio) — install it with `brew install ffmpeg`."
|
|
286
|
+
: "python3 + ffmpeg present.",
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// --- Python package into the managed venv ---------------------------------
|
|
291
|
+
const pkgStep = await ensureVenvPackage(deps, spec);
|
|
292
|
+
record(pkgStep);
|
|
293
|
+
if (pkgStep.status === "failed") {
|
|
294
|
+
return fail(`Package install failed: ${pkgStep.detail}`);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// --- Warm-pull the model (onnx-asr only; best-effort, never fatal) --------
|
|
298
|
+
if (opts.skipModel) {
|
|
299
|
+
record({ name: "model-warm-pull", status: "skipped", detail: "Skipped by request." });
|
|
300
|
+
} else {
|
|
301
|
+
record(await warmPullModel(deps, spec, opts.model ?? spec.defaultModel));
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// --- Verify: a runnable binary via the same ladder the daemon uses --------
|
|
305
|
+
const binPath = resolveBin(spec, deps);
|
|
306
|
+
const runnable = !!binPath && deps.existsImpl(binPath);
|
|
307
|
+
const venv = pythonVenvDir(deps.env);
|
|
308
|
+
if (!runnable) {
|
|
309
|
+
const detail = `install ran but no runnable \`${spec.bin}\` was found (looked at ${join(venv, "bin", spec.bin)} and PATH).`;
|
|
310
|
+
record({ name: "verify", status: "failed", detail });
|
|
311
|
+
return fail(`${spec.provider} install did not produce a runnable binary. ${detail}`);
|
|
312
|
+
}
|
|
313
|
+
record({ name: "verify", status: "ok", detail: `runnable binary at ${binPath}.` });
|
|
314
|
+
|
|
315
|
+
const summary = `${spec.provider} installed and verified runnable (${binPath}).`;
|
|
316
|
+
deps.log(summary);
|
|
317
|
+
return {
|
|
318
|
+
ok: true,
|
|
319
|
+
provider: spec.provider,
|
|
320
|
+
steps,
|
|
321
|
+
summary,
|
|
322
|
+
binPath,
|
|
323
|
+
venv: binPath!.startsWith(venv) ? venv : "",
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** sudo-if-needed prefix: `[]` when root, `["sudo"]` otherwise (when sudo exists). */
|
|
328
|
+
function sudoPrefix(deps: PythonInstallDeps): {
|
|
329
|
+
prefix: string[];
|
|
330
|
+
haveRoot: boolean;
|
|
331
|
+
haveSudo: boolean;
|
|
332
|
+
} {
|
|
333
|
+
const haveRoot = deps.uid() === 0;
|
|
334
|
+
const haveSudo = deps.which("sudo") !== null;
|
|
335
|
+
return { prefix: haveRoot ? [] : haveSudo ? ["sudo"] : [], haveRoot, haveSudo };
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* Ensure python3 (+venv +pip) and ffmpeg on Debian/Ubuntu via apt. Idempotent:
|
|
340
|
+
* already-present binaries skip apt entirely; apt itself no-ops on installed
|
|
341
|
+
* packages. Needs root or sudo; instructs when neither is available. Ported
|
|
342
|
+
* from scribe's `ensureLinuxSystemDeps`.
|
|
343
|
+
*/
|
|
344
|
+
async function ensureLinuxSystemDeps(deps: PythonInstallDeps): Promise<PythonInstallStep> {
|
|
345
|
+
const pyOk = deps.which("python3") !== null;
|
|
346
|
+
const ffOk = deps.which("ffmpeg") !== null;
|
|
347
|
+
if (pyOk && ffOk) {
|
|
348
|
+
return { name: "system-deps", status: "skipped", detail: "python3 + ffmpeg already present." };
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
const apt = deps.which("apt-get") ?? deps.which("apt");
|
|
352
|
+
if (apt === null) {
|
|
353
|
+
return {
|
|
354
|
+
name: "system-deps",
|
|
355
|
+
status: "failed",
|
|
356
|
+
detail:
|
|
357
|
+
"No apt found and python3/ffmpeg missing. Install them with your package manager, then re-run.",
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
const { prefix, haveRoot, haveSudo } = sudoPrefix(deps);
|
|
362
|
+
if (!haveRoot && !haveSudo) {
|
|
363
|
+
return {
|
|
364
|
+
name: "system-deps",
|
|
365
|
+
status: "failed",
|
|
366
|
+
detail:
|
|
367
|
+
"Need root or sudo to apt-install python3/python3-venv/ffmpeg. Re-run as root, or run: `sudo apt-get install -y python3 python3-venv python3-pip ffmpeg`, then re-run this command.",
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
const pkgs = ["python3", "python3-venv", "python3-pip", "ffmpeg"];
|
|
372
|
+
deps.log(`Installing system deps via apt: ${pkgs.join(" ")} …`);
|
|
373
|
+
// `apt-get update` is best-effort — a transient mirror failure shouldn't
|
|
374
|
+
// abort if the cache is already warm; report and continue.
|
|
375
|
+
const update = await deps.run([...prefix, "apt-get", "update"]);
|
|
376
|
+
if (update.exitCode !== 0) {
|
|
377
|
+
deps.log(`apt-get update returned ${update.exitCode} (continuing): ${update.stderr.trim()}`);
|
|
378
|
+
}
|
|
379
|
+
const install = await deps.run([...prefix, "apt-get", "install", "-y", ...pkgs]);
|
|
380
|
+
if (install.exitCode !== 0) {
|
|
381
|
+
return {
|
|
382
|
+
name: "system-deps",
|
|
383
|
+
status: "failed",
|
|
384
|
+
detail: `apt-get install failed (exit ${install.exitCode}): ${install.stderr.trim() || install.stdout.trim()}`,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
return { name: "system-deps", status: "ok", detail: `Installed: ${pkgs.join(", ")}.` };
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Install the provider's python package into the managed venv. Idempotent: a
|
|
392
|
+
* binary already resolvable (env override / venv / PATH) skips the install.
|
|
393
|
+
*/
|
|
394
|
+
async function ensureVenvPackage(
|
|
395
|
+
deps: PythonInstallDeps,
|
|
396
|
+
spec: PythonProviderSpec,
|
|
397
|
+
): Promise<PythonInstallStep> {
|
|
398
|
+
const existing = resolveBin(spec, deps);
|
|
399
|
+
if (existing && deps.existsImpl(existing)) {
|
|
400
|
+
return {
|
|
401
|
+
name: "package",
|
|
402
|
+
status: "skipped",
|
|
403
|
+
detail: `\`${spec.bin}\` already runnable at ${existing} — nothing to install.`,
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const venv = pythonVenvDir(deps.env);
|
|
408
|
+
const pip = join(venv, "bin", "pip");
|
|
409
|
+
|
|
410
|
+
// Create the venv (idempotent: re-running `python3 -m venv` on an existing
|
|
411
|
+
// venv is harmless).
|
|
412
|
+
const mk = await deps.run(["python3", "-m", "venv", venv]);
|
|
413
|
+
if (mk.exitCode !== 0) {
|
|
414
|
+
return {
|
|
415
|
+
name: "package",
|
|
416
|
+
status: "failed",
|
|
417
|
+
detail: `Could not create venv at ${venv} (exit ${mk.exitCode}): ${mk.stderr.trim() || mk.stdout.trim()}. Is python3-venv installed?`,
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
deps.log(`Installing ${spec.pipTarget} into ${venv} via pip (this can take a few minutes) …`);
|
|
422
|
+
const install = await deps.run([pip, "install", spec.pipTarget]);
|
|
423
|
+
if (install.exitCode !== 0) {
|
|
424
|
+
return {
|
|
425
|
+
name: "package",
|
|
426
|
+
status: "failed",
|
|
427
|
+
detail: `pip install ${spec.pipTarget} failed (exit ${install.exitCode}): ${install.stderr.trim().slice(-800) || install.stdout.trim().slice(-800)}`,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
return {
|
|
431
|
+
name: "package",
|
|
432
|
+
status: "ok",
|
|
433
|
+
detail: `Installed ${spec.pipTarget} into ${venv} (binary: ${join(venv, "bin", spec.bin)}).`,
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Warm-pull the model so the first real transcription isn't a long cold
|
|
439
|
+
* download. onnx-asr only — parakeet-mlx fetches its model implicitly on
|
|
440
|
+
* first use and has no separate pull verb (scribe's proven split).
|
|
441
|
+
* Best-effort: always `ok`/`skipped`, never `failed`.
|
|
442
|
+
*
|
|
443
|
+
* FRAGILE (inherited from scribe): assumes `onnx-asr <model> --help` triggers
|
|
444
|
+
* the model fetch+cache without an audio file. Worst case is a silent no-op —
|
|
445
|
+
* the model still lazy-loads on the first transcription — but verify against
|
|
446
|
+
* the real onnx-asr CLI when bumping it.
|
|
447
|
+
*/
|
|
448
|
+
async function warmPullModel(
|
|
449
|
+
deps: PythonInstallDeps,
|
|
450
|
+
spec: PythonProviderSpec,
|
|
451
|
+
model: string,
|
|
452
|
+
): Promise<PythonInstallStep> {
|
|
453
|
+
if (spec.provider !== "onnx-asr") {
|
|
454
|
+
return {
|
|
455
|
+
name: "model-warm-pull",
|
|
456
|
+
status: "skipped",
|
|
457
|
+
detail: `${spec.provider} downloads its model (~${spec.approxDiskMb}MB) on first transcription — no separate warm-pull.`,
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
const bin = resolveBin(spec, deps) ?? join(pythonVenvDir(deps.env), "bin", spec.bin);
|
|
461
|
+
deps.log(`Warm-pulling model ${model} (~${spec.approxDiskMb}MB one-time download) …`);
|
|
462
|
+
const r = await deps.run([bin, model, "--help"]);
|
|
463
|
+
if (r.exitCode === 0) {
|
|
464
|
+
return { name: "model-warm-pull", status: "ok", detail: `Model ${model} cached.` };
|
|
465
|
+
}
|
|
466
|
+
return {
|
|
467
|
+
name: "model-warm-pull",
|
|
468
|
+
status: "skipped",
|
|
469
|
+
detail: `Could not warm-pull ${model} now (exit ${r.exitCode}); it will download on the first transcription.`,
|
|
470
|
+
};
|
|
471
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { describe, test, expect } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
planInstall,
|
|
4
|
+
selectModelForRam,
|
|
5
|
+
selectAsset,
|
|
6
|
+
isSharedLibFile,
|
|
7
|
+
MODELS,
|
|
8
|
+
TRANSCRIBE_CPP_VERSION,
|
|
9
|
+
} from "./install.ts";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Install-planning tests (scribe-fold Phase 2a). Pure selection logic — the
|
|
13
|
+
* RAM-tier matrix (scribe#82 fix), the OS/arch → asset mapping, and the full
|
|
14
|
+
* plan. No network, no fs.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const GB = 2 ** 30;
|
|
18
|
+
|
|
19
|
+
describe("selectModelForRam — the tier matrix", () => {
|
|
20
|
+
test("<1GB → refuse (null; steer to remote)", () => {
|
|
21
|
+
expect(selectModelForRam(0.5 * GB)).toBeNull();
|
|
22
|
+
expect(selectModelForRam(0)).toBeNull();
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("1–2GB → whisper-tiny.en", () => {
|
|
26
|
+
expect(selectModelForRam(1 * GB)!.name).toBe("whisper-tiny.en");
|
|
27
|
+
expect(selectModelForRam(1.9 * GB)!.name).toBe("whisper-tiny.en");
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("2–4GB → whisper-small.en (NOT Parakeet — the scribe#82 floor fix)", () => {
|
|
31
|
+
expect(selectModelForRam(2 * GB)!.name).toBe("whisper-small.en");
|
|
32
|
+
expect(selectModelForRam(3.9 * GB)!.name).toBe("whisper-small.en");
|
|
33
|
+
// The scribe#82 regression: 2GB must NOT pick parakeet (it OOMs there).
|
|
34
|
+
expect(selectModelForRam(2 * GB)!.name).not.toContain("parakeet");
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("≥4GB → parakeet-tdt-0.6b-v3", () => {
|
|
38
|
+
expect(selectModelForRam(4 * GB)!.name).toBe("parakeet-tdt-0.6b-v3");
|
|
39
|
+
expect(selectModelForRam(16 * GB)!.name).toBe("parakeet-tdt-0.6b-v3");
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("boundaries are inclusive at the tier floor", () => {
|
|
43
|
+
expect(selectModelForRam(1 * GB)!.name).toBe("whisper-tiny.en");
|
|
44
|
+
expect(selectModelForRam(2 * GB)!.name).toBe("whisper-small.en");
|
|
45
|
+
expect(selectModelForRam(4 * GB)!.name).toBe("parakeet-tdt-0.6b-v3");
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
describe("selectAsset — OS/arch → prebuilt release asset", () => {
|
|
50
|
+
test("linux x64", () => {
|
|
51
|
+
const a = selectAsset("linux", "x64")!;
|
|
52
|
+
expect(a.asset).toBe(`transcribe-native-${TRANSCRIBE_CPP_VERSION}-linux-x86_64-cpu-vulkan.tar.gz`);
|
|
53
|
+
expect(a.url).toContain("releases/download");
|
|
54
|
+
});
|
|
55
|
+
test("linux arm64", () => {
|
|
56
|
+
expect(selectAsset("linux", "arm64")!.asset).toContain("linux-aarch64");
|
|
57
|
+
});
|
|
58
|
+
test("macos arm64 → metal", () => {
|
|
59
|
+
expect(selectAsset("darwin", "arm64")!.asset).toContain("macos-arm64-metal");
|
|
60
|
+
});
|
|
61
|
+
test("macos x64", () => {
|
|
62
|
+
expect(selectAsset("darwin", "x64")!.asset).toContain("macos-x86_64");
|
|
63
|
+
});
|
|
64
|
+
test("unsupported platform/arch → null", () => {
|
|
65
|
+
expect(selectAsset("win32", "x64")).toBeNull();
|
|
66
|
+
expect(selectAsset("linux", "riscv64")).toBeNull();
|
|
67
|
+
expect(selectAsset("freebsd", "x64")).toBeNull();
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
describe("planInstall", () => {
|
|
72
|
+
test("supported host: asset + tier model + footprint", () => {
|
|
73
|
+
const plan = planInstall({ platform: "linux", arch: "x64", totalRamBytes: 8 * GB });
|
|
74
|
+
expect(plan.supported).toBe(true);
|
|
75
|
+
expect(plan.asset!.asset).toContain("linux-x86_64");
|
|
76
|
+
expect(plan.model!.name).toBe("parakeet-tdt-0.6b-v3");
|
|
77
|
+
expect(plan.footprintMb).toBe(MODELS["parakeet-tdt-0.6b-v3"]!.approxSizeMb);
|
|
78
|
+
expect(plan.totalRamGb).toBe(8);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test("2GB box picks whisper-small, not parakeet (scribe#82)", () => {
|
|
82
|
+
const plan = planInstall({ platform: "linux", arch: "arm64", totalRamBytes: 2 * GB });
|
|
83
|
+
expect(plan.supported).toBe(true);
|
|
84
|
+
expect(plan.model!.name).toBe("whisper-small.en");
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("<1GB → refused with a remote steer (not a hard 'unsupported')", () => {
|
|
88
|
+
const plan = planInstall({ platform: "linux", arch: "x64", totalRamBytes: 0.5 * GB });
|
|
89
|
+
expect(plan.supported).toBe(false);
|
|
90
|
+
expect(plan.refused).toBe(true);
|
|
91
|
+
expect(plan.asset).toBeDefined(); // platform IS supported; RAM isn't
|
|
92
|
+
expect(plan.reason).toContain("scribe-http");
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("unsupported platform → not supported, steer to remote, no asset", () => {
|
|
96
|
+
const plan = planInstall({ platform: "win32", arch: "x64", totalRamBytes: 16 * GB });
|
|
97
|
+
expect(plan.supported).toBe(false);
|
|
98
|
+
expect(plan.refused).toBeUndefined();
|
|
99
|
+
expect(plan.asset).toBeUndefined();
|
|
100
|
+
expect(plan.reason).toContain("scribe-http");
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("--model override wins over the RAM tier", () => {
|
|
104
|
+
const plan = planInstall({
|
|
105
|
+
platform: "darwin",
|
|
106
|
+
arch: "arm64",
|
|
107
|
+
totalRamBytes: 16 * GB,
|
|
108
|
+
overrideModel: "whisper-tiny.en",
|
|
109
|
+
});
|
|
110
|
+
expect(plan.supported).toBe(true);
|
|
111
|
+
expect(plan.model!.name).toBe("whisper-tiny.en");
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("--model override with an unknown id → not supported, lists valid models", () => {
|
|
115
|
+
const plan = planInstall({
|
|
116
|
+
platform: "linux",
|
|
117
|
+
arch: "x64",
|
|
118
|
+
totalRamBytes: 8 * GB,
|
|
119
|
+
overrideModel: "whisper-huge",
|
|
120
|
+
});
|
|
121
|
+
expect(plan.supported).toBe(false);
|
|
122
|
+
expect(plan.reason).toContain("whisper-tiny.en");
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("override lets a 1GB box request a big model (operator's call)", () => {
|
|
126
|
+
const plan = planInstall({
|
|
127
|
+
platform: "linux",
|
|
128
|
+
arch: "x64",
|
|
129
|
+
totalRamBytes: 1 * GB,
|
|
130
|
+
overrideModel: "parakeet-tdt-0.6b-v3",
|
|
131
|
+
});
|
|
132
|
+
expect(plan.supported).toBe(true);
|
|
133
|
+
expect(plan.model!.name).toBe("parakeet-tdt-0.6b-v3");
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe("MODELS registry", () => {
|
|
138
|
+
test("each model has a resolvable HuggingFace URL + a gguf file", () => {
|
|
139
|
+
for (const m of Object.values(MODELS)) {
|
|
140
|
+
expect(m.file).toMatch(/\.gguf$/);
|
|
141
|
+
expect(m.url).toContain("huggingface.co/handy-computer");
|
|
142
|
+
expect(m.url).toContain(m.file);
|
|
143
|
+
expect(m.approxSizeMb).toBeGreaterThan(0);
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test("whisper-tiny.en footprint reflects the real Q5_K_M size (~42MB)", () => {
|
|
148
|
+
// Live-verified: whisper-tiny.en-Q5_K_M.gguf is 44,135,072 B ≈ 42MB (was 35).
|
|
149
|
+
expect(MODELS["whisper-tiny.en"]!.approxSizeMb).toBe(42);
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
describe("isSharedLibFile — what we keep from the v0.1.1 library tarball", () => {
|
|
154
|
+
test("matches macOS + Linux shared libs (incl. versioned .so)", () => {
|
|
155
|
+
expect(isSharedLibFile("libtranscribe.dylib")).toBe(true);
|
|
156
|
+
expect(isSharedLibFile("libggml-metal.dylib")).toBe(true);
|
|
157
|
+
expect(isSharedLibFile("libtranscribe.so")).toBe(true);
|
|
158
|
+
expect(isSharedLibFile("libggml.so.1")).toBe(true);
|
|
159
|
+
expect(isSharedLibFile("libggml.so.0.9.4")).toBe(true);
|
|
160
|
+
});
|
|
161
|
+
test("rejects non-libs (the CLI, manifests, licenses)", () => {
|
|
162
|
+
expect(isSharedLibFile("transcribe-cli")).toBe(false);
|
|
163
|
+
expect(isSharedLibFile("contract.json")).toBe(false);
|
|
164
|
+
expect(isSharedLibFile("LICENSE")).toBe(false);
|
|
165
|
+
expect(isSharedLibFile("model.gguf")).toBe(false);
|
|
166
|
+
});
|
|
167
|
+
});
|