@openparachute/vault 0.6.5-rc.2 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.parachute/module.json +1 -0
- package/core/src/core.test.ts +7 -7
- package/core/src/do-param-cap.test.ts +161 -0
- package/core/src/links.ts +8 -13
- package/core/src/mcp.ts +1 -1
- package/core/src/notes.ts +33 -15
- package/core/src/onboarding.ts +14 -296
- package/core/src/schema.ts +5 -5
- package/core/src/seed-packs.test.ts +357 -0
- package/core/src/seed-packs.ts +823 -0
- package/core/src/sql-in.test.ts +58 -0
- package/core/src/sql-in.ts +76 -0
- package/core/src/tag-hierarchy.ts +1 -1
- package/core/src/tag-schemas.ts +2 -2
- package/core/src/transcription/provider.ts +141 -0
- package/core/src/types.ts +1 -1
- package/core/src/vault-projection.ts +1 -1
- package/core/src/wikilinks.ts +10 -4
- package/package.json +1 -1
- package/src/add-pack.test.ts +142 -0
- package/src/admin-spa.ts +2 -1
- package/src/auth.ts +1 -1
- package/src/cli.ts +806 -7
- package/src/export-watch.ts +1 -1
- package/src/live-frame-parity.test.ts +201 -0
- package/src/module-manifest.ts +8 -0
- package/src/onboarding-seed.test.ts +188 -40
- package/src/onboarding-seed.ts +41 -46
- package/src/routes.ts +20 -3
- package/src/routing.test.ts +2 -2
- package/src/routing.ts +18 -6
- package/src/self-register.test.ts +19 -0
- package/src/self-register.ts +6 -1
- package/src/server.ts +133 -31
- package/src/services-manifest.ts +8 -0
- package/src/subscriptions.ts +100 -42
- package/src/tag-scope.ts +3 -3
- package/src/test-support/live-frame-corpus.ts +72 -0
- package/src/token-store.ts +2 -2
- package/src/transcription/build.test.ts +305 -0
- package/src/transcription/build.ts +332 -0
- package/src/transcription/capability.test.ts +118 -0
- package/src/transcription/capability.ts +96 -0
- package/src/transcription/download.test.ts +101 -0
- package/src/transcription/download.ts +71 -0
- package/src/transcription/install-python.test.ts +366 -0
- package/src/transcription/install-python.ts +471 -0
- package/src/transcription/install.test.ts +167 -0
- package/src/transcription/install.ts +296 -0
- package/src/transcription/providers/onnx-asr.test.ts +229 -0
- package/src/transcription/providers/onnx-asr.ts +239 -0
- package/src/transcription/providers/parakeet-mlx.test.ts +290 -0
- package/src/transcription/providers/parakeet-mlx.ts +242 -0
- package/src/transcription/providers/scribe-http.test.ts +195 -0
- package/src/transcription/providers/scribe-http.ts +144 -0
- package/src/transcription/providers/transcribe-cpp.test.ts +314 -0
- package/src/transcription/providers/transcribe-cpp.ts +293 -0
- package/src/transcription/select.test.ts +299 -0
- package/src/transcription/select.ts +397 -0
- package/src/transcription/tiers.test.ts +197 -0
- package/src/transcription/tiers.ts +184 -0
- package/src/transcription-worker.test.ts +44 -0
- package/src/transcription-worker.ts +57 -122
- package/src/vault-create.test.ts +43 -10
- package/src/vault.test.ts +49 -1
- package/src/ws-server.ts +408 -0
- package/src/ws-subscribe.test.ts +474 -0
- package/src/ws-subscribe.ts +242 -0
- package/src/subscribe.test.ts +0 -609
- package/src/subscribe.ts +0 -248
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Building `transcribe-cli` from source at install time (scribe-fold Phase 2b).
|
|
3
|
+
*
|
|
4
|
+
* transcribe.cpp v0.1.1 ships a **library** (`libtranscribe.{dylib,so}` +
|
|
5
|
+
* `libggml*`), NOT a prebuilt `transcribe-cli` executable — see
|
|
6
|
+
* `install.ts` + the provider-seam design doc. This module closes that gap: it
|
|
7
|
+
* fetches the CLI's source (`examples/cli/main.cpp` + `examples/common/wav.cpp`
|
|
8
|
+
* + the public headers) from the transcribe.cpp repo pinned at the v0.1.1 tag
|
|
9
|
+
* commit, applies the **vault#534 backend-init patch** (see
|
|
10
|
+
* `patchMainCppBackendInit` below — temporary until upstream ships the fix),
|
|
11
|
+
* then compiles it with the host C++ compiler against the extracted prebuilt
|
|
12
|
+
* dylibs. The result is a ~127KB driver that links the prebuilt
|
|
13
|
+
* `libtranscribe` at runtime — **no ggml / Metal rebuild**.
|
|
14
|
+
*
|
|
15
|
+
* ## The proven recipe (live-verified 2026-07-03, macOS arm64 M4)
|
|
16
|
+
*
|
|
17
|
+
* c++ -std=c++17 -O2 \
|
|
18
|
+
* -I<srcDir>/include \
|
|
19
|
+
* -I<srcDir>/examples/common \
|
|
20
|
+
* <srcDir>/examples/cli/main.cpp \
|
|
21
|
+
* <srcDir>/examples/common/wav.cpp \
|
|
22
|
+
* -L<libsDir> -ltranscribe \
|
|
23
|
+
* -Wl,-rpath,@loader_path/../libs \ # Linux: $ORIGIN/../libs
|
|
24
|
+
* -o <binDir>/transcribe-cli
|
|
25
|
+
*
|
|
26
|
+
* The rpath is load-bearing: `libtranscribe.dylib`'s install_name is
|
|
27
|
+
* `@rpath/libtranscribe.dylib` (with no rpath of its own), so the CLI carries
|
|
28
|
+
* `@loader_path/../libs` to resolve it from a sibling `libs/` dir. In turn
|
|
29
|
+
* `libtranscribe` references `libggml*` via `@loader_path/libggml*.dylib`, so
|
|
30
|
+
* having all dylibs in the same `libs/` dir is enough — the CLI never needs to
|
|
31
|
+
* know about libggml. On Linux the analogous `$ORIGIN/../libs` is passed as a
|
|
32
|
+
* single argv element (we spawn the compiler directly, NOT through a shell, so
|
|
33
|
+
* `$ORIGIN` is NOT expanded — it lands literally in the ELF RUNPATH for the
|
|
34
|
+
* runtime loader to expand).
|
|
35
|
+
*
|
|
36
|
+
* ## Testability
|
|
37
|
+
*
|
|
38
|
+
* `buildTranscribeCli` is a pure orchestrator over injected side effects
|
|
39
|
+
* (`which` / `fetchSource` / `readFile` / `writeFile` / `compile` / `exists` /
|
|
40
|
+
* `removeBin`), mirroring the provider's `SpawnRunner` seam. Tests exercise
|
|
41
|
+
* every branch — toolchain absent, fetch failure, patch failure, compile
|
|
42
|
+
* failure, success — with no network and no real compiler. The real compile is
|
|
43
|
+
* exercised only by the install-time live verify.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
import { join } from "path";
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Pinned source ref for the CLI build: the transcribe.cpp **v0.1.1 tag commit**.
|
|
50
|
+
* Kept in lockstep with `TRANSCRIBE_CPP_VERSION` in `install.ts` — bump both
|
|
51
|
+
* together (a newer CLI source may need different headers or flags, and the
|
|
52
|
+
* provider's output parsing is validated against a specific version).
|
|
53
|
+
*/
|
|
54
|
+
export const TRANSCRIBE_CPP_SOURCE_REF = "d89ecb75062e8457681c563994675dc60e31db80";
|
|
55
|
+
|
|
56
|
+
const RAW_BASE = `https://raw.githubusercontent.com/handy-computer/transcribe.cpp/${TRANSCRIBE_CPP_SOURCE_REF}`;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Repo-relative source files needed to build `transcribe-cli`, laid out under a
|
|
60
|
+
* local source dir so the include paths resolve exactly as upstream expects.
|
|
61
|
+
* `main.cpp` includes `transcribe.h` + `transcribe/{parakeet,voxtral_realtime,
|
|
62
|
+
* whisper}.h` + `wav.h`; `wav.cpp` includes `wav.h` + `dr_wav.h`. We fetch the
|
|
63
|
+
* complete `include/` header set (the two extra sub-headers are tiny and keep
|
|
64
|
+
* the tree self-consistent). Verified sufficient by a real compile at this ref.
|
|
65
|
+
*/
|
|
66
|
+
export const CLI_SOURCE_FILES: readonly string[] = [
|
|
67
|
+
"include/transcribe.h",
|
|
68
|
+
"include/transcribe/extensions.h",
|
|
69
|
+
"include/transcribe/moonshine_streaming.h",
|
|
70
|
+
"include/transcribe/parakeet.h",
|
|
71
|
+
"include/transcribe/voxtral_realtime.h",
|
|
72
|
+
"include/transcribe/whisper.h",
|
|
73
|
+
"examples/common/wav.h",
|
|
74
|
+
"examples/common/wav.cpp",
|
|
75
|
+
"examples/common/dr_wav.h",
|
|
76
|
+
"examples/cli/main.cpp",
|
|
77
|
+
];
|
|
78
|
+
|
|
79
|
+
/** Full raw.githubusercontent URL for a repo-relative path at the pinned ref. */
|
|
80
|
+
export function cliSourceUrl(repoPath: string): string {
|
|
81
|
+
return `${RAW_BASE}/${repoPath}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// ---------------------------------------------------------------------------
|
|
85
|
+
// Build-time source patch: unconditional backend init (vault#534 blocker 2)
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
/** Marker comment baked into the injected code — makes the patch idempotent
|
|
89
|
+
* and greppable in a fetched source tree. */
|
|
90
|
+
export const BACKEND_INIT_PATCH_MARKER = "[parachute-vault vault#534 backend-init patch]";
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* The anchor line in the pinned `examples/cli/main.cpp` the patch injects
|
|
94
|
+
* before: the batch-mode comment that opens the inference half of `main()`,
|
|
95
|
+
* directly AFTER the log-sink install (so backend plugin-load logs route
|
|
96
|
+
* through the CLI's log callback) and after the `--list-devices` early return.
|
|
97
|
+
* Unique at the pinned ref (verified); if the pin ever advances and this
|
|
98
|
+
* anchor drifts, the build fails LOUDLY (stage "patch") so the patch gets
|
|
99
|
+
* re-evaluated against the new source instead of silently shipping a CLI that
|
|
100
|
+
* can't transcribe on Linux.
|
|
101
|
+
*/
|
|
102
|
+
export const BACKEND_INIT_PATCH_ANCHOR =
|
|
103
|
+
" // Batch mode: --batch reads a file list, one wav path per line. Loads";
|
|
104
|
+
|
|
105
|
+
const BACKEND_INIT_PATCH_BLOCK = ` // ${BACKEND_INIT_PATCH_MARKER}
|
|
106
|
+
// Upstream (at the pinned ${TRANSCRIBE_CPP_SOURCE_REF.slice(0, 12)}) only calls
|
|
107
|
+
// transcribe_init_backends_default() in the --list-devices path. On Linux
|
|
108
|
+
// the CPU backends are dlopen plugins (libggml-cpu-*.so), so with no init
|
|
109
|
+
// the ggml device registry is empty and EVERY transcription fails
|
|
110
|
+
// ("whisper: failed to initialize CPU backend", exit 1). macOS is masked:
|
|
111
|
+
// its libtranscribe.dylib direct-links libggml-cpu, which self-registers
|
|
112
|
+
// at library load. Init the default backends unconditionally before any
|
|
113
|
+
// inference (batch or single-file). Verified fix on real Linux containers,
|
|
114
|
+
// both arches — see vault#534. DROP THIS PATCH when upstream ships the fix
|
|
115
|
+
// and the source pin advances past it.
|
|
116
|
+
transcribe_init_backends_default();
|
|
117
|
+
|
|
118
|
+
`;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Apply the vault#534 backend-init patch to the fetched `main.cpp` source.
|
|
122
|
+
* Pure (string → string) so tests pin the behavior without a filesystem.
|
|
123
|
+
* Idempotent: an already-patched source is returned unchanged. Throws with a
|
|
124
|
+
* clear operator-facing message when the anchor is missing — the caller
|
|
125
|
+
* surfaces it as a `stage: "patch"` build failure (fail loudly; never compile
|
|
126
|
+
* an unpatched CLI on the assumption it'll work).
|
|
127
|
+
*/
|
|
128
|
+
export function patchMainCppBackendInit(source: string): string {
|
|
129
|
+
if (source.includes(BACKEND_INIT_PATCH_MARKER)) return source;
|
|
130
|
+
const idx = source.indexOf(BACKEND_INIT_PATCH_ANCHOR);
|
|
131
|
+
if (idx === -1) {
|
|
132
|
+
throw new Error(
|
|
133
|
+
`backend-init patch anchor not found in examples/cli/main.cpp at ${TRANSCRIBE_CPP_SOURCE_REF.slice(0, 12)} — ` +
|
|
134
|
+
`the pinned source no longer matches the vault#534 patch. If the pin advanced, check whether upstream ` +
|
|
135
|
+
`now calls transcribe_init_backends_default() unconditionally (then drop the patch); otherwise re-anchor it.`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
return source.slice(0, idx) + BACKEND_INIT_PATCH_BLOCK + source.slice(idx);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** C++ compiler binaries we probe for, in preference order. */
|
|
142
|
+
export const CXX_CANDIDATES = ["c++", "clang++", "g++"] as const;
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Resolve a C++ compiler path via the injected `which` (production:
|
|
146
|
+
* `Bun.which`). Returns the first candidate found, or `null` when none is on
|
|
147
|
+
* PATH — the caller keeps the current provider and prints a platform hint.
|
|
148
|
+
*/
|
|
149
|
+
export function detectCompiler(which: (cmd: string) => string | null | undefined): string | null {
|
|
150
|
+
for (const c of CXX_CANDIDATES) {
|
|
151
|
+
const p = which(c);
|
|
152
|
+
if (p) return p;
|
|
153
|
+
}
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** A short "install a compiler" hint tailored to the host platform. */
|
|
158
|
+
export function compilerInstallHint(platform: string): string {
|
|
159
|
+
if (platform === "darwin") return "install the Xcode command-line tools: xcode-select --install";
|
|
160
|
+
if (platform === "linux") {
|
|
161
|
+
return "install a C++ toolchain: apt install build-essential (Debian/Ubuntu) | dnf install gcc-c++ (Fedora/RHEL)";
|
|
162
|
+
}
|
|
163
|
+
return "install a C++ compiler (c++/clang++/g++) on PATH";
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export interface CompileCommandInput {
|
|
167
|
+
/** Host platform (`process.platform`) — selects the rpath flavor. */
|
|
168
|
+
platform: string;
|
|
169
|
+
/** Resolved compiler path (`detectCompiler`). */
|
|
170
|
+
compiler: string;
|
|
171
|
+
/** The local source dir the CLI sources were fetched into. */
|
|
172
|
+
srcDir: string;
|
|
173
|
+
/** The extracted prebuilt dylibs dir to link `-ltranscribe` from. */
|
|
174
|
+
libsDir: string;
|
|
175
|
+
/** Where the compiled `transcribe-cli` lands. */
|
|
176
|
+
outPath: string;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Build the compiler argv (the proven recipe). Pure — a test pins the shape,
|
|
181
|
+
* mirroring `buildTranscribeArgs` for the provider. The rpath resolves the
|
|
182
|
+
* sibling `libs/` dir from `bin/`: `@loader_path/../libs` (macOS) /
|
|
183
|
+
* `$ORIGIN/../libs` (Linux, passed literally — no shell expansion).
|
|
184
|
+
*/
|
|
185
|
+
export function buildCompileCommand(input: CompileCommandInput): string[] {
|
|
186
|
+
const includeDir = join(input.srcDir, "include");
|
|
187
|
+
const commonDir = join(input.srcDir, "examples", "common");
|
|
188
|
+
const mainCpp = join(input.srcDir, "examples", "cli", "main.cpp");
|
|
189
|
+
const wavCpp = join(input.srcDir, "examples", "common", "wav.cpp");
|
|
190
|
+
const rpath = input.platform === "darwin" ? "@loader_path/../libs" : "$ORIGIN/../libs";
|
|
191
|
+
return [
|
|
192
|
+
input.compiler,
|
|
193
|
+
"-std=c++17",
|
|
194
|
+
"-O2",
|
|
195
|
+
`-I${includeDir}`,
|
|
196
|
+
`-I${commonDir}`,
|
|
197
|
+
mainCpp,
|
|
198
|
+
wavCpp,
|
|
199
|
+
`-L${input.libsDir}`,
|
|
200
|
+
"-ltranscribe",
|
|
201
|
+
`-Wl,-rpath,${rpath}`,
|
|
202
|
+
"-o",
|
|
203
|
+
input.outPath,
|
|
204
|
+
];
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Result of running the compiler: exit code + captured stderr. */
|
|
208
|
+
export interface CompileResult {
|
|
209
|
+
exitCode: number;
|
|
210
|
+
stderr: string;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Injected side effects for `buildTranscribeCli` (tests pass mocks). */
|
|
214
|
+
export interface BuildCliDeps {
|
|
215
|
+
/** Host platform (`process.platform`). */
|
|
216
|
+
platform: string;
|
|
217
|
+
/** Compiler probe (production: `Bun.which`). */
|
|
218
|
+
which: (cmd: string) => string | null | undefined;
|
|
219
|
+
/** Fetch `files` (repo-relative) into `srcDir`. Throws on network failure. */
|
|
220
|
+
fetchSource: (files: readonly string[], srcDir: string) => Promise<void>;
|
|
221
|
+
/** Read a fetched source file as UTF-8 (production: `fs.readFileSync`). */
|
|
222
|
+
readFile: (p: string) => string;
|
|
223
|
+
/** Write a patched source file (production: `fs.writeFileSync`). */
|
|
224
|
+
writeFile: (p: string, content: string) => void;
|
|
225
|
+
/** Run the compiler argv. */
|
|
226
|
+
compile: (cmd: string[]) => CompileResult | Promise<CompileResult>;
|
|
227
|
+
/** Existence probe (production: `fs.existsSync`). */
|
|
228
|
+
exists: (p: string) => boolean;
|
|
229
|
+
/** Remove a stale / half-built binary (production: `fs.rmSync{force}`). */
|
|
230
|
+
removeBin: (p: string) => void;
|
|
231
|
+
/** Move a file into place (production: `fs.renameSync`). Used to promote the
|
|
232
|
+
* freshly-built binary atomically, so a failed rebuild never clobbers a
|
|
233
|
+
* previously working binary. */
|
|
234
|
+
rename: (from: string, to: string) => void;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export interface BuildCliInput {
|
|
238
|
+
/** Local dir to fetch the CLI sources into. */
|
|
239
|
+
srcDir: string;
|
|
240
|
+
/** The extracted prebuilt dylibs dir. */
|
|
241
|
+
libsDir: string;
|
|
242
|
+
/** Output binary path. */
|
|
243
|
+
binPath: string;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** The outcome of a build attempt — a discriminated union the CLI branches on. */
|
|
247
|
+
export type CliBuildResult =
|
|
248
|
+
| { ok: true; binPath: string; compiler: string; command: string }
|
|
249
|
+
| {
|
|
250
|
+
ok: false;
|
|
251
|
+
/** Which stage failed — drives the operator guidance. */
|
|
252
|
+
stage: "toolchain" | "fetch" | "patch" | "compile";
|
|
253
|
+
message: string;
|
|
254
|
+
compiler?: string;
|
|
255
|
+
/** The compile argv (as a string) when we got far enough to build one. */
|
|
256
|
+
command?: string;
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Fetch the CLI source at the pinned ref and compile it against the extracted
|
|
261
|
+
* dylibs. Non-throwing: every failure is a typed `{ ok: false }` so the caller
|
|
262
|
+
* keeps the current provider and reports honestly. Never leaves a half-built
|
|
263
|
+
* binary — a nonzero compile (or a missing output) removes `binPath` so
|
|
264
|
+
* `available()` can't mistake a partial artifact for a runnable CLI.
|
|
265
|
+
*/
|
|
266
|
+
export async function buildTranscribeCli(
|
|
267
|
+
input: BuildCliInput,
|
|
268
|
+
deps: BuildCliDeps,
|
|
269
|
+
): Promise<CliBuildResult> {
|
|
270
|
+
const compiler = detectCompiler(deps.which);
|
|
271
|
+
if (!compiler) {
|
|
272
|
+
return {
|
|
273
|
+
ok: false,
|
|
274
|
+
stage: "toolchain",
|
|
275
|
+
message: `no C++ compiler on PATH (looked for ${CXX_CANDIDATES.join(", ")}) — ${compilerInstallHint(deps.platform)}`,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
try {
|
|
280
|
+
await deps.fetchSource(CLI_SOURCE_FILES, input.srcDir);
|
|
281
|
+
} catch (err) {
|
|
282
|
+
return {
|
|
283
|
+
ok: false,
|
|
284
|
+
stage: "fetch",
|
|
285
|
+
compiler,
|
|
286
|
+
message: `failed to fetch transcribe-cli source at ${TRANSCRIBE_CPP_SOURCE_REF}: ${err instanceof Error ? err.message : String(err)}`,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Apply the vault#534 backend-init source patch BEFORE compiling — without
|
|
291
|
+
// it the built CLI exits 1 on every transcription on Linux (dlopen'd CPU
|
|
292
|
+
// backends are never registered). A missing anchor is a loud, typed failure,
|
|
293
|
+
// never a silent skip.
|
|
294
|
+
const mainCppPath = join(input.srcDir, "examples", "cli", "main.cpp");
|
|
295
|
+
try {
|
|
296
|
+
deps.writeFile(mainCppPath, patchMainCppBackendInit(deps.readFile(mainCppPath)));
|
|
297
|
+
} catch (err) {
|
|
298
|
+
return {
|
|
299
|
+
ok: false,
|
|
300
|
+
stage: "patch",
|
|
301
|
+
compiler,
|
|
302
|
+
message: err instanceof Error ? err.message : String(err),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const common = { platform: deps.platform, compiler, srcDir: input.srcDir, libsDir: input.libsDir };
|
|
307
|
+
// The reported command targets the FINAL binPath (what an operator would run
|
|
308
|
+
// by hand to reproduce). The actual compile writes to a temp sibling and is
|
|
309
|
+
// promoted on success — so a failed rebuild (e.g. `--force` that then fails)
|
|
310
|
+
// never clobbers a previously working binary, and no half-built artifact is
|
|
311
|
+
// ever left where available() would treat it as runnable. The temp is a
|
|
312
|
+
// sibling in the same dir, so the @loader_path/../libs rpath is unaffected.
|
|
313
|
+
const command = buildCompileCommand({ ...common, outPath: input.binPath }).join(" ");
|
|
314
|
+
const tmpOut = `${input.binPath}.building`;
|
|
315
|
+
const argv = buildCompileCommand({ ...common, outPath: tmpOut });
|
|
316
|
+
|
|
317
|
+
deps.removeBin(tmpOut); // clear any stale temp from a prior interrupted build
|
|
318
|
+
const res = await deps.compile(argv);
|
|
319
|
+
if (res.exitCode !== 0 || !deps.exists(tmpOut)) {
|
|
320
|
+
deps.removeBin(tmpOut); // drop any partial output; the existing binary stands
|
|
321
|
+
return {
|
|
322
|
+
ok: false,
|
|
323
|
+
stage: "compile",
|
|
324
|
+
compiler,
|
|
325
|
+
command,
|
|
326
|
+
message: (res.stderr || "").trim() || `compiler exited ${res.exitCode}`,
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
deps.rename(tmpOut, input.binPath); // atomic promote over any prior binary
|
|
331
|
+
return { ok: true, binPath: input.binPath, compiler, command };
|
|
332
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { describe, test, expect, afterEach } from "bun:test";
|
|
2
|
+
import { resolveTranscriptionCapability, defaultTranscriptionProvider } from "./capability.ts";
|
|
3
|
+
import { ScribeHttpProvider } from "./providers/scribe-http.ts";
|
|
4
|
+
import { TranscribeCppProvider } from "./providers/transcribe-cpp.ts";
|
|
5
|
+
import type { TranscriptionProvider } from "../../core/src/transcription/provider.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Capability-flag tests (scribe-fold Phase 1). The vault landing surfaces
|
|
9
|
+
* `transcription: { enabled, provider? }`; `enabled` iff a provider is
|
|
10
|
+
* configured AND available. Notes gates its mic on this.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** A minimal fake provider so tests don't depend on live scribe discovery. */
|
|
14
|
+
function fakeProvider(available: boolean, name = "fake"): TranscriptionProvider {
|
|
15
|
+
return {
|
|
16
|
+
name,
|
|
17
|
+
available: async () => (available ? { ok: true } : { ok: false, reason: "off" }),
|
|
18
|
+
transcribe: async () => ({ text: "" }),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
describe("resolveTranscriptionCapability", () => {
|
|
23
|
+
test("enabled:true + provider name when the provider is available", async () => {
|
|
24
|
+
const cap = await resolveTranscriptionCapability(fakeProvider(true, "scribe-http"));
|
|
25
|
+
expect(cap).toEqual({ enabled: true, provider: "scribe-http" });
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("enabled:false + no provider name when the provider is unavailable", async () => {
|
|
29
|
+
const cap = await resolveTranscriptionCapability(fakeProvider(false));
|
|
30
|
+
expect(cap).toEqual({ enabled: false });
|
|
31
|
+
expect(cap.provider).toBeUndefined();
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("no provider configured (scribe-http with no URL) resolves to disabled, no throw", async () => {
|
|
35
|
+
// The "no provider configured" path: a scribe-http provider whose URL is
|
|
36
|
+
// undefined reports unavailable rather than throwing, so the landing never
|
|
37
|
+
// crashes when scribe is absent.
|
|
38
|
+
const cap = await resolveTranscriptionCapability(new ScribeHttpProvider({ url: undefined }));
|
|
39
|
+
expect(cap.enabled).toBe(false);
|
|
40
|
+
expect(cap.provider).toBeUndefined();
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("a configured scribe-http provider resolves to enabled: scribe-http", async () => {
|
|
44
|
+
const cap = await resolveTranscriptionCapability(new ScribeHttpProvider({ url: "http://scribe.test" }));
|
|
45
|
+
expect(cap).toEqual({ enabled: true, provider: "scribe-http" });
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("never omits minutes_remaining as an unmetered self-host concern", async () => {
|
|
49
|
+
const cap = await resolveTranscriptionCapability(fakeProvider(true));
|
|
50
|
+
expect("minutes_remaining" in cap).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("a transcribe-cpp provider (installed) resolves to enabled: transcribe-cpp", async () => {
|
|
54
|
+
// existsImpl stubs the binary + model as present so available() is ok
|
|
55
|
+
// without touching disk.
|
|
56
|
+
const p = new TranscribeCppProvider({
|
|
57
|
+
binPath: "/tc/transcribe-cli",
|
|
58
|
+
modelPath: "/tc/model.gguf",
|
|
59
|
+
existsImpl: () => true,
|
|
60
|
+
});
|
|
61
|
+
const cap = await resolveTranscriptionCapability(p);
|
|
62
|
+
expect(cap).toEqual({ enabled: true, provider: "transcribe-cpp" });
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("a transcribe-cpp provider (not installed) resolves to disabled", async () => {
|
|
66
|
+
const p = new TranscribeCppProvider({ binPath: "/tc/transcribe-cli", modelPath: "/tc/model.gguf", existsImpl: () => false });
|
|
67
|
+
const cap = await resolveTranscriptionCapability(p);
|
|
68
|
+
expect(cap.enabled).toBe(false);
|
|
69
|
+
expect(cap.provider).toBeUndefined();
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* `defaultTranscriptionProvider` honors `TRANSCRIPTION_PROVIDER` so the
|
|
75
|
+
* capability flag reflects whichever provider is configured (scribe-fold 2a).
|
|
76
|
+
*/
|
|
77
|
+
describe("defaultTranscriptionProvider — provider selection", () => {
|
|
78
|
+
const saved = process.env.TRANSCRIPTION_PROVIDER;
|
|
79
|
+
afterEach(() => {
|
|
80
|
+
if (saved === undefined) delete process.env.TRANSCRIPTION_PROVIDER;
|
|
81
|
+
else process.env.TRANSCRIPTION_PROVIDER = saved;
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("default (unset) → the scribe-http provider", () => {
|
|
85
|
+
delete process.env.TRANSCRIPTION_PROVIDER;
|
|
86
|
+
expect(defaultTranscriptionProvider().name).toBe("scribe-http");
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("TRANSCRIPTION_PROVIDER=transcribe-cpp → the transcribe-cpp provider", () => {
|
|
90
|
+
process.env.TRANSCRIPTION_PROVIDER = "transcribe-cpp";
|
|
91
|
+
expect(defaultTranscriptionProvider().name).toBe("transcribe-cpp");
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("TRANSCRIPTION_PROVIDER=parakeet-mlx → the parakeet-mlx provider (2b)", () => {
|
|
95
|
+
process.env.TRANSCRIPTION_PROVIDER = "parakeet-mlx";
|
|
96
|
+
expect(defaultTranscriptionProvider().name).toBe("parakeet-mlx");
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("TRANSCRIPTION_PROVIDER=onnx-asr → the onnx-asr provider (2b)", () => {
|
|
100
|
+
process.env.TRANSCRIPTION_PROVIDER = "onnx-asr";
|
|
101
|
+
expect(defaultTranscriptionProvider().name).toBe("onnx-asr");
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("an unconfigured python provider resolves to disabled (no throw)", async () => {
|
|
105
|
+
// No venv/PATH binary in a fresh env → available() is ok:false and the
|
|
106
|
+
// landing capability is {enabled:false} rather than a crash.
|
|
107
|
+
process.env.TRANSCRIPTION_PROVIDER = "parakeet-mlx";
|
|
108
|
+
const prevBin = process.env.PARAKEET_MLX_BIN;
|
|
109
|
+
process.env.PARAKEET_MLX_BIN = "/definitely/not/a/real/parakeet-mlx";
|
|
110
|
+
try {
|
|
111
|
+
const cap = await resolveTranscriptionCapability(defaultTranscriptionProvider());
|
|
112
|
+
expect(cap.enabled).toBe(false);
|
|
113
|
+
} finally {
|
|
114
|
+
if (prevBin === undefined) delete process.env.PARAKEET_MLX_BIN;
|
|
115
|
+
else process.env.PARAKEET_MLX_BIN = prevBin;
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
});
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcription capability flag (scribe-fold Phase 1).
|
|
3
|
+
*
|
|
4
|
+
* The vault landing (`GET /vault/<name>/api/vault`) surfaces a `transcription`
|
|
5
|
+
* capability so a surface (Notes) can gate its microphone affordance on
|
|
6
|
+
* whether transcription is actually possible — distinct from the
|
|
7
|
+
* `auto_transcribe.enabled` POLICY toggle (which defaults on even when no
|
|
8
|
+
* provider is reachable). `enabled` is true only when a provider is configured
|
|
9
|
+
* AND its `available()` probe passes.
|
|
10
|
+
*
|
|
11
|
+
* `minutes_remaining` is deliberately omitted: it's a cloud/plan concern, and
|
|
12
|
+
* self-host is unmetered. The cloud voice build, whose Workers-AI provider
|
|
13
|
+
* implements the same `TranscriptionProvider` interface, will add metering at
|
|
14
|
+
* that layer.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { TranscriptionProvider } from "../../core/src/transcription/provider.ts";
|
|
18
|
+
import { ScribeHttpProvider } from "./providers/scribe-http.ts";
|
|
19
|
+
import { TranscribeCppProvider } from "./providers/transcribe-cpp.ts";
|
|
20
|
+
import { ParakeetMlxProvider } from "./providers/parakeet-mlx.ts";
|
|
21
|
+
import { OnnxAsrProvider } from "./providers/onnx-asr.ts";
|
|
22
|
+
import { getCachedScribeUrl } from "../scribe-discovery.ts";
|
|
23
|
+
import { resolveScribeAuthToken } from "../scribe-env.ts";
|
|
24
|
+
import {
|
|
25
|
+
resolveTranscriptionProviderName,
|
|
26
|
+
resolveTranscribeCppPaths,
|
|
27
|
+
resolveParakeetMlxBin,
|
|
28
|
+
resolveParakeetMlxModel,
|
|
29
|
+
resolveOnnxAsrBin,
|
|
30
|
+
resolveOnnxAsrModel,
|
|
31
|
+
} from "./select.ts";
|
|
32
|
+
|
|
33
|
+
export interface TranscriptionCapability {
|
|
34
|
+
/** True when a provider is configured AND available. Notes gates the mic on this. */
|
|
35
|
+
enabled: boolean;
|
|
36
|
+
/** The active provider's name (e.g. "scribe-http"). Omitted when disabled. */
|
|
37
|
+
provider?: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Build the default provider from live per-process config, honoring
|
|
42
|
+
* `TRANSCRIPTION_PROVIDER` (scribe-fold Phase 2a) so the capability flag
|
|
43
|
+
* reflects whichever provider is actually configured:
|
|
44
|
+
*
|
|
45
|
+
* - `transcribe-cpp` → the local provider, resolving the installed binary +
|
|
46
|
+
* GGUF model paths; `available()` is `false` until `transcription install`
|
|
47
|
+
* has run.
|
|
48
|
+
* - `parakeet-mlx` / `onnx-asr` (scribe-fold Phase 2b) → the Python-based
|
|
49
|
+
* local providers, resolving the binary via env override → managed venv →
|
|
50
|
+
* PATH; `available()` is `false` until a runnable binary exists.
|
|
51
|
+
* - `scribe-http` (default) → the remote provider (URL from
|
|
52
|
+
* `SCRIBE_URL`/services.json, bearer from `SCRIBE_AUTH_TOKEN`). When scribe
|
|
53
|
+
* isn't discoverable the URL is undefined and it reports itself unavailable
|
|
54
|
+
* rather than throwing.
|
|
55
|
+
*/
|
|
56
|
+
export function defaultTranscriptionProvider(): TranscriptionProvider {
|
|
57
|
+
const name = resolveTranscriptionProviderName();
|
|
58
|
+
if (name === "transcribe-cpp") {
|
|
59
|
+
const paths = resolveTranscribeCppPaths();
|
|
60
|
+
return new TranscribeCppProvider({ binPath: paths.binPath, modelPath: paths.modelPath });
|
|
61
|
+
}
|
|
62
|
+
if (name === "parakeet-mlx") {
|
|
63
|
+
return new ParakeetMlxProvider({
|
|
64
|
+
binPath: resolveParakeetMlxBin(),
|
|
65
|
+
model: resolveParakeetMlxModel(),
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
if (name === "onnx-asr") {
|
|
69
|
+
return new OnnxAsrProvider({
|
|
70
|
+
binPath: resolveOnnxAsrBin(),
|
|
71
|
+
model: resolveOnnxAsrModel(),
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
return new ScribeHttpProvider({
|
|
75
|
+
url: getCachedScribeUrl(),
|
|
76
|
+
token: resolveScribeAuthToken(),
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Resolve the transcription capability for the vault landing. `enabled` is the
|
|
82
|
+
* provider's `available()` result; `provider` is its name when enabled. A
|
|
83
|
+
* "no provider configured" deployment resolves cleanly to `{ enabled: false }`
|
|
84
|
+
* (no throw), so the landing never crashes when scribe is absent.
|
|
85
|
+
*
|
|
86
|
+
* The `provider` arg is an injection seam for tests + a future explicit
|
|
87
|
+
* provider selection; production omits it and uses the default scribe-http
|
|
88
|
+
* provider built from live config.
|
|
89
|
+
*/
|
|
90
|
+
export async function resolveTranscriptionCapability(
|
|
91
|
+
provider: TranscriptionProvider = defaultTranscriptionProvider(),
|
|
92
|
+
): Promise<TranscriptionCapability> {
|
|
93
|
+
const avail = await provider.available();
|
|
94
|
+
if (!avail.ok) return { enabled: false };
|
|
95
|
+
return { enabled: true, provider: provider.name };
|
|
96
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { describe, test, expect, afterAll } from "bun:test";
|
|
2
|
+
import { existsSync, mkdtempSync, readFileSync, rmSync } from "fs";
|
|
3
|
+
import { tmpdir } from "os";
|
|
4
|
+
import { join } from "path";
|
|
5
|
+
import { downloadTo } from "./download.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `downloadTo` tests (vault#534 blocker 1). The manual body→FileSink pump
|
|
9
|
+
* replaced `Bun.write(dest, resp)`, which hangs forever on Linux for large
|
|
10
|
+
* responses. These run the pump against an in-process `Bun.serve` (no
|
|
11
|
+
* subprocess involved — see CLAUDE.md's spawnSync note; plain in-process
|
|
12
|
+
* fetches are fine) and pin: bytes land intact, HTTP errors throw, and a
|
|
13
|
+
* mid-stream failure never leaves a partial file behind.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const dir = mkdtempSync(join(tmpdir(), "vault-download-test-"));
|
|
17
|
+
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
|
18
|
+
|
|
19
|
+
// 4MB of deterministic non-trivial bytes — large enough to stream in many
|
|
20
|
+
// chunks (the hang was specific to the large-response path).
|
|
21
|
+
const PAYLOAD = new Uint8Array(4 * 1024 * 1024);
|
|
22
|
+
for (let i = 0; i < PAYLOAD.length; i++) PAYLOAD[i] = (i * 31 + (i >> 8)) & 0xff;
|
|
23
|
+
|
|
24
|
+
function serve(handler: (req: Request) => Response | Promise<Response>) {
|
|
25
|
+
const server = Bun.serve({ port: 0, fetch: handler });
|
|
26
|
+
return { server, url: `http://127.0.0.1:${server.port}` };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe("downloadTo", () => {
|
|
30
|
+
test("streams a multi-MB body to disk byte-for-byte", async () => {
|
|
31
|
+
const { server, url } = serve(() => new Response(PAYLOAD));
|
|
32
|
+
const dest = join(dir, "ok.bin");
|
|
33
|
+
try {
|
|
34
|
+
await downloadTo(`${url}/file`, dest);
|
|
35
|
+
const got = readFileSync(dest);
|
|
36
|
+
expect(got.byteLength).toBe(PAYLOAD.byteLength);
|
|
37
|
+
expect(Buffer.from(got).equals(Buffer.from(PAYLOAD))).toBe(true);
|
|
38
|
+
} finally {
|
|
39
|
+
server.stop(true);
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("non-OK status throws with the status + URL, writes nothing", async () => {
|
|
44
|
+
const { server, url } = serve(() => new Response("nope", { status: 404 }));
|
|
45
|
+
const dest = join(dir, "missing.bin");
|
|
46
|
+
try {
|
|
47
|
+
expect(downloadTo(`${url}/gone`, dest)).rejects.toThrow(/download failed \(404\)/);
|
|
48
|
+
expect(existsSync(dest)).toBe(false);
|
|
49
|
+
} finally {
|
|
50
|
+
server.stop(true);
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("truncated download throws AND removes the partial file", async () => {
|
|
55
|
+
// Raw TCP fixture: a real content-length response (how GitHub releases +
|
|
56
|
+
// HuggingFace serve these binaries) whose connection dies mid-body.
|
|
57
|
+
// Bun.serve can't fixture this — it re-frames streamed responses as
|
|
58
|
+
// chunked (dropping content-length), and Bun's fetch ends a truncated
|
|
59
|
+
// chunked stream CLEANLY, so truncation is only detectable on the
|
|
60
|
+
// content-length path `downloadTo` verifies.
|
|
61
|
+
const partial = PAYLOAD.slice(0, 64 * 1024);
|
|
62
|
+
const listener = Bun.listen({
|
|
63
|
+
hostname: "127.0.0.1",
|
|
64
|
+
port: 0,
|
|
65
|
+
socket: {
|
|
66
|
+
data(socket) {
|
|
67
|
+
socket.write(
|
|
68
|
+
`HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: ${PAYLOAD.byteLength}\r\n\r\n`,
|
|
69
|
+
);
|
|
70
|
+
socket.write(partial);
|
|
71
|
+
socket.end(); // die after 64KB of a declared 4MB
|
|
72
|
+
},
|
|
73
|
+
open() {},
|
|
74
|
+
close() {},
|
|
75
|
+
error() {},
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
const dest = join(dir, "partial.bin");
|
|
79
|
+
try {
|
|
80
|
+
let threw = false;
|
|
81
|
+
try {
|
|
82
|
+
await downloadTo(`http://127.0.0.1:${listener.port}/flaky`, dest);
|
|
83
|
+
} catch (err) {
|
|
84
|
+
threw = true;
|
|
85
|
+
// The truncation surfaces differently per platform: on macOS the body
|
|
86
|
+
// stream errors mid-pump (wrapped "failed after N bytes") or the
|
|
87
|
+
// short read trips the content-length check ("truncated"); on Linux
|
|
88
|
+
// Bun's fetch() itself rejects on the early socket close. All are
|
|
89
|
+
// loud failures — the invariants are THROWS + NO PARTIAL FILE.
|
|
90
|
+
expect(String(err)).toMatch(
|
|
91
|
+
/failed after \d+ bytes|truncated|socket connection was closed|connection closed/i,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
expect(threw).toBe(true);
|
|
95
|
+
// The half-written file must not survive to be mistaken for a good artifact.
|
|
96
|
+
expect(existsSync(dest)).toBe(false);
|
|
97
|
+
} finally {
|
|
98
|
+
listener.stop(true);
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
});
|