@openparachute/vault 0.6.5-rc.9 → 0.7.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/.parachute/module.json +1 -0
  2. package/core/src/contract-concurrency.test.ts +117 -0
  3. package/core/src/contract-taxonomy.test.ts +119 -0
  4. package/core/src/contract-typed-index.test.ts +106 -0
  5. package/core/src/core.test.ts +7 -7
  6. package/core/src/mcp.ts +1 -1
  7. package/core/src/schema.ts +5 -5
  8. package/core/src/seed-packs.test.ts +222 -56
  9. package/core/src/seed-packs.ts +334 -70
  10. package/core/src/tag-hierarchy.ts +1 -1
  11. package/core/src/tag-schemas.ts +2 -2
  12. package/core/src/types.ts +1 -1
  13. package/package.json +1 -1
  14. package/src/admin-spa.ts +2 -1
  15. package/src/auth.ts +1 -1
  16. package/src/cli.ts +317 -53
  17. package/src/contract-errors.test.ts +98 -0
  18. package/src/contract-honest-queries.test.ts +100 -0
  19. package/src/contract-search.test.ts +127 -0
  20. package/src/export-watch.ts +1 -1
  21. package/src/live-frame-parity.test.ts +201 -0
  22. package/src/module-manifest.ts +8 -0
  23. package/src/onboarding-seed.test.ts +82 -20
  24. package/src/onboarding-seed.ts +2 -2
  25. package/src/routes.ts +3 -3
  26. package/src/routing.test.ts +2 -2
  27. package/src/routing.ts +18 -6
  28. package/src/self-register.test.ts +19 -0
  29. package/src/self-register.ts +6 -1
  30. package/src/server.ts +56 -0
  31. package/src/services-manifest.ts +8 -0
  32. package/src/subscriptions.ts +100 -42
  33. package/src/tag-scope.ts +3 -3
  34. package/src/test-support/live-frame-corpus.ts +72 -0
  35. package/src/token-store.ts +2 -2
  36. package/src/transcription/build.test.ts +86 -5
  37. package/src/transcription/build.ts +87 -7
  38. package/src/transcription/capability.test.ts +25 -0
  39. package/src/transcription/capability.ts +27 -2
  40. package/src/transcription/download.test.ts +101 -0
  41. package/src/transcription/download.ts +71 -0
  42. package/src/transcription/install-python.test.ts +366 -0
  43. package/src/transcription/install-python.ts +471 -0
  44. package/src/transcription/providers/onnx-asr.test.ts +229 -0
  45. package/src/transcription/providers/onnx-asr.ts +239 -0
  46. package/src/transcription/providers/parakeet-mlx.test.ts +290 -0
  47. package/src/transcription/providers/parakeet-mlx.ts +242 -0
  48. package/src/transcription/select.test.ts +166 -1
  49. package/src/transcription/select.ts +234 -1
  50. package/src/transcription/tiers.test.ts +197 -0
  51. package/src/transcription/tiers.ts +184 -0
  52. package/src/vault-create.test.ts +19 -14
  53. package/src/vault.test.ts +1 -1
  54. package/src/ws-server.ts +408 -0
  55. package/src/ws-subscribe.test.ts +474 -0
  56. package/src/ws-subscribe.ts +242 -0
  57. package/src/subscribe.test.ts +0 -609
  58. package/src/subscribe.ts +0 -248
@@ -6,6 +6,9 @@ import {
6
6
  detectCompiler,
7
7
  compilerInstallHint,
8
8
  cliSourceUrl,
9
+ patchMainCppBackendInit,
10
+ BACKEND_INIT_PATCH_ANCHOR,
11
+ BACKEND_INIT_PATCH_MARKER,
9
12
  CLI_SOURCE_FILES,
10
13
  CXX_CANDIDATES,
11
14
  TRANSCRIBE_CPP_SOURCE_REF,
@@ -17,10 +20,26 @@ import {
17
20
  * transcribe-cli build tests (scribe-fold Phase 2b). Pure recipe shape +
18
21
  * fully-mocked orchestration — NO network, NO real compiler (the real compile
19
22
  * is exercised only by the install-time live verify). Covers the toolchain-
20
- * absent, fetch-failure, compile-failure, and success branches, and the
21
- * "never leave a half-built binary" invariant.
23
+ * absent, fetch-failure, patch-failure, compile-failure, and success branches,
24
+ * and the "never leave a half-built binary" invariant.
22
25
  */
23
26
 
27
+ /** A minimal stand-in for the pinned main.cpp: log-sink install + the batch-
28
+ * mode comment the vault#534 patch anchors on. */
29
+ const MAIN_CPP_FIXTURE = [
30
+ "int main(int argc, char ** argv) {",
31
+ " if (!args.quiet) {",
32
+ " transcribe_log_set(log_cb, nullptr);",
33
+ " }",
34
+ "",
35
+ BACKEND_INIT_PATCH_ANCHOR,
36
+ " // the model ONCE and reuses the context across all files.",
37
+ " if (!args.batch_file.empty()) {",
38
+ " }",
39
+ "}",
40
+ "",
41
+ ].join("\n");
42
+
24
43
  describe("detectCompiler", () => {
25
44
  test("returns the first candidate found (c++ preferred)", () => {
26
45
  const which = (c: string) => (c === "c++" ? "/usr/bin/c++" : null);
@@ -106,15 +125,31 @@ describe("buildCompileCommand — the proven recipe shape", () => {
106
125
 
107
126
  /** Build a deps object with call-recording mocks; overrides win. */
108
127
  function mkDeps(over: Partial<BuildCliDeps> = {}): BuildCliDeps & {
109
- calls: { fetch: number; compile: number; removed: string[]; renamed: Array<[string, string]> };
128
+ calls: {
129
+ fetch: number;
130
+ compile: number;
131
+ removed: string[];
132
+ renamed: Array<[string, string]>;
133
+ written: Array<[string, string]>;
134
+ };
110
135
  } {
111
- const calls = { fetch: 0, compile: 0, removed: [] as string[], renamed: [] as Array<[string, string]> };
136
+ const calls = {
137
+ fetch: 0,
138
+ compile: 0,
139
+ removed: [] as string[],
140
+ renamed: [] as Array<[string, string]>,
141
+ written: [] as Array<[string, string]>,
142
+ };
112
143
  const deps: BuildCliDeps = {
113
144
  platform: "darwin",
114
145
  which: (c) => (c === "c++" ? "/usr/bin/c++" : null),
115
146
  fetchSource: async () => {
116
147
  calls.fetch++;
117
148
  },
149
+ readFile: () => MAIN_CPP_FIXTURE,
150
+ writeFile: (p, content) => {
151
+ calls.written.push([p, content]);
152
+ },
118
153
  compile: async (): Promise<CompileResult> => {
119
154
  calls.compile++;
120
155
  return { exitCode: 0, stderr: "" };
@@ -134,8 +169,36 @@ function mkDeps(over: Partial<BuildCliDeps> = {}): BuildCliDeps & {
134
169
  const INPUT = { srcDir: "/t/.src", libsDir: "/t/libs", binPath: "/t/bin/transcribe-cli" };
135
170
  const TMP = `${INPUT.binPath}.building`; // the temp output the build compiles to
136
171
 
172
+ describe("patchMainCppBackendInit — the vault#534 backend-init source patch", () => {
173
+ test("injects transcribe_init_backends_default() before the batch-mode anchor", () => {
174
+ const patched = patchMainCppBackendInit(MAIN_CPP_FIXTURE);
175
+ expect(patched).toContain("transcribe_init_backends_default();");
176
+ expect(patched).toContain(BACKEND_INIT_PATCH_MARKER);
177
+ // The init call lands BEFORE the inference paths (the anchor) and AFTER
178
+ // the log-sink install, so backend plugin-load logs route through the sink.
179
+ const initAt = patched.indexOf("transcribe_init_backends_default();");
180
+ expect(initAt).toBeGreaterThan(patched.indexOf("transcribe_log_set(log_cb, nullptr);"));
181
+ expect(initAt).toBeLessThan(patched.indexOf(BACKEND_INIT_PATCH_ANCHOR));
182
+ // Everything else is preserved verbatim: removing the injected block
183
+ // restores the original source exactly.
184
+ const [before, after] = patched.split(/ \/\/ \[parachute-vault[\s\S]*?transcribe_init_backends_default\(\);\n\n/);
185
+ expect(`${before}${after}`).toBe(MAIN_CPP_FIXTURE);
186
+ });
187
+
188
+ test("idempotent: an already-patched source is returned unchanged", () => {
189
+ const once = patchMainCppBackendInit(MAIN_CPP_FIXTURE);
190
+ expect(patchMainCppBackendInit(once)).toBe(once);
191
+ });
192
+
193
+ test("anchor missing ⇒ throws a loud, actionable error naming the pin + vault#534", () => {
194
+ expect(() => patchMainCppBackendInit("int main() { return 0; }\n")).toThrow(
195
+ /anchor not found[\s\S]*vault#534/,
196
+ );
197
+ });
198
+ });
199
+
137
200
  describe("buildTranscribeCli — orchestration branches", () => {
138
- test("SUCCESS: fetch → compile ok → temp exists ⇒ promote (rename) + { ok:true }", async () => {
201
+ test("SUCCESS: fetch → patch → compile ok → temp exists ⇒ promote (rename) + { ok:true }", async () => {
139
202
  const deps = mkDeps();
140
203
  const r = await buildTranscribeCli(INPUT, deps);
141
204
  expect(r.ok).toBe(true);
@@ -149,11 +212,29 @@ describe("buildTranscribeCli — orchestration branches", () => {
149
212
  }
150
213
  expect(deps.calls.fetch).toBe(1);
151
214
  expect(deps.calls.compile).toBe(1);
215
+ // the vault#534 backend-init patch was written into the fetched main.cpp
216
+ expect(deps.calls.written).toHaveLength(1);
217
+ const [patchedPath, patchedContent] = deps.calls.written[0]!;
218
+ expect(patchedPath).toBe(join(INPUT.srcDir, "examples", "cli", "main.cpp"));
219
+ expect(patchedContent).toContain("transcribe_init_backends_default();");
152
220
  // only the TEMP is cleared pre-compile; the fresh temp is renamed into place
153
221
  expect(deps.calls.removed).toEqual([TMP]);
154
222
  expect(deps.calls.renamed).toEqual([[TMP, INPUT.binPath]]);
155
223
  });
156
224
 
225
+ test("PATCH FAILURE: anchor missing in fetched source ⇒ { ok:false, stage:'patch' }, no write, no compile", async () => {
226
+ const deps = mkDeps({ readFile: () => "int main() { return 0; }\n" });
227
+ const r = await buildTranscribeCli(INPUT, deps);
228
+ expect(r.ok).toBe(false);
229
+ if (!r.ok) {
230
+ expect(r.stage).toBe("patch");
231
+ expect(r.message).toContain("anchor not found");
232
+ expect(r.message).toContain("vault#534");
233
+ }
234
+ expect(deps.calls.written).toEqual([]);
235
+ expect(deps.calls.compile).toBe(0);
236
+ });
237
+
157
238
  test("TOOLCHAIN ABSENT: no compiler ⇒ { ok:false, stage:'toolchain' }, no fetch/compile", async () => {
158
239
  const deps = mkDeps({ which: () => null });
159
240
  const r = await buildTranscribeCli(INPUT, deps);
@@ -6,8 +6,10 @@
6
6
  * `install.ts` + the provider-seam design doc. This module closes that gap: it
7
7
  * fetches the CLI's source (`examples/cli/main.cpp` + `examples/common/wav.cpp`
8
8
  * + the public headers) from the transcribe.cpp repo pinned at the v0.1.1 tag
9
- * commit, then compiles it with the host C++ compiler against the extracted
10
- * prebuilt dylibs. The result is a ~127KB driver that links the prebuilt
9
+ * commit, applies the **vault#534 backend-init patch** (see
10
+ * `patchMainCppBackendInit` below temporary until upstream ships the fix),
11
+ * then compiles it with the host C++ compiler against the extracted prebuilt
12
+ * dylibs. The result is a ~127KB driver that links the prebuilt
11
13
  * `libtranscribe` at runtime — **no ggml / Metal rebuild**.
12
14
  *
13
15
  * ## The proven recipe (live-verified 2026-07-03, macOS arm64 M4)
@@ -34,10 +36,11 @@
34
36
  * ## Testability
35
37
  *
36
38
  * `buildTranscribeCli` is a pure orchestrator over injected side effects
37
- * (`which` / `fetchSource` / `compile` / `exists` / `removeBin`), mirroring the
38
- * provider's `SpawnRunner` seam. Tests exercise every branch — toolchain
39
- * absent, fetch failure, compile failure, success — with no network and no real
40
- * compiler. The real compile is exercised only by the install-time live verify.
39
+ * (`which` / `fetchSource` / `readFile` / `writeFile` / `compile` / `exists` /
40
+ * `removeBin`), mirroring the provider's `SpawnRunner` seam. Tests exercise
41
+ * every branch — toolchain absent, fetch failure, patch failure, compile
42
+ * failure, success with no network and no real compiler. The real compile is
43
+ * exercised only by the install-time live verify.
41
44
  */
42
45
 
43
46
  import { join } from "path";
@@ -78,6 +81,63 @@ export function cliSourceUrl(repoPath: string): string {
78
81
  return `${RAW_BASE}/${repoPath}`;
79
82
  }
80
83
 
84
+ // ---------------------------------------------------------------------------
85
+ // Build-time source patch: unconditional backend init (vault#534 blocker 2)
86
+ // ---------------------------------------------------------------------------
87
+
88
+ /** Marker comment baked into the injected code — makes the patch idempotent
89
+ * and greppable in a fetched source tree. */
90
+ export const BACKEND_INIT_PATCH_MARKER = "[parachute-vault vault#534 backend-init patch]";
91
+
92
+ /**
93
+ * The anchor line in the pinned `examples/cli/main.cpp` the patch injects
94
+ * before: the batch-mode comment that opens the inference half of `main()`,
95
+ * directly AFTER the log-sink install (so backend plugin-load logs route
96
+ * through the CLI's log callback) and after the `--list-devices` early return.
97
+ * Unique at the pinned ref (verified); if the pin ever advances and this
98
+ * anchor drifts, the build fails LOUDLY (stage "patch") so the patch gets
99
+ * re-evaluated against the new source instead of silently shipping a CLI that
100
+ * can't transcribe on Linux.
101
+ */
102
+ export const BACKEND_INIT_PATCH_ANCHOR =
103
+ " // Batch mode: --batch reads a file list, one wav path per line. Loads";
104
+
105
+ const BACKEND_INIT_PATCH_BLOCK = ` // ${BACKEND_INIT_PATCH_MARKER}
106
+ // Upstream (at the pinned ${TRANSCRIBE_CPP_SOURCE_REF.slice(0, 12)}) only calls
107
+ // transcribe_init_backends_default() in the --list-devices path. On Linux
108
+ // the CPU backends are dlopen plugins (libggml-cpu-*.so), so with no init
109
+ // the ggml device registry is empty and EVERY transcription fails
110
+ // ("whisper: failed to initialize CPU backend", exit 1). macOS is masked:
111
+ // its libtranscribe.dylib direct-links libggml-cpu, which self-registers
112
+ // at library load. Init the default backends unconditionally before any
113
+ // inference (batch or single-file). Verified fix on real Linux containers,
114
+ // both arches — see vault#534. DROP THIS PATCH when upstream ships the fix
115
+ // and the source pin advances past it.
116
+ transcribe_init_backends_default();
117
+
118
+ `;
119
+
120
+ /**
121
+ * Apply the vault#534 backend-init patch to the fetched `main.cpp` source.
122
+ * Pure (string → string) so tests pin the behavior without a filesystem.
123
+ * Idempotent: an already-patched source is returned unchanged. Throws with a
124
+ * clear operator-facing message when the anchor is missing — the caller
125
+ * surfaces it as a `stage: "patch"` build failure (fail loudly; never compile
126
+ * an unpatched CLI on the assumption it'll work).
127
+ */
128
+ export function patchMainCppBackendInit(source: string): string {
129
+ if (source.includes(BACKEND_INIT_PATCH_MARKER)) return source;
130
+ const idx = source.indexOf(BACKEND_INIT_PATCH_ANCHOR);
131
+ if (idx === -1) {
132
+ throw new Error(
133
+ `backend-init patch anchor not found in examples/cli/main.cpp at ${TRANSCRIBE_CPP_SOURCE_REF.slice(0, 12)} — ` +
134
+ `the pinned source no longer matches the vault#534 patch. If the pin advanced, check whether upstream ` +
135
+ `now calls transcribe_init_backends_default() unconditionally (then drop the patch); otherwise re-anchor it.`,
136
+ );
137
+ }
138
+ return source.slice(0, idx) + BACKEND_INIT_PATCH_BLOCK + source.slice(idx);
139
+ }
140
+
81
141
  /** C++ compiler binaries we probe for, in preference order. */
82
142
  export const CXX_CANDIDATES = ["c++", "clang++", "g++"] as const;
83
143
 
@@ -158,6 +218,10 @@ export interface BuildCliDeps {
158
218
  which: (cmd: string) => string | null | undefined;
159
219
  /** Fetch `files` (repo-relative) into `srcDir`. Throws on network failure. */
160
220
  fetchSource: (files: readonly string[], srcDir: string) => Promise<void>;
221
+ /** Read a fetched source file as UTF-8 (production: `fs.readFileSync`). */
222
+ readFile: (p: string) => string;
223
+ /** Write a patched source file (production: `fs.writeFileSync`). */
224
+ writeFile: (p: string, content: string) => void;
161
225
  /** Run the compiler argv. */
162
226
  compile: (cmd: string[]) => CompileResult | Promise<CompileResult>;
163
227
  /** Existence probe (production: `fs.existsSync`). */
@@ -185,7 +249,7 @@ export type CliBuildResult =
185
249
  | {
186
250
  ok: false;
187
251
  /** Which stage failed — drives the operator guidance. */
188
- stage: "toolchain" | "fetch" | "compile";
252
+ stage: "toolchain" | "fetch" | "patch" | "compile";
189
253
  message: string;
190
254
  compiler?: string;
191
255
  /** The compile argv (as a string) when we got far enough to build one. */
@@ -223,6 +287,22 @@ export async function buildTranscribeCli(
223
287
  };
224
288
  }
225
289
 
290
+ // Apply the vault#534 backend-init source patch BEFORE compiling — without
291
+ // it the built CLI exits 1 on every transcription on Linux (dlopen'd CPU
292
+ // backends are never registered). A missing anchor is a loud, typed failure,
293
+ // never a silent skip.
294
+ const mainCppPath = join(input.srcDir, "examples", "cli", "main.cpp");
295
+ try {
296
+ deps.writeFile(mainCppPath, patchMainCppBackendInit(deps.readFile(mainCppPath)));
297
+ } catch (err) {
298
+ return {
299
+ ok: false,
300
+ stage: "patch",
301
+ compiler,
302
+ message: err instanceof Error ? err.message : String(err),
303
+ };
304
+ }
305
+
226
306
  const common = { platform: deps.platform, compiler, srcDir: input.srcDir, libsDir: input.libsDir };
227
307
  // The reported command targets the FINAL binPath (what an operator would run
228
308
  // by hand to reproduce). The actual compile writes to a temp sibling and is
@@ -90,4 +90,29 @@ describe("defaultTranscriptionProvider — provider selection", () => {
90
90
  process.env.TRANSCRIPTION_PROVIDER = "transcribe-cpp";
91
91
  expect(defaultTranscriptionProvider().name).toBe("transcribe-cpp");
92
92
  });
93
+
94
+ test("TRANSCRIPTION_PROVIDER=parakeet-mlx → the parakeet-mlx provider (2b)", () => {
95
+ process.env.TRANSCRIPTION_PROVIDER = "parakeet-mlx";
96
+ expect(defaultTranscriptionProvider().name).toBe("parakeet-mlx");
97
+ });
98
+
99
+ test("TRANSCRIPTION_PROVIDER=onnx-asr → the onnx-asr provider (2b)", () => {
100
+ process.env.TRANSCRIPTION_PROVIDER = "onnx-asr";
101
+ expect(defaultTranscriptionProvider().name).toBe("onnx-asr");
102
+ });
103
+
104
+ test("an unconfigured python provider resolves to disabled (no throw)", async () => {
105
+ // No venv/PATH binary in a fresh env → available() is ok:false and the
106
+ // landing capability is {enabled:false} rather than a crash.
107
+ process.env.TRANSCRIPTION_PROVIDER = "parakeet-mlx";
108
+ const prevBin = process.env.PARAKEET_MLX_BIN;
109
+ process.env.PARAKEET_MLX_BIN = "/definitely/not/a/real/parakeet-mlx";
110
+ try {
111
+ const cap = await resolveTranscriptionCapability(defaultTranscriptionProvider());
112
+ expect(cap.enabled).toBe(false);
113
+ } finally {
114
+ if (prevBin === undefined) delete process.env.PARAKEET_MLX_BIN;
115
+ else process.env.PARAKEET_MLX_BIN = prevBin;
116
+ }
117
+ });
93
118
  });
@@ -17,9 +17,18 @@
17
17
  import type { TranscriptionProvider } from "../../core/src/transcription/provider.ts";
18
18
  import { ScribeHttpProvider } from "./providers/scribe-http.ts";
19
19
  import { TranscribeCppProvider } from "./providers/transcribe-cpp.ts";
20
+ import { ParakeetMlxProvider } from "./providers/parakeet-mlx.ts";
21
+ import { OnnxAsrProvider } from "./providers/onnx-asr.ts";
20
22
  import { getCachedScribeUrl } from "../scribe-discovery.ts";
21
23
  import { resolveScribeAuthToken } from "../scribe-env.ts";
22
- import { resolveTranscriptionProviderName, resolveTranscribeCppPaths } from "./select.ts";
24
+ import {
25
+ resolveTranscriptionProviderName,
26
+ resolveTranscribeCppPaths,
27
+ resolveParakeetMlxBin,
28
+ resolveParakeetMlxModel,
29
+ resolveOnnxAsrBin,
30
+ resolveOnnxAsrModel,
31
+ } from "./select.ts";
23
32
 
24
33
  export interface TranscriptionCapability {
25
34
  /** True when a provider is configured AND available. Notes gates the mic on this. */
@@ -36,16 +45,32 @@ export interface TranscriptionCapability {
36
45
  * - `transcribe-cpp` → the local provider, resolving the installed binary +
37
46
  * GGUF model paths; `available()` is `false` until `transcription install`
38
47
  * has run.
48
+ * - `parakeet-mlx` / `onnx-asr` (scribe-fold Phase 2b) → the Python-based
49
+ * local providers, resolving the binary via env override → managed venv →
50
+ * PATH; `available()` is `false` until a runnable binary exists.
39
51
  * - `scribe-http` (default) → the remote provider (URL from
40
52
  * `SCRIBE_URL`/services.json, bearer from `SCRIBE_AUTH_TOKEN`). When scribe
41
53
  * isn't discoverable the URL is undefined and it reports itself unavailable
42
54
  * rather than throwing.
43
55
  */
44
56
  export function defaultTranscriptionProvider(): TranscriptionProvider {
45
- if (resolveTranscriptionProviderName() === "transcribe-cpp") {
57
+ const name = resolveTranscriptionProviderName();
58
+ if (name === "transcribe-cpp") {
46
59
  const paths = resolveTranscribeCppPaths();
47
60
  return new TranscribeCppProvider({ binPath: paths.binPath, modelPath: paths.modelPath });
48
61
  }
62
+ if (name === "parakeet-mlx") {
63
+ return new ParakeetMlxProvider({
64
+ binPath: resolveParakeetMlxBin(),
65
+ model: resolveParakeetMlxModel(),
66
+ });
67
+ }
68
+ if (name === "onnx-asr") {
69
+ return new OnnxAsrProvider({
70
+ binPath: resolveOnnxAsrBin(),
71
+ model: resolveOnnxAsrModel(),
72
+ });
73
+ }
49
74
  return new ScribeHttpProvider({
50
75
  url: getCachedScribeUrl(),
51
76
  token: resolveScribeAuthToken(),
@@ -0,0 +1,101 @@
1
+ import { describe, test, expect, afterAll } from "bun:test";
2
+ import { existsSync, mkdtempSync, readFileSync, rmSync } from "fs";
3
+ import { tmpdir } from "os";
4
+ import { join } from "path";
5
+ import { downloadTo } from "./download.ts";
6
+
7
+ /**
8
+ * `downloadTo` tests (vault#534 blocker 1). The manual body→FileSink pump
9
+ * replaced `Bun.write(dest, resp)`, which hangs forever on Linux for large
10
+ * responses. These run the pump against an in-process `Bun.serve` (no
11
+ * subprocess involved — see CLAUDE.md's spawnSync note; plain in-process
12
+ * fetches are fine) and pin: bytes land intact, HTTP errors throw, and a
13
+ * mid-stream failure never leaves a partial file behind.
14
+ */
15
+
16
+ const dir = mkdtempSync(join(tmpdir(), "vault-download-test-"));
17
+ afterAll(() => rmSync(dir, { recursive: true, force: true }));
18
+
19
+ // 4MB of deterministic non-trivial bytes — large enough to stream in many
20
+ // chunks (the hang was specific to the large-response path).
21
+ const PAYLOAD = new Uint8Array(4 * 1024 * 1024);
22
+ for (let i = 0; i < PAYLOAD.length; i++) PAYLOAD[i] = (i * 31 + (i >> 8)) & 0xff;
23
+
24
+ function serve(handler: (req: Request) => Response | Promise<Response>) {
25
+ const server = Bun.serve({ port: 0, fetch: handler });
26
+ return { server, url: `http://127.0.0.1:${server.port}` };
27
+ }
28
+
29
+ describe("downloadTo", () => {
30
+ test("streams a multi-MB body to disk byte-for-byte", async () => {
31
+ const { server, url } = serve(() => new Response(PAYLOAD));
32
+ const dest = join(dir, "ok.bin");
33
+ try {
34
+ await downloadTo(`${url}/file`, dest);
35
+ const got = readFileSync(dest);
36
+ expect(got.byteLength).toBe(PAYLOAD.byteLength);
37
+ expect(Buffer.from(got).equals(Buffer.from(PAYLOAD))).toBe(true);
38
+ } finally {
39
+ server.stop(true);
40
+ }
41
+ });
42
+
43
+ test("non-OK status throws with the status + URL, writes nothing", async () => {
44
+ const { server, url } = serve(() => new Response("nope", { status: 404 }));
45
+ const dest = join(dir, "missing.bin");
46
+ try {
47
+ expect(downloadTo(`${url}/gone`, dest)).rejects.toThrow(/download failed \(404\)/);
48
+ expect(existsSync(dest)).toBe(false);
49
+ } finally {
50
+ server.stop(true);
51
+ }
52
+ });
53
+
54
+ test("truncated download throws AND removes the partial file", async () => {
55
+ // Raw TCP fixture: a real content-length response (how GitHub releases +
56
+ // HuggingFace serve these binaries) whose connection dies mid-body.
57
+ // Bun.serve can't fixture this — it re-frames streamed responses as
58
+ // chunked (dropping content-length), and Bun's fetch ends a truncated
59
+ // chunked stream CLEANLY, so truncation is only detectable on the
60
+ // content-length path `downloadTo` verifies.
61
+ const partial = PAYLOAD.slice(0, 64 * 1024);
62
+ const listener = Bun.listen({
63
+ hostname: "127.0.0.1",
64
+ port: 0,
65
+ socket: {
66
+ data(socket) {
67
+ socket.write(
68
+ `HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: ${PAYLOAD.byteLength}\r\n\r\n`,
69
+ );
70
+ socket.write(partial);
71
+ socket.end(); // die after 64KB of a declared 4MB
72
+ },
73
+ open() {},
74
+ close() {},
75
+ error() {},
76
+ },
77
+ });
78
+ const dest = join(dir, "partial.bin");
79
+ try {
80
+ let threw = false;
81
+ try {
82
+ await downloadTo(`http://127.0.0.1:${listener.port}/flaky`, dest);
83
+ } catch (err) {
84
+ threw = true;
85
+ // The truncation surfaces differently per platform: on macOS the body
86
+ // stream errors mid-pump (wrapped "failed after N bytes") or the
87
+ // short read trips the content-length check ("truncated"); on Linux
88
+ // Bun's fetch() itself rejects on the early socket close. All are
89
+ // loud failures — the invariants are THROWS + NO PARTIAL FILE.
90
+ expect(String(err)).toMatch(
91
+ /failed after \d+ bytes|truncated|socket connection was closed|connection closed/i,
92
+ );
93
+ }
94
+ expect(threw).toBe(true);
95
+ // The half-written file must not survive to be mistaken for a good artifact.
96
+ expect(existsSync(dest)).toBe(false);
97
+ } finally {
98
+ listener.stop(true);
99
+ }
100
+ });
101
+ });
@@ -0,0 +1,71 @@
1
+ /**
2
+ * Streaming file download for `transcription install` (tarball, GGUF models,
3
+ * CLI sources).
4
+ *
5
+ * ## Why a manual pump instead of `Bun.write(dest, resp)` (vault#534 blocker 1)
6
+ *
7
+ * `Bun.write(dest, response)` HANGS FOREVER on Linux when streaming a large
8
+ * Response body to disk — zero bytes land, no error, no timeout. Reproduced
9
+ * deterministically in real Linux containers on BOTH arches (aarch64 native +
10
+ * x86_64 emulated) across bun 1.2.23 / 1.3.13 / 1.3.14, while the identical
11
+ * code works on macOS. In the same environment `curl` pulls the same URL at
12
+ * full speed and `await resp.arrayBuffer()` gets all 26MB in ~2s — the bug is
13
+ * specifically Bun's Response→file streaming fast path. Manually pumping
14
+ * `resp.body` chunks into a `Bun.file(dest).writer()` moves the same 26MB in
15
+ * <1s and works on both platforms, so that's what we do everywhere (no
16
+ * platform gate). See vault#534 for the full container verification.
17
+ */
18
+
19
+ import { rmSync } from "fs";
20
+
21
+ /**
22
+ * Download `url` to `dest` (follows redirects). Streams chunk-by-chunk — never
23
+ * buffers the whole body (models run up to ~660MB). On any mid-stream failure
24
+ * the partial file is removed so a retry never trusts a truncated artifact.
25
+ * When the server sent an honest `content-length` (no content-encoding
26
+ * transform), a byte-count mismatch is treated as a failed download too.
27
+ */
28
+ export async function downloadTo(url: string, dest: string): Promise<void> {
29
+ const resp = await fetch(url, { redirect: "follow" });
30
+ if (!resp.ok) throw new Error(`download failed (${resp.status}) for ${url}`);
31
+ if (!resp.body) throw new Error(`download failed (empty response body) for ${url}`);
32
+
33
+ // content-length is only the on-the-wire byte count when no transfer
34
+ // decompression happened; fetch auto-decodes content-encoding'd bodies, so
35
+ // only enforce the size check for identity responses (the normal case for
36
+ // release tarballs + GGUFs — both already-compressed binary).
37
+ const encoding = resp.headers.get("content-encoding");
38
+ const lengthHeader = resp.headers.get("content-length");
39
+ const expectedBytes =
40
+ (!encoding || encoding === "identity") && lengthHeader && /^\d+$/.test(lengthHeader)
41
+ ? Number(lengthHeader)
42
+ : null;
43
+
44
+ const sink = Bun.file(dest).writer();
45
+ let written = 0;
46
+ try {
47
+ for await (const chunk of resp.body) {
48
+ sink.write(chunk);
49
+ written += chunk.byteLength;
50
+ }
51
+ await sink.end();
52
+ } catch (err) {
53
+ // Flush/close what we can, then remove the partial file — a half-written
54
+ // tarball/model must never survive to be mistaken for a good artifact.
55
+ try {
56
+ await sink.end();
57
+ } catch {
58
+ // best-effort close; the rm below is what matters
59
+ }
60
+ rmSync(dest, { force: true });
61
+ const msg = err instanceof Error ? err.message : String(err);
62
+ throw new Error(`download of ${url} failed after ${written} bytes: ${msg}`);
63
+ }
64
+
65
+ if (expectedBytes !== null && written !== expectedBytes) {
66
+ rmSync(dest, { force: true });
67
+ throw new Error(
68
+ `download of ${url} was truncated: got ${written} bytes, expected ${expectedBytes} (content-length)`,
69
+ );
70
+ }
71
+ }