@openparachute/vault 0.7.6 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -16
- package/core/src/attachment/policy.test.ts +7 -0
- package/core/src/attachment/policy.ts +9 -0
- package/core/src/attachment-tickets-tool.test.ts +15 -0
- package/core/src/conformance.test.ts +78 -0
- package/core/src/conformance.ts +34 -5
- package/core/src/connection-pragmas.test.ts +27 -1
- package/core/src/core.test.ts +88 -3
- package/core/src/cursor.ts +2 -0
- package/core/src/do-param-cap.test.ts +167 -0
- package/core/src/lede.test.ts +60 -0
- package/core/src/mcp-manifest.ts +15 -1
- package/core/src/mcp.ts +37 -5
- package/core/src/notes.ts +120 -48
- package/core/src/query-operators.ts +87 -5
- package/core/src/query-warnings.ts +11 -3
- package/core/src/schema.ts +32 -0
- package/core/src/seed-packs.ts +74 -5
- package/core/src/sql-in.ts +32 -2
- package/core/src/store.ts +16 -49
- package/core/src/test-preload.ts +48 -3
- package/core/src/types.ts +13 -1
- package/core/src/wikilinks.test.ts +57 -0
- package/core/src/wikilinks.ts +63 -29
- package/package.json +1 -1
- package/src/attachment-tickets.test.ts +62 -0
- package/src/attachment-tickets.ts +2 -2
- package/src/cli.ts +36 -8
- package/src/config.ts +34 -1
- package/src/contract-honest-queries.test.ts +33 -1
- package/src/contract-search.test.ts +47 -0
- package/src/embedding/select.ts +16 -3
- package/src/live-match.test.ts +8 -0
- package/src/live-match.ts +15 -0
- package/src/mcp-http.test.ts +12 -0
- package/src/mcp-http.ts +1 -0
- package/src/mcp-tools.ts +51 -17
- package/src/mirror-routes.test.ts +22 -31
- package/src/onboarding-seed.test.ts +68 -0
- package/src/routes.ts +88 -25
- package/src/routing.test.ts +24 -0
- package/src/routing.ts +2 -0
- package/src/subscriptions.ts +18 -2
- package/src/tag-scope-note-tags.test.ts +476 -0
- package/src/tag-scope.ts +73 -5
- package/src/test-home-isolation.test.ts +137 -0
- package/src/test-support/spawn.ts +12 -0
- package/src/transcription/download.test.ts +187 -1
- package/src/transcription/download.ts +149 -2
- package/src/transcription/install-python.test.ts +23 -2
- package/src/transcription/install-python.ts +13 -4
- package/src/vault.test.ts +39 -4
- package/src/version.test.ts +8 -0
- package/src/ws-server.ts +12 -2
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The suite must never write into a live Parachute install.
|
|
3
|
+
*
|
|
4
|
+
* On 2026-08-22 it did: two `bun test` runs created ~158 real vault
|
|
5
|
+
* directories under `~/.parachute/vault/data/` on a developer box —
|
|
6
|
+
* `tagscope-*`, `mint-*`, `retier-*`, `ledger-*`, and a `solo` vault from the
|
|
7
|
+
* mirror-routes tests. `core/src/test-preload.ts` existed and was loaded; it
|
|
8
|
+
* simply declined to act, because it only assigned PARACHUTE_HOME when the
|
|
9
|
+
* variable was *unset*, and the box's shell profile exported
|
|
10
|
+
* `PARACHUTE_HOME="$HOME/.parachute"`. The isolation was conditional on the
|
|
11
|
+
* one condition that mattered being false.
|
|
12
|
+
*
|
|
13
|
+
* These tests pin both layers of the fix:
|
|
14
|
+
* 1. the preload overrides an inherited PARACHUTE_HOME unconditionally
|
|
15
|
+
* 2. `configDirPath()` refuses the `~/.parachute` fallback under
|
|
16
|
+
* NODE_ENV=test, so a run that never loaded the preload (wrong cwd →
|
|
17
|
+
* bunfig.toml not found) fails loudly instead of writing real vaults
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { describe, test, expect } from "bun:test";
|
|
21
|
+
import { mkdtempSync, rmSync, existsSync } from "fs";
|
|
22
|
+
import { homedir, tmpdir } from "os";
|
|
23
|
+
import { join, resolve } from "path";
|
|
24
|
+
|
|
25
|
+
import { runSubprocess } from "./test-support/spawn.ts";
|
|
26
|
+
|
|
27
|
+
const REPO_ROOT = resolve(import.meta.dir, "..");
|
|
28
|
+
|
|
29
|
+
describe("test-home isolation", () => {
|
|
30
|
+
test("this very process is not pointed at the real ~/.parachute", () => {
|
|
31
|
+
// The in-process invariant. If the preload stops loading, or stops
|
|
32
|
+
// overriding, this is the first thing that fails.
|
|
33
|
+
const real = join(homedir(), ".parachute");
|
|
34
|
+
expect(process.env.PARACHUTE_HOME).toBeTruthy();
|
|
35
|
+
expect(process.env.PARACHUTE_HOME).not.toBe(real);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("the preload overrides an inherited PARACHUTE_HOME rather than deferring to it", async () => {
|
|
39
|
+
const sandbox = mkdtempSync(join(tmpdir(), "test-home-isolation-"));
|
|
40
|
+
try {
|
|
41
|
+
// Stand in for the live install: a HOME whose `.parachute` we watch, and
|
|
42
|
+
// the exported PARACHUTE_HOME a login shell would hand `bun test`. This
|
|
43
|
+
// is the exact shape of the 2026-08-22 environment.
|
|
44
|
+
const inherited = join(sandbox, ".parachute");
|
|
45
|
+
const probe = join(sandbox, "probe.test.ts");
|
|
46
|
+
await Bun.write(
|
|
47
|
+
probe,
|
|
48
|
+
[
|
|
49
|
+
`import { test } from "bun:test";`,
|
|
50
|
+
`import { writeVaultConfig } from ${JSON.stringify(join(REPO_ROOT, "src/config.ts"))};`,
|
|
51
|
+
`import { getVaultStore, closeAllStores } from ${JSON.stringify(join(REPO_ROOT, "src/vault-store.ts"))};`,
|
|
52
|
+
`test("writes a vault wherever config.ts points", async () => {`,
|
|
53
|
+
` console.log("RESOLVED_HOME=" + process.env.PARACHUTE_HOME);`,
|
|
54
|
+
` writeVaultConfig({ name: "isolation-probe", api_keys: [], created_at: new Date().toISOString() });`,
|
|
55
|
+
` getVaultStore("isolation-probe");`,
|
|
56
|
+
` closeAllStores();`,
|
|
57
|
+
`});`,
|
|
58
|
+
].join("\n"),
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
// cwd = repo root so bunfig.toml's `[test] preload` applies, which is
|
|
62
|
+
// the normal way the suite runs.
|
|
63
|
+
const res = await runSubprocess({
|
|
64
|
+
cmd: ["bun", "test", probe],
|
|
65
|
+
cwd: REPO_ROOT,
|
|
66
|
+
env: { HOME: sandbox, PARACHUTE_HOME: inherited },
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const out = res.stdout + res.stderr;
|
|
70
|
+
// Assert the child actually got as far as writing a vault — without
|
|
71
|
+
// this, a child that crashed on startup would satisfy every "nothing
|
|
72
|
+
// landed in the live install" assertion below for the wrong reason.
|
|
73
|
+
expect(res.exitCode).toBe(0);
|
|
74
|
+
expect(out).not.toContain(`RESOLVED_HOME=${inherited}`);
|
|
75
|
+
expect(out).toContain("RESOLVED_HOME=");
|
|
76
|
+
// The load-bearing assertion: nothing landed in the "live" install.
|
|
77
|
+
expect(existsSync(join(inherited, "vault", "data", "isolation-probe"))).toBe(false);
|
|
78
|
+
expect(existsSync(inherited)).toBe(false);
|
|
79
|
+
} finally {
|
|
80
|
+
rmSync(sandbox, { recursive: true, force: true });
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("runSubprocess propagates the live PARACHUTE_HOME to the child", async () => {
|
|
85
|
+
// Bun gives a child the environ the parent *started* with, not the live
|
|
86
|
+
// `process.env` — so the preload's runtime assignment is invisible to any
|
|
87
|
+
// child spawned without an explicit env, and that child resolves its home
|
|
88
|
+
// to the real ~/.parachute. `runSubprocess` spreads `process.env`, which
|
|
89
|
+
// is the only reason CLI tests are safe; pin it.
|
|
90
|
+
const res = await runSubprocess({
|
|
91
|
+
cmd: ["bun", "-e", `console.log("CHILD_PH=" + process.env.PARACHUTE_HOME)`],
|
|
92
|
+
cwd: REPO_ROOT,
|
|
93
|
+
});
|
|
94
|
+
expect(res.stdout.trim()).toBe(`CHILD_PH=${process.env.PARACHUTE_HOME}`);
|
|
95
|
+
expect(res.stdout).not.toContain(`CHILD_PH=${join(homedir(), ".parachute")}`);
|
|
96
|
+
expect(res.stdout).not.toContain("CHILD_PH=undefined");
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("configDirPath() refuses the ~/.parachute fallback under NODE_ENV=test", async () => {
|
|
100
|
+
// Simulates the other way the isolation can be lost: the preload never
|
|
101
|
+
// loads (bunfig.toml is resolved from the cwd, so running the suite by
|
|
102
|
+
// path from a parent directory skips it) and PARACHUTE_HOME is unset.
|
|
103
|
+
const res = await runSubprocess({
|
|
104
|
+
cmd: [
|
|
105
|
+
"bun",
|
|
106
|
+
"-e",
|
|
107
|
+
`const { vaultDir } = await import(${JSON.stringify(join(REPO_ROOT, "src/config.ts"))}); console.log(vaultDir("x"));`,
|
|
108
|
+
],
|
|
109
|
+
cwd: REPO_ROOT,
|
|
110
|
+
env: { PARACHUTE_HOME: undefined, NODE_ENV: "test" },
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
const out = res.stdout + res.stderr;
|
|
114
|
+
expect(res.exitCode).not.toBe(0);
|
|
115
|
+
expect(out).toContain("refusing to default to the live install");
|
|
116
|
+
// The message has to be actionable for an npm consumer whose test runner
|
|
117
|
+
// set NODE_ENV=test, not just for a contributor in this repo.
|
|
118
|
+
expect(out).toContain("Set PARACHUTE_HOME explicitly");
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test("configDirPath() still falls back to ~/.parachute outside tests", async () => {
|
|
122
|
+
// The guard is test-only: real installs resolve the home the way they
|
|
123
|
+
// always have.
|
|
124
|
+
const res = await runSubprocess({
|
|
125
|
+
cmd: [
|
|
126
|
+
"bun",
|
|
127
|
+
"-e",
|
|
128
|
+
`const { vaultDir } = await import(${JSON.stringify(join(REPO_ROOT, "src/config.ts"))}); console.log(vaultDir("x"));`,
|
|
129
|
+
],
|
|
130
|
+
cwd: REPO_ROOT,
|
|
131
|
+
env: { PARACHUTE_HOME: undefined, NODE_ENV: undefined, HOME: "/tmp/not-a-real-home" },
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
expect(res.exitCode).toBe(0);
|
|
135
|
+
expect(res.stdout.trim()).toBe(join("/tmp/not-a-real-home", ".parachute", "vault", "data", "x"));
|
|
136
|
+
});
|
|
137
|
+
});
|
|
@@ -18,6 +18,18 @@
|
|
|
18
18
|
* `runSubprocess` uses async `Bun.spawn` + `await proc.exited`, so the
|
|
19
19
|
* parent's event loop keeps servicing requests while the child runs.
|
|
20
20
|
*
|
|
21
|
+
* THE OTHER RULE — always give a spawned child an explicit `env`.
|
|
22
|
+
*
|
|
23
|
+
* Bun hands a child the environ the parent process *started* with, not the
|
|
24
|
+
* live `process.env`: assignments made at runtime are invisible to it. That
|
|
25
|
+
* includes the temp `PARACHUTE_HOME` that `core/src/test-preload.ts` sets for
|
|
26
|
+
* the whole suite, so a child spawned with no `env` resolves its parachute
|
|
27
|
+
* home to the developer's real `~/.parachute` (verified with Bun 1.3.14).
|
|
28
|
+
* `runSubprocess` handles this for you — it spreads the live `process.env`
|
|
29
|
+
* below and merges `opts.env` over it. Hand-rolled `Bun.spawnSync` calls must
|
|
30
|
+
* do the same; `src/version.test.ts` was silently pointing its CLI child at
|
|
31
|
+
* the real install until the config.ts NODE_ENV=test guard caught it.
|
|
32
|
+
*
|
|
21
33
|
* NOTE on what's *fine*: `Bun.spawnSync` is safe in tests that don't run
|
|
22
34
|
* an in-test server the child talks to over HTTP — e.g. shelling out to
|
|
23
35
|
* `git` to build a fixture repo (`mirror-*.test.ts`), or a `Bun.serve`
|
|
@@ -2,7 +2,12 @@ import { describe, test, expect, afterAll } from "bun:test";
|
|
|
2
2
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from "fs";
|
|
3
3
|
import { tmpdir } from "os";
|
|
4
4
|
import { join } from "path";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
downloadTo,
|
|
7
|
+
ensureDownloaded,
|
|
8
|
+
fetchUpstreamDigest,
|
|
9
|
+
sha256OfFile,
|
|
10
|
+
} from "./download.ts";
|
|
6
11
|
|
|
7
12
|
/**
|
|
8
13
|
* `downloadTo` tests (vault#534 blocker 1). The manual body→FileSink pump
|
|
@@ -99,3 +104,184 @@ describe("downloadTo", () => {
|
|
|
99
104
|
}
|
|
100
105
|
});
|
|
101
106
|
});
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Checksum + idempotency (vault#531). The digest shape under test is the real
|
|
110
|
+
* one: HuggingFace serves LFS-backed `resolve/` URLs as a 302 carrying
|
|
111
|
+
* `x-linked-etag` = the plain sha256 of the bytes. Verified against a live
|
|
112
|
+
* download of ggml-tiny.en.bin (77,704,715 bytes) — header and
|
|
113
|
+
* `shasum -a 256` agreed exactly — and fixtured here so the suite stays
|
|
114
|
+
* offline.
|
|
115
|
+
*/
|
|
116
|
+
const PAYLOAD_SHA = new Bun.CryptoHasher("sha256").update(PAYLOAD).digest("hex");
|
|
117
|
+
|
|
118
|
+
describe("downloadTo — sha256 verification", () => {
|
|
119
|
+
test("accepts a body whose hash matches", async () => {
|
|
120
|
+
const { server, url } = serve(() => new Response(PAYLOAD));
|
|
121
|
+
const dest = join(dir, "sha-ok.bin");
|
|
122
|
+
try {
|
|
123
|
+
await downloadTo(`${url}/file`, dest, { sha256: PAYLOAD_SHA });
|
|
124
|
+
expect(readFileSync(dest).byteLength).toBe(PAYLOAD.byteLength);
|
|
125
|
+
} finally {
|
|
126
|
+
server.stop(true);
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("a complete-but-corrupt body throws AND removes the file", async () => {
|
|
131
|
+
// Length is right, bytes are wrong — invisible to the content-length
|
|
132
|
+
// check, which is exactly the gap a checksum closes.
|
|
133
|
+
const corrupt = new Uint8Array(PAYLOAD);
|
|
134
|
+
corrupt[corrupt.length - 1] ^= 0xff;
|
|
135
|
+
const { server, url } = serve(() => new Response(corrupt));
|
|
136
|
+
const dest = join(dir, "sha-bad.bin");
|
|
137
|
+
try {
|
|
138
|
+
expect(downloadTo(`${url}/file`, dest, { sha256: PAYLOAD_SHA })).rejects.toThrow(
|
|
139
|
+
/failed checksum: sha256 [0-9a-f]{64}, expected/,
|
|
140
|
+
);
|
|
141
|
+
} finally {
|
|
142
|
+
server.stop(true);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("no sha256 given → unchanged behaviour, body accepted", async () => {
|
|
147
|
+
const { server, url } = serve(() => new Response(PAYLOAD));
|
|
148
|
+
const dest = join(dir, "sha-none.bin");
|
|
149
|
+
try {
|
|
150
|
+
await downloadTo(`${url}/file`, dest);
|
|
151
|
+
expect(existsSync(dest)).toBe(true);
|
|
152
|
+
} finally {
|
|
153
|
+
server.stop(true);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
describe("fetchUpstreamDigest", () => {
|
|
159
|
+
function redirectingServer(headers: Record<string, string>) {
|
|
160
|
+
return serve((req) => {
|
|
161
|
+
if (new URL(req.url).pathname === "/target") return new Response(PAYLOAD);
|
|
162
|
+
return new Response("", { status: 302, headers: { location: "/target", ...headers } });
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
test("reads the sha256 off a HuggingFace-style x-linked-etag 302", async () => {
|
|
167
|
+
const { server, url } = redirectingServer({ "x-linked-etag": `"${PAYLOAD_SHA}"` });
|
|
168
|
+
try {
|
|
169
|
+
const digest = await fetchUpstreamDigest(`${url}/model.bin`);
|
|
170
|
+
expect(digest?.sha256).toBe(PAYLOAD_SHA);
|
|
171
|
+
} finally {
|
|
172
|
+
server.stop(true);
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test("tolerates a weak-etag prefix", async () => {
|
|
177
|
+
const { server, url } = redirectingServer({ "x-linked-etag": `W/"${PAYLOAD_SHA}"` });
|
|
178
|
+
try {
|
|
179
|
+
expect((await fetchUpstreamDigest(`${url}/model.bin`))?.sha256).toBe(PAYLOAD_SHA);
|
|
180
|
+
} finally {
|
|
181
|
+
server.stop(true);
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
test("returns null for an opaque etag that is not a sha256", async () => {
|
|
186
|
+
// A GitHub-release-style cache token must never be mistaken for a digest.
|
|
187
|
+
const { server, url } = redirectingServer({ "x-linked-etag": '"0x8DA1B2C3D4E5F60"' });
|
|
188
|
+
try {
|
|
189
|
+
expect(await fetchUpstreamDigest(`${url}/model.bin`)).toBeNull();
|
|
190
|
+
} finally {
|
|
191
|
+
server.stop(true);
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
test("returns null when the host publishes no digest at all", async () => {
|
|
196
|
+
const { server, url } = serve(() => new Response(PAYLOAD));
|
|
197
|
+
try {
|
|
198
|
+
expect(await fetchUpstreamDigest(`${url}/asset.tar.gz`)).toBeNull();
|
|
199
|
+
} finally {
|
|
200
|
+
server.stop(true);
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test("returns null (does not throw) when the probe cannot connect", async () => {
|
|
205
|
+
expect(await fetchUpstreamDigest("http://127.0.0.1:1/nope")).toBeNull();
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
describe("ensureDownloaded — partial-download idempotency (vault#531)", () => {
|
|
210
|
+
function digestServer() {
|
|
211
|
+
return serve((req) => {
|
|
212
|
+
if (new URL(req.url).pathname === "/target") return new Response(PAYLOAD);
|
|
213
|
+
return new Response("", {
|
|
214
|
+
status: 302,
|
|
215
|
+
headers: { location: "/target", "x-linked-etag": `"${PAYLOAD_SHA}"` },
|
|
216
|
+
});
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
test("fetches when absent and reports it verified", async () => {
|
|
221
|
+
const { server, url } = digestServer();
|
|
222
|
+
const dest = join(dir, "ens-new.bin");
|
|
223
|
+
try {
|
|
224
|
+
const r = await ensureDownloaded(`${url}/model.bin`, dest);
|
|
225
|
+
expect(r.outcome).toBe("downloaded");
|
|
226
|
+
expect(r.verified).toBe(true);
|
|
227
|
+
expect(await sha256OfFile(dest)).toBe(PAYLOAD_SHA);
|
|
228
|
+
} finally {
|
|
229
|
+
server.stop(true);
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
test("reuses an intact existing file without re-downloading", async () => {
|
|
234
|
+
const { server, url } = digestServer();
|
|
235
|
+
const dest = join(dir, "ens-good.bin");
|
|
236
|
+
try {
|
|
237
|
+
await Bun.write(dest, PAYLOAD);
|
|
238
|
+
const r = await ensureDownloaded(`${url}/model.bin`, dest);
|
|
239
|
+
expect(r.outcome).toBe("reused");
|
|
240
|
+
expect(r.verified).toBe(true);
|
|
241
|
+
} finally {
|
|
242
|
+
server.stop(true);
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
test("REPAIRS a corrupt file that a plain existsSync check would have skipped", async () => {
|
|
247
|
+
// The bug: install skipped anything that existed. A truncated model
|
|
248
|
+
// survived every non---force re-run.
|
|
249
|
+
const { server, url } = digestServer();
|
|
250
|
+
const dest = join(dir, "ens-corrupt.bin");
|
|
251
|
+
try {
|
|
252
|
+
await Bun.write(dest, PAYLOAD.slice(0, 1024)); // truncated leftover
|
|
253
|
+
const r = await ensureDownloaded(`${url}/model.bin`, dest);
|
|
254
|
+
expect(r.outcome).toBe("repaired");
|
|
255
|
+
expect(await sha256OfFile(dest)).toBe(PAYLOAD_SHA);
|
|
256
|
+
} finally {
|
|
257
|
+
server.stop(true);
|
|
258
|
+
}
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
test("force re-downloads even an intact file", async () => {
|
|
262
|
+
const { server, url } = digestServer();
|
|
263
|
+
const dest = join(dir, "ens-force.bin");
|
|
264
|
+
try {
|
|
265
|
+
await Bun.write(dest, PAYLOAD);
|
|
266
|
+
const r = await ensureDownloaded(`${url}/model.bin`, dest, { force: true });
|
|
267
|
+
expect(r.outcome).toBe("downloaded");
|
|
268
|
+
} finally {
|
|
269
|
+
server.stop(true);
|
|
270
|
+
}
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
test("no published digest → an existing file is reused unverified, as before", async () => {
|
|
274
|
+
const { server, url } = serve(() => new Response(PAYLOAD));
|
|
275
|
+
const dest = join(dir, "ens-nodigest.bin");
|
|
276
|
+
try {
|
|
277
|
+
await Bun.write(dest, PAYLOAD.slice(0, 512));
|
|
278
|
+
const r = await ensureDownloaded(`${url}/asset.tar.gz`, dest);
|
|
279
|
+
expect(r.outcome).toBe("reused");
|
|
280
|
+
expect(r.verified).toBe(false);
|
|
281
|
+
// Unchanged: with nothing to check against we must not guess.
|
|
282
|
+
expect(readFileSync(dest).byteLength).toBe(512);
|
|
283
|
+
} finally {
|
|
284
|
+
server.stop(true);
|
|
285
|
+
}
|
|
286
|
+
});
|
|
287
|
+
});
|
|
@@ -16,7 +16,83 @@
|
|
|
16
16
|
* platform gate). See vault#534 for the full container verification.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import { rmSync } from "fs";
|
|
19
|
+
import { existsSync, rmSync } from "fs";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A content hash the UPSTREAM host publishes for an artifact, discovered at
|
|
23
|
+
* download time rather than pinned in this repo.
|
|
24
|
+
*/
|
|
25
|
+
export interface UpstreamDigest {
|
|
26
|
+
/** Lowercase hex sha256 of the file's bytes. */
|
|
27
|
+
sha256: string;
|
|
28
|
+
/** Where we learned it, for error messages. */
|
|
29
|
+
source: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Ask the host whether it publishes a content digest for `url`, WITHOUT
|
|
34
|
+
* downloading the body.
|
|
35
|
+
*
|
|
36
|
+
* HuggingFace — where every GGUF model in `models.ts` / `install.ts` comes
|
|
37
|
+
* from — serves LFS-backed `resolve/` URLs as a 302 whose `x-linked-etag`
|
|
38
|
+
* header is the git-LFS oid: the plain sha256 of the file's bytes. Verified
|
|
39
|
+
* against a real download (`ggml-tiny.en.bin`, 77,704,715 bytes): the header
|
|
40
|
+
* and `shasum -a 256` of the fetched file agree exactly. `x-linked-size`
|
|
41
|
+
* likewise matches the byte count.
|
|
42
|
+
*
|
|
43
|
+
* The redirect TARGET's own `etag` is a different thing — HF's Xet CAS hash,
|
|
44
|
+
* not a sha256 of the content — so the digest has to be read off the 302
|
|
45
|
+
* itself. That's why this uses `redirect: "manual"` instead of reading
|
|
46
|
+
* headers off the followed response.
|
|
47
|
+
*
|
|
48
|
+
* Returns `null` for any host that publishes nothing (GitHub release assets,
|
|
49
|
+
* today). A missing digest is NOT an error: it degrades to the size check
|
|
50
|
+
* `downloadTo` already performs. Network failures here are swallowed for the
|
|
51
|
+
* same reason — a checksum probe must never be the thing that breaks an
|
|
52
|
+
* install.
|
|
53
|
+
*/
|
|
54
|
+
export async function fetchUpstreamDigest(url: string): Promise<UpstreamDigest | null> {
|
|
55
|
+
try {
|
|
56
|
+
const resp = await fetch(url, { method: "GET", redirect: "manual", headers: { range: "bytes=0-0" } });
|
|
57
|
+
// Consume/discard: with `redirect: manual` the 302 body is a short stub,
|
|
58
|
+
// and the range header keeps a non-redirecting 200 to a single byte.
|
|
59
|
+
try { await resp.arrayBuffer(); } catch { /* nothing worth reporting */ }
|
|
60
|
+
const linked = resp.headers.get("x-linked-etag") ?? resp.headers.get("x-linked-etag".toUpperCase());
|
|
61
|
+
const hex = normalizeSha256Etag(linked);
|
|
62
|
+
if (hex) return { sha256: hex, source: "HuggingFace x-linked-etag (git-LFS oid)" };
|
|
63
|
+
return null;
|
|
64
|
+
} catch {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* An ETag is quoted and may carry a `W/` weak prefix. Accept it only when what
|
|
71
|
+
* remains is exactly 64 hex characters — anything else is some other host's
|
|
72
|
+
* opaque cache token, not a sha256, and must not be treated as one.
|
|
73
|
+
*/
|
|
74
|
+
function normalizeSha256Etag(raw: string | null): string | null {
|
|
75
|
+
if (!raw) return null;
|
|
76
|
+
const stripped = raw.trim().replace(/^W\//i, "").replace(/^"|"$/g, "").toLowerCase();
|
|
77
|
+
return /^[0-9a-f]{64}$/.test(stripped) ? stripped : null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Streaming sha256 of a file already on disk — never buffers it whole. */
|
|
81
|
+
export async function sha256OfFile(path: string): Promise<string> {
|
|
82
|
+
const hasher = new Bun.CryptoHasher("sha256");
|
|
83
|
+
const stream = Bun.file(path).stream();
|
|
84
|
+
for await (const chunk of stream) hasher.update(chunk);
|
|
85
|
+
return hasher.digest("hex");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export interface DownloadOptions {
|
|
89
|
+
/**
|
|
90
|
+
* Expected lowercase-hex sha256. When set, the hash is computed AS THE BYTES
|
|
91
|
+
* STREAM (no second pass, no buffering) and a mismatch removes the file and
|
|
92
|
+
* throws.
|
|
93
|
+
*/
|
|
94
|
+
sha256?: string | null;
|
|
95
|
+
}
|
|
20
96
|
|
|
21
97
|
/**
|
|
22
98
|
* Download `url` to `dest` (follows redirects). Streams chunk-by-chunk — never
|
|
@@ -24,8 +100,13 @@ import { rmSync } from "fs";
|
|
|
24
100
|
* the partial file is removed so a retry never trusts a truncated artifact.
|
|
25
101
|
* When the server sent an honest `content-length` (no content-encoding
|
|
26
102
|
* transform), a byte-count mismatch is treated as a failed download too.
|
|
103
|
+
*
|
|
104
|
+
* With `opts.sha256`, content integrity is checked as well as length — a
|
|
105
|
+
* corrupted-but-complete transfer (a proxy that mangles bytes, a truncation
|
|
106
|
+
* the length check can't see because no `content-length` was sent) is caught
|
|
107
|
+
* and the file removed.
|
|
27
108
|
*/
|
|
28
|
-
export async function downloadTo(url: string, dest: string): Promise<void> {
|
|
109
|
+
export async function downloadTo(url: string, dest: string, opts: DownloadOptions = {}): Promise<void> {
|
|
29
110
|
const resp = await fetch(url, { redirect: "follow" });
|
|
30
111
|
if (!resp.ok) throw new Error(`download failed (${resp.status}) for ${url}`);
|
|
31
112
|
if (!resp.body) throw new Error(`download failed (empty response body) for ${url}`);
|
|
@@ -41,11 +122,15 @@ export async function downloadTo(url: string, dest: string): Promise<void> {
|
|
|
41
122
|
? Number(lengthHeader)
|
|
42
123
|
: null;
|
|
43
124
|
|
|
125
|
+
const wantSha = opts.sha256 ? opts.sha256.trim().toLowerCase() : null;
|
|
126
|
+
const hasher = wantSha ? new Bun.CryptoHasher("sha256") : null;
|
|
127
|
+
|
|
44
128
|
const sink = Bun.file(dest).writer();
|
|
45
129
|
let written = 0;
|
|
46
130
|
try {
|
|
47
131
|
for await (const chunk of resp.body) {
|
|
48
132
|
sink.write(chunk);
|
|
133
|
+
hasher?.update(chunk);
|
|
49
134
|
written += chunk.byteLength;
|
|
50
135
|
}
|
|
51
136
|
await sink.end();
|
|
@@ -68,4 +153,66 @@ export async function downloadTo(url: string, dest: string): Promise<void> {
|
|
|
68
153
|
`download of ${url} was truncated: got ${written} bytes, expected ${expectedBytes} (content-length)`,
|
|
69
154
|
);
|
|
70
155
|
}
|
|
156
|
+
|
|
157
|
+
if (wantSha && hasher) {
|
|
158
|
+
const got = hasher.digest("hex");
|
|
159
|
+
if (got !== wantSha) {
|
|
160
|
+
rmSync(dest, { force: true });
|
|
161
|
+
throw new Error(
|
|
162
|
+
`download of ${url} failed checksum: sha256 ${got}, expected ${wantSha}`,
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Fetch `url` to `dest` **idempotently** — the policy `transcription install`
|
|
170
|
+
* needs for a re-run (vault#531).
|
|
171
|
+
*
|
|
172
|
+
* The hole this closes: the install path skipped any file that merely EXISTED
|
|
173
|
+
* (`existsSync(dest) && !force`). `downloadTo` removes partials it notices, but
|
|
174
|
+
* it can't notice everything — a `kill -9` mid-write, a full disk, or a server
|
|
175
|
+
* that sent no `content-length` all leave a complete-looking file behind. From
|
|
176
|
+
* then on every non-`--force` re-run reported "already present — skipping" and
|
|
177
|
+
* the operator got a corrupt model forever, with no signal and no path out
|
|
178
|
+
* short of knowing to pass `--force`.
|
|
179
|
+
*
|
|
180
|
+
* So: when the artifact is already on disk and the host publishes a digest, we
|
|
181
|
+
* VERIFY it before trusting it, and re-download on mismatch. When the host
|
|
182
|
+
* publishes nothing, behaviour is unchanged (skip) — this can only ever turn a
|
|
183
|
+
* silent corruption into a repair, never a good file into a re-download.
|
|
184
|
+
*
|
|
185
|
+
* Returns what happened so the caller can print something honest.
|
|
186
|
+
*/
|
|
187
|
+
export type EnsureOutcome = "downloaded" | "reused" | "repaired";
|
|
188
|
+
|
|
189
|
+
export async function ensureDownloaded(
|
|
190
|
+
url: string,
|
|
191
|
+
dest: string,
|
|
192
|
+
opts: {
|
|
193
|
+
force?: boolean;
|
|
194
|
+
/**
|
|
195
|
+
* Called just before bytes are actually pulled, so the caller can print a
|
|
196
|
+
* progress line only when there IS a transfer. `reason` distinguishes a
|
|
197
|
+
* first fetch from replacing a bad file.
|
|
198
|
+
*/
|
|
199
|
+
onBeforeDownload?: (reason: "missing" | "forced" | "corrupt") => void;
|
|
200
|
+
} = {},
|
|
201
|
+
): Promise<{ outcome: EnsureOutcome; verified: boolean }> {
|
|
202
|
+
const digest = await fetchUpstreamDigest(url);
|
|
203
|
+
|
|
204
|
+
if (existsSync(dest) && !opts.force) {
|
|
205
|
+
if (!digest) return { outcome: "reused", verified: false };
|
|
206
|
+
const actual = await sha256OfFile(dest);
|
|
207
|
+
if (actual === digest.sha256) return { outcome: "reused", verified: true };
|
|
208
|
+
// Present but wrong — the corrupt-and-skipped case. Replace it.
|
|
209
|
+
rmSync(dest, { force: true });
|
|
210
|
+
opts.onBeforeDownload?.("corrupt");
|
|
211
|
+
await downloadTo(url, dest, { sha256: digest.sha256 });
|
|
212
|
+
return { outcome: "repaired", verified: true };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
opts.onBeforeDownload?.(existsSync(dest) ? "forced" : "missing");
|
|
216
|
+
await downloadTo(url, dest, { sha256: digest?.sha256 ?? null });
|
|
217
|
+
return { outcome: "downloaded", verified: digest !== null };
|
|
71
218
|
}
|
|
@@ -328,7 +328,8 @@ describe("installPythonBackend — warm-pull + verify", () => {
|
|
|
328
328
|
files,
|
|
329
329
|
run: async (cmd) => {
|
|
330
330
|
if (cmd.some((c) => c.endsWith("/pip"))) files.add(venvBin("onnx-asr"));
|
|
331
|
-
|
|
331
|
+
// Warm-pull is `<bin> <model> --help`; verify is `<bin> --help`.
|
|
332
|
+
if (cmd.includes("--help") && cmd.length > 2) return { exitCode: 1, stdout: "", stderr: "offline" };
|
|
332
333
|
return { exitCode: 0, stdout: "", stderr: "" };
|
|
333
334
|
},
|
|
334
335
|
});
|
|
@@ -349,7 +350,8 @@ describe("installPythonBackend — warm-pull + verify", () => {
|
|
|
349
350
|
const out = await installPythonBackend(deps, { provider: "parakeet-mlx" });
|
|
350
351
|
expect(out.ok).toBe(true);
|
|
351
352
|
expect(out.steps.find((s) => s.name === "model-warm-pull")!.status).toBe("skipped");
|
|
352
|
-
expect(runs.some((r) => r.includes("--help"))).toBe(false);
|
|
353
|
+
expect(runs.some((r) => r.length > 2 && r.includes("--help"))).toBe(false);
|
|
354
|
+
expect(runs.some((r) => r.length === 2 && r[1] === "--help")).toBe(true);
|
|
353
355
|
});
|
|
354
356
|
|
|
355
357
|
test("HONEST verify: pip 'succeeds' but no binary appears → ok:false, no activation signal", async () => {
|
|
@@ -363,4 +365,23 @@ describe("installPythonBackend — warm-pull + verify", () => {
|
|
|
363
365
|
expect(out.binPath).toBeUndefined();
|
|
364
366
|
expect(out.steps.find((s) => s.name === "verify")!.status).toBe("failed");
|
|
365
367
|
});
|
|
368
|
+
|
|
369
|
+
test("HONEST verify: binary exists but --help fails → ok:false (vault#539)", async () => {
|
|
370
|
+
const files = new Set<string>([venvBin("onnx-asr")]);
|
|
371
|
+
const { deps } = makeDeps({
|
|
372
|
+
path: { python3: "/usr/bin/python3", ffmpeg: "/usr/bin/ffmpeg" },
|
|
373
|
+
files,
|
|
374
|
+
run: async (cmd) => {
|
|
375
|
+
if (cmd.length === 2 && cmd[1] === "--help") {
|
|
376
|
+
return { exitCode: 127, stdout: "", stderr: "No such file or directory: python3" };
|
|
377
|
+
}
|
|
378
|
+
return { exitCode: 0, stdout: "", stderr: "" };
|
|
379
|
+
},
|
|
380
|
+
});
|
|
381
|
+
const out = await installPythonBackend(deps, { provider: "onnx-asr", skipModel: true });
|
|
382
|
+
expect(out.ok).toBe(false);
|
|
383
|
+
expect(out.binPath).toBeUndefined();
|
|
384
|
+
expect(out.steps.find((s) => s.name === "verify")!.status).toBe("failed");
|
|
385
|
+
expect(out.steps.find((s) => s.name === "verify")!.detail).toMatch(/--help/);
|
|
386
|
+
});
|
|
366
387
|
});
|
|
@@ -301,16 +301,25 @@ export async function installPythonBackend(
|
|
|
301
301
|
record(await warmPullModel(deps, spec, opts.model ?? spec.defaultModel));
|
|
302
302
|
}
|
|
303
303
|
|
|
304
|
-
// --- Verify:
|
|
304
|
+
// --- Verify: EXECUTE the binary (--help), not just existsSync (vault#539) --
|
|
305
|
+
// Same honesty as probeTranscribeCliRunnable: a venv entrypoint can exist
|
|
306
|
+
// and still fail (moved PARACHUTE_HOME, deleted interpreter, half-finished
|
|
307
|
+
// pip). Per-request capability gates stay spawn-free; this is install-time.
|
|
305
308
|
const binPath = resolveBin(spec, deps);
|
|
306
|
-
const runnable = !!binPath && deps.existsImpl(binPath);
|
|
307
309
|
const venv = pythonVenvDir(deps.env);
|
|
308
|
-
if (!
|
|
310
|
+
if (!binPath || !deps.existsImpl(binPath)) {
|
|
309
311
|
const detail = `install ran but no runnable \`${spec.bin}\` was found (looked at ${join(venv, "bin", spec.bin)} and PATH).`;
|
|
310
312
|
record({ name: "verify", status: "failed", detail });
|
|
311
313
|
return fail(`${spec.provider} install did not produce a runnable binary. ${detail}`);
|
|
312
314
|
}
|
|
313
|
-
|
|
315
|
+
const help = await deps.run([binPath, "--help"]);
|
|
316
|
+
if (help.exitCode !== 0) {
|
|
317
|
+
const reason = help.stderr.trim().split("\n")[0]?.slice(0, 200) || `exit ${help.exitCode}`;
|
|
318
|
+
const detail = `\`${spec.bin} --help\` failed (${reason}). A present file is not a runnable provider.`;
|
|
319
|
+
record({ name: "verify", status: "failed", detail });
|
|
320
|
+
return fail(`${spec.provider} binary exists but is not runnable. ${detail}`);
|
|
321
|
+
}
|
|
322
|
+
record({ name: "verify", status: "ok", detail: `runnable binary at ${binPath} (verified --help).` });
|
|
314
323
|
|
|
315
324
|
const summary = `${spec.provider} installed and verified runnable (${binPath}).`;
|
|
316
325
|
deps.log(summary);
|