@openparachute/vault 0.6.5-rc.2 → 0.6.5-rc.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/core/src/do-param-cap.test.ts +161 -0
- package/core/src/links.ts +8 -13
- package/core/src/notes.ts +33 -15
- package/core/src/onboarding.ts +14 -296
- package/core/src/seed-packs.test.ts +191 -0
- package/core/src/seed-packs.ts +559 -0
- package/core/src/sql-in.test.ts +58 -0
- package/core/src/sql-in.ts +76 -0
- package/core/src/transcription/provider.ts +141 -0
- package/core/src/vault-projection.ts +1 -1
- package/core/src/wikilinks.ts +10 -4
- package/package.json +1 -1
- package/src/add-pack.test.ts +142 -0
- package/src/cli.ts +542 -7
- package/src/onboarding-seed.test.ts +126 -40
- package/src/onboarding-seed.ts +41 -46
- package/src/routes.ts +17 -0
- package/src/server.ts +77 -31
- package/src/transcription/build.test.ts +224 -0
- package/src/transcription/build.ts +252 -0
- package/src/transcription/capability.test.ts +93 -0
- package/src/transcription/capability.ts +71 -0
- package/src/transcription/install.test.ts +167 -0
- package/src/transcription/install.ts +296 -0
- package/src/transcription/providers/scribe-http.test.ts +195 -0
- package/src/transcription/providers/scribe-http.ts +144 -0
- package/src/transcription/providers/transcribe-cpp.test.ts +314 -0
- package/src/transcription/providers/transcribe-cpp.ts +293 -0
- package/src/transcription/select.test.ts +134 -0
- package/src/transcription/select.ts +164 -0
- package/src/transcription-worker.test.ts +44 -0
- package/src/transcription-worker.ts +57 -122
- package/src/vault-create.test.ts +38 -10
- package/src/vault.test.ts +48 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `transcribe-cpp` — the local, no-Python transcription provider
|
|
3
|
+
* (scribe-fold Phase 2a).
|
|
4
|
+
*
|
|
5
|
+
* Turns audio into text by subprocessing the prebuilt `transcribe-cli` from
|
|
6
|
+
* transcribe.cpp (github.com/handy-computer/transcribe.cpp — CJ Pais /
|
|
7
|
+
* Mozilla.ai, MIT). transcribe.cpp's in-process N-API binding crashes on Bun,
|
|
8
|
+
* so we drive the CLI as a subprocess — structurally like the `scribe-http`
|
|
9
|
+
* provider, but local. It's the recommended low-RAM / no-Python provider,
|
|
10
|
+
* opt-in via `parachute-vault transcription install` + `TRANSCRIPTION_PROVIDER`.
|
|
11
|
+
*
|
|
12
|
+
* ## The audio path
|
|
13
|
+
*
|
|
14
|
+
* `transcribe-cli` STRICTLY requires **16kHz mono WAV** input — it rejects other
|
|
15
|
+
* sample rates (a 44.1kHz .wav exits non-zero; verified against v0.1.1). So we
|
|
16
|
+
* ALWAYS transcode the incoming audio through `ffmpeg -ar 16000 -ac 1` first,
|
|
17
|
+
* even when it's already a `.wav` — we can't trust an incoming WAV's sample
|
|
18
|
+
* rate. The install verb prints an ffmpeg hint when it's missing; here a missing
|
|
19
|
+
* ffmpeg surfaces as a terminal `ffmpeg_missing` error the operator must fix.
|
|
20
|
+
*
|
|
21
|
+
* ## Reading the output
|
|
22
|
+
*
|
|
23
|
+
* `transcribe-cli` prints a human-readable report to STDOUT (audio/model/run
|
|
24
|
+
* lines, a `text: <transcript>` line, then per-segment lines) and sends its
|
|
25
|
+
* library `[info]`/`[debug]` logs to STDERR. The transcript is the single
|
|
26
|
+
* `text:` line; `text: (empty)` is its no-result sentinel. See
|
|
27
|
+
* `parseTranscribeCliOutput`.
|
|
28
|
+
*
|
|
29
|
+
* ## Error mapping (mirrors scribe-http's terminal-vs-retriable contract)
|
|
30
|
+
*
|
|
31
|
+
* - binary / model not installed → non-retriable `missing_provider` (operator
|
|
32
|
+
* runs `transcription install`); never spawns.
|
|
33
|
+
* - ffmpeg not found → non-retriable `ffmpeg_missing`.
|
|
34
|
+
* - ffmpeg transcode non-zero exit → non-retriable `transcode_failed`
|
|
35
|
+
* (a re-run on the same undecodable blob won't help).
|
|
36
|
+
* - `transcribe-cli` non-zero exit → RETRIABLE `transcribe_cli_error` (could
|
|
37
|
+
* be a transient resource blip; the worker backs off, and a deterministic
|
|
38
|
+
* failure still terminates after maxAttempts).
|
|
39
|
+
* - empty/unparseable stdout → plain `Error` (retriable via the worker),
|
|
40
|
+
* matching scribe-http's "missing text field".
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
import { randomUUID } from "crypto";
|
|
44
|
+
import { existsSync } from "fs";
|
|
45
|
+
import { writeFile, unlink } from "fs/promises";
|
|
46
|
+
import { tmpdir } from "os";
|
|
47
|
+
import { join } from "path";
|
|
48
|
+
import {
|
|
49
|
+
TranscriptionError,
|
|
50
|
+
type TranscriptionProvider,
|
|
51
|
+
type TranscribeInput,
|
|
52
|
+
type TranscribeResult,
|
|
53
|
+
type ProviderAvailability,
|
|
54
|
+
} from "../../../core/src/transcription/provider.ts";
|
|
55
|
+
|
|
56
|
+
/** transcribe-cli can be slow on CPU for long audio; generous default. */
|
|
57
|
+
const DEFAULT_TIMEOUT_MS = 600_000;
|
|
58
|
+
|
|
59
|
+
/** Result of running a subprocess: exit code + captured streams (as text). */
|
|
60
|
+
export interface SubprocessResult {
|
|
61
|
+
exitCode: number;
|
|
62
|
+
stdout: string;
|
|
63
|
+
stderr: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Injection seam for the subprocess call. Production uses `defaultSpawnRunner`
|
|
68
|
+
* (Bun.spawn); tests pass a mock so no real binary runs. A launch failure
|
|
69
|
+
* (binary not found) resolves to `exitCode: 127` rather than throwing, so the
|
|
70
|
+
* provider maps it to a friendly terminal error.
|
|
71
|
+
*/
|
|
72
|
+
export type SpawnRunner = (
|
|
73
|
+
cmd: string[],
|
|
74
|
+
opts?: { timeoutMs?: number },
|
|
75
|
+
) => Promise<SubprocessResult>;
|
|
76
|
+
|
|
77
|
+
/** Production spawn runner over `Bun.spawn`. */
|
|
78
|
+
export const defaultSpawnRunner: SpawnRunner = async (cmd, opts) => {
|
|
79
|
+
let proc: ReturnType<typeof Bun.spawn>;
|
|
80
|
+
try {
|
|
81
|
+
proc = Bun.spawn(cmd, { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
|
|
82
|
+
} catch (err) {
|
|
83
|
+
// ENOENT etc. — the binary couldn't be launched at all.
|
|
84
|
+
return { exitCode: 127, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
|
|
85
|
+
}
|
|
86
|
+
let timedOut = false;
|
|
87
|
+
const timer = opts?.timeoutMs
|
|
88
|
+
? setTimeout(() => {
|
|
89
|
+
timedOut = true;
|
|
90
|
+
proc.kill();
|
|
91
|
+
}, opts.timeoutMs)
|
|
92
|
+
: null;
|
|
93
|
+
try {
|
|
94
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
95
|
+
new Response(proc.stdout as ReadableStream).text(),
|
|
96
|
+
new Response(proc.stderr as ReadableStream).text(),
|
|
97
|
+
proc.exited,
|
|
98
|
+
]);
|
|
99
|
+
if (timedOut) {
|
|
100
|
+
return { exitCode: 124, stdout, stderr: `${stderr}\n[transcribe-cli timed out]` };
|
|
101
|
+
}
|
|
102
|
+
return { exitCode, stdout, stderr };
|
|
103
|
+
} finally {
|
|
104
|
+
if (timer) clearTimeout(timer);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
export interface TranscribeCppProviderOpts {
|
|
109
|
+
/** Path to the `transcribe-cli` binary. Undefined ⇒ unavailable + terminal. */
|
|
110
|
+
binPath?: string;
|
|
111
|
+
/** Path to the GGUF model. Undefined ⇒ unavailable + terminal. */
|
|
112
|
+
modelPath?: string;
|
|
113
|
+
/** ffmpeg binary (default "ffmpeg"; resolved on PATH). */
|
|
114
|
+
ffmpegPath?: string;
|
|
115
|
+
/** Per-transcription timeout (ms). Default 600s. */
|
|
116
|
+
timeoutMs?: number;
|
|
117
|
+
/** Subprocess runner (tests inject a mock). Default `defaultSpawnRunner`. */
|
|
118
|
+
spawn?: SpawnRunner;
|
|
119
|
+
/** Existence probe (tests inject). Default `fs.existsSync`. */
|
|
120
|
+
existsImpl?: (p: string) => boolean;
|
|
121
|
+
/** Scratch dir for the temp audio/wav files. Default `os.tmpdir()`. */
|
|
122
|
+
tmpDir?: string;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Build the `transcribe-cli` argv. Upstream usage:
|
|
127
|
+
* `transcribe-cli -m <model.gguf> <audio.wav>`
|
|
128
|
+
* (model via `-m`, audio as a positional). Exported so a test pins the shape.
|
|
129
|
+
*/
|
|
130
|
+
export function buildTranscribeArgs(binPath: string, modelPath: string, wavPath: string): string[] {
|
|
131
|
+
return [binPath, "-m", modelPath, wavPath];
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Build the ffmpeg transcode argv → 16kHz mono WAV. Exported for tests. */
|
|
135
|
+
export function buildFfmpegArgs(ffmpegPath: string, inputPath: string, wavPath: string): string[] {
|
|
136
|
+
return [ffmpegPath, "-nostdin", "-y", "-i", inputPath, "-ar", "16000", "-ac", "1", wavPath];
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Extract the transcript from `transcribe-cli`'s stdout report. The CLI prints a
|
|
141
|
+
* human report whose transcript is the single `text: <transcript>` line at
|
|
142
|
+
* column 0 (library `[info]`/`[debug]` logs go to STDERR, and the indented
|
|
143
|
+
* per-segment lines start with a bracket, not `text:`). We return the first
|
|
144
|
+
* `text:` line trimmed; the `text: (empty)` no-result sentinel maps to "" so the
|
|
145
|
+
* caller treats it as a retriable empty-output failure. Returns "" when no
|
|
146
|
+
* `text:` line is present. Validated against transcribe.cpp v0.1.1. Exported for
|
|
147
|
+
* tests.
|
|
148
|
+
*/
|
|
149
|
+
export function parseTranscribeCliOutput(stdout: string): string {
|
|
150
|
+
for (const line of stdout.split("\n")) {
|
|
151
|
+
const m = line.match(/^text:\s?(.*)$/);
|
|
152
|
+
if (m) {
|
|
153
|
+
const t = m[1]!.trim();
|
|
154
|
+
return t === "(empty)" ? "" : t;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return "";
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export class TranscribeCppProvider implements TranscriptionProvider {
|
|
161
|
+
readonly name = "transcribe-cpp";
|
|
162
|
+
|
|
163
|
+
private readonly binPath: string | undefined;
|
|
164
|
+
private readonly modelPath: string | undefined;
|
|
165
|
+
private readonly ffmpegPath: string;
|
|
166
|
+
private readonly timeoutMs: number;
|
|
167
|
+
private readonly spawn: SpawnRunner;
|
|
168
|
+
private readonly existsImpl: (p: string) => boolean;
|
|
169
|
+
private readonly tmpDir: string;
|
|
170
|
+
/** Once available, stays available for this process — avoids re-statting. */
|
|
171
|
+
private cachedAvailable = false;
|
|
172
|
+
|
|
173
|
+
constructor(opts: TranscribeCppProviderOpts = {}) {
|
|
174
|
+
this.binPath = opts.binPath;
|
|
175
|
+
this.modelPath = opts.modelPath;
|
|
176
|
+
this.ffmpegPath = opts.ffmpegPath ?? "ffmpeg";
|
|
177
|
+
this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
178
|
+
this.spawn = opts.spawn ?? defaultSpawnRunner;
|
|
179
|
+
this.existsImpl = opts.existsImpl ?? existsSync;
|
|
180
|
+
this.tmpDir = opts.tmpDir ?? tmpdir();
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Cheap readiness probe — a runnable `transcribe-cli` binary AND the model
|
|
185
|
+
* both present on disk. NEVER spawns (it gates the capability flag on every
|
|
186
|
+
* `/api/vault` GET). Memoizes the once-true result so repeated landing reads
|
|
187
|
+
* don't re-stat.
|
|
188
|
+
*
|
|
189
|
+
* The binary check is load-bearing for honesty: transcribe.cpp v0.1.1 ships a
|
|
190
|
+
* *library*, not a prebuilt CLI, so `transcription install` can leave the libs
|
|
191
|
+
* + model in place with no runnable CLI. `binPath` resolves to
|
|
192
|
+
* `TRANSCRIBE_CPP_BIN` or a located binary; when neither exists we report
|
|
193
|
+
* `ok:false` (capability `enabled:false`) rather than claiming a capability we
|
|
194
|
+
* can't deliver.
|
|
195
|
+
*/
|
|
196
|
+
async available(): Promise<ProviderAvailability> {
|
|
197
|
+
if (this.cachedAvailable) return { ok: true };
|
|
198
|
+
if (!this.binPath || !this.existsImpl(this.binPath)) {
|
|
199
|
+
return {
|
|
200
|
+
ok: false,
|
|
201
|
+
reason:
|
|
202
|
+
"no runnable transcribe-cli (run `parachute-vault transcription install` to build one against the prebuilt libs — needs a C++ compiler — or set TRANSCRIBE_CPP_BIN; see `parachute-vault transcription status`)",
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
if (!this.modelPath || !this.existsImpl(this.modelPath)) {
|
|
206
|
+
return {
|
|
207
|
+
ok: false,
|
|
208
|
+
reason: "no GGUF model installed (run `parachute-vault transcription install`)",
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
this.cachedAvailable = true;
|
|
212
|
+
return { ok: true };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async transcribe(input: TranscribeInput): Promise<TranscribeResult> {
|
|
216
|
+
if (!this.binPath || !this.existsImpl(this.binPath)) {
|
|
217
|
+
throw new TranscriptionError("transcribe-cli not installed", {
|
|
218
|
+
code: "missing_provider",
|
|
219
|
+
retriable: false,
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
if (!this.modelPath || !this.existsImpl(this.modelPath)) {
|
|
223
|
+
throw new TranscriptionError("no GGUF model installed for transcribe-cpp", {
|
|
224
|
+
code: "missing_provider",
|
|
225
|
+
retriable: false,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const id = randomUUID();
|
|
230
|
+
const inExt = extFor(input.filename, input.mimeType);
|
|
231
|
+
const inputPath = join(this.tmpDir, `parachute-tc-${id}.${inExt}`);
|
|
232
|
+
const wavPath = join(this.tmpDir, `parachute-tc-${id}.16k.wav`);
|
|
233
|
+
const cleanup: string[] = [inputPath, wavPath];
|
|
234
|
+
|
|
235
|
+
try {
|
|
236
|
+
await writeFile(inputPath, input.audio);
|
|
237
|
+
|
|
238
|
+
// transcribe-cli STRICTLY requires 16kHz mono WAV and rejects other sample
|
|
239
|
+
// rates (a 44.1kHz .wav exits non-zero). ALWAYS transcode — even an
|
|
240
|
+
// incoming .wav, whose sample rate we can't trust — via ffmpeg.
|
|
241
|
+
const ff = await this.spawn(buildFfmpegArgs(this.ffmpegPath, inputPath, wavPath), {
|
|
242
|
+
timeoutMs: this.timeoutMs,
|
|
243
|
+
});
|
|
244
|
+
if (ff.exitCode !== 0) {
|
|
245
|
+
const missing = ff.exitCode === 127;
|
|
246
|
+
throw new TranscriptionError(
|
|
247
|
+
missing
|
|
248
|
+
? "ffmpeg not found — install it to transcode audio to 16kHz mono WAV for transcribe-cpp (apt install ffmpeg / brew install ffmpeg)"
|
|
249
|
+
: `ffmpeg transcode failed (exit ${ff.exitCode}): ${ff.stderr.trim().slice(0, 500)}`,
|
|
250
|
+
{ code: missing ? "ffmpeg_missing" : "transcode_failed", retriable: false },
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const res = await this.spawn(buildTranscribeArgs(this.binPath, this.modelPath, wavPath), {
|
|
255
|
+
timeoutMs: this.timeoutMs,
|
|
256
|
+
});
|
|
257
|
+
if (res.exitCode !== 0) {
|
|
258
|
+
// Launch failure (127) means the binary vanished under us despite the
|
|
259
|
+
// existsSync above — terminal. Any other non-zero is retriable so a
|
|
260
|
+
// transient resource blip backs off; a deterministic failure still
|
|
261
|
+
// terminates after the worker's maxAttempts.
|
|
262
|
+
if (res.exitCode === 127) {
|
|
263
|
+
throw new TranscriptionError(`transcribe-cli could not be launched: ${res.stderr.trim()}`, {
|
|
264
|
+
code: "missing_provider",
|
|
265
|
+
retriable: false,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
throw new TranscriptionError(
|
|
269
|
+
`transcribe-cli exited ${res.exitCode}: ${res.stderr.trim().slice(0, 500)}`,
|
|
270
|
+
{ code: "transcribe_cli_error", retriable: true },
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const text = parseTranscribeCliOutput(res.stdout);
|
|
275
|
+
if (!text) {
|
|
276
|
+
// Empty/unparseable — plain Error is treated as retriable by the worker.
|
|
277
|
+
throw new Error("transcribe-cli produced no transcript text");
|
|
278
|
+
}
|
|
279
|
+
return { text };
|
|
280
|
+
} finally {
|
|
281
|
+
await Promise.all(cleanup.map((p) => unlink(p).catch(() => {})));
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Choose a temp-file extension from the filename or mime type. */
|
|
287
|
+
function extFor(filename: string, mimeType: string): string {
|
|
288
|
+
const fromName = filename.split(".").pop();
|
|
289
|
+
if (fromName && fromName.length <= 5 && /^[a-z0-9]+$/i.test(fromName)) return fromName.toLowerCase();
|
|
290
|
+
const sub = mimeType.split("/")[1]?.split(";")[0]?.trim();
|
|
291
|
+
if (sub && /^[a-z0-9]+$/i.test(sub)) return sub.toLowerCase();
|
|
292
|
+
return "bin";
|
|
293
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { describe, test, expect, afterEach } from "bun:test";
|
|
2
|
+
import { mkdirSync, writeFileSync, rmSync } from "fs";
|
|
3
|
+
import { join } from "path";
|
|
4
|
+
import { tmpdir } from "os";
|
|
5
|
+
import {
|
|
6
|
+
resolveTranscriptionProviderName,
|
|
7
|
+
resolveTranscribeCppPaths,
|
|
8
|
+
transcribeCppInstalled,
|
|
9
|
+
readManifest,
|
|
10
|
+
transcriptionHomeDir,
|
|
11
|
+
} from "./select.ts";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Provider-selection + path-resolution tests (scribe-fold Phase 2a). All env is
|
|
15
|
+
* passed explicitly so the shared bun-test process isn't polluted.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const silent = { warn: () => {} };
|
|
19
|
+
|
|
20
|
+
describe("resolveTranscriptionProviderName", () => {
|
|
21
|
+
test("unset → scribe-http (behavior-preserving default)", () => {
|
|
22
|
+
expect(resolveTranscriptionProviderName({}, silent)).toBe("scribe-http");
|
|
23
|
+
});
|
|
24
|
+
test("blank → scribe-http", () => {
|
|
25
|
+
expect(resolveTranscriptionProviderName({ TRANSCRIPTION_PROVIDER: " " }, silent)).toBe("scribe-http");
|
|
26
|
+
});
|
|
27
|
+
test("explicit transcribe-cpp", () => {
|
|
28
|
+
expect(resolveTranscriptionProviderName({ TRANSCRIPTION_PROVIDER: "transcribe-cpp" }, silent)).toBe(
|
|
29
|
+
"transcribe-cpp",
|
|
30
|
+
);
|
|
31
|
+
});
|
|
32
|
+
test("explicit scribe-http", () => {
|
|
33
|
+
expect(resolveTranscriptionProviderName({ TRANSCRIPTION_PROVIDER: "scribe-http" }, silent)).toBe(
|
|
34
|
+
"scribe-http",
|
|
35
|
+
);
|
|
36
|
+
});
|
|
37
|
+
test("unknown value → warns + falls back to scribe-http", () => {
|
|
38
|
+
let warned = false;
|
|
39
|
+
const name = resolveTranscriptionProviderName(
|
|
40
|
+
{ TRANSCRIPTION_PROVIDER: "whisper-magic" },
|
|
41
|
+
{ warn: () => (warned = true) },
|
|
42
|
+
);
|
|
43
|
+
expect(name).toBe("scribe-http");
|
|
44
|
+
expect(warned).toBe(true);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe("transcriptionHomeDir + resolveTranscribeCppPaths — PARACHUTE_HOME + overrides", () => {
|
|
49
|
+
test("honors PARACHUTE_HOME", () => {
|
|
50
|
+
const dir = transcriptionHomeDir({ PARACHUTE_HOME: "/custom/home" });
|
|
51
|
+
expect(dir).toBe(join("/custom/home", "transcription"));
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("default binPath/modelsDir under the transcription dir", () => {
|
|
55
|
+
const paths = resolveTranscribeCppPaths({ PARACHUTE_HOME: "/h" });
|
|
56
|
+
expect(paths.binPath).toBe(join("/h", "transcription", "bin", "transcribe-cli"));
|
|
57
|
+
expect(paths.modelsDir).toBe(join("/h", "transcription", "models"));
|
|
58
|
+
expect(paths.manifestPath).toBe(join("/h", "transcription", "install.json"));
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("TRANSCRIBE_CPP_BIN + TRANSCRIBE_CPP_MODEL env overrides win", () => {
|
|
62
|
+
const paths = resolveTranscribeCppPaths({
|
|
63
|
+
PARACHUTE_HOME: "/h",
|
|
64
|
+
TRANSCRIBE_CPP_BIN: "/opt/tc",
|
|
65
|
+
TRANSCRIBE_CPP_MODEL: "/models/custom.gguf",
|
|
66
|
+
});
|
|
67
|
+
expect(paths.binPath).toBe("/opt/tc");
|
|
68
|
+
expect(paths.modelPath).toBe("/models/custom.gguf");
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("model path resolves from the install manifest when no env override", () => {
|
|
72
|
+
const home = mkTmpHome();
|
|
73
|
+
writeManifest(home, { modelFile: "whisper-small.en-Q5_K_M.gguf", model: "whisper-small.en" });
|
|
74
|
+
const paths = resolveTranscribeCppPaths({ PARACHUTE_HOME: home });
|
|
75
|
+
expect(paths.modelPath).toBe(join(home, "transcription", "models", "whisper-small.en-Q5_K_M.gguf"));
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("no manifest, no env → modelPath undefined", () => {
|
|
79
|
+
const home = mkTmpHome();
|
|
80
|
+
expect(resolveTranscribeCppPaths({ PARACHUTE_HOME: home }).modelPath).toBeUndefined();
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
describe("transcribeCppInstalled", () => {
|
|
85
|
+
test("true only when both binary AND model exist on disk", () => {
|
|
86
|
+
const home = mkTmpHome();
|
|
87
|
+
const tcDir = join(home, "transcription");
|
|
88
|
+
mkdirSync(join(tcDir, "bin"), { recursive: true });
|
|
89
|
+
mkdirSync(join(tcDir, "models"), { recursive: true });
|
|
90
|
+
// Neither present yet.
|
|
91
|
+
expect(transcribeCppInstalled(resolveTranscribeCppPaths({ PARACHUTE_HOME: home }))).toBe(false);
|
|
92
|
+
|
|
93
|
+
// Binary only → still false.
|
|
94
|
+
writeFileSync(join(tcDir, "bin", "transcribe-cli"), "#!/bin/sh\n");
|
|
95
|
+
writeManifest(home, { modelFile: "m.gguf", model: "whisper-tiny.en" });
|
|
96
|
+
expect(transcribeCppInstalled(resolveTranscribeCppPaths({ PARACHUTE_HOME: home }))).toBe(false);
|
|
97
|
+
|
|
98
|
+
// Model too → true.
|
|
99
|
+
writeFileSync(join(tcDir, "models", "m.gguf"), "gguf");
|
|
100
|
+
expect(transcribeCppInstalled(resolveTranscribeCppPaths({ PARACHUTE_HOME: home }))).toBe(true);
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
describe("readManifest", () => {
|
|
105
|
+
test("returns null for a missing/unreadable manifest", () => {
|
|
106
|
+
expect(readManifest(join(tmpdir(), "does-not-exist-xyz.json"))).toBeNull();
|
|
107
|
+
});
|
|
108
|
+
test("parses a written manifest", () => {
|
|
109
|
+
const home = mkTmpHome();
|
|
110
|
+
writeManifest(home, { model: "whisper-tiny.en", modelFile: "t.gguf" });
|
|
111
|
+
const m = readManifest(join(home, "transcription", "install.json"));
|
|
112
|
+
expect(m?.model).toBe("whisper-tiny.en");
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
// --- helpers ---------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
const tmpHomes: string[] = [];
|
|
119
|
+
afterEach(() => {
|
|
120
|
+
for (const h of tmpHomes.splice(0)) rmSync(h, { recursive: true, force: true });
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
function mkTmpHome(): string {
|
|
124
|
+
const home = join(tmpdir(), `tc-select-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
125
|
+
mkdirSync(home, { recursive: true });
|
|
126
|
+
tmpHomes.push(home);
|
|
127
|
+
return home;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function writeManifest(home: string, fields: Record<string, unknown>): void {
|
|
131
|
+
const tcDir = join(home, "transcription");
|
|
132
|
+
mkdirSync(tcDir, { recursive: true });
|
|
133
|
+
writeFileSync(join(tcDir, "install.json"), JSON.stringify(fields));
|
|
134
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcription provider selection + `transcribe-cpp` path resolution
|
|
3
|
+
* (scribe-fold Phase 2a).
|
|
4
|
+
*
|
|
5
|
+
* Vault now ships TWO transcription providers behind the Phase 1
|
|
6
|
+
* `TranscriptionProvider` seam:
|
|
7
|
+
*
|
|
8
|
+
* - `scribe-http` — the remote/compat provider (the DEFAULT; existing scribe
|
|
9
|
+
* installs are unchanged);
|
|
10
|
+
* - `transcribe-cpp` — a local, no-Python provider that subprocesses the
|
|
11
|
+
* prebuilt `transcribe-cli` (opt-in via `transcription install` + config).
|
|
12
|
+
*
|
|
13
|
+
* The active provider is chosen by the `TRANSCRIPTION_PROVIDER` env var
|
|
14
|
+
* (persisted in `~/.parachute/vault/.env`), resolved here so the worker boot
|
|
15
|
+
* (`server.ts`) and the capability flag (`capability.ts`) agree on one source
|
|
16
|
+
* of truth. Unset ⇒ `scribe-http`, so no config change means no behavior
|
|
17
|
+
* change.
|
|
18
|
+
*
|
|
19
|
+
* The `transcribe-cli` binary, GGUF model, and runtime shared libraries live
|
|
20
|
+
* under `$PARACHUTE_HOME/transcription/` (parallel to the vault's other
|
|
21
|
+
* ecosystem state), written there by `transcription install`. NOTE:
|
|
22
|
+
* transcribe.cpp v0.1.1 ships a *library*, not a prebuilt CLI, so after an
|
|
23
|
+
* install the `libs/` + `models/` are present but `bin/transcribe-cli` may be
|
|
24
|
+
* absent until a CLI is built or `TRANSCRIBE_CPP_BIN` is set — the readiness
|
|
25
|
+
* checks below gate on the binary existing. Paths are resolved per-call so
|
|
26
|
+
* `PARACHUTE_HOME` overrides (tests, Docker) apply.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { join } from "path";
|
|
30
|
+
import { homedir } from "os";
|
|
31
|
+
import { existsSync, readFileSync } from "fs";
|
|
32
|
+
|
|
33
|
+
export const TRANSCRIPTION_PROVIDERS = ["scribe-http", "transcribe-cpp"] as const;
|
|
34
|
+
export type TranscriptionProviderName = (typeof TRANSCRIPTION_PROVIDERS)[number];
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Resolve the configured provider name. `TRANSCRIPTION_PROVIDER` selects it;
|
|
38
|
+
* unset (or blank) ⇒ `scribe-http` (the behavior-preserving default). An
|
|
39
|
+
* unrecognized value warns once and falls back to `scribe-http` rather than
|
|
40
|
+
* failing boot — a typo shouldn't take transcription offline hard.
|
|
41
|
+
*/
|
|
42
|
+
export function resolveTranscriptionProviderName(
|
|
43
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
44
|
+
logger: { warn?: (...args: unknown[]) => void } = console,
|
|
45
|
+
): TranscriptionProviderName {
|
|
46
|
+
const raw = env.TRANSCRIPTION_PROVIDER?.trim();
|
|
47
|
+
if (!raw) return "scribe-http";
|
|
48
|
+
if ((TRANSCRIPTION_PROVIDERS as readonly string[]).includes(raw)) {
|
|
49
|
+
return raw as TranscriptionProviderName;
|
|
50
|
+
}
|
|
51
|
+
logger.warn?.(
|
|
52
|
+
`[transcribe] unknown TRANSCRIPTION_PROVIDER="${raw}" — falling back to scribe-http. ` +
|
|
53
|
+
`Valid values: ${TRANSCRIPTION_PROVIDERS.join(", ")}.`,
|
|
54
|
+
);
|
|
55
|
+
return "scribe-http";
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The ecosystem root (shared with `config.ts`'s `configDirPath`), per-call so
|
|
59
|
+
* `PARACHUTE_HOME` overrides apply in tests / Docker. */
|
|
60
|
+
function ecosystemRoot(env: NodeJS.ProcessEnv): string {
|
|
61
|
+
return env.PARACHUTE_HOME ?? join(homedir(), ".parachute");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** The cache dir for transcribe-cpp's binary + models: `<root>/transcription/`. */
|
|
65
|
+
export function transcriptionHomeDir(env: NodeJS.ProcessEnv = process.env): string {
|
|
66
|
+
return join(ecosystemRoot(env), "transcription");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Resolved filesystem locations for the transcribe-cpp install. */
|
|
70
|
+
export interface TranscribeCppPaths {
|
|
71
|
+
/** `<root>/transcription/`. */
|
|
72
|
+
dir: string;
|
|
73
|
+
/** `<dir>/bin/`. */
|
|
74
|
+
binDir: string;
|
|
75
|
+
/** The `transcribe-cli` binary path (env `TRANSCRIBE_CPP_BIN` overrides).
|
|
76
|
+
* May not exist on disk — v0.1.1 ships a library, not a CLI. */
|
|
77
|
+
binPath: string;
|
|
78
|
+
/** `<dir>/libs/` — the extracted runtime shared libraries (libtranscribe +
|
|
79
|
+
* libggml) the CLI links against. */
|
|
80
|
+
libsDir: string;
|
|
81
|
+
/** `<dir>/models/`. */
|
|
82
|
+
modelsDir: string;
|
|
83
|
+
/** `<dir>/install.json` — the install manifest. */
|
|
84
|
+
manifestPath: string;
|
|
85
|
+
/**
|
|
86
|
+
* The active GGUF model path, or `undefined` when nothing is installed.
|
|
87
|
+
* Env `TRANSCRIBE_CPP_MODEL` overrides; otherwise resolved from the
|
|
88
|
+
* manifest's `modelFile` under `modelsDir`.
|
|
89
|
+
*/
|
|
90
|
+
modelPath: string | undefined;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Persisted `install.json` shape (written by `transcription install`). */
|
|
94
|
+
export interface TranscribeCppManifest {
|
|
95
|
+
version: string;
|
|
96
|
+
asset: string;
|
|
97
|
+
/** Binary filename under `bin/` (usually "transcribe-cli"). */
|
|
98
|
+
binFile: string;
|
|
99
|
+
/** Model id (e.g. "whisper-small.en"). */
|
|
100
|
+
model: string;
|
|
101
|
+
/** GGUF filename under `models/`. */
|
|
102
|
+
modelFile: string;
|
|
103
|
+
/** Runtime shared-library filenames extracted under `libs/` (libtranscribe +
|
|
104
|
+
* libggml). v0.1.1 ships these instead of a CLI. */
|
|
105
|
+
libFiles?: string[];
|
|
106
|
+
/** When the `transcribe-cli` was built from source at install, the pinned
|
|
107
|
+
* upstream source ref it was compiled from. Absent when no CLI was built
|
|
108
|
+
* (build skipped/failed, or an operator-supplied `TRANSCRIBE_CPP_BIN`). */
|
|
109
|
+
binBuiltFrom?: string;
|
|
110
|
+
os: string;
|
|
111
|
+
arch: string;
|
|
112
|
+
ram_gb: number;
|
|
113
|
+
installedAt: string;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Resolve the transcribe-cpp binary + model paths. Env overrides
|
|
118
|
+
* (`TRANSCRIBE_CPP_BIN`, `TRANSCRIBE_CPP_MODEL`) win; otherwise the binary is
|
|
119
|
+
* `<dir>/bin/transcribe-cli` and the model comes from the install manifest.
|
|
120
|
+
*/
|
|
121
|
+
export function resolveTranscribeCppPaths(
|
|
122
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
123
|
+
): TranscribeCppPaths {
|
|
124
|
+
const dir = transcriptionHomeDir(env);
|
|
125
|
+
const binDir = join(dir, "bin");
|
|
126
|
+
const libsDir = join(dir, "libs");
|
|
127
|
+
const modelsDir = join(dir, "models");
|
|
128
|
+
const manifestPath = join(dir, "install.json");
|
|
129
|
+
|
|
130
|
+
const binPath = env.TRANSCRIBE_CPP_BIN?.trim() || join(binDir, "transcribe-cli");
|
|
131
|
+
|
|
132
|
+
let modelPath = env.TRANSCRIBE_CPP_MODEL?.trim() || undefined;
|
|
133
|
+
if (!modelPath) {
|
|
134
|
+
const manifest = readManifest(manifestPath);
|
|
135
|
+
if (manifest?.modelFile) modelPath = join(modelsDir, manifest.modelFile);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return { dir, binDir, binPath, libsDir, modelsDir, manifestPath, modelPath };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Read + parse the install manifest, or `null` when absent/unreadable. */
|
|
142
|
+
export function readManifest(manifestPath: string): TranscribeCppManifest | null {
|
|
143
|
+
try {
|
|
144
|
+
if (!existsSync(manifestPath)) return null;
|
|
145
|
+
const parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as TranscribeCppManifest;
|
|
146
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
147
|
+
} catch {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Cheap, spawn-free readiness check: a runnable `transcribe-cli` binary AND a
|
|
154
|
+
* model both exist on disk. Used by the worker-boot gate (`server.ts`) and
|
|
155
|
+
* mirrors the provider's `available()`. Because v0.1.1 ships a library (no CLI),
|
|
156
|
+
* this stays `false` after an install until a CLI is built or
|
|
157
|
+
* `TRANSCRIBE_CPP_BIN` points at one — so the worker never starts only to
|
|
158
|
+
* terminal-fail every item.
|
|
159
|
+
*/
|
|
160
|
+
export function transcribeCppInstalled(
|
|
161
|
+
paths: TranscribeCppPaths = resolveTranscribeCppPaths(),
|
|
162
|
+
): boolean {
|
|
163
|
+
return existsSync(paths.binPath) && !!paths.modelPath && existsSync(paths.modelPath);
|
|
164
|
+
}
|
|
@@ -140,6 +140,50 @@ describe("transcription worker", () => {
|
|
|
140
140
|
expect(att.metadata?.transcript).toBe("hello world transcript");
|
|
141
141
|
});
|
|
142
142
|
|
|
143
|
+
test("injected provider (scribe-fold 2a): worker routes audio→text through it, not scribe-http", async () => {
|
|
144
|
+
const note = await store.createNote(
|
|
145
|
+
"# 🎙️ Voice memo\n\n_Transcript pending._\n",
|
|
146
|
+
{ id: "n1cpp", metadata: { transcribe_stub: true } },
|
|
147
|
+
);
|
|
148
|
+
seedAudio("memos/cpp.webm");
|
|
149
|
+
await store.addAttachment(note.id, "memos/cpp.webm", "audio/webm", {
|
|
150
|
+
transcribe_status: "pending",
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
// A fake local provider (stands in for transcribe-cpp). If the worker used
|
|
154
|
+
// it, the note gets this text; a `fetchImpl` that throws proves scribe-http
|
|
155
|
+
// was NOT called.
|
|
156
|
+
let called = 0;
|
|
157
|
+
const provider = {
|
|
158
|
+
name: "transcribe-cpp",
|
|
159
|
+
available: async () => ({ ok: true }),
|
|
160
|
+
transcribe: async () => {
|
|
161
|
+
called++;
|
|
162
|
+
return { text: "local transcript" };
|
|
163
|
+
},
|
|
164
|
+
};
|
|
165
|
+
const worker = startTranscriptionWorker({
|
|
166
|
+
vaultList: () => ["default"],
|
|
167
|
+
getStore: () => store as unknown as Store,
|
|
168
|
+
resolveAssetsDir: () => assetsRoot,
|
|
169
|
+
provider,
|
|
170
|
+
fetchImpl: (() => {
|
|
171
|
+
throw new Error("scribe-http must not be called when a provider is injected");
|
|
172
|
+
}) as unknown as typeof fetch,
|
|
173
|
+
pollIntervalMs: 10_000_000,
|
|
174
|
+
logger: silentLogger,
|
|
175
|
+
});
|
|
176
|
+
try {
|
|
177
|
+
expect(await worker.tick()).toBe(1);
|
|
178
|
+
} finally {
|
|
179
|
+
await worker.stop();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
expect(called).toBe(1);
|
|
183
|
+
const updated = await store.getNote("n1cpp");
|
|
184
|
+
expect(updated!.content).toBe("# 🎙️ Voice memo\n\nlocal transcript\n");
|
|
185
|
+
});
|
|
186
|
+
|
|
143
187
|
test("no-clobber: stub flag absent → does not touch note content", async () => {
|
|
144
188
|
await store.createNote("my own edit", { id: "n2" });
|
|
145
189
|
seedAudio("memos/b.webm");
|