@oh-my-pi/pi-coding-agent 16.1.13 → 16.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/cli.js +5547 -2759
- package/dist/types/config/settings-schema.d.ts +44 -14
- package/dist/types/eval/__tests__/julia-prelude.test.d.ts +1 -0
- package/dist/types/eval/backend-helpers.d.ts +23 -0
- package/dist/types/eval/executor-base.d.ts +118 -0
- package/dist/types/eval/index.d.ts +2 -0
- package/dist/types/eval/jl/executor.d.ts +44 -0
- package/dist/types/eval/jl/index.d.ts +11 -0
- package/dist/types/eval/jl/kernel.d.ts +28 -0
- package/dist/types/eval/jl/prelude.d.ts +1 -0
- package/dist/types/eval/jl/runtime.d.ts +22 -0
- package/dist/types/eval/kernel-base.d.ts +105 -0
- package/dist/types/eval/py/kernel.d.ts +3 -61
- package/dist/types/eval/rb/executor.d.ts +77 -0
- package/dist/types/eval/rb/index.d.ts +11 -0
- package/dist/types/eval/rb/kernel.d.ts +31 -0
- package/dist/types/eval/rb/prelude.d.ts +1 -0
- package/dist/types/eval/rb/runtime.d.ts +23 -0
- package/dist/types/eval/runtime-env.d.ts +24 -0
- package/dist/types/eval/types.d.ts +3 -3
- package/dist/types/extensibility/extensions/runner.d.ts +2 -15
- package/dist/types/extensibility/hooks/loader.d.ts +1 -25
- package/dist/types/extensibility/session-handler-types.d.ts +23 -0
- package/dist/types/jsonrpc/message-framing.d.ts +35 -0
- package/dist/types/mnemopi/embed-client.d.ts +7 -24
- package/dist/types/modes/components/chat-transcript-builder.d.ts +1 -1
- package/dist/types/modes/components/custom-editor.d.ts +11 -0
- package/dist/types/modes/components/selector-helpers.d.ts +53 -0
- package/dist/types/modes/interactive-mode.d.ts +0 -2
- package/dist/types/modes/theme/theme.d.ts +8 -1
- package/dist/types/modes/types.d.ts +0 -2
- package/dist/types/modes/utils/copy-targets.d.ts +1 -1
- package/dist/types/modes/utils/interactive-context-helpers.d.ts +14 -0
- package/dist/types/modes/utils/transcript-render-helpers.d.ts +54 -0
- package/dist/types/sdk.d.ts +1 -1
- package/dist/types/session/agent-session.d.ts +2 -2
- package/dist/types/stt/asr-client.d.ts +3 -29
- package/dist/types/subprocess/worker-client.d.ts +149 -0
- package/dist/types/subprocess/worker-runtime.d.ts +107 -0
- package/dist/types/tiny/title-client.d.ts +14 -34
- package/dist/types/tools/eval-backends.d.ts +6 -3
- package/dist/types/tools/eval-render.d.ts +6 -5
- package/dist/types/tools/eval.d.ts +13 -15
- package/dist/types/tools/index.d.ts +3 -3
- package/dist/types/tts/tts-client.d.ts +3 -28
- package/dist/types/tui/code-cell.d.ts +7 -0
- package/dist/types/utils/file-display-mode.d.ts +1 -1
- package/dist/types/web/parallel.d.ts +6 -0
- package/package.json +12 -12
- package/src/config/settings-schema.ts +50 -18
- package/src/config/settings.ts +5 -0
- package/src/dap/client.ts +13 -107
- package/src/eval/__tests__/julia-prelude.test.ts +77 -0
- package/src/eval/backend-helpers.ts +48 -0
- package/src/eval/executor-base.ts +425 -0
- package/src/eval/index.ts +2 -0
- package/src/eval/jl/executor.ts +540 -0
- package/src/eval/jl/index.ts +54 -0
- package/src/eval/jl/kernel.ts +235 -0
- package/src/eval/jl/prelude.jl +930 -0
- package/src/eval/jl/prelude.ts +3 -0
- package/src/eval/jl/runner.jl +634 -0
- package/src/eval/jl/runtime.ts +118 -0
- package/src/eval/js/index.ts +3 -14
- package/src/eval/kernel-base.ts +569 -0
- package/src/eval/py/executor.ts +43 -252
- package/src/eval/py/index.ts +9 -20
- package/src/eval/py/kernel.ts +29 -544
- package/src/eval/rb/executor.ts +504 -0
- package/src/eval/rb/index.ts +54 -0
- package/src/eval/rb/kernel.ts +230 -0
- package/src/eval/rb/prelude.rb +721 -0
- package/src/eval/rb/prelude.ts +3 -0
- package/src/eval/rb/runner.rb +474 -0
- package/src/eval/rb/runtime.ts +132 -0
- package/src/eval/runtime-env.ts +104 -0
- package/src/eval/types.ts +3 -3
- package/src/extensibility/extensions/runner.ts +4 -11
- package/src/extensibility/hooks/loader.ts +3 -21
- package/src/extensibility/session-handler-types.ts +21 -0
- package/src/internal-urls/docs-index.generated.txt +1 -1
- package/src/jsonrpc/message-framing.ts +142 -0
- package/src/lsp/client.ts +13 -109
- package/src/mnemopi/embed-client.ts +43 -198
- package/src/modes/components/agent-dashboard.ts +17 -40
- package/src/modes/components/chat-transcript-builder.ts +18 -102
- package/src/modes/components/custom-editor.ts +19 -0
- package/src/modes/components/extensions/extension-dashboard.ts +4 -13
- package/src/modes/components/extensions/extension-list.ts +16 -44
- package/src/modes/components/history-search.ts +4 -16
- package/src/modes/components/selector-helpers.ts +129 -0
- package/src/modes/components/settings-selector.ts +7 -5
- package/src/modes/components/tree-selector.ts +13 -18
- package/src/modes/controllers/event-controller.ts +3 -9
- package/src/modes/controllers/input-controller.ts +32 -54
- package/src/modes/interactive-mode.ts +5 -7
- package/src/modes/theme/theme.ts +35 -0
- package/src/modes/types.ts +0 -2
- package/src/modes/utils/copy-targets.ts +3 -2
- package/src/modes/utils/interactive-context-helpers.ts +27 -0
- package/src/modes/utils/transcript-render-helpers.ts +157 -0
- package/src/modes/utils/ui-helpers.ts +21 -126
- package/src/prompts/system/system-prompt.md +10 -10
- package/src/prompts/tools/bash.md +0 -1
- package/src/prompts/tools/eval.md +6 -4
- package/src/prompts/tools/find.md +0 -4
- package/src/prompts/tools/read.md +1 -2
- package/src/prompts/tools/replace.md +1 -1
- package/src/prompts/tools/search.md +0 -1
- package/src/sdk.ts +13 -7
- package/src/session/agent-session.ts +9 -7
- package/src/stt/asr-client.ts +35 -215
- package/src/stt/asr-worker.ts +29 -181
- package/src/subprocess/worker-client.ts +297 -0
- package/src/subprocess/worker-runtime.ts +277 -0
- package/src/task/executor.ts +4 -4
- package/src/tiny/title-client.ts +53 -219
- package/src/tiny/worker.ts +29 -180
- package/src/tools/eval-backends.ts +10 -3
- package/src/tools/eval-render.ts +17 -8
- package/src/tools/eval.ts +187 -22
- package/src/tools/index.ts +51 -28
- package/src/tts/tts-client.ts +38 -206
- package/src/tts/tts-worker.ts +23 -97
- package/src/tui/code-cell.ts +12 -1
- package/src/utils/file-display-mode.ts +2 -3
- package/src/web/parallel.ts +43 -42
- package/src/web/search/providers/parallel.ts +10 -99
package/src/stt/asr-client.ts
CHANGED
|
@@ -1,26 +1,21 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import { logger } from "@oh-my-pi/pi-utils";
|
|
2
|
+
import {
|
|
3
|
+
createUnavailableWorker,
|
|
4
|
+
createWorkerHandle,
|
|
5
|
+
createWorkerSubprocess,
|
|
6
|
+
logWorkerMessage,
|
|
7
|
+
resolveWorkerSpawnCmd,
|
|
8
|
+
SMOKE_TEST_TIMEOUT_MS,
|
|
9
|
+
type SpawnedSubprocess,
|
|
10
|
+
smokeTestWorker,
|
|
11
|
+
spawnWorkerOrUnavailable,
|
|
12
|
+
type WorkerHandle,
|
|
13
|
+
} from "../subprocess/worker-client";
|
|
14
|
+
import { tinyWorkerEnv } from "../tiny/title-client";
|
|
6
15
|
import { safeSend } from "../utils/ipc";
|
|
7
16
|
import type { SttProgressEvent, SttWorkerInbound, SttWorkerOutbound } from "./asr-protocol";
|
|
8
17
|
import type { SttModelKey } from "./models";
|
|
9
18
|
|
|
10
|
-
/**
|
|
11
|
-
* Abstraction over the speech-recognition subprocess. Modelled as a worker
|
|
12
|
-
* interface so the parent composes lifecycle, ping/pong, and request/response
|
|
13
|
-
* correlation uniformly; the runtime implementation is a Bun child process so
|
|
14
|
-
* `onnxruntime-node`'s NAPI finalizer never runs inside the main agent address
|
|
15
|
-
* space — that destructor segfaults Bun on shutdown (issue #1606).
|
|
16
|
-
*/
|
|
17
|
-
interface WorkerHandle {
|
|
18
|
-
send(message: SttWorkerInbound): void;
|
|
19
|
-
onMessage(handler: (message: SttWorkerOutbound) => void): () => void;
|
|
20
|
-
onError(handler: (error: Error) => void): () => void;
|
|
21
|
-
terminate(): Promise<void>;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
19
|
type PendingRequest =
|
|
25
20
|
| { kind: "transcribe"; modelKey: SttModelKey; resolve: (text: string) => void; reject: (error: Error) => void }
|
|
26
21
|
| { kind: "download"; modelKey: SttModelKey; resolve: (ok: boolean) => void };
|
|
@@ -64,205 +59,50 @@ interface StreamState {
|
|
|
64
59
|
finish: (apply: () => void) => void;
|
|
65
60
|
}
|
|
66
61
|
|
|
67
|
-
// Cold-starting the worker subprocess from a compiled binary (decompress +
|
|
68
|
-
// module graph load) is slow on contended CI runners; the probe only needs to
|
|
69
|
-
// prove the worker spawns and ponges, so a generous bound removes the flake.
|
|
70
|
-
const SMOKE_TEST_TIMEOUT_MS = 30_000;
|
|
71
|
-
|
|
72
62
|
/**
|
|
73
63
|
* Hidden subcommand on the main CLI that boots the speech-recognition worker in
|
|
74
64
|
* the spawned subprocess. Kept in sync with the dispatch in `cli.ts`.
|
|
75
65
|
*/
|
|
76
66
|
export const STT_WORKER_ARG = "__omp_worker_stt";
|
|
77
67
|
|
|
78
|
-
function readTinyModelSetting(key: "providers.tinyModelDevice" | "providers.tinyModelDtype"): string | undefined {
|
|
79
|
-
try {
|
|
80
|
-
const value = settings.get(key);
|
|
81
|
-
return typeof value === "string" ? value : undefined;
|
|
82
|
-
} catch {
|
|
83
|
-
// Settings may be uninitialized (e.g. `omp --smoke-test`); fall back to env/default.
|
|
84
|
-
return undefined;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/**
|
|
89
|
-
* Env handed to the speech subprocess. The `PI_TINY_DEVICE` / `PI_TINY_DTYPE`
|
|
90
|
-
* env vars win; otherwise the persisted `providers.tinyModelDevice` /
|
|
91
|
-
* `providers.tinyModelDtype` settings are mapped onto those vars so the
|
|
92
|
-
* subprocess's env-based resolution picks them up (shared with tiny models).
|
|
93
|
-
*/
|
|
94
|
-
function sttWorkerEnv(): Record<string, string> {
|
|
95
|
-
const overlay = tinyWorkerEnvOverlay(
|
|
96
|
-
$env,
|
|
97
|
-
readTinyModelSetting("providers.tinyModelDevice"),
|
|
98
|
-
readTinyModelSetting("providers.tinyModelDtype"),
|
|
99
|
-
);
|
|
100
|
-
const base = $env as Record<string, string | undefined>;
|
|
101
|
-
const merged: Record<string, string> = {};
|
|
102
|
-
for (const key in base) {
|
|
103
|
-
const value = base[key];
|
|
104
|
-
if (typeof value === "string") merged[key] = value;
|
|
105
|
-
}
|
|
106
|
-
for (const key in overlay) merged[key] = overlay[key];
|
|
107
|
-
return merged;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
interface SttWorkerSpawnCommand {
|
|
111
|
-
cmd: string[];
|
|
112
|
-
cwd?: string;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/**
|
|
116
|
-
* Resolve the command used to relaunch the agent CLI into stt-worker mode. In a
|
|
117
|
-
* compiled binary the entry point is the binary itself; otherwise re-enter the
|
|
118
|
-
* declared worker-host entry with a cwd-relative script path (Bun's subprocess
|
|
119
|
-
* IPC is more reliable that way under `bun test`), falling back to this
|
|
120
|
-
* package's own `src/cli.ts` when no host entry is declared.
|
|
121
|
-
*/
|
|
122
|
-
function sttWorkerSpawnCmd(): SttWorkerSpawnCommand {
|
|
123
|
-
if (isCompiledBinary()) return { cmd: [process.execPath, STT_WORKER_ARG] };
|
|
124
|
-
const hostEntry = workerHostEntry();
|
|
125
|
-
if (hostEntry) {
|
|
126
|
-
return { cmd: [process.execPath, path.basename(hostEntry), STT_WORKER_ARG], cwd: path.dirname(hostEntry) };
|
|
127
|
-
}
|
|
128
|
-
const packageRoot = path.resolve(import.meta.dir, "..", "..");
|
|
129
|
-
return { cmd: [process.execPath, "src/cli.ts", STT_WORKER_ARG], cwd: packageRoot };
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
interface SpawnedSubprocess {
|
|
133
|
-
proc: Subprocess<"ignore", "ignore", "ignore">;
|
|
134
|
-
inbound: Set<(message: SttWorkerOutbound) => void>;
|
|
135
|
-
errors: Set<(error: Error) => void>;
|
|
136
|
-
/**
|
|
137
|
-
* Flipped to `true` right before the parent SIGKILLs the child so `onExit`
|
|
138
|
-
* can distinguish the expected hard-kill from a crash/OOM/external signal.
|
|
139
|
-
*/
|
|
140
|
-
intentionalExit: { value: boolean };
|
|
141
|
-
}
|
|
142
|
-
|
|
143
68
|
/**
|
|
144
69
|
* Spawn the speech worker as a subprocess. Exported for tests and the smoke
|
|
145
70
|
* probe; production callers go through {@link spawnSttWorker}.
|
|
146
71
|
*/
|
|
147
|
-
export function createSttSubprocess(): SpawnedSubprocess {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
const proc = Bun.spawn({
|
|
153
|
-
cmd: spawnCommand.cmd,
|
|
154
|
-
cwd: spawnCommand.cwd,
|
|
155
|
-
env: sttWorkerEnv(),
|
|
156
|
-
stdin: "ignore",
|
|
157
|
-
stdout: "ignore",
|
|
158
|
-
stderr: "ignore",
|
|
159
|
-
serialization: "advanced",
|
|
160
|
-
windowsHide: true,
|
|
161
|
-
ipc(message) {
|
|
162
|
-
for (const handler of inbound) handler(message as SttWorkerOutbound);
|
|
163
|
-
},
|
|
164
|
-
onExit(_proc, exitCode, signalCode) {
|
|
165
|
-
if (exitCode === 0) return;
|
|
166
|
-
// Swallow only the expected SIGKILL from `terminate()`; every other
|
|
167
|
-
// signal exit is a real worker death that must fault in-flight
|
|
168
|
-
// requests so callers don't await forever.
|
|
169
|
-
if (exitCode === null && intentionalExit.value) return;
|
|
170
|
-
const reason = exitCode !== null ? `code ${exitCode}` : `signal ${signalCode ?? "unknown"}`;
|
|
171
|
-
const err = new Error(`stt subprocess exited with ${reason}`);
|
|
172
|
-
for (const handler of errors) handler(err);
|
|
173
|
-
},
|
|
72
|
+
export function createSttSubprocess(): SpawnedSubprocess<SttWorkerOutbound> {
|
|
73
|
+
return createWorkerSubprocess<SttWorkerOutbound>({
|
|
74
|
+
spawnCommand: resolveWorkerSpawnCmd(STT_WORKER_ARG),
|
|
75
|
+
env: tinyWorkerEnv(),
|
|
76
|
+
exitLabel: "stt subprocess",
|
|
174
77
|
});
|
|
175
|
-
// Don't keep the parent event loop alive on an idle worker; dispose calls
|
|
176
|
-
// `terminate()` explicitly. Bun's test runner can starve IPC delivery for
|
|
177
|
-
// unref'd subprocesses, so keep it referenced under tests.
|
|
178
|
-
if (!isBunTestRuntime()) proc.unref();
|
|
179
|
-
return { proc, inbound, errors, intentionalExit };
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
function wrapSubprocess({ proc, inbound, errors, intentionalExit }: SpawnedSubprocess): WorkerHandle {
|
|
183
|
-
return {
|
|
184
|
-
send(message) {
|
|
185
|
-
safeSend(proc, message, "stt");
|
|
186
|
-
},
|
|
187
|
-
onMessage(handler) {
|
|
188
|
-
inbound.add(handler);
|
|
189
|
-
return () => inbound.delete(handler);
|
|
190
|
-
},
|
|
191
|
-
onError(handler) {
|
|
192
|
-
errors.add(handler);
|
|
193
|
-
return () => errors.delete(handler);
|
|
194
|
-
},
|
|
195
|
-
async terminate() {
|
|
196
|
-
// SIGKILL: the whole point of subprocess isolation is that the parent
|
|
197
|
-
// never runs `onnxruntime-node`'s NAPI finalizer. Hard-kill instead —
|
|
198
|
-
// the model lives in process memory and the OS reclaims everything.
|
|
199
|
-
intentionalExit.value = true;
|
|
200
|
-
try {
|
|
201
|
-
proc.kill("SIGKILL");
|
|
202
|
-
} catch {
|
|
203
|
-
// Already gone.
|
|
204
|
-
}
|
|
205
|
-
},
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
function spawnInlineUnavailableWorker(error: unknown): WorkerHandle {
|
|
210
|
-
const listeners = new Set<(message: SttWorkerOutbound) => void>();
|
|
211
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
212
|
-
const emit = (message: SttWorkerOutbound): void => {
|
|
213
|
-
for (const listener of listeners) listener(message);
|
|
214
|
-
};
|
|
215
|
-
return {
|
|
216
|
-
send(message) {
|
|
217
|
-
queueMicrotask(() => {
|
|
218
|
-
if (message.type === "ping") {
|
|
219
|
-
emit({ type: "pong", id: message.id });
|
|
220
|
-
return;
|
|
221
|
-
}
|
|
222
|
-
emit({ type: "error", id: message.id, error: errorMessage });
|
|
223
|
-
});
|
|
224
|
-
},
|
|
225
|
-
onMessage(handler) {
|
|
226
|
-
listeners.add(handler);
|
|
227
|
-
return () => listeners.delete(handler);
|
|
228
|
-
},
|
|
229
|
-
onError() {
|
|
230
|
-
return () => {};
|
|
231
|
-
},
|
|
232
|
-
async terminate() {
|
|
233
|
-
listeners.clear();
|
|
234
|
-
},
|
|
235
|
-
};
|
|
236
78
|
}
|
|
237
79
|
|
|
238
|
-
function
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
error: error instanceof Error ? error.message : String(error),
|
|
244
|
-
});
|
|
245
|
-
return spawnInlineUnavailableWorker(error);
|
|
246
|
-
}
|
|
80
|
+
function wrapSubprocess(
|
|
81
|
+
spawned: SpawnedSubprocess<SttWorkerOutbound>,
|
|
82
|
+
): WorkerHandle<SttWorkerInbound, SttWorkerOutbound> {
|
|
83
|
+
const { proc } = spawned;
|
|
84
|
+
return createWorkerHandle<SttWorkerInbound, SttWorkerOutbound>(spawned, message => safeSend(proc, message, "stt"));
|
|
247
85
|
}
|
|
248
86
|
|
|
249
|
-
function
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
87
|
+
function spawnSttWorker(): WorkerHandle<SttWorkerInbound, SttWorkerOutbound> {
|
|
88
|
+
return spawnWorkerOrUnavailable(
|
|
89
|
+
() => wrapSubprocess(createSttSubprocess()),
|
|
90
|
+
createUnavailableWorker<SttWorkerInbound, SttWorkerOutbound>,
|
|
91
|
+
"stt worker spawn failed; speech-to-text disabled",
|
|
92
|
+
);
|
|
253
93
|
}
|
|
254
94
|
|
|
255
95
|
export class SttClient {
|
|
256
|
-
#worker: WorkerHandle | null = null;
|
|
96
|
+
#worker: WorkerHandle<SttWorkerInbound, SttWorkerOutbound> | null = null;
|
|
257
97
|
#unsubscribeMessage: (() => void) | null = null;
|
|
258
98
|
#unsubscribeError: (() => void) | null = null;
|
|
259
99
|
#pending = new Map<string, PendingRequest>();
|
|
260
100
|
#streams = new Map<string, StreamState>();
|
|
261
101
|
#progressListeners = new Set<(event: SttProgressEvent) => void>();
|
|
262
102
|
#nextRequestId = 0;
|
|
263
|
-
#spawnWorker: () => WorkerHandle
|
|
103
|
+
#spawnWorker: () => WorkerHandle<SttWorkerInbound, SttWorkerOutbound>;
|
|
264
104
|
|
|
265
|
-
constructor(spawnWorker: () => WorkerHandle = spawnSttWorker) {
|
|
105
|
+
constructor(spawnWorker: () => WorkerHandle<SttWorkerInbound, SttWorkerOutbound> = spawnSttWorker) {
|
|
266
106
|
this.#spawnWorker = spawnWorker;
|
|
267
107
|
}
|
|
268
108
|
|
|
@@ -407,7 +247,7 @@ export class SttClient {
|
|
|
407
247
|
}
|
|
408
248
|
}
|
|
409
249
|
|
|
410
|
-
#ensureWorker(): WorkerHandle {
|
|
250
|
+
#ensureWorker(): WorkerHandle<SttWorkerInbound, SttWorkerOutbound> {
|
|
411
251
|
if (this.#worker) return this.#worker;
|
|
412
252
|
const worker = this.#spawnWorker();
|
|
413
253
|
this.#worker = worker;
|
|
@@ -497,25 +337,5 @@ export async function smokeTestSttWorker({
|
|
|
497
337
|
}: {
|
|
498
338
|
timeoutMs?: number;
|
|
499
339
|
} = {}): Promise<void> {
|
|
500
|
-
|
|
501
|
-
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
|
502
|
-
const timer = setTimeout(() => reject(new Error(`stt worker did not pong within ${timeoutMs}ms`)), timeoutMs);
|
|
503
|
-
const unsubscribeMessage = handle.onMessage(message => {
|
|
504
|
-
if (message.type === "pong") {
|
|
505
|
-
resolve();
|
|
506
|
-
return;
|
|
507
|
-
}
|
|
508
|
-
if (message.type === "log") return;
|
|
509
|
-
reject(new Error(`stt worker: expected pong, got ${JSON.stringify(message)}`));
|
|
510
|
-
});
|
|
511
|
-
const unsubscribeError = handle.onError(reject);
|
|
512
|
-
try {
|
|
513
|
-
handle.send({ type: "ping", id: "smoke" } satisfies SttWorkerInbound);
|
|
514
|
-
await promise;
|
|
515
|
-
} finally {
|
|
516
|
-
clearTimeout(timer);
|
|
517
|
-
unsubscribeMessage();
|
|
518
|
-
unsubscribeError();
|
|
519
|
-
await handle.terminate();
|
|
520
|
-
}
|
|
340
|
+
await smokeTestWorker(wrapSubprocess(createSttSubprocess()), "stt worker", timeoutMs);
|
|
521
341
|
}
|
package/src/stt/asr-worker.ts
CHANGED
|
@@ -10,14 +10,23 @@ import type {
|
|
|
10
10
|
import {
|
|
11
11
|
ensureRuntimeInstalled,
|
|
12
12
|
getTinyModelsCacheDir,
|
|
13
|
-
installRuntimeModuleResolver,
|
|
14
13
|
isCompiledBinary,
|
|
15
14
|
resolveRuntimeModule,
|
|
16
15
|
} from "@oh-my-pi/pi-utils";
|
|
17
16
|
import packageJson from "../../package.json" with { type: "json" };
|
|
17
|
+
import {
|
|
18
|
+
errorMessage,
|
|
19
|
+
errorText,
|
|
20
|
+
getTransformersVersionSpec,
|
|
21
|
+
loadTransformersRuntime,
|
|
22
|
+
MemoizedRuntime,
|
|
23
|
+
replayCachedReady,
|
|
24
|
+
sendLog,
|
|
25
|
+
sendProgress,
|
|
26
|
+
} from "../subprocess/worker-runtime";
|
|
18
27
|
import { resolveTinyModelDevicePreference, type TinyModelDevice, tinyModelDeviceLoadOrder } from "../tiny/device";
|
|
19
28
|
import { resolveTinyModelDtypeOverride, type TinyModelDtype } from "../tiny/dtype";
|
|
20
|
-
import type {
|
|
29
|
+
import type { SttTransport, SttWorkerInbound } from "./asr-protocol";
|
|
21
30
|
import { type EndpointerEvent, StreamEndpointer } from "./endpointer";
|
|
22
31
|
import {
|
|
23
32
|
getSttModelSpec,
|
|
@@ -28,9 +37,7 @@ import {
|
|
|
28
37
|
} from "./models";
|
|
29
38
|
|
|
30
39
|
const ASR_TASK = "automatic-speech-recognition";
|
|
31
|
-
const TRANSFORMERS_PACKAGE = "@huggingface/transformers";
|
|
32
40
|
const SHERPA_PACKAGE = "sherpa-onnx-node";
|
|
33
|
-
const COMPILED_TRANSFORMERS_VERSION = process.env.PI_TINY_TRANSFORMERS_VERSION;
|
|
34
41
|
// Whisper long-form decoding: split into 30s windows with 5s overlap so audio of
|
|
35
42
|
// any length transcribes without exceeding the 30s receptive field.
|
|
36
43
|
const CHUNK_LENGTH_S = 30;
|
|
@@ -136,34 +143,8 @@ function runOnModel<T>(work: () => Promise<T>): Promise<T> {
|
|
|
136
143
|
);
|
|
137
144
|
return run;
|
|
138
145
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
let cachedTransformersVersionSpec: string | undefined;
|
|
143
|
-
function resolveTransformersVersionSpec(): string {
|
|
144
|
-
const manifest = packageJson as {
|
|
145
|
-
optionalDependencies?: Record<string, string>;
|
|
146
|
-
dependencies?: Record<string, string>;
|
|
147
|
-
};
|
|
148
|
-
const versionSpec =
|
|
149
|
-
manifest.optionalDependencies?.[TRANSFORMERS_PACKAGE] ?? manifest.dependencies?.[TRANSFORMERS_PACKAGE];
|
|
150
|
-
if (!versionSpec) throw new Error(`${TRANSFORMERS_PACKAGE} is missing from package.json optionalDependencies`);
|
|
151
|
-
if (!versionSpec.startsWith("catalog:")) return versionSpec;
|
|
152
|
-
if (COMPILED_TRANSFORMERS_VERSION) return COMPILED_TRANSFORMERS_VERSION;
|
|
153
|
-
const installed = sourceRequire(`${TRANSFORMERS_PACKAGE}/package.json`) as { version: string };
|
|
154
|
-
return installed.version;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
/**
|
|
158
|
-
* Lazily resolve (and memoize) the transformers version spec. In the `catalog:`
|
|
159
|
-
* case this `require`s the installed package manifest, so defer it to the
|
|
160
|
-
* compiled-binary runtime-install path (only reached on a real transcribe /
|
|
161
|
-
* download) — loading this worker for a smoke ping never triggers the resolve.
|
|
162
|
-
*/
|
|
163
|
-
function getTransformersVersionSpec(): string {
|
|
164
|
-
cachedTransformersVersionSpec ??= resolveTransformersVersionSpec();
|
|
165
|
-
return cachedTransformersVersionSpec;
|
|
166
|
-
}
|
|
146
|
+
const transformersRuntime = new MemoizedRuntime<TransformersRuntime>();
|
|
147
|
+
const sherpaRuntime = new MemoizedRuntime<SherpaRuntime>();
|
|
167
148
|
|
|
168
149
|
let cachedSherpaVersionSpec: string | undefined;
|
|
169
150
|
function resolveSherpaVersionSpec(): string {
|
|
@@ -181,23 +162,6 @@ function getSherpaVersionSpec(): string {
|
|
|
181
162
|
return cachedSherpaVersionSpec;
|
|
182
163
|
}
|
|
183
164
|
|
|
184
|
-
function errorText(error: unknown): string {
|
|
185
|
-
return error instanceof Error ? (error.stack ?? error.message) : String(error);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
function errorMessage(error: unknown): string {
|
|
189
|
-
return error instanceof Error ? error.message : String(error);
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
function sendLog(
|
|
193
|
-
transport: SttTransport,
|
|
194
|
-
level: "debug" | "warn" | "error",
|
|
195
|
-
msg: string,
|
|
196
|
-
meta?: Record<string, unknown>,
|
|
197
|
-
): void {
|
|
198
|
-
transport.send({ type: "log", level, msg, meta });
|
|
199
|
-
}
|
|
200
|
-
|
|
201
165
|
function getSttRuntimeDir(): string {
|
|
202
166
|
const key = getTransformersVersionSpec().replace(/[^A-Za-z0-9._-]/g, "_");
|
|
203
167
|
return path.join(path.dirname(getTinyModelsCacheDir()), "stt-runtime", `transformers-${key}`);
|
|
@@ -208,73 +172,6 @@ function getSherpaRuntimeDir(): string {
|
|
|
208
172
|
return path.join(path.dirname(getTinyModelsCacheDir()), "stt-runtime", `sherpa-${key}`);
|
|
209
173
|
}
|
|
210
174
|
|
|
211
|
-
function sendRuntimeInstallProgress(
|
|
212
|
-
transport: SttTransport,
|
|
213
|
-
requestId: string,
|
|
214
|
-
modelKey: SttModelKey,
|
|
215
|
-
status: "initiate" | "download" | "done",
|
|
216
|
-
name: string,
|
|
217
|
-
): void {
|
|
218
|
-
transport.send({ type: "progress", id: requestId, event: { modelKey, status, name } });
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/**
|
|
222
|
-
* Prepare the freshly-installed compiled runtime for loading: stub `sharp` (the
|
|
223
|
-
* speech pipeline is audio-only, so the native image codec is dead weight) and
|
|
224
|
-
* patch the module resolver so Transformers.js's bare requires resolve against
|
|
225
|
-
* the cache. Returns the absolute Transformers.js entrypoint to `require`.
|
|
226
|
-
*/
|
|
227
|
-
async function prepareCompiledRuntime(runtimeDir: string): Promise<string> {
|
|
228
|
-
const nodeModules = path.join(runtimeDir, "node_modules");
|
|
229
|
-
const sharpStub = path.join(runtimeDir, "omp-sharp-stub.cjs");
|
|
230
|
-
await Bun.write(sharpStub, "module.exports = {};\n");
|
|
231
|
-
installRuntimeModuleResolver({ runtimeNodeModules: nodeModules, stubs: { sharp: sharpStub } });
|
|
232
|
-
const entry = resolveRuntimeModule(nodeModules, TRANSFORMERS_PACKAGE);
|
|
233
|
-
if (!entry) throw new Error(`Unable to resolve ${TRANSFORMERS_PACKAGE} in compiled runtime at ${nodeModules}`);
|
|
234
|
-
return entry;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function configureTransformers(transformers: TransformersRuntime): TransformersRuntime {
|
|
238
|
-
transformers.env.cacheDir = getTinyModelsCacheDir();
|
|
239
|
-
transformers.env.allowLocalModels = false;
|
|
240
|
-
transformers.env.logLevel = transformers.LogLevel.ERROR;
|
|
241
|
-
return transformers;
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
async function loadTransformers(
|
|
245
|
-
transport: SttTransport,
|
|
246
|
-
requestId: string,
|
|
247
|
-
modelKey: SttModelKey,
|
|
248
|
-
): Promise<TransformersRuntime> {
|
|
249
|
-
if (transformersRuntime) return transformersRuntime;
|
|
250
|
-
transformersRuntime = (async () => {
|
|
251
|
-
if (!isCompiledBinary()) return configureTransformers(sourceRequire(TRANSFORMERS_PACKAGE) as TransformersRuntime);
|
|
252
|
-
const runtimeDir = await ensureRuntimeInstalled({
|
|
253
|
-
runtimeDir: getSttRuntimeDir(),
|
|
254
|
-
install: {
|
|
255
|
-
dependencies: { [TRANSFORMERS_PACKAGE]: getTransformersVersionSpec() },
|
|
256
|
-
trustedDependencies: ["onnxruntime-node"],
|
|
257
|
-
},
|
|
258
|
-
probePackage: TRANSFORMERS_PACKAGE,
|
|
259
|
-
onPhase: phase =>
|
|
260
|
-
sendRuntimeInstallProgress(
|
|
261
|
-
transport,
|
|
262
|
-
requestId,
|
|
263
|
-
modelKey,
|
|
264
|
-
phase,
|
|
265
|
-
`${TRANSFORMERS_PACKAGE}@${getTransformersVersionSpec()}`,
|
|
266
|
-
),
|
|
267
|
-
});
|
|
268
|
-
const entry = await prepareCompiledRuntime(runtimeDir);
|
|
269
|
-
const require_ = createRequire(entry);
|
|
270
|
-
return configureTransformers(require_(entry) as TransformersRuntime);
|
|
271
|
-
})().catch(error => {
|
|
272
|
-
transformersRuntime = null;
|
|
273
|
-
throw error;
|
|
274
|
-
});
|
|
275
|
-
return transformersRuntime;
|
|
276
|
-
}
|
|
277
|
-
|
|
278
175
|
/**
|
|
279
176
|
* Resolve the native `sherpa-onnx-node` module. In a compiled binary the addon
|
|
280
177
|
* (plus its per-platform prebuilt `sherpa-onnx.node` + bundled onnxruntime
|
|
@@ -283,69 +180,25 @@ async function loadTransformers(
|
|
|
283
180
|
* is enough — no module-resolver patch or bare-require stubbing is needed.
|
|
284
181
|
* Memoized so the runtime loads once per process.
|
|
285
182
|
*/
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
requestId: string,
|
|
289
|
-
modelKey: SttModelKey,
|
|
290
|
-
): Promise<SherpaRuntime> {
|
|
291
|
-
if (sherpaRuntime) return sherpaRuntime;
|
|
292
|
-
sherpaRuntime = (async () => {
|
|
183
|
+
function loadSherpaRuntime(transport: SttTransport, requestId: string, modelKey: SttModelKey): Promise<SherpaRuntime> {
|
|
184
|
+
return sherpaRuntime.load(async () => {
|
|
293
185
|
if (!isCompiledBinary()) return sourceRequire(SHERPA_PACKAGE) as SherpaRuntime;
|
|
294
186
|
const runtimeDir = await ensureRuntimeInstalled({
|
|
295
187
|
runtimeDir: getSherpaRuntimeDir(),
|
|
296
188
|
install: { dependencies: { [SHERPA_PACKAGE]: getSherpaVersionSpec() } },
|
|
297
189
|
probePackage: SHERPA_PACKAGE,
|
|
298
190
|
onPhase: phase =>
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
requestId,
|
|
302
|
-
modelKey,
|
|
303
|
-
|
|
304
|
-
`${SHERPA_PACKAGE}@${getSherpaVersionSpec()}`,
|
|
305
|
-
),
|
|
191
|
+
transport.send({
|
|
192
|
+
type: "progress",
|
|
193
|
+
id: requestId,
|
|
194
|
+
event: { modelKey, status: phase, name: `${SHERPA_PACKAGE}@${getSherpaVersionSpec()}` },
|
|
195
|
+
}),
|
|
306
196
|
});
|
|
307
197
|
const nodeModules = path.join(runtimeDir, "node_modules");
|
|
308
198
|
const entry = resolveRuntimeModule(nodeModules, SHERPA_PACKAGE);
|
|
309
199
|
if (!entry) throw new Error(`Unable to resolve ${SHERPA_PACKAGE} in compiled runtime at ${nodeModules}`);
|
|
310
200
|
return createRequire(entry)(entry) as SherpaRuntime;
|
|
311
|
-
})().catch(error => {
|
|
312
|
-
sherpaRuntime = null;
|
|
313
|
-
throw error;
|
|
314
201
|
});
|
|
315
|
-
return sherpaRuntime;
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
function toProgressEvent(modelKey: SttModelKey, info: ProgressInfo): SttProgressEvent {
|
|
319
|
-
if (info.status === "ready") {
|
|
320
|
-
return { modelKey, status: info.status, task: info.task, model: info.model };
|
|
321
|
-
}
|
|
322
|
-
if (info.status === "progress_total") {
|
|
323
|
-
return {
|
|
324
|
-
modelKey,
|
|
325
|
-
status: info.status,
|
|
326
|
-
name: info.name,
|
|
327
|
-
progress: info.progress,
|
|
328
|
-
loaded: info.loaded,
|
|
329
|
-
total: info.total,
|
|
330
|
-
files: info.files,
|
|
331
|
-
};
|
|
332
|
-
}
|
|
333
|
-
if (info.status === "progress") {
|
|
334
|
-
return {
|
|
335
|
-
modelKey,
|
|
336
|
-
status: info.status,
|
|
337
|
-
name: info.name,
|
|
338
|
-
file: info.file,
|
|
339
|
-
progress: info.progress,
|
|
340
|
-
loaded: info.loaded,
|
|
341
|
-
total: info.total,
|
|
342
|
-
};
|
|
343
|
-
}
|
|
344
|
-
return { modelKey, status: info.status, name: info.name, file: info.file };
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
function sendProgress(transport: SttTransport, id: string, modelKey: SttModelKey, info: ProgressInfo): void {
|
|
348
|
-
transport.send({ type: "progress", id, event: toProgressEvent(modelKey, info) });
|
|
349
202
|
}
|
|
350
203
|
|
|
351
204
|
async function loadPipelineOnDevice(
|
|
@@ -407,7 +260,13 @@ async function loadTransformersModel(
|
|
|
407
260
|
transport: SttTransport,
|
|
408
261
|
requestId: string,
|
|
409
262
|
): Promise<LoadedModel> {
|
|
410
|
-
const transformers = await
|
|
263
|
+
const transformers = await loadTransformersRuntime(
|
|
264
|
+
transformersRuntime,
|
|
265
|
+
transport,
|
|
266
|
+
requestId,
|
|
267
|
+
modelKey,
|
|
268
|
+
getSttRuntimeDir,
|
|
269
|
+
);
|
|
411
270
|
const startedAt = performance.now();
|
|
412
271
|
const { pipeline, device } = await loadPipelineWithDeviceFallback(
|
|
413
272
|
transformers,
|
|
@@ -548,19 +407,8 @@ async function loadSherpaModel(
|
|
|
548
407
|
async function loadModel(modelKey: SttModelKey, transport: SttTransport, requestId: string): Promise<LoadedModel> {
|
|
549
408
|
const spec = getSttModelSpec(modelKey);
|
|
550
409
|
if (!spec) throw new Error(`Unknown stt model: ${modelKey}`);
|
|
551
|
-
const cached = models.
|
|
552
|
-
if (cached)
|
|
553
|
-
void cached
|
|
554
|
-
.then(() => {
|
|
555
|
-
transport.send({
|
|
556
|
-
type: "progress",
|
|
557
|
-
id: requestId,
|
|
558
|
-
event: { modelKey, status: "ready", task: ASR_TASK, model: spec.repo },
|
|
559
|
-
});
|
|
560
|
-
})
|
|
561
|
-
.catch(() => undefined);
|
|
562
|
-
return cached;
|
|
563
|
-
}
|
|
410
|
+
const cached = replayCachedReady(models, modelKey, transport, requestId, ASR_TASK, spec.repo);
|
|
411
|
+
if (cached) return cached;
|
|
564
412
|
|
|
565
413
|
const loading =
|
|
566
414
|
spec.engine === "sherpa"
|