@pellux/goodvibes-tui 1.26.0 → 1.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/docs/foundation-artifacts/operator-contract.json +3756 -1110
- package/package.json +8 -7
- package/scripts/postinstall.js +57 -1
- package/src/audio/activation-sound.ts +117 -0
- package/src/audio/capture.ts +112 -0
- package/src/audio/onnxruntime-assets.d.ts +24 -0
- package/src/audio/voice-capture-wiring.ts +119 -0
- package/src/audio/voice-input-session.ts +181 -0
- package/src/audio/wake-inference.ts +182 -0
- package/src/audio/wake-runtime.ts +341 -0
- package/src/cli/bundle-command.ts +4 -1
- package/src/cli/management-utils.ts +4 -1
- package/src/core/voice-capture-status.ts +53 -0
- package/src/core/voice-stt-gateway.ts +75 -0
- package/src/core/wake-provision-runner.ts +86 -0
- package/src/core/wake-provision-status.ts +170 -0
- package/src/daemon/cli.ts +38 -1
- package/src/daemon/provision-wake-model.ts +81 -0
- package/src/input/command-registry.ts +6 -0
- package/src/input/commands/experience-runtime.ts +37 -3
- package/src/input/handler-shortcuts.ts +8 -0
- package/src/input/keybindings.ts +10 -1
- package/src/input/onboarding/onboarding-feature-units.ts +5 -5
- package/src/input/settings-modal-types.ts +14 -1
- package/src/main.ts +12 -14
- package/src/renderer/process-indicator.ts +66 -0
- package/src/renderer/settings-modal-helpers.ts +4 -2
- package/src/renderer/shell-surface.ts +22 -3
- package/src/runtime/bootstrap-core.ts +7 -7
- package/src/runtime/bootstrap.ts +1 -1
- package/src/runtime/device-posture-composition.ts +110 -0
- package/src/runtime/disposal-wiring.ts +16 -0
- package/src/runtime/runtime-services-types.ts +17 -0
- package/src/runtime/services.ts +81 -4
- package/src/runtime/voice-setup-services.ts +50 -4
- package/src/shell/voice-capture-shell.ts +69 -0
- package/src/verification/verification-ledger.ts +59 -20
- package/src/version.ts +1 -1
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* wake-inference.ts — the terminal's inference runtime for the wake engine.
|
|
3
|
+
*
|
|
4
|
+
* The SDK engine never imports an inference runtime; it declares the shape of a
|
|
5
|
+
* session ({@link WakeInferenceSession}) and takes one from the host. This is
|
|
6
|
+
* that host half: onnxruntime-web on its WASM backend, which serves the browser
|
|
7
|
+
* tab and bun/node from the same package with no native module to build.
|
|
8
|
+
*
|
|
9
|
+
* WHY THE TWO ASSETS ARE COPIED TO DISK
|
|
10
|
+
*
|
|
11
|
+
* onnxruntime-web loads its WASM glue by DYNAMIC IMPORT of a path, and a
|
|
12
|
+
* `bun build --compile` binary cannot satisfy that: the runtime asks for
|
|
13
|
+
* `/$bunfs/root/ort-wasm-simd-threaded.mjs`, which is not something bun embeds
|
|
14
|
+
* for a dynamic path import, and session creation fails at runtime with
|
|
15
|
+
* "Cannot find module". So both assets are imported with `with { type: 'file' }`
|
|
16
|
+
* — embedded in the binary, handed to us as a path — written into a directory
|
|
17
|
+
* this surface owns, and `ort.env.wasm.wasmPaths` is pointed at that directory.
|
|
18
|
+
* The same code path runs from source, where the imports resolve to the real
|
|
19
|
+
* files in node_modules and the copy is a no-op after the first launch.
|
|
20
|
+
*
|
|
21
|
+
* The copy is content-checked, not timestamp-checked: an identical byte length
|
|
22
|
+
* and identical bytes mean the file already on disk is the one this build wants,
|
|
23
|
+
* so nothing is rewritten. A different build writes its own bytes over the top
|
|
24
|
+
* rather than trusting a stale extraction from a previous version.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
28
|
+
import { join } from 'node:path';
|
|
29
|
+
import * as ort from 'onnxruntime-web/wasm';
|
|
30
|
+
import glueAssetPath from 'onnxruntime-web/ort-wasm-simd-threaded.mjs' with { type: 'file' };
|
|
31
|
+
import wasmAssetPath from 'onnxruntime-web/ort-wasm-simd-threaded.wasm' with { type: 'file' };
|
|
32
|
+
import type {
|
|
33
|
+
WakeInferenceSession,
|
|
34
|
+
WakeModelHandle,
|
|
35
|
+
WakeRuntimeSettings,
|
|
36
|
+
WakeTensor,
|
|
37
|
+
} from '@pellux/goodvibes-sdk/platform/voice';
|
|
38
|
+
import { WakeWordEngine } from '@pellux/goodvibes-sdk/platform/voice';
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The runtime asks for its glue and its wasm by these exact names, relative to
|
|
42
|
+
* `ort.env.wasm.wasmPaths` — so the extracted copies must keep them.
|
|
43
|
+
*/
|
|
44
|
+
const ORT_ASSETS = [
|
|
45
|
+
{ source: glueAssetPath, name: 'ort-wasm-simd-threaded.mjs' },
|
|
46
|
+
{ source: wasmAssetPath, name: 'ort-wasm-simd-threaded.wasm' },
|
|
47
|
+
] as const;
|
|
48
|
+
|
|
49
|
+
/** Set once, before the first session is created; changing it afterwards has no effect. */
|
|
50
|
+
let configuredAssetDirectory: string | null = null;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Copy the onnxruntime-web assets into `directory` and return the value
|
|
54
|
+
* `ort.env.wasm.wasmPaths` wants — a directory prefix WITH its trailing slash,
|
|
55
|
+
* which the runtime concatenates a file name onto. Without the slash it looks
|
|
56
|
+
* for a sibling of the directory and reports a missing module.
|
|
57
|
+
*/
|
|
58
|
+
export function extractOnnxRuntimeAssets(directory: string): string {
|
|
59
|
+
mkdirSync(directory, { recursive: true });
|
|
60
|
+
for (const asset of ORT_ASSETS) {
|
|
61
|
+
const target = join(directory, asset.name);
|
|
62
|
+
const bytes = readFileSync(asset.source);
|
|
63
|
+
let existing: Buffer | null = null;
|
|
64
|
+
try {
|
|
65
|
+
existing = readFileSync(target);
|
|
66
|
+
} catch {
|
|
67
|
+
// Absent, or unreadable: either way it gets written below.
|
|
68
|
+
}
|
|
69
|
+
if (existing !== null && existing.equals(bytes)) continue;
|
|
70
|
+
writeFileSync(target, bytes);
|
|
71
|
+
}
|
|
72
|
+
return `${directory}/`;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Point onnxruntime-web at the extracted assets and pin its threading.
|
|
77
|
+
*
|
|
78
|
+
* `numThreads = 1` is not a conservatism: the engine costs ~3.5 ms per 80 ms
|
|
79
|
+
* frame single-threaded, comfortably inside its budget, while a threaded WASM
|
|
80
|
+
* build needs a cross-origin-isolated environment to get its workers and would
|
|
81
|
+
* spend the difference failing to start them.
|
|
82
|
+
*/
|
|
83
|
+
export function configureOnnxRuntime(assetDirectory: string): void {
|
|
84
|
+
if (configuredAssetDirectory === assetDirectory) return;
|
|
85
|
+
ort.env.wasm.wasmPaths = extractOnnxRuntimeAssets(assetDirectory);
|
|
86
|
+
ort.env.wasm.numThreads = 1;
|
|
87
|
+
ort.env.logLevel = 'error';
|
|
88
|
+
configuredAssetDirectory = assetDirectory;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Adapt an onnxruntime session to the SDK's session shape.
|
|
93
|
+
*
|
|
94
|
+
* The engine reads `inputNames[0]` / `outputNames[0]` rather than hard-coded
|
|
95
|
+
* names, so the pinned classifier (`onnx::Flatten_0` -> `output`) and the
|
|
96
|
+
* speech-embedding front end (`input_1` -> `embedding`) both work through this
|
|
97
|
+
* one adapter with nothing per-model here.
|
|
98
|
+
*/
|
|
99
|
+
export function adaptOnnxSession(session: ort.InferenceSession): WakeInferenceSession {
|
|
100
|
+
return {
|
|
101
|
+
inputNames: session.inputNames,
|
|
102
|
+
outputNames: session.outputNames,
|
|
103
|
+
async run(feeds: Readonly<Record<string, WakeTensor>>): Promise<Readonly<Record<string, WakeTensor>>> {
|
|
104
|
+
const inputs: Record<string, ort.Tensor> = {};
|
|
105
|
+
for (const [name, tensor] of Object.entries(feeds)) {
|
|
106
|
+
inputs[name] = new ort.Tensor('float32', tensor.data, [...tensor.dims]);
|
|
107
|
+
}
|
|
108
|
+
const outputs = await session.run(inputs);
|
|
109
|
+
const result: Record<string, WakeTensor> = {};
|
|
110
|
+
for (const [name, tensor] of Object.entries(outputs)) {
|
|
111
|
+
// Every tensor the wake graphs produce is float32; the cast narrows the
|
|
112
|
+
// runtime's union of typed-array data types to the one this path sees.
|
|
113
|
+
result[name] = { data: tensor.data as Float32Array, dims: [...tensor.dims] };
|
|
114
|
+
}
|
|
115
|
+
return result;
|
|
116
|
+
},
|
|
117
|
+
release: () => session.release(),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Load one .onnx file into an adapted session. */
|
|
122
|
+
export async function loadWakeSession(modelPath: string): Promise<WakeInferenceSession> {
|
|
123
|
+
const bytes = readFileSync(modelPath);
|
|
124
|
+
const session = await ort.InferenceSession.create(new Uint8Array(bytes), {
|
|
125
|
+
executionProviders: ['wasm'],
|
|
126
|
+
});
|
|
127
|
+
return adaptOnnxSession(session);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface WakeEngineFactoryOptions {
|
|
131
|
+
/** Directory this surface owns for the extracted onnxruntime assets. */
|
|
132
|
+
readonly assetDirectory: string;
|
|
133
|
+
/** The provisioned speech-embedding front end. */
|
|
134
|
+
readonly embeddingPath: string;
|
|
135
|
+
/** The provisioned classifiers, in configuration order. */
|
|
136
|
+
readonly models: readonly { readonly id: string; readonly path: string }[];
|
|
137
|
+
readonly settings: Pick<WakeRuntimeSettings, 'tuning' | 'preRollMs' | 'vadThreshold'>;
|
|
138
|
+
/**
|
|
139
|
+
* The provisioned speech gate, present only when `voice.wake.vadThreshold` asks
|
|
140
|
+
* for one. Absent means the engine scores every frame, which is what the
|
|
141
|
+
* shipped default of 0 asks for — not a gate that failed to load, because that
|
|
142
|
+
* case is a startup blocker upstream of here.
|
|
143
|
+
*/
|
|
144
|
+
readonly vadPath?: string;
|
|
145
|
+
/** Where a classifier that fails to run is reported (the engine skips it rather than dying). */
|
|
146
|
+
readonly warn: (message: string, meta?: Readonly<Record<string, unknown>>) => void;
|
|
147
|
+
/** Injected in tests: a stub session per path, so no real model file is read. */
|
|
148
|
+
readonly loadSession?: (modelPath: string) => Promise<WakeInferenceSession>;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Build the `createEngine` callback the SDK listener drives.
|
|
153
|
+
*
|
|
154
|
+
* The listener calls it on EVERY start including a restart, because a restart
|
|
155
|
+
* exists to recover from a runtime that died — so the sessions are created here
|
|
156
|
+
* per call rather than captured once.
|
|
157
|
+
*/
|
|
158
|
+
export function createWakeEngineFactory(options: WakeEngineFactoryOptions): () => Promise<WakeWordEngine> {
|
|
159
|
+
const load = options.loadSession ?? loadWakeSession;
|
|
160
|
+
return async () => {
|
|
161
|
+
if (options.loadSession === undefined) configureOnnxRuntime(options.assetDirectory);
|
|
162
|
+
const embedding = await load(options.embeddingPath);
|
|
163
|
+
const models: WakeModelHandle[] = [];
|
|
164
|
+
for (const model of options.models) {
|
|
165
|
+
models.push({ id: model.id, session: await load(model.path) });
|
|
166
|
+
}
|
|
167
|
+
// Loaded per start, like every other session here: a restart exists to
|
|
168
|
+
// recover from a runtime that died, and a gate session carried over from the
|
|
169
|
+
// dead run would be the one piece that did not recover.
|
|
170
|
+
const vad = options.vadPath !== undefined && options.settings.vadThreshold > 0
|
|
171
|
+
? { session: await load(options.vadPath), threshold: options.settings.vadThreshold }
|
|
172
|
+
: undefined;
|
|
173
|
+
return new WakeWordEngine({
|
|
174
|
+
embedding,
|
|
175
|
+
models,
|
|
176
|
+
tuning: options.settings.tuning,
|
|
177
|
+
preRollMs: options.settings.preRollMs,
|
|
178
|
+
...(vad !== undefined ? { vad } : {}),
|
|
179
|
+
warn: options.warn,
|
|
180
|
+
});
|
|
181
|
+
};
|
|
182
|
+
}
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* wake-runtime.ts — wake-word detection on the terminal.
|
|
3
|
+
*
|
|
4
|
+
* The SDK owns the policy that would otherwise be rewritten per surface: the
|
|
5
|
+
* front end, the patience/cooldown rules, keeping ONE stream open across a wake
|
|
6
|
+
* so the utterance that follows is not clipped, resetting the engine afterwards
|
|
7
|
+
* so the command just spoken is not scored again, and the supervisor's restart
|
|
8
|
+
* and latch decisions. What is local is everything that touches this machine —
|
|
9
|
+
* the recorder subprocess (capture.ts), the inference runtime (wake-inference.ts),
|
|
10
|
+
* transcription over the daemon (core/voice-stt-gateway.ts), the activation sound, and
|
|
11
|
+
* the shell's indicator row.
|
|
12
|
+
*
|
|
13
|
+
* TWO THINGS ARE DELIBERATELY NOT AUTOMATIC
|
|
14
|
+
*
|
|
15
|
+
* - **ENABLING NEVER DOWNLOADS.** The model ships with the installation, so on a
|
|
16
|
+
* normal machine `start()` below finds the artifacts already there and needs
|
|
17
|
+
* nothing else — that is the whole point of provisioning at install time. What
|
|
18
|
+
* it must never do is FETCH: the check here is read-only and content-verified,
|
|
19
|
+
* and a host whose artifacts are missing or torn gets told which and gets the
|
|
20
|
+
* recovery command named, instead of a switch that silently pulls 6 MB. A
|
|
21
|
+
* download is an install-time or boot-time act with a receipt; a toggle is not.
|
|
22
|
+
* - **Disabled means no device is opened at all.** `settings.active` is the only
|
|
23
|
+
* thing consulted before opening a microphone, and the SDK listener re-checks
|
|
24
|
+
* it and refuses without touching the capture opener. A configuration that is
|
|
25
|
+
* off must never produce a microphone permission prompt.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
29
|
+
import { join } from 'node:path';
|
|
30
|
+
import {
|
|
31
|
+
WakeListener,
|
|
32
|
+
encodeWavPcm16,
|
|
33
|
+
resolveManagedWakePaths,
|
|
34
|
+
resolveWakeModelFiles,
|
|
35
|
+
resolveWakeRuntimeSettings,
|
|
36
|
+
retainedClipFileName,
|
|
37
|
+
utteranceToAudioArtifact,
|
|
38
|
+
wakeProvisionStatus,
|
|
39
|
+
type AudioCaptureOpener,
|
|
40
|
+
type CapturedUtterance,
|
|
41
|
+
type WakeListenerState,
|
|
42
|
+
type WakeRuntimeSettings,
|
|
43
|
+
type WakeProvisionStatus,
|
|
44
|
+
type WakeListenerOptions,
|
|
45
|
+
} from '@pellux/goodvibes-sdk/platform/voice';
|
|
46
|
+
import type { VoiceCaptureIndicatorState } from '../core/voice-capture-status.ts';
|
|
47
|
+
import { describeWakeBlockers, describeWakeLimitations, terminalWakeCapabilities } from '../core/wake-provision-status.ts';
|
|
48
|
+
import { createWakeEngineFactory } from './wake-inference.ts';
|
|
49
|
+
import type { VoiceSttGateway } from '../core/voice-stt-gateway.ts';
|
|
50
|
+
import { describeTranscriptionFailure } from '../core/voice-stt-gateway.ts';
|
|
51
|
+
|
|
52
|
+
/** Indicator kind per listener phase. `idle`/`stopped` render nothing. */
|
|
53
|
+
const PHASE_INDICATOR: Partial<Record<WakeListenerState['phase'], VoiceCaptureIndicatorState['kind']>> = {
|
|
54
|
+
starting: 'wake-listening',
|
|
55
|
+
listening: 'wake-listening',
|
|
56
|
+
'capturing-utterance': 'wake-capturing',
|
|
57
|
+
restarting: 'wake-restarting',
|
|
58
|
+
latched: 'wake-latched',
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
export interface WakeRuntimeDeps {
|
|
62
|
+
/** Reads a config key. The live subscription below watches the two enablement rows. */
|
|
63
|
+
readonly readConfig: (key: string) => unknown;
|
|
64
|
+
/** Runtime toggling: `voice.wake.enabled` and `voice.wake.surfaces.tui`. */
|
|
65
|
+
readonly subscribeConfig: (key: string, listener: () => void) => () => void;
|
|
66
|
+
readonly openCapture: AudioCaptureOpener;
|
|
67
|
+
/** Managed root the wake tree hangs off — `<managed>/wake` (SDK resolveManagedWakePaths). */
|
|
68
|
+
readonly managedRoot: string;
|
|
69
|
+
/** Directory this surface owns for the extracted onnxruntime assets. */
|
|
70
|
+
readonly assetDirectory: string;
|
|
71
|
+
/** Resolves the speech-to-text gateway, or explains why there is none. */
|
|
72
|
+
readonly resolveTranscriber: () => { readonly available: true; readonly gateway: VoiceSttGateway } | { readonly available: false; readonly reason: string };
|
|
73
|
+
/** Plays the resolved activation sound at the moment of a wake. */
|
|
74
|
+
readonly playActivationSound: (sound: WakeRuntimeSettings['activationSound']) => void;
|
|
75
|
+
/** Submits the recognised text as a turn (`voice.wake.autoSubmit` on). */
|
|
76
|
+
readonly submitTurn: (text: string) => void;
|
|
77
|
+
/** Writes the recognised text into the composer draft (`voice.wake.autoSubmit` off). */
|
|
78
|
+
readonly writeDraft: (text: string) => void;
|
|
79
|
+
readonly notify: (message: string) => void;
|
|
80
|
+
readonly render: () => void;
|
|
81
|
+
/** Names the retained clip so the SDK sweeper can reap it when this session ends. */
|
|
82
|
+
readonly sessionId: string;
|
|
83
|
+
readonly warn: (message: string, meta?: Readonly<Record<string, unknown>>) => void;
|
|
84
|
+
/** Injected in tests: a stub inference session per model path. */
|
|
85
|
+
readonly loadSession?: WakeRuntimeTestSeams['loadSession'];
|
|
86
|
+
/** Injected in tests: skips the on-disk provisioning check. */
|
|
87
|
+
readonly provisionStatus?: WakeRuntimeTestSeams['provisionStatus'];
|
|
88
|
+
/** Injected in tests: the suppression stage, without real WebAssembly. */
|
|
89
|
+
readonly createNoiseSuppression?: WakeRuntimeTestSeams['createNoiseSuppression'];
|
|
90
|
+
readonly now?: () => number;
|
|
91
|
+
readonly setTimeout?: (handler: () => void, ms: number) => unknown;
|
|
92
|
+
readonly clearTimeout?: (handle: unknown) => void;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* What this runtime reads out of provisioning: whether the wake models are usable
|
|
97
|
+
* at all, and whether the SPEECH GATE's own artifact is there — they are separate
|
|
98
|
+
* answers, because `voice.wake.vadThreshold` defaults to 0 and a detector runs
|
|
99
|
+
* perfectly well with no gate provisioned.
|
|
100
|
+
*/
|
|
101
|
+
export interface WakeProvisionSnapshot {
|
|
102
|
+
readonly ready: boolean;
|
|
103
|
+
readonly reason: string | null;
|
|
104
|
+
readonly vadReady: boolean;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The two seams a test replaces so no model file and no runtime are needed. */
|
|
108
|
+
export interface WakeRuntimeTestSeams {
|
|
109
|
+
readonly loadSession: Parameters<typeof createWakeEngineFactory>[0]['loadSession'];
|
|
110
|
+
/**
|
|
111
|
+
* Builds the suppression stage. The SDK's listener wraps the opener and defaults
|
|
112
|
+
* to the embedded speexdsp filter; a test replaces it so the WIRING is checkable
|
|
113
|
+
* without compiling WebAssembly, while the filter's numbers stay the SDK's to
|
|
114
|
+
* assert.
|
|
115
|
+
*/
|
|
116
|
+
readonly createNoiseSuppression: NonNullable<WakeListenerOptions['createNoiseSuppression']>;
|
|
117
|
+
readonly provisionStatus: (managedRoot: string) => WakeProvisionSnapshot;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface WakeRuntime {
|
|
121
|
+
/**
|
|
122
|
+
* Apply the current configuration: start the detector when it should run, stop
|
|
123
|
+
* it when it should not. Called once at startup and on every change to the two
|
|
124
|
+
* enablement rows.
|
|
125
|
+
*/
|
|
126
|
+
refresh(): Promise<void>;
|
|
127
|
+
/** Current indicator row, or null when nothing is listening. */
|
|
128
|
+
status(): VoiceCaptureIndicatorState | null;
|
|
129
|
+
/** The resolved settings, for a status surface. */
|
|
130
|
+
settings(): WakeRuntimeSettings;
|
|
131
|
+
/** Stop and release the device. Registered in the shell's teardown registry. */
|
|
132
|
+
stop(): Promise<void>;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Wire wake detection. Nothing is opened until `refresh()` finds it active. */
|
|
136
|
+
/**
|
|
137
|
+
* Provisioning state for the capability read. Shares the injected seam `start`
|
|
138
|
+
* uses, so a test drives both from one stub, and is read per call rather than
|
|
139
|
+
* cached: a user provisions the speech gate while the session is running, and a
|
|
140
|
+
* snapshot taken at startup would keep reporting it unavailable afterwards.
|
|
141
|
+
*/
|
|
142
|
+
function provisionStatusFor(deps: WakeRuntimeDeps): WakeProvisionSnapshot {
|
|
143
|
+
return (deps.provisionStatus ?? ((root: string) => wakeProvisionStatus({ managedRoot: root })))(deps.managedRoot);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export function wireWakeRuntime(deps: WakeRuntimeDeps): WakeRuntime {
|
|
147
|
+
let listener: WakeListener | null = null;
|
|
148
|
+
let phase: WakeListenerState['phase'] = 'idle';
|
|
149
|
+
let deviceLabel: string | null = null;
|
|
150
|
+
let detail: string | undefined;
|
|
151
|
+
|
|
152
|
+
const resolve = (): WakeRuntimeSettings => resolveWakeRuntimeSettings(
|
|
153
|
+
deps.readConfig,
|
|
154
|
+
'tui',
|
|
155
|
+
terminalWakeCapabilities(provisionStatusFor(deps)),
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
const status = (): VoiceCaptureIndicatorState | null => {
|
|
159
|
+
const kind = PHASE_INDICATOR[phase];
|
|
160
|
+
if (kind === undefined) return null;
|
|
161
|
+
return { kind, deviceLabel, indicator: resolve().indicator, ...(detail !== undefined ? { detail } : {}) };
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const retainClip = (utterance: CapturedUtterance): void => {
|
|
165
|
+
const retainedDir = resolveManagedWakePaths(deps.managedRoot).retainedDir;
|
|
166
|
+
try {
|
|
167
|
+
mkdirSync(retainedDir, { recursive: true });
|
|
168
|
+
// The SDK owns the filename, and it is load-bearing rather than cosmetic: the
|
|
169
|
+
// recovery sweeper reads the owning session id from the first `--`-delimited
|
|
170
|
+
// segment, so a name invented here would get a live session's clips reaped as
|
|
171
|
+
// orphans (or leave them unreaped forever).
|
|
172
|
+
const target = join(retainedDir, retainedClipFileName(deps.sessionId, (deps.now ?? Date.now)()));
|
|
173
|
+
writeFileSync(target, encodeWavPcm16(utterance.samples, utterance.sampleRate));
|
|
174
|
+
} catch (error) {
|
|
175
|
+
deps.warn('retaining the wake clip failed', { error: error instanceof Error ? error.message : String(error) });
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
const transcribeUtterance = async (utterance: CapturedUtterance, settings: WakeRuntimeSettings): Promise<void> => {
|
|
180
|
+
if (settings.retainAudio === 'session-temp') retainClip(utterance);
|
|
181
|
+
if (utterance.silent) {
|
|
182
|
+
deps.notify('[Wake] Heard the wake phrase but nothing above the silence floor followed it.');
|
|
183
|
+
deps.render();
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
const resolution = deps.resolveTranscriber();
|
|
187
|
+
if (!resolution.available) {
|
|
188
|
+
deps.notify(`[Wake] Captured the utterance after the wake phrase but could not transcribe it: ${resolution.reason}`);
|
|
189
|
+
deps.render();
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
try {
|
|
193
|
+
const text = (await resolution.gateway.transcribe(utteranceToAudioArtifact(utterance))).trim();
|
|
194
|
+
if (text.length === 0) {
|
|
195
|
+
deps.notify('[Wake] Speech-to-text returned no words for what followed the wake phrase.');
|
|
196
|
+
deps.render();
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
if (settings.autoSubmit) deps.submitTurn(text);
|
|
200
|
+
else {
|
|
201
|
+
deps.writeDraft(text);
|
|
202
|
+
deps.notify('[Wake] Transcript placed in the composer (voice.wake.autoSubmit is off — press Enter to send it).');
|
|
203
|
+
deps.render();
|
|
204
|
+
}
|
|
205
|
+
} catch (error) {
|
|
206
|
+
deps.notify(`[Wake] Transcription failed: ${describeTranscriptionFailure(error)}`);
|
|
207
|
+
deps.render();
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
const start = async (settings: WakeRuntimeSettings): Promise<void> => {
|
|
212
|
+
const provision = provisionStatusFor(deps);
|
|
213
|
+
if (!provision.ready) {
|
|
214
|
+
deps.notify(
|
|
215
|
+
`[Wake] voice.wake.enabled is on, but the wake models are not provisioned (${provision.reason ?? 'not-provisioned'}), `
|
|
216
|
+
+ 'so nothing is listening. Installing goodvibes provisions them and a running daemon retries at boot, so this '
|
|
217
|
+
+ 'means the download has not succeeded yet — run /voice wake setup to fetch and verify them now.',
|
|
218
|
+
);
|
|
219
|
+
deps.render();
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
const paths = resolveManagedWakePaths(deps.managedRoot);
|
|
223
|
+
// The SDK resolves `voice.wake.models` to files: the pinned id inside the
|
|
224
|
+
// managed tree, any other id against voice.wake.customModelDir — and when that
|
|
225
|
+
// row is empty, against the managed `custom` directory, which is the fallback
|
|
226
|
+
// the row's description promises and the one a host would otherwise get wrong
|
|
227
|
+
// by looking in the process's working directory.
|
|
228
|
+
const modelFiles = resolveWakeModelFiles(settings.modelIds, {
|
|
229
|
+
managedRoot: deps.managedRoot,
|
|
230
|
+
customModelDir: settings.customModelDir,
|
|
231
|
+
});
|
|
232
|
+
const unpinned = modelFiles.filter((model) => !model.pinned);
|
|
233
|
+
if (unpinned.length > 0) {
|
|
234
|
+
// Worth saying out loud: these bytes were never checksum-verified.
|
|
235
|
+
deps.notify(`[Wake] Loading ${unpinned.length} custom model${unpinned.length === 1 ? '' : 's'} that ${unpinned.length === 1 ? 'is' : 'are'} not checksum-pinned: ${unpinned.map((model) => `${model.id} (${model.path})`).join(', ')}`);
|
|
236
|
+
}
|
|
237
|
+
const active = new WakeListener({
|
|
238
|
+
settings,
|
|
239
|
+
openCapture: deps.openCapture,
|
|
240
|
+
...(deps.createNoiseSuppression !== undefined ? { createNoiseSuppression: deps.createNoiseSuppression } : {}),
|
|
241
|
+
createEngine: createWakeEngineFactory({
|
|
242
|
+
assetDirectory: deps.assetDirectory,
|
|
243
|
+
embeddingPath: paths.embeddingPath,
|
|
244
|
+
models: modelFiles.map((model) => ({ id: model.id, path: model.path })),
|
|
245
|
+
settings,
|
|
246
|
+
// The speech gate, when the row asks for one. `voice.wake.vadThreshold`
|
|
247
|
+
// above 0 with the artifact missing never reaches here — it is a blocker
|
|
248
|
+
// from resolveWakeRuntimeSettings, so the detector does not start at all
|
|
249
|
+
// rather than starting with the gate quietly absent.
|
|
250
|
+
...(settings.vadThreshold > 0 ? { vadPath: paths.vadPath } : {}),
|
|
251
|
+
warn: deps.warn,
|
|
252
|
+
...(deps.loadSession !== undefined ? { loadSession: deps.loadSession } : {}),
|
|
253
|
+
}),
|
|
254
|
+
handlers: {
|
|
255
|
+
onStateChange: (state) => {
|
|
256
|
+
phase = state.phase;
|
|
257
|
+
deviceLabel = state.deviceLabel;
|
|
258
|
+
if (state.phase === 'latched') detail = state.latchReason ?? undefined;
|
|
259
|
+
deps.render();
|
|
260
|
+
},
|
|
261
|
+
onWake: (event) => {
|
|
262
|
+
deps.playActivationSound(event.activationSound);
|
|
263
|
+
deps.render();
|
|
264
|
+
},
|
|
265
|
+
onUtterance: (utterance) => { void transcribeUtterance(utterance, settings); },
|
|
266
|
+
onFailure: (error, restarting, failureDetail) => {
|
|
267
|
+
detail = failureDetail;
|
|
268
|
+
// Reported to the user, not only logged: a detector that stopped
|
|
269
|
+
// listening is exactly the thing a silent log entry hides.
|
|
270
|
+
deps.notify(restarting
|
|
271
|
+
? `[Wake] The capture stream ended (${error.message}) — ${failureDetail}.`
|
|
272
|
+
: `[Wake] The wake-word detector stopped: ${failureDetail}. It stays off until voice.wake.enabled is turned off and on again.`);
|
|
273
|
+
deps.render();
|
|
274
|
+
},
|
|
275
|
+
},
|
|
276
|
+
warn: deps.warn,
|
|
277
|
+
...(deps.now !== undefined ? { now: deps.now } : {}),
|
|
278
|
+
...(deps.setTimeout !== undefined ? { setTimeout: deps.setTimeout } : {}),
|
|
279
|
+
...(deps.clearTimeout !== undefined ? { clearTimeout: deps.clearTimeout } : {}),
|
|
280
|
+
});
|
|
281
|
+
listener = active;
|
|
282
|
+
const outcome = await active.start();
|
|
283
|
+
if (!outcome.started) {
|
|
284
|
+
listener = null;
|
|
285
|
+
phase = 'idle';
|
|
286
|
+
deps.notify(`[Wake] The wake-word detector did not start (${outcome.refusal}): ${outcome.detail}`);
|
|
287
|
+
} else {
|
|
288
|
+
const limitations = describeWakeLimitations(settings);
|
|
289
|
+
if (limitations.length > 0) {
|
|
290
|
+
deps.notify([`[Wake] Listening on ${outcome.deviceLabel}, with these rows not in force:`, ...limitations].join('\n'));
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
deps.render();
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
const stop = async (): Promise<void> => {
|
|
297
|
+
const active = listener;
|
|
298
|
+
listener = null;
|
|
299
|
+
phase = 'idle';
|
|
300
|
+
deviceLabel = null;
|
|
301
|
+
detail = undefined;
|
|
302
|
+
if (active !== null) await active.stop();
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
return {
|
|
306
|
+
refresh: async () => {
|
|
307
|
+
const settings = resolve();
|
|
308
|
+
if (!settings.active) {
|
|
309
|
+
await stop();
|
|
310
|
+
if (settings.enabled && settings.surfaceEnabled && settings.blockers.length > 0) {
|
|
311
|
+
// Blocked, not off: the user asked for this and is owed the reason.
|
|
312
|
+
deps.notify(['[Wake] The wake-word detector cannot start:', ...describeWakeBlockers(settings)].join('\n'));
|
|
313
|
+
}
|
|
314
|
+
deps.render();
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
if (listener !== null) return;
|
|
318
|
+
await start(settings);
|
|
319
|
+
},
|
|
320
|
+
status,
|
|
321
|
+
settings: resolve,
|
|
322
|
+
stop,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Subscribe to the two rows that turn detection on and off at runtime, and apply
|
|
328
|
+
* the current configuration once. Returns the unsubscribers for the shell's
|
|
329
|
+
* teardown registry, plus a final release so a live recorder subprocess cannot
|
|
330
|
+
* outlive the process on any exit path.
|
|
331
|
+
*/
|
|
332
|
+
export function startWakeRuntime(runtime: WakeRuntime, deps: Pick<WakeRuntimeDeps, 'subscribeConfig'>): readonly (() => void)[] {
|
|
333
|
+
const apply = (): void => { void runtime.refresh(); };
|
|
334
|
+
const unsubs = [
|
|
335
|
+
deps.subscribeConfig('voice.wake.enabled', apply),
|
|
336
|
+
deps.subscribeConfig('voice.wake.surfaces.tui', apply),
|
|
337
|
+
() => { void runtime.stop(); },
|
|
338
|
+
];
|
|
339
|
+
apply();
|
|
340
|
+
return unsubs;
|
|
341
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
|
-
import { RuntimeEventBus } from '@/runtime/index.ts';
|
|
3
|
+
import { RuntimeEventBus, configureRuntimeEventBusDefaults, runtimeEventBusOptionsFrom } from '@/runtime/index.ts';
|
|
4
4
|
import { createShellPathService } from '@/runtime/index.ts';
|
|
5
5
|
import { listProviderRuntimeSnapshots } from '@pellux/goodvibes-sdk/platform/providers';
|
|
6
6
|
import { createRuntimeServices } from '../runtime/services.ts';
|
|
@@ -83,6 +83,9 @@ function readAuthPosture(runtime: CliCommandRuntime) {
|
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
async function buildProviderReadiness(runtime: CliCommandRuntime) {
|
|
86
|
+
// Point the bus listener cap at runtime.eventBus.maxListeners before the
|
|
87
|
+
// first bus exists, so every bus this process builds later uses it.
|
|
88
|
+
configureRuntimeEventBusDefaults(runtimeEventBusOptionsFrom((key) => runtime.configManager.get(key)));
|
|
86
89
|
const runtimeBus = new RuntimeEventBus();
|
|
87
90
|
const runtimeStore = createRuntimeStore();
|
|
88
91
|
const services = createRuntimeServices({
|
|
@@ -24,7 +24,7 @@ import { refreshMemoryRecallSnapshot } from '../runtime/orchestrator-core-servic
|
|
|
24
24
|
import { createRuntimeServices } from '../runtime/services.ts';
|
|
25
25
|
import { createRuntimeStore } from '../runtime/store/index.ts';
|
|
26
26
|
import type { RuntimeServices } from '../runtime/services.ts';
|
|
27
|
-
import { RuntimeEventBus, type TurnEvent, createShellPathService } from '@/runtime/index.ts';
|
|
27
|
+
import { RuntimeEventBus, type TurnEvent, createShellPathService, configureRuntimeEventBusDefaults, runtimeEventBusOptionsFrom } from '@/runtime/index.ts';
|
|
28
28
|
import { summarizeError } from '@pellux/goodvibes-sdk/platform/utils';
|
|
29
29
|
import { resolveRuntimeEndpointBinding } from './endpoints.ts';
|
|
30
30
|
import { applyRuntimeEndpointFlagOverrides } from './config-overrides.ts';
|
|
@@ -225,6 +225,9 @@ export async function withRuntimeServices<T>(
|
|
|
225
225
|
runtime: CliCommandRuntime,
|
|
226
226
|
fn: (services: RuntimeServices) => Promise<T> | T,
|
|
227
227
|
): Promise<T> {
|
|
228
|
+
// Point the bus listener cap at runtime.eventBus.maxListeners before the
|
|
229
|
+
// first bus exists, so every bus this process builds later uses it.
|
|
230
|
+
configureRuntimeEventBusDefaults(runtimeEventBusOptionsFrom((key) => runtime.configManager.get(key)));
|
|
228
231
|
const runtimeBus = new RuntimeEventBus();
|
|
229
232
|
const runtimeStore = createRuntimeStore();
|
|
230
233
|
const services = createRuntimeServices({
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* voice-capture-status.ts — what the shell shows while a microphone is open.
|
|
3
|
+
*
|
|
4
|
+
* A capture indicator is not decoration. Two of the three states here exist
|
|
5
|
+
* because the alternative is a user who cannot tell whether their microphone is
|
|
6
|
+
* live: wake detection holds a device open for as long as the feature is on, and
|
|
7
|
+
* push-to-talk holds one open between two keypresses. `voice.wake.indicator`
|
|
8
|
+
* governs how prominent the detector's row is; a push-to-talk recording is
|
|
9
|
+
* always shown, because the user is mid-act and waiting for it.
|
|
10
|
+
*
|
|
11
|
+
* The shape lives here rather than in the renderer or in src/audio so both can
|
|
12
|
+
* use it: the audio layer must not import shell-UI (the architecture check
|
|
13
|
+
* enforces that), and the renderer must not own capture state.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** What a voice-capture row is reporting. */
|
|
17
|
+
export type VoiceCaptureIndicatorKind =
|
|
18
|
+
/** Push-to-talk: the user pressed the key and is speaking now. */
|
|
19
|
+
'recording'
|
|
20
|
+
/** Push-to-talk: the device is being opened (a permission prompt takes real time). */
|
|
21
|
+
| 'requesting'
|
|
22
|
+
/** Push-to-talk: capture ended and the audio is being transcribed. */
|
|
23
|
+
| 'transcribing'
|
|
24
|
+
/** Wake detection is listening and scoring frames. */
|
|
25
|
+
| 'wake-listening'
|
|
26
|
+
/** A wake confirmed; the utterance that followed it is being captured. */
|
|
27
|
+
| 'wake-capturing'
|
|
28
|
+
/** The detector's stream died and a restart is scheduled. */
|
|
29
|
+
| 'wake-restarting'
|
|
30
|
+
/** The supervisor gave up; the detector is off until the feature is toggled. */
|
|
31
|
+
| 'wake-latched';
|
|
32
|
+
|
|
33
|
+
/** One live capture row. Absent (null) means no microphone is open and no row renders. */
|
|
34
|
+
export interface VoiceCaptureIndicatorState {
|
|
35
|
+
readonly kind: VoiceCaptureIndicatorKind;
|
|
36
|
+
/** What opened the device, e.g. `parecord`; null before anything is open. */
|
|
37
|
+
readonly deviceLabel: string | null;
|
|
38
|
+
/**
|
|
39
|
+
* Prominence for the wake rows, from `voice.wake.indicator`. `off` suppresses
|
|
40
|
+
* them entirely. Push-to-talk rows ignore it: the user asked for that capture
|
|
41
|
+
* one keypress ago and is owed the confirmation.
|
|
42
|
+
*/
|
|
43
|
+
readonly indicator: 'off' | 'statusline' | 'banner';
|
|
44
|
+
/** Extra words for the row — a restart delay, a latch reason, an elapsed time. */
|
|
45
|
+
readonly detail?: string | undefined;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** True when this state should produce a footer row at all. */
|
|
49
|
+
export function voiceCaptureRowVisible(state: VoiceCaptureIndicatorState | null): boolean {
|
|
50
|
+
if (state === null) return false;
|
|
51
|
+
const isWakeRow = state.kind.startsWith('wake-');
|
|
52
|
+
return !isWakeRow || state.indicator !== 'off';
|
|
53
|
+
}
|