@seams/voice-capture 0.0.0-stage → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +201 -0
- package/README.md +174 -2
- package/dist/assets/NOTICE.txt +3 -0
- package/dist/assets/SILERO_LICENSE.txt +21 -0
- package/dist/assets/ort-wasm-simd-threaded.mjs +59 -0
- package/dist/assets/ort-wasm-simd-threaded.wasm +0 -0
- package/dist/assets/silero-vad.onnx +0 -0
- package/dist/assets/voice-activity-worklet.js +21 -0
- package/dist/browser/card.js +800 -0
- package/dist/browser/chunks/microphone-CU5gQG6w.js +1 -0
- package/dist/browser/chunks/ort.wasm.bundle.min-CWqlff_l.js +3 -0
- package/dist/browser/index.js +1 -0
- package/dist/esm/card/index.js +6 -0
- package/dist/esm/card/phraseCard.js +459 -0
- package/dist/esm/card/prompt.js +1074 -0
- package/dist/esm/card/ribbon.js +489 -0
- package/dist/esm/card/surface.js +224 -0
- package/dist/esm/card/types.js +0 -0
- package/dist/esm/card/visual.js +2 -0
- package/dist/esm/chunks/visual-Di3ojTED.js +1520 -0
- package/dist/esm/config.js +42 -0
- package/dist/esm/index.js +8 -0
- package/dist/esm/inference.js +193 -0
- package/dist/esm/microphone.js +428 -0
- package/dist/esm/microphoneInput.js +117 -0
- package/dist/esm/speechPause.js +33 -0
- package/dist/esm/voiceActivity.js +69 -0
- package/dist/esm/wav.js +36 -0
- package/dist/example/example.js +71 -0
- package/dist/example/index.html +29 -0
- package/dist/hosting.d.mts +4 -0
- package/dist/hosting.mjs +26 -0
- package/dist/types/card/index.d.ts +6 -0
- package/dist/types/card/phraseCard.d.ts +72 -0
- package/dist/types/card/prompt.d.ts +122 -0
- package/dist/types/card/ribbon.d.ts +62 -0
- package/dist/types/card/surface.d.ts +28 -0
- package/dist/types/card/types.d.ts +65 -0
- package/dist/types/card/visual.d.ts +25 -0
- package/dist/types/config.d.ts +47 -0
- package/dist/types/index.d.ts +7 -0
- package/dist/types/inference.d.ts +47 -0
- package/dist/types/microphone.d.ts +81 -0
- package/dist/types/microphoneInput.d.ts +19 -0
- package/dist/types/speechPause.d.ts +8 -0
- package/dist/types/voiceActivity.d.ts +15 -0
- package/dist/types/wav.d.ts +9 -0
- package/package.json +57 -4
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
//#region src/config.ts
|
|
2
|
+
const DEFAULTS = {
|
|
3
|
+
assetBase: null,
|
|
4
|
+
microphonePreference: null,
|
|
5
|
+
styleNonce: null,
|
|
6
|
+
label: "Voice capture",
|
|
7
|
+
place: "this page",
|
|
8
|
+
site: "this site",
|
|
9
|
+
recoveryHint: "Reload the page and try again.",
|
|
10
|
+
cacheName: "seams-voice-capture-models-v1"
|
|
11
|
+
};
|
|
12
|
+
let current = DEFAULTS;
|
|
13
|
+
/** Sets where the assets are and how the page names itself. Call it once, before a recording. */
|
|
14
|
+
function configureVoiceCapture(config) {
|
|
15
|
+
const base = new URL(config.assetBase, globalThis.document?.baseURI);
|
|
16
|
+
if (!base.pathname.endsWith("/")) base.pathname += "/";
|
|
17
|
+
current = {
|
|
18
|
+
assetBase: base,
|
|
19
|
+
microphonePreference: config.microphonePreference ?? null,
|
|
20
|
+
styleNonce: config.styleNonce ?? null,
|
|
21
|
+
label: config.label ?? DEFAULTS.label,
|
|
22
|
+
place: config.place ?? DEFAULTS.place,
|
|
23
|
+
site: config.site ?? DEFAULTS.site,
|
|
24
|
+
recoveryHint: config.recoveryHint ?? DEFAULTS.recoveryHint,
|
|
25
|
+
cacheName: config.cacheName ?? DEFAULTS.cacheName
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function captureConfig() {
|
|
29
|
+
return current;
|
|
30
|
+
}
|
|
31
|
+
/** The nonce for a style element, or undefined when the page has none. */
|
|
32
|
+
function styleNonce() {
|
|
33
|
+
const nonce = current.styleNonce;
|
|
34
|
+
return (typeof nonce === "function" ? nonce() : nonce) ?? void 0;
|
|
35
|
+
}
|
|
36
|
+
/** The URL of one asset file. It throws before `configureVoiceCapture` set the folder. */
|
|
37
|
+
function assetUrl(fileName) {
|
|
38
|
+
if (!current.assetBase) throw new Error("Call configureVoiceCapture({ assetBase }) before you load a voice asset.");
|
|
39
|
+
return new URL(fileName, current.assetBase);
|
|
40
|
+
}
|
|
41
|
+
//#endregion
|
|
42
|
+
export { assetUrl, captureConfig, configureVoiceCapture, styleNonce };
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { configureVoiceCapture } from "./config.js";
|
|
2
|
+
import { BrowserInferenceModel, VoiceAssetLoadError, voiceInferenceEngine } from "./inference.js";
|
|
3
|
+
import { createVoiceActivity, levelDetector, voiceActivityModel } from "./voiceActivity.js";
|
|
4
|
+
import { endPauseMs, expectedSpeechMs } from "./speechPause.js";
|
|
5
|
+
import { MAX_SAMPLES, SAMPLE_RATE, pcm16Wav } from "./wav.js";
|
|
6
|
+
import { VoiceMicrophoneInterruptedError, openVoiceMicrophone, rememberVoiceMicrophone, useSystemVoiceMicrophone } from "./microphoneInput.js";
|
|
7
|
+
import { InsufficientVoiceAudioError, recordVoice, requireVoiceForeground, requireVoiceUserAction, voiceQualityMessage } from "./microphone.js";
|
|
8
|
+
export { BrowserInferenceModel, InsufficientVoiceAudioError, MAX_SAMPLES, SAMPLE_RATE, VoiceAssetLoadError, VoiceMicrophoneInterruptedError, configureVoiceCapture, createVoiceActivity, endPauseMs, expectedSpeechMs, levelDetector, openVoiceMicrophone, pcm16Wav, recordVoice, rememberVoiceMicrophone, requireVoiceForeground, requireVoiceUserAction, useSystemVoiceMicrophone, voiceActivityModel, voiceInferenceEngine, voiceQualityMessage };
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { assetUrl, captureConfig } from "./config.js";
|
|
2
|
+
//#region src/inference.ts
|
|
3
|
+
const DOWNLOAD_IDLE_TIMEOUT_MS = 3e4;
|
|
4
|
+
var VoiceAssetLoadError = class extends Error {
|
|
5
|
+
constructor(kind, message, options) {
|
|
6
|
+
super(message, options);
|
|
7
|
+
this.kind = kind;
|
|
8
|
+
this.name = "VoiceAssetLoadError";
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
async function openModelCache() {
|
|
12
|
+
try {
|
|
13
|
+
return await caches.open(captureConfig().cacheName);
|
|
14
|
+
} catch {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
var BrowserInferenceAsset = class {
|
|
19
|
+
pending = null;
|
|
20
|
+
progress = { kind: "checking_cache" };
|
|
21
|
+
listeners = /* @__PURE__ */ new Set();
|
|
22
|
+
constructor(asset) {
|
|
23
|
+
this.asset = asset;
|
|
24
|
+
}
|
|
25
|
+
get size() {
|
|
26
|
+
return this.asset.bytes;
|
|
27
|
+
}
|
|
28
|
+
async download(onProgress) {
|
|
29
|
+
if (onProgress) {
|
|
30
|
+
this.listeners.add(onProgress);
|
|
31
|
+
onProgress(this.progress);
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
if (!this.pending) this.pending = this.bytes().catch(this.failed);
|
|
35
|
+
const bytes = await this.pending;
|
|
36
|
+
this.publish({ kind: "ready" });
|
|
37
|
+
return bytes;
|
|
38
|
+
} finally {
|
|
39
|
+
if (onProgress) this.listeners.delete(onProgress);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
failed = (error) => {
|
|
43
|
+
this.pending = null;
|
|
44
|
+
this.progress = { kind: "checking_cache" };
|
|
45
|
+
throw error;
|
|
46
|
+
};
|
|
47
|
+
publish(progress) {
|
|
48
|
+
this.progress = progress;
|
|
49
|
+
for (const listener of this.listeners) listener(progress);
|
|
50
|
+
}
|
|
51
|
+
async valid(bytes) {
|
|
52
|
+
if (bytes.byteLength !== this.asset.bytes) return false;
|
|
53
|
+
const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes));
|
|
54
|
+
let hex = "";
|
|
55
|
+
for (const byte of digest) hex += byte.toString(16).padStart(2, "0");
|
|
56
|
+
return hex === this.asset.sha256;
|
|
57
|
+
}
|
|
58
|
+
async bytes() {
|
|
59
|
+
const { fileName, sha256 } = this.asset;
|
|
60
|
+
this.publish({ kind: "checking_cache" });
|
|
61
|
+
const cache = await openModelCache();
|
|
62
|
+
const cacheKey = new URL(`${fileName}?sha256=${sha256}`, assetUrl("")).href;
|
|
63
|
+
try {
|
|
64
|
+
const response = await cache?.match(cacheKey);
|
|
65
|
+
if (response) {
|
|
66
|
+
const bytes = await response.arrayBuffer();
|
|
67
|
+
if (await this.valid(bytes)) return bytes;
|
|
68
|
+
await cache?.delete(cacheKey);
|
|
69
|
+
}
|
|
70
|
+
} catch {}
|
|
71
|
+
let bytes;
|
|
72
|
+
try {
|
|
73
|
+
bytes = await this.fetchBytes();
|
|
74
|
+
} catch (error) {
|
|
75
|
+
if (!(error instanceof VoiceAssetLoadError) || error.kind !== "connection") throw error;
|
|
76
|
+
bytes = await this.fetchBytes();
|
|
77
|
+
}
|
|
78
|
+
if (cache) try {
|
|
79
|
+
await cache.put(cacheKey, new Response(bytes, { headers: { "Content-Type": "application/octet-stream" } }));
|
|
80
|
+
} catch {}
|
|
81
|
+
return bytes;
|
|
82
|
+
}
|
|
83
|
+
async fetchBytes() {
|
|
84
|
+
const { fileName } = this.asset;
|
|
85
|
+
const label = typeof this.asset.label === "function" ? this.asset.label() : this.asset.label;
|
|
86
|
+
const controller = new AbortController();
|
|
87
|
+
let idle = setTimeout(abortDownload.bind(null, controller), DOWNLOAD_IDLE_TIMEOUT_MS);
|
|
88
|
+
this.publish({
|
|
89
|
+
kind: "downloading",
|
|
90
|
+
received: 0
|
|
91
|
+
});
|
|
92
|
+
try {
|
|
93
|
+
const response = await fetch(assetUrl(fileName), {
|
|
94
|
+
credentials: "omit",
|
|
95
|
+
cache: "reload",
|
|
96
|
+
signal: controller.signal
|
|
97
|
+
});
|
|
98
|
+
if (!response.ok || !response.body) {
|
|
99
|
+
const transient = [
|
|
100
|
+
408,
|
|
101
|
+
429,
|
|
102
|
+
500,
|
|
103
|
+
502,
|
|
104
|
+
503,
|
|
105
|
+
504
|
|
106
|
+
].includes(response.status);
|
|
107
|
+
throw new VoiceAssetLoadError(transient ? "connection" : "unavailable", transient ? `${label} download was interrupted. Check your connection and retry.` : `${label} is unavailable from ${captureConfig().site}. ${captureConfig().recoveryHint}`);
|
|
108
|
+
}
|
|
109
|
+
const reader = response.body.getReader();
|
|
110
|
+
const bytes = new Uint8Array(this.asset.bytes);
|
|
111
|
+
let received = 0;
|
|
112
|
+
try {
|
|
113
|
+
while (true) {
|
|
114
|
+
const chunk = await reader.read();
|
|
115
|
+
if (chunk.done) break;
|
|
116
|
+
clearTimeout(idle);
|
|
117
|
+
idle = setTimeout(abortDownload.bind(null, controller), DOWNLOAD_IDLE_TIMEOUT_MS);
|
|
118
|
+
if (received + chunk.value.length > bytes.length) throw new VoiceAssetLoadError("integrity", `${label} size is invalid. Retry loading.`);
|
|
119
|
+
bytes.set(chunk.value, received);
|
|
120
|
+
received += chunk.value.length;
|
|
121
|
+
this.publish({
|
|
122
|
+
kind: "downloading",
|
|
123
|
+
received
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
} finally {
|
|
127
|
+
await reader.cancel().catch(ignoreCancelledDownload);
|
|
128
|
+
reader.releaseLock();
|
|
129
|
+
}
|
|
130
|
+
if (received !== bytes.length || !await this.valid(bytes.buffer)) throw new VoiceAssetLoadError("integrity", `${label} integrity check failed. Retry loading.`);
|
|
131
|
+
return bytes.buffer;
|
|
132
|
+
} catch (error) {
|
|
133
|
+
if (error instanceof VoiceAssetLoadError) throw error;
|
|
134
|
+
throw new VoiceAssetLoadError("connection", `${label} download was interrupted. Check your connection and retry.`, { cause: error });
|
|
135
|
+
} finally {
|
|
136
|
+
clearTimeout(idle);
|
|
137
|
+
controller.abort();
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
function abortDownload(controller) {
|
|
142
|
+
controller.abort();
|
|
143
|
+
}
|
|
144
|
+
function ignoreCancelledDownload() {}
|
|
145
|
+
const voiceInferenceEngine = new BrowserInferenceAsset({
|
|
146
|
+
fileName: "ort-wasm-simd-threaded.wasm",
|
|
147
|
+
sha256: "3398c10d07d229bd91b364548e130e0e51a8e5704b88c7c083ebbeb78842dee2",
|
|
148
|
+
bytes: 14239897,
|
|
149
|
+
label: () => `${captureConfig().label} engine`
|
|
150
|
+
});
|
|
151
|
+
var BrowserInferenceModel = class {
|
|
152
|
+
asset;
|
|
153
|
+
session = null;
|
|
154
|
+
constructor(asset) {
|
|
155
|
+
this.asset = new BrowserInferenceAsset(asset);
|
|
156
|
+
}
|
|
157
|
+
prepare() {
|
|
158
|
+
if (!this.session) this.session = this.load().catch(this.failed);
|
|
159
|
+
return this.session;
|
|
160
|
+
}
|
|
161
|
+
/** Runs the model on one row of samples, as its `samples` input. Dispose the outputs. */
|
|
162
|
+
async run(samples) {
|
|
163
|
+
const session = await this.prepare();
|
|
164
|
+
const { Tensor } = await import("onnxruntime-web/wasm");
|
|
165
|
+
const input = new Tensor("float32", samples, [1, samples.length]);
|
|
166
|
+
try {
|
|
167
|
+
return await session.run({ samples: input });
|
|
168
|
+
} finally {
|
|
169
|
+
input.dispose();
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
failed = (error) => {
|
|
173
|
+
this.session = null;
|
|
174
|
+
throw error;
|
|
175
|
+
};
|
|
176
|
+
async load() {
|
|
177
|
+
try {
|
|
178
|
+
const ort = await import("onnxruntime-web/wasm");
|
|
179
|
+
ort.env.wasm.numThreads = 1;
|
|
180
|
+
ort.env.wasm.wasmPaths = assetUrl("").href;
|
|
181
|
+
ort.env.wasm.wasmBinary = await voiceInferenceEngine.download();
|
|
182
|
+
const bytes = await this.asset.download();
|
|
183
|
+
return await ort.InferenceSession.create(bytes, { executionProviders: ["wasm"] });
|
|
184
|
+
} catch (error) {
|
|
185
|
+
if (error instanceof VoiceAssetLoadError) throw error;
|
|
186
|
+
const { label, recoveryHint } = captureConfig();
|
|
187
|
+
console.error(`[${label}] inference engine could not start`, error);
|
|
188
|
+
throw new VoiceAssetLoadError("initialization", `${label} could not start. Retry loading. If it still fails, ${recoveryHint.charAt(0).toLowerCase()}${recoveryHint.slice(1)}`, { cause: error });
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
//#endregion
|
|
193
|
+
export { BrowserInferenceModel, VoiceAssetLoadError, voiceInferenceEngine };
|
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
import { assetUrl, captureConfig } from "./config.js";
|
|
2
|
+
import { createVoiceActivity, levelDetector } from "./voiceActivity.js";
|
|
3
|
+
import { endPauseMs } from "./speechPause.js";
|
|
4
|
+
import { SAMPLE_RATE } from "./wav.js";
|
|
5
|
+
import { VoiceMicrophoneInterruptedError, openVoiceMicrophone, rememberVoiceMicrophone } from "./microphoneInput.js";
|
|
6
|
+
//#region src/microphone.ts
|
|
7
|
+
function voiceQualityMessage(quality) {
|
|
8
|
+
if (quality.noise === "high") return "Background noise is high. Move closer to the microphone and try again.";
|
|
9
|
+
return `Audio too brief or quiet: ${(quality.voicedMs / 1e3).toFixed(2)} of ${(quality.requiredVoicedMs / 1e3).toFixed(2)} seconds detected.`;
|
|
10
|
+
}
|
|
11
|
+
/** Carries a typed capture rejection across the SDK's exception-based UI action boundary. */
|
|
12
|
+
var InsufficientVoiceAudioError = class extends Error {
|
|
13
|
+
constructor(quality) {
|
|
14
|
+
super(voiceQualityMessage(quality));
|
|
15
|
+
this.quality = quality;
|
|
16
|
+
this.name = "InsufficientVoiceAudioError";
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
const LEVEL_THRESHOLD = .006;
|
|
20
|
+
const REQUIRED_VOICED_MS = 600;
|
|
21
|
+
const MAX_RECORDING_MS = 8e3;
|
|
22
|
+
const SPEECH_START_MS = 100;
|
|
23
|
+
const VOICE_FRAME_MS = 32;
|
|
24
|
+
/** Throws unless the page is in front and the person just acted on it (a click or a key). */
|
|
25
|
+
function requireVoiceUserAction() {
|
|
26
|
+
requireVoiceForeground();
|
|
27
|
+
if (!navigator.userActivation.isActive) {
|
|
28
|
+
const { label, place } = captureConfig();
|
|
29
|
+
throw new Error(`Open ${place} and select Record to use ${label}.`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/** Throws unless the page is visible and has the focus. */
|
|
33
|
+
function requireVoiceForeground() {
|
|
34
|
+
if (document.visibilityState !== "visible" || !document.hasFocus()) {
|
|
35
|
+
const { label, place } = captureConfig();
|
|
36
|
+
throw new Error(`Keep ${place} open and focused to use ${label}.`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function cancelled() {
|
|
40
|
+
return /* @__PURE__ */ new Error(`${captureConfig().label} recording was cancelled.`);
|
|
41
|
+
}
|
|
42
|
+
function detectorOf(options) {
|
|
43
|
+
return options.detector === "level" ? async () => levelDetector : createVoiceActivity;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Records one utterance at 16 kHz. The recording ends after a pause, or after 8 s. A recording
|
|
47
|
+
* with less than 0.6 s of speech is `insufficient_audio`. The samples hold the speech with 0.2 s
|
|
48
|
+
* of margin on each side.
|
|
49
|
+
*/
|
|
50
|
+
async function recordVoice(signal, onCapture, options = {}) {
|
|
51
|
+
const expectedSpeechMs = options.expectedSpeechMs ?? null;
|
|
52
|
+
const createDetector = detectorOf(options);
|
|
53
|
+
if (signal.aborted) throw cancelled();
|
|
54
|
+
const attemptId = crypto.randomUUID();
|
|
55
|
+
onCapture({ kind: "opening" });
|
|
56
|
+
const microphone = await openVoiceMicrophone(signal);
|
|
57
|
+
const { stream } = microphone;
|
|
58
|
+
try {
|
|
59
|
+
if (signal.aborted) throw cancelled();
|
|
60
|
+
const clip = await new VoiceRecording(stream, signal, onCapture, expectedSpeechMs, createDetector).result.finally(onCapture.bind(null, { kind: "processing" }));
|
|
61
|
+
if (signal.aborted) throw cancelled();
|
|
62
|
+
const decoded = await new OfflineAudioContext(1, SAMPLE_RATE, SAMPLE_RATE).decodeAudioData(clip.bytes);
|
|
63
|
+
const renderer = new OfflineAudioContext(1, Math.ceil(decoded.duration * SAMPLE_RATE), SAMPLE_RATE);
|
|
64
|
+
const source = renderer.createBufferSource();
|
|
65
|
+
source.buffer = decoded;
|
|
66
|
+
source.connect(renderer.destination);
|
|
67
|
+
source.start();
|
|
68
|
+
try {
|
|
69
|
+
const rendered = await renderer.startRendering();
|
|
70
|
+
try {
|
|
71
|
+
if (signal.aborted) throw cancelled();
|
|
72
|
+
const samples = rendered.getChannelData(0);
|
|
73
|
+
const measurement = await measureAudio(samples, signal, await createDetector());
|
|
74
|
+
const quality = {
|
|
75
|
+
attemptId,
|
|
76
|
+
capturedAtMs: Date.now(),
|
|
77
|
+
durationMs: rendered.duration * 1e3,
|
|
78
|
+
recordingMs: clip.recordingMs,
|
|
79
|
+
endedBy: clip.endedBy,
|
|
80
|
+
voicedMs: measurement.voicedMs,
|
|
81
|
+
peakFrameRms: measurement.peakFrameRms,
|
|
82
|
+
levelThreshold: LEVEL_THRESHOLD,
|
|
83
|
+
requiredVoicedMs: REQUIRED_VOICED_MS,
|
|
84
|
+
microphone: microphone.label,
|
|
85
|
+
microphoneSelection: microphone.selection,
|
|
86
|
+
processing: microphone.processing,
|
|
87
|
+
backgroundRms: measurement.backgroundRms,
|
|
88
|
+
speechToBackgroundDb: measurement.speechToBackgroundDb,
|
|
89
|
+
noise: measurement.noise
|
|
90
|
+
};
|
|
91
|
+
onCapture({
|
|
92
|
+
kind: "measured",
|
|
93
|
+
quality
|
|
94
|
+
});
|
|
95
|
+
if (measurement.kind === "insufficient_audio") return {
|
|
96
|
+
kind: "insufficient_audio",
|
|
97
|
+
quality
|
|
98
|
+
};
|
|
99
|
+
const speech = samples.slice(Math.max(0, measurement.first - 3200), Math.min(samples.length, measurement.last + 3200));
|
|
100
|
+
await rememberVoiceMicrophone(microphone);
|
|
101
|
+
if (signal.aborted) {
|
|
102
|
+
speech.fill(0);
|
|
103
|
+
signal.throwIfAborted();
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
kind: "captured",
|
|
107
|
+
samples: speech,
|
|
108
|
+
quality,
|
|
109
|
+
lastSoundAtMs: clip.lastSoundAtMs
|
|
110
|
+
};
|
|
111
|
+
} finally {
|
|
112
|
+
wipeAudioBuffer(rendered);
|
|
113
|
+
}
|
|
114
|
+
} finally {
|
|
115
|
+
wipeAudioBuffer(decoded);
|
|
116
|
+
}
|
|
117
|
+
} finally {
|
|
118
|
+
for (const track of stream.getTracks()) track.stop();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
var VoiceRecording = class {
|
|
122
|
+
recorder;
|
|
123
|
+
chunks = [];
|
|
124
|
+
timer = null;
|
|
125
|
+
completion = deferred();
|
|
126
|
+
startedAtMs = performance.now();
|
|
127
|
+
ended = null;
|
|
128
|
+
activity = {
|
|
129
|
+
kind: "waiting",
|
|
130
|
+
activeMs: 0
|
|
131
|
+
};
|
|
132
|
+
audioContext = null;
|
|
133
|
+
worklet = null;
|
|
134
|
+
detector = null;
|
|
135
|
+
pending = Promise.resolve();
|
|
136
|
+
queuedFrames = 0;
|
|
137
|
+
closed = false;
|
|
138
|
+
result = this.completion.promise;
|
|
139
|
+
constructor(stream, signal, onCapture, expectedSpeechMs, createDetector) {
|
|
140
|
+
this.stream = stream;
|
|
141
|
+
this.signal = signal;
|
|
142
|
+
this.onCapture = onCapture;
|
|
143
|
+
this.expectedSpeechMs = expectedSpeechMs;
|
|
144
|
+
this.createDetector = createDetector;
|
|
145
|
+
this.recorder = new MediaRecorder(stream);
|
|
146
|
+
this.recorder.addEventListener("dataavailable", this.data);
|
|
147
|
+
this.recorder.addEventListener("stop", this.stopped);
|
|
148
|
+
this.recorder.addEventListener("error", this.failed);
|
|
149
|
+
for (const track of stream.getAudioTracks()) {
|
|
150
|
+
track.addEventListener("ended", this.interrupted);
|
|
151
|
+
track.addEventListener("mute", this.interrupted);
|
|
152
|
+
}
|
|
153
|
+
signal.addEventListener("abort", this.cancelled, { once: true });
|
|
154
|
+
this.timer = setTimeout(this.startupFailed, MAX_RECORDING_MS);
|
|
155
|
+
this.start();
|
|
156
|
+
}
|
|
157
|
+
async start() {
|
|
158
|
+
try {
|
|
159
|
+
await this.startPauseDetection();
|
|
160
|
+
this.signal.throwIfAborted();
|
|
161
|
+
this.startedAtMs = performance.now();
|
|
162
|
+
if (this.closed) return;
|
|
163
|
+
this.recorder.start();
|
|
164
|
+
if (this.timer !== null) clearTimeout(this.timer);
|
|
165
|
+
this.timer = setTimeout(this.stop.bind(this, "limit"), MAX_RECORDING_MS);
|
|
166
|
+
this.onCapture({ kind: "listening" });
|
|
167
|
+
} catch (error) {
|
|
168
|
+
this.cleanup();
|
|
169
|
+
this.completion.reject(error instanceof Error ? error : /* @__PURE__ */ new Error("Speech detection could not start."));
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
data = (event) => {
|
|
173
|
+
this.chunks.push(event.data);
|
|
174
|
+
};
|
|
175
|
+
stop = (endedBy) => {
|
|
176
|
+
if (this.recorder.state === "inactive") return;
|
|
177
|
+
this.ended = {
|
|
178
|
+
endedBy,
|
|
179
|
+
recordingMs: performance.now() - this.startedAtMs,
|
|
180
|
+
lastSoundAtMs: this.activity.kind === "speaking" ? this.activity.lastSoundAtMs : null
|
|
181
|
+
};
|
|
182
|
+
this.recorder.stop();
|
|
183
|
+
this.stopTracks();
|
|
184
|
+
};
|
|
185
|
+
cancelled = () => {
|
|
186
|
+
if (this.recorder.state === "inactive") {
|
|
187
|
+
this.cleanup();
|
|
188
|
+
this.completion.reject(cancelled());
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
this.stop("stream_ended");
|
|
192
|
+
};
|
|
193
|
+
startupFailed = () => {
|
|
194
|
+
this.cleanup();
|
|
195
|
+
this.completion.reject(/* @__PURE__ */ new Error("Speech detection startup timed out. Please try again."));
|
|
196
|
+
};
|
|
197
|
+
detectorFailed = (error) => {
|
|
198
|
+
this.cleanup();
|
|
199
|
+
this.completion.reject(error instanceof Error ? error : /* @__PURE__ */ new Error("Speech detection failed."));
|
|
200
|
+
};
|
|
201
|
+
failed = () => {
|
|
202
|
+
this.cleanup();
|
|
203
|
+
this.completion.reject(/* @__PURE__ */ new Error(`${captureConfig().label} could not record this microphone.`));
|
|
204
|
+
};
|
|
205
|
+
interrupted = () => {
|
|
206
|
+
this.cleanup();
|
|
207
|
+
this.completion.reject(new VoiceMicrophoneInterruptedError());
|
|
208
|
+
};
|
|
209
|
+
stopped = () => {
|
|
210
|
+
const end = this.ended;
|
|
211
|
+
this.cleanup();
|
|
212
|
+
if (this.signal.aborted) {
|
|
213
|
+
this.completion.reject(cancelled());
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
if (end === null) {
|
|
217
|
+
this.completion.reject(new VoiceMicrophoneInterruptedError());
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
new Blob(this.chunks, { type: this.recorder.mimeType }).arrayBuffer().then(this.complete.bind(this, end), this.completion.reject.bind(this.completion));
|
|
221
|
+
};
|
|
222
|
+
complete = (end, bytes) => {
|
|
223
|
+
if (this.signal.aborted) {
|
|
224
|
+
this.completion.reject(cancelled());
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
this.completion.resolve({
|
|
228
|
+
bytes,
|
|
229
|
+
...end
|
|
230
|
+
});
|
|
231
|
+
};
|
|
232
|
+
async startPauseDetection() {
|
|
233
|
+
const context = new AudioContext({ sampleRate: SAMPLE_RATE });
|
|
234
|
+
this.audioContext = context;
|
|
235
|
+
const resumed = context.resume();
|
|
236
|
+
this.detector = await this.createDetector();
|
|
237
|
+
this.signal.throwIfAborted();
|
|
238
|
+
if (context.sampleRate !== 16e3) throw new Error("Speech detection requires 16 kHz audio.");
|
|
239
|
+
await context.audioWorklet.addModule(assetUrl("voice-activity-worklet.js"));
|
|
240
|
+
this.signal.throwIfAborted();
|
|
241
|
+
const worklet = new AudioWorkletNode(context, "voice-activity-frames");
|
|
242
|
+
this.worklet = worklet;
|
|
243
|
+
worklet.port.onmessage = this.receiveFrame;
|
|
244
|
+
context.createMediaStreamSource(this.stream).connect(worklet);
|
|
245
|
+
worklet.connect(context.destination);
|
|
246
|
+
await resumed;
|
|
247
|
+
}
|
|
248
|
+
receiveFrame = (event) => {
|
|
249
|
+
if (!(event.data instanceof Float32Array) || event.data.length !== 512) {
|
|
250
|
+
this.failed();
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
if (this.closed || this.recorder.state !== "recording") {
|
|
254
|
+
event.data.fill(0);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
if (++this.queuedFrames > 32) {
|
|
258
|
+
event.data.fill(0);
|
|
259
|
+
this.failed();
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
this.pending = this.pending.then(this.processFrame.bind(this, event.data)).catch(this.detectorFailed);
|
|
263
|
+
};
|
|
264
|
+
async processFrame(frame) {
|
|
265
|
+
const detector = this.detector;
|
|
266
|
+
try {
|
|
267
|
+
if (this.closed || !detector) return;
|
|
268
|
+
const probability = await detector.probability(frame);
|
|
269
|
+
if (this.closed) {
|
|
270
|
+
detector.clear();
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
let energy = 0;
|
|
274
|
+
for (const sample of frame) energy += sample * sample;
|
|
275
|
+
const rms = Math.sqrt(energy / frame.length);
|
|
276
|
+
const speech = probability >= .5 && rms >= LEVEL_THRESHOLD;
|
|
277
|
+
this.updateActivity(speech);
|
|
278
|
+
this.onCapture({
|
|
279
|
+
kind: "activity",
|
|
280
|
+
rms,
|
|
281
|
+
speech
|
|
282
|
+
});
|
|
283
|
+
} finally {
|
|
284
|
+
frame.fill(0);
|
|
285
|
+
this.queuedFrames -= 1;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
updateActivity(audible) {
|
|
289
|
+
const now = performance.now();
|
|
290
|
+
switch (this.activity.kind) {
|
|
291
|
+
case "waiting": {
|
|
292
|
+
const activeMs = audible ? this.activity.activeMs + VOICE_FRAME_MS : 0;
|
|
293
|
+
this.activity = activeMs >= SPEECH_START_MS ? {
|
|
294
|
+
kind: "speaking",
|
|
295
|
+
lastSoundAtMs: now,
|
|
296
|
+
voicedMs: activeMs
|
|
297
|
+
} : {
|
|
298
|
+
kind: "waiting",
|
|
299
|
+
activeMs
|
|
300
|
+
};
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
case "speaking": {
|
|
304
|
+
const { lastSoundAtMs, voicedMs } = this.activity;
|
|
305
|
+
if (audible) {
|
|
306
|
+
this.activity = {
|
|
307
|
+
kind: "speaking",
|
|
308
|
+
lastSoundAtMs: now,
|
|
309
|
+
voicedMs: voicedMs + VOICE_FRAME_MS
|
|
310
|
+
};
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
if (now - lastSoundAtMs >= endPauseMs(voicedMs, this.expectedSpeechMs)) this.stop("pause");
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
default: return assertNeverActivity(this.activity);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
stopPauseDetection = () => {
|
|
320
|
+
this.closed = true;
|
|
321
|
+
if (this.worklet) {
|
|
322
|
+
this.worklet.port.onmessage = null;
|
|
323
|
+
this.worklet.port.close();
|
|
324
|
+
this.worklet.disconnect();
|
|
325
|
+
}
|
|
326
|
+
this.worklet = null;
|
|
327
|
+
this.detector?.clear();
|
|
328
|
+
this.detector = null;
|
|
329
|
+
const context = this.audioContext;
|
|
330
|
+
this.audioContext = null;
|
|
331
|
+
if (context) context.close().catch(ignoreClosedContext);
|
|
332
|
+
};
|
|
333
|
+
stopTracks() {
|
|
334
|
+
for (const track of this.stream.getTracks()) track.stop();
|
|
335
|
+
}
|
|
336
|
+
cleanup() {
|
|
337
|
+
for (const track of this.stream.getAudioTracks()) {
|
|
338
|
+
track.removeEventListener("ended", this.interrupted);
|
|
339
|
+
track.removeEventListener("mute", this.interrupted);
|
|
340
|
+
}
|
|
341
|
+
this.stopPauseDetection();
|
|
342
|
+
this.stopTracks();
|
|
343
|
+
if (this.timer !== null) clearTimeout(this.timer);
|
|
344
|
+
this.signal.removeEventListener("abort", this.cancelled);
|
|
345
|
+
this.recorder.removeEventListener("dataavailable", this.data);
|
|
346
|
+
this.recorder.removeEventListener("stop", this.stopped);
|
|
347
|
+
this.recorder.removeEventListener("error", this.failed);
|
|
348
|
+
}
|
|
349
|
+
};
|
|
350
|
+
async function measureAudio(samples, signal, detector) {
|
|
351
|
+
let first = -1;
|
|
352
|
+
let last = 0;
|
|
353
|
+
let voicedSamples = 0;
|
|
354
|
+
let peakFrameRms = 0;
|
|
355
|
+
let backgroundEnergy = 0;
|
|
356
|
+
let backgroundSamples = 0;
|
|
357
|
+
let speechEnergy = 0;
|
|
358
|
+
const frame = new Float32Array(512);
|
|
359
|
+
try {
|
|
360
|
+
for (let offset = 0; offset < samples.length; offset += frame.length) {
|
|
361
|
+
signal.throwIfAborted();
|
|
362
|
+
frame.fill(0);
|
|
363
|
+
const count = Math.min(frame.length, samples.length - offset);
|
|
364
|
+
frame.set(samples.subarray(offset, offset + count));
|
|
365
|
+
let energy = 0;
|
|
366
|
+
for (const sample of frame) energy += sample * sample;
|
|
367
|
+
const rms = Math.sqrt(energy / count);
|
|
368
|
+
peakFrameRms = Math.max(peakFrameRms, rms);
|
|
369
|
+
if (!(await detector.probability(frame) >= .5 && rms >= LEVEL_THRESHOLD)) {
|
|
370
|
+
backgroundEnergy += energy;
|
|
371
|
+
backgroundSamples += count;
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
if (first < 0) first = offset;
|
|
375
|
+
last = offset + count;
|
|
376
|
+
voicedSamples += count;
|
|
377
|
+
speechEnergy += energy;
|
|
378
|
+
}
|
|
379
|
+
} finally {
|
|
380
|
+
frame.fill(0);
|
|
381
|
+
detector.clear();
|
|
382
|
+
}
|
|
383
|
+
const backgroundRms = Math.sqrt(backgroundEnergy / Math.max(1, backgroundSamples));
|
|
384
|
+
const speechRms = Math.sqrt(speechEnergy / Math.max(1, voicedSamples));
|
|
385
|
+
const speechToBackgroundDb = voicedSamples && backgroundSamples ? 20 * Math.log10(Math.max(speechRms, 1e-8) / Math.max(backgroundRms, 1e-8)) : null;
|
|
386
|
+
const noise = {
|
|
387
|
+
backgroundRms,
|
|
388
|
+
speechToBackgroundDb,
|
|
389
|
+
noise: backgroundRms >= LEVEL_THRESHOLD && (speechToBackgroundDb === null || speechToBackgroundDb < 10) ? "high" : backgroundSamples ? "normal" : "unknown"
|
|
390
|
+
};
|
|
391
|
+
const voicedMs = voicedSamples / SAMPLE_RATE * 1e3;
|
|
392
|
+
if (first < 0 || voicedMs < REQUIRED_VOICED_MS) return {
|
|
393
|
+
kind: "insufficient_audio",
|
|
394
|
+
voicedMs,
|
|
395
|
+
peakFrameRms,
|
|
396
|
+
...noise
|
|
397
|
+
};
|
|
398
|
+
return {
|
|
399
|
+
kind: "captured",
|
|
400
|
+
first,
|
|
401
|
+
last,
|
|
402
|
+
voicedMs,
|
|
403
|
+
peakFrameRms,
|
|
404
|
+
...noise
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
function wipeAudioBuffer(buffer) {
|
|
408
|
+
for (let channel = 0; channel < buffer.numberOfChannels; channel += 1) buffer.getChannelData(channel).fill(0);
|
|
409
|
+
}
|
|
410
|
+
function ignoreClosedContext() {}
|
|
411
|
+
/** A promise with its two settle functions. */
|
|
412
|
+
function deferred() {
|
|
413
|
+
let resolve;
|
|
414
|
+
let reject;
|
|
415
|
+
return {
|
|
416
|
+
promise: new Promise((onResolve, onReject) => {
|
|
417
|
+
resolve = onResolve;
|
|
418
|
+
reject = onReject;
|
|
419
|
+
}),
|
|
420
|
+
resolve,
|
|
421
|
+
reject
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
function assertNeverActivity(activity) {
|
|
425
|
+
throw new Error(`Unexpected speech activity: ${String(activity)}`);
|
|
426
|
+
}
|
|
427
|
+
//#endregion
|
|
428
|
+
export { InsufficientVoiceAudioError, recordVoice, requireVoiceForeground, requireVoiceUserAction, voiceQualityMessage };
|