@seams/voice-capture 0.0.0-stage → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +201 -0
- package/README.md +174 -2
- package/dist/assets/NOTICE.txt +3 -0
- package/dist/assets/SILERO_LICENSE.txt +21 -0
- package/dist/assets/ort-wasm-simd-threaded.mjs +59 -0
- package/dist/assets/ort-wasm-simd-threaded.wasm +0 -0
- package/dist/assets/silero-vad.onnx +0 -0
- package/dist/assets/voice-activity-worklet.js +21 -0
- package/dist/browser/card.js +800 -0
- package/dist/browser/chunks/microphone-CU5gQG6w.js +1 -0
- package/dist/browser/chunks/ort.wasm.bundle.min-CWqlff_l.js +3 -0
- package/dist/browser/index.js +1 -0
- package/dist/esm/card/index.js +6 -0
- package/dist/esm/card/phraseCard.js +459 -0
- package/dist/esm/card/prompt.js +1074 -0
- package/dist/esm/card/ribbon.js +489 -0
- package/dist/esm/card/surface.js +224 -0
- package/dist/esm/card/types.js +0 -0
- package/dist/esm/card/visual.js +2 -0
- package/dist/esm/chunks/visual-Di3ojTED.js +1520 -0
- package/dist/esm/config.js +42 -0
- package/dist/esm/index.js +8 -0
- package/dist/esm/inference.js +193 -0
- package/dist/esm/microphone.js +428 -0
- package/dist/esm/microphoneInput.js +117 -0
- package/dist/esm/speechPause.js +33 -0
- package/dist/esm/voiceActivity.js +69 -0
- package/dist/esm/wav.js +36 -0
- package/dist/example/example.js +71 -0
- package/dist/example/index.html +29 -0
- package/dist/hosting.d.mts +4 -0
- package/dist/hosting.mjs +26 -0
- package/dist/types/card/index.d.ts +6 -0
- package/dist/types/card/phraseCard.d.ts +72 -0
- package/dist/types/card/prompt.d.ts +122 -0
- package/dist/types/card/ribbon.d.ts +62 -0
- package/dist/types/card/surface.d.ts +28 -0
- package/dist/types/card/types.d.ts +65 -0
- package/dist/types/card/visual.d.ts +25 -0
- package/dist/types/config.d.ts +47 -0
- package/dist/types/index.d.ts +7 -0
- package/dist/types/inference.d.ts +47 -0
- package/dist/types/microphone.d.ts +81 -0
- package/dist/types/microphoneInput.d.ts +19 -0
- package/dist/types/speechPause.d.ts +8 -0
- package/dist/types/voiceActivity.d.ts +15 -0
- package/dist/types/wav.d.ts +9 -0
- package/package.json +57 -4
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { captureConfig } from "./config.js";
|
|
2
|
+
//#region src/microphoneInput.ts
|
|
3
|
+
let preference = { kind: "unread" };
|
|
4
|
+
function concreteDeviceId(value) {
|
|
5
|
+
return typeof value === "string" && value.length > 0 && value !== "default" && value !== "communications";
|
|
6
|
+
}
|
|
7
|
+
async function readPreference() {
|
|
8
|
+
if (preference.kind !== "unread") return preference;
|
|
9
|
+
let saved = { kind: "system_default" };
|
|
10
|
+
try {
|
|
11
|
+
const raw = await captureConfig().microphonePreference?.read();
|
|
12
|
+
if (typeof raw === "object" && raw !== null && "kind" in raw && raw.kind === "device") {
|
|
13
|
+
const deviceId = "deviceId" in raw ? raw.deviceId : null;
|
|
14
|
+
if (concreteDeviceId(deviceId)) saved = {
|
|
15
|
+
kind: "device",
|
|
16
|
+
deviceId
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
} catch {}
|
|
20
|
+
if (preference.kind === "unread") preference = saved;
|
|
21
|
+
return preference;
|
|
22
|
+
}
|
|
23
|
+
function constraints(preference) {
|
|
24
|
+
const audio = {
|
|
25
|
+
channelCount: 1,
|
|
26
|
+
echoCancellation: true,
|
|
27
|
+
noiseSuppression: true,
|
|
28
|
+
autoGainControl: true
|
|
29
|
+
};
|
|
30
|
+
if (preference.kind === "device") audio.deviceId = { exact: preference.deviceId };
|
|
31
|
+
return {
|
|
32
|
+
audio,
|
|
33
|
+
video: false
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/** Select the concrete input behind a browser's moving default-device alias. */
|
|
37
|
+
async function resolvedPreference(settings) {
|
|
38
|
+
if (concreteDeviceId(settings.deviceId)) return {
|
|
39
|
+
kind: "device",
|
|
40
|
+
deviceId: settings.deviceId
|
|
41
|
+
};
|
|
42
|
+
if (settings.groupId) try {
|
|
43
|
+
const devices = await navigator.mediaDevices.enumerateDevices();
|
|
44
|
+
for (const device of devices) if (device.kind === "audioinput" && device.groupId === settings.groupId && concreteDeviceId(device.deviceId)) return {
|
|
45
|
+
kind: "device",
|
|
46
|
+
deviceId: device.deviceId
|
|
47
|
+
};
|
|
48
|
+
} catch {}
|
|
49
|
+
return { kind: "system_default" };
|
|
50
|
+
}
|
|
51
|
+
async function openVoiceMicrophone(signal) {
|
|
52
|
+
signal.throwIfAborted();
|
|
53
|
+
const preference = await readPreference();
|
|
54
|
+
signal.throwIfAborted();
|
|
55
|
+
let stream;
|
|
56
|
+
let selection = preference.kind === "device" ? "preferred" : "default";
|
|
57
|
+
try {
|
|
58
|
+
stream = await navigator.mediaDevices.getUserMedia(constraints(preference));
|
|
59
|
+
} catch (error) {
|
|
60
|
+
signal.throwIfAborted();
|
|
61
|
+
if (preference.kind !== "device" || !(error instanceof DOMException) || error.name !== "OverconstrainedError" && error.name !== "NotFoundError") throw error;
|
|
62
|
+
stream = await navigator.mediaDevices.getUserMedia(constraints({ kind: "system_default" }));
|
|
63
|
+
selection = "replacement";
|
|
64
|
+
}
|
|
65
|
+
const stop = stopMicrophone.bind(null, stream);
|
|
66
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
67
|
+
try {
|
|
68
|
+
signal.throwIfAborted();
|
|
69
|
+
const track = stream.getAudioTracks()[0];
|
|
70
|
+
if (!track || track.readyState !== "live") throw new VoiceMicrophoneInterruptedError();
|
|
71
|
+
const settings = track.getSettings();
|
|
72
|
+
const resolved = await resolvedPreference(settings);
|
|
73
|
+
signal.throwIfAborted();
|
|
74
|
+
if (track.readyState !== "live") throw new VoiceMicrophoneInterruptedError();
|
|
75
|
+
return {
|
|
76
|
+
stream,
|
|
77
|
+
preference: resolved,
|
|
78
|
+
selection,
|
|
79
|
+
label: track.label || "Default microphone",
|
|
80
|
+
processing: {
|
|
81
|
+
noiseSuppression: settings.noiseSuppression ?? null,
|
|
82
|
+
echoCancellation: settings.echoCancellation ?? null,
|
|
83
|
+
autoGainControl: settings.autoGainControl ?? null,
|
|
84
|
+
sampleRate: settings.sampleRate ?? null
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
} catch (error) {
|
|
88
|
+
stop();
|
|
89
|
+
throw error;
|
|
90
|
+
} finally {
|
|
91
|
+
signal.removeEventListener("abort", stop);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
function stopMicrophone(stream) {
|
|
95
|
+
for (const track of stream.getTracks()) track.stop();
|
|
96
|
+
}
|
|
97
|
+
async function rememberVoiceMicrophone(microphone) {
|
|
98
|
+
if (microphone.preference.kind !== "device" || microphone.selection === "replacement") return;
|
|
99
|
+
await savePreference(microphone.preference);
|
|
100
|
+
}
|
|
101
|
+
async function useSystemVoiceMicrophone() {
|
|
102
|
+
await savePreference({ kind: "system_default" });
|
|
103
|
+
}
|
|
104
|
+
async function savePreference(selected) {
|
|
105
|
+
preference = selected;
|
|
106
|
+
try {
|
|
107
|
+
await captureConfig().microphonePreference?.save(selected);
|
|
108
|
+
} catch {}
|
|
109
|
+
}
|
|
110
|
+
var VoiceMicrophoneInterruptedError = class extends Error {
|
|
111
|
+
constructor() {
|
|
112
|
+
super("The microphone disconnected or stopped. Check your microphone and try again.");
|
|
113
|
+
this.name = "VoiceMicrophoneInterruptedError";
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
//#endregion
|
|
117
|
+
export { VoiceMicrophoneInterruptedError, openVoiceMicrophone, rememberVoiceMicrophone, useSystemVoiceMicrophone };
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
//#region src/speechPause.ts
|
|
2
|
+
/** The silence that ends a recording when the expected sentence is unknown or not yet said. */
|
|
3
|
+
const END_PAUSE_MS = 1500;
|
|
4
|
+
/**
|
|
5
|
+
* The silence that ends a recording once the voiced time covers the expected sentence. A pause
|
|
6
|
+
* inside the sentence comes before that point, so it still gets `END_PAUSE_MS`.
|
|
7
|
+
*/
|
|
8
|
+
const SHORT_END_PAUSE_MS = 800;
|
|
9
|
+
/**
|
|
10
|
+
* Voiced time of one spoken word at a fast pace. Normal speech is about 300 ms a word, and
|
|
11
|
+
* pauses between words are not voiced, so most speakers pass this before the last word ends.
|
|
12
|
+
*/
|
|
13
|
+
const VOICED_MS_PER_WORD = 200;
|
|
14
|
+
/** The silence that ends a recording after `voicedMs` of speech. */
|
|
15
|
+
function endPauseMs(voicedMs, expectedSpeechMs) {
|
|
16
|
+
return expectedSpeechMs !== null && voicedMs >= expectedSpeechMs ? SHORT_END_PAUSE_MS : END_PAUSE_MS;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* The least voiced time of the whole `sentence` as people read it aloud. Digits, dots and dashes
|
|
20
|
+
* are words of their own: "Send 0.01 NEAR to seams-v1.testnet" is read as 13 words ("send zero
|
|
21
|
+
* point zero one near to seams dash v one dot testnet").
|
|
22
|
+
*/
|
|
23
|
+
function expectedSpeechMs(sentence) {
|
|
24
|
+
return sentence.trim().replace(/[.!?,;:]+$/, "").split(/\s+/).filter(Boolean).reduce((count, word) => count + spokenWords(word), 0) * VOICED_MS_PER_WORD;
|
|
25
|
+
}
|
|
26
|
+
function spokenWords(word) {
|
|
27
|
+
const letters = word.match(/[A-Za-z]+/g)?.length ?? 0;
|
|
28
|
+
const digits = word.match(/[0-9]/g)?.length ?? 0;
|
|
29
|
+
const marks = word.match(/[.\-_@]/g)?.length ?? 0;
|
|
30
|
+
return Math.max(1, letters + digits + marks);
|
|
31
|
+
}
|
|
32
|
+
//#endregion
|
|
33
|
+
export { endPauseMs, expectedSpeechMs };
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { BrowserInferenceModel } from "./inference.js";
|
|
2
|
+
//#region src/voiceActivity.ts
|
|
3
|
+
const voiceActivityModel = new BrowserInferenceModel({
|
|
4
|
+
fileName: "silero-vad.onnx",
|
|
5
|
+
sha256: "2623a2953f6ff3d2c1e61740c6cdb7168133479b267dfef114a4a3cc5bdd788f",
|
|
6
|
+
bytes: 2327524,
|
|
7
|
+
label: "Speech detection"
|
|
8
|
+
});
|
|
9
|
+
/** The speech detection model. It downloads the model and the model engine on first use. */
|
|
10
|
+
async function createVoiceActivity() {
|
|
11
|
+
return new VoiceActivity(await voiceActivityModel.prepare(), await import("onnxruntime-web/wasm"));
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* A detector with no model and no download: every frame counts as speech, so only the level
|
|
15
|
+
* check of the recorder decides. It cannot tell speech from other loud sound.
|
|
16
|
+
*/
|
|
17
|
+
const levelDetector = {
|
|
18
|
+
probability: async () => 1,
|
|
19
|
+
clear: () => {}
|
|
20
|
+
};
|
|
21
|
+
var VoiceActivity = class {
|
|
22
|
+
state = new Float32Array(256);
|
|
23
|
+
context = new Float32Array(64);
|
|
24
|
+
constructor(session, ort) {
|
|
25
|
+
this.session = session;
|
|
26
|
+
this.ort = ort;
|
|
27
|
+
}
|
|
28
|
+
async probability(frame) {
|
|
29
|
+
if (frame.length !== 512) throw new Error("Speech detection requires a 512-sample frame.");
|
|
30
|
+
const samples = new Float32Array(576);
|
|
31
|
+
samples.set(this.context);
|
|
32
|
+
samples.set(frame, 64);
|
|
33
|
+
const input = new this.ort.Tensor("float32", samples, [1, 576]);
|
|
34
|
+
const state = new this.ort.Tensor("float32", this.state, [
|
|
35
|
+
2,
|
|
36
|
+
1,
|
|
37
|
+
128
|
|
38
|
+
]);
|
|
39
|
+
const sr = new this.ort.Tensor("int64", BigInt64Array.of(16000n), []);
|
|
40
|
+
try {
|
|
41
|
+
const outputs = await this.session.run({
|
|
42
|
+
input,
|
|
43
|
+
state,
|
|
44
|
+
sr
|
|
45
|
+
});
|
|
46
|
+
try {
|
|
47
|
+
const probability = outputs.output?.data;
|
|
48
|
+
const nextState = outputs.stateN?.data;
|
|
49
|
+
if (!(probability instanceof Float32Array) || probability.length !== 1 || !Number.isFinite(probability[0]) || probability[0] < 0 || probability[0] > 1 || !(nextState instanceof Float32Array) || nextState.length !== 256) throw new Error("Speech detection returned an invalid result.");
|
|
50
|
+
this.state.set(nextState);
|
|
51
|
+
this.context.set(frame.subarray(448));
|
|
52
|
+
return probability[0];
|
|
53
|
+
} finally {
|
|
54
|
+
for (const output of Object.values(outputs)) output.dispose();
|
|
55
|
+
}
|
|
56
|
+
} finally {
|
|
57
|
+
input.dispose();
|
|
58
|
+
state.dispose();
|
|
59
|
+
sr.dispose();
|
|
60
|
+
samples.fill(0);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
clear() {
|
|
64
|
+
this.state.fill(0);
|
|
65
|
+
this.context.fill(0);
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
//#endregion
|
|
69
|
+
export { createVoiceActivity, levelDetector, voiceActivityModel };
|
package/dist/esm/wav.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
//#region src/wav.ts
|
|
2
|
+
const SAMPLE_RATE = 16e3;
|
|
3
|
+
/** The Speak-to-Sign verifier refuses audio longer than 8 s. */
|
|
4
|
+
const MAX_SAMPLES = 8 * SAMPLE_RATE;
|
|
5
|
+
/**
|
|
6
|
+
* 16 kHz mono PCM16 with the plain 44-byte header that the Speak-to-Sign verifier writes for its
|
|
7
|
+
* canonical WAV, so the SHA-256 of these bytes is the audio digest of a verdict. Samples after
|
|
8
|
+
* 8 s are left out.
|
|
9
|
+
*/
|
|
10
|
+
function pcm16Wav(samples) {
|
|
11
|
+
const count = Math.min(samples.length, MAX_SAMPLES);
|
|
12
|
+
const bytes = new Uint8Array(44 + count * 2);
|
|
13
|
+
const view = new DataView(bytes.buffer);
|
|
14
|
+
writeAscii(bytes, 0, "RIFF");
|
|
15
|
+
view.setUint32(4, 36 + count * 2, true);
|
|
16
|
+
writeAscii(bytes, 8, "WAVEfmt ");
|
|
17
|
+
view.setUint32(16, 16, true);
|
|
18
|
+
view.setUint16(20, 1, true);
|
|
19
|
+
view.setUint16(22, 1, true);
|
|
20
|
+
view.setUint32(24, SAMPLE_RATE, true);
|
|
21
|
+
view.setUint32(28, SAMPLE_RATE * 2, true);
|
|
22
|
+
view.setUint16(32, 2, true);
|
|
23
|
+
view.setUint16(34, 16, true);
|
|
24
|
+
writeAscii(bytes, 36, "data");
|
|
25
|
+
view.setUint32(40, count * 2, true);
|
|
26
|
+
for (let index = 0; index < count; index += 1) {
|
|
27
|
+
const sample = Math.max(-1, Math.min(1, samples[index] ?? 0));
|
|
28
|
+
view.setInt16(44 + index * 2, Math.round(sample < 0 ? sample * 32768 : sample * 32767), true);
|
|
29
|
+
}
|
|
30
|
+
return bytes;
|
|
31
|
+
}
|
|
32
|
+
function writeAscii(bytes, offset, text) {
|
|
33
|
+
for (let index = 0; index < text.length; index += 1) bytes[offset + index] = text.charCodeAt(index);
|
|
34
|
+
}
|
|
35
|
+
//#endregion
|
|
36
|
+
export { MAX_SAMPLES, SAMPLE_RATE, pcm16Wav };
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// The example of @seams/voice-capture: one sentence on the phrase card, and three enrollment
|
|
2
|
+
// phrases. It imports the built single-file package, so it needs no bundler.
|
|
3
|
+
|
|
4
|
+
import { configureVoiceCapture, pcm16Wav, SAMPLE_RATE } from '../browser/index.js';
|
|
5
|
+
import { openPhraseCard } from '../browser/card.js';
|
|
6
|
+
|
|
7
|
+
configureVoiceCapture({ assetBase: '../assets/' });
|
|
8
|
+
|
|
9
|
+
const ENROLLMENT_PHRASES = [
|
|
10
|
+
'My voice is my signature for this wallet.',
|
|
11
|
+
'I send money only when I say so.',
|
|
12
|
+
'Seams, please learn how I sound today.',
|
|
13
|
+
];
|
|
14
|
+
|
|
15
|
+
const log = document.querySelector('#log');
|
|
16
|
+
const parent = document.querySelector('#card');
|
|
17
|
+
const detector = () => (document.querySelector('#level').checked ? 'level' : 'model');
|
|
18
|
+
|
|
19
|
+
/** Shows one sentence, records it and returns its WAV, or null when the person closed the card. */
|
|
20
|
+
async function recordSentence(sentence, signal) {
|
|
21
|
+
const surface = openPhraseCard({ sentence, signal, detector: detector(), frame: { parent } });
|
|
22
|
+
try {
|
|
23
|
+
const recording = await surface.record();
|
|
24
|
+
if (recording.kind === 'cancelled') return null;
|
|
25
|
+
const seconds = recording.samples.length / SAMPLE_RATE;
|
|
26
|
+
const wav = pcm16Wav(recording.samples);
|
|
27
|
+
// The caller owns the samples: clear them after use.
|
|
28
|
+
recording.samples.fill(0);
|
|
29
|
+
// An application sends the WAV to its verifier here, and shows the answer on the same card.
|
|
30
|
+
void surface.show({
|
|
31
|
+
kind: 'check',
|
|
32
|
+
label: 'Checking',
|
|
33
|
+
caption: 'Your application checks the recording',
|
|
34
|
+
details: ['This example checks nothing.'],
|
|
35
|
+
segments: ['done', 'run'],
|
|
36
|
+
});
|
|
37
|
+
await new Promise((resolve) => setTimeout(resolve, 900));
|
|
38
|
+
await surface.show({
|
|
39
|
+
kind: 'result',
|
|
40
|
+
label: '',
|
|
41
|
+
headline: 'Recorded',
|
|
42
|
+
caption: sentence,
|
|
43
|
+
detail: `${seconds.toFixed(1)} s, ${wav.length} bytes of WAV`,
|
|
44
|
+
action: { label: 'Done' },
|
|
45
|
+
line: 'seal',
|
|
46
|
+
});
|
|
47
|
+
return { seconds, bytes: wav.length };
|
|
48
|
+
} finally {
|
|
49
|
+
surface.close();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
document.querySelector('#sentence').addEventListener('click', async () => {
|
|
54
|
+
const result = await recordSentence('Send two NEAR to alice.testnet', new AbortController().signal);
|
|
55
|
+
log.textContent = result
|
|
56
|
+
? `One sentence: ${result.seconds.toFixed(1)} s, ${result.bytes} bytes.`
|
|
57
|
+
: 'The recording was cancelled.';
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
document.querySelector('#enroll').addEventListener('click', async () => {
|
|
61
|
+
const lines = [];
|
|
62
|
+
for (const phrase of ENROLLMENT_PHRASES) {
|
|
63
|
+
const result = await recordSentence(phrase, new AbortController().signal);
|
|
64
|
+
if (!result) {
|
|
65
|
+
lines.push('The enrollment was cancelled.');
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
lines.push(`"${phrase}": ${result.seconds.toFixed(1)} s, ${result.bytes} bytes.`);
|
|
69
|
+
}
|
|
70
|
+
log.textContent = lines.join('\n');
|
|
71
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en" data-theme="light">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8" />
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
6
|
+
<title>voice-capture example</title>
|
|
7
|
+
<style>
|
|
8
|
+
body { margin: 0; padding: 2rem 1rem; font-family: system-ui, sans-serif; background: #f5f6f8; color: #14161a; }
|
|
9
|
+
main { max-width: 26.25rem; margin: 0 auto; display: grid; gap: 1rem; }
|
|
10
|
+
.row { display: flex; gap: 0.5rem; flex-wrap: wrap; }
|
|
11
|
+
button.start { padding: 0.6rem 1rem; border-radius: 0.75rem; border: 1px solid #c9ced6; background: #fff; font: inherit; cursor: pointer; }
|
|
12
|
+
pre { margin: 0; padding: 0.75rem; background: #fff; border: 1px solid #dfe3e8; border-radius: 0.75rem; white-space: pre-wrap; font-size: 0.8rem; }
|
|
13
|
+
</style>
|
|
14
|
+
</head>
|
|
15
|
+
<body>
|
|
16
|
+
<main>
|
|
17
|
+
<h1>voice-capture</h1>
|
|
18
|
+
<p>This page uses only <code>@seams/voice-capture</code>. The audio stays in the page.</p>
|
|
19
|
+
<label><input type="checkbox" id="level" /> Record with no model download (the detector <code>level</code>)</label>
|
|
20
|
+
<div class="row">
|
|
21
|
+
<button class="start" id="sentence">Record one sentence</button>
|
|
22
|
+
<button class="start" id="enroll">Record three enrollment phrases</button>
|
|
23
|
+
</div>
|
|
24
|
+
<pre id="log" aria-live="polite">Nothing recorded yet.</pre>
|
|
25
|
+
<div id="card"></div>
|
|
26
|
+
</main>
|
|
27
|
+
<script type="module" src="./example.js"></script>
|
|
28
|
+
</body>
|
|
29
|
+
</html>
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Every file of the asset folder. */
|
|
2
|
+
export declare const voiceCaptureAssets: readonly string[];
|
|
3
|
+
/** Copies the asset files into `outputDirectory` and returns their names. */
|
|
4
|
+
export declare function copyVoiceCaptureAssets(outputDirectory: string): Promise<readonly string[]>;
|
package/dist/hosting.mjs
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// The files that a page must serve from the folder that it gives to configureVoiceCapture.
|
|
2
|
+
|
|
3
|
+
import { cp, mkdir } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
|
|
7
|
+
const assets = fileURLToPath(new URL('./assets/', import.meta.url));
|
|
8
|
+
|
|
9
|
+
/** Every file of the asset folder. The model engine is the file of the pinned onnxruntime-web. */
|
|
10
|
+
export const voiceCaptureAssets = Object.freeze([
|
|
11
|
+
'silero-vad.onnx',
|
|
12
|
+
'voice-activity-worklet.js',
|
|
13
|
+
'ort-wasm-simd-threaded.mjs',
|
|
14
|
+
'ort-wasm-simd-threaded.wasm',
|
|
15
|
+
'NOTICE.txt',
|
|
16
|
+
'SILERO_LICENSE.txt',
|
|
17
|
+
]);
|
|
18
|
+
|
|
19
|
+
/** Copies the asset files into `outputDirectory` and returns their names. */
|
|
20
|
+
export async function copyVoiceCaptureAssets(outputDirectory) {
|
|
21
|
+
await mkdir(outputDirectory, { recursive: true });
|
|
22
|
+
for (const fileName of voiceCaptureAssets) {
|
|
23
|
+
await cp(path.join(assets, fileName), path.join(outputDirectory, fileName));
|
|
24
|
+
}
|
|
25
|
+
return voiceCaptureAssets;
|
|
26
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { openPhraseCard, type PhraseCardFrame, type PhraseCardInput } from './surface';
|
|
2
|
+
export type { PhraseCheckView, PhraseChoice, PhraseRecording, PhraseResultView, PhraseSegmentState, PhraseSurface, PhraseText, PhraseView, } from './types';
|
|
3
|
+
export { leaveSurface, VoicePrompt, type PhraseDisplay, type PromptChange } from './prompt';
|
|
4
|
+
export { PhraseCard, type HandedOverRows, type PromptCopy } from './phraseCard';
|
|
5
|
+
export { VoiceRibbon, type VoiceCheckingStyle } from './ribbon';
|
|
6
|
+
export { VoiceVisual } from './visual';
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { PhraseView } from './types';
|
|
2
|
+
import type { VoicePrompt } from './prompt';
|
|
3
|
+
export type PromptCopy = Readonly<{
|
|
4
|
+
label: string;
|
|
5
|
+
sentence: string;
|
|
6
|
+
message: string;
|
|
7
|
+
action: string;
|
|
8
|
+
}>;
|
|
9
|
+
/** What a card that takes another card's place shows at first: that card's rows. */
|
|
10
|
+
export type HandedOverRows = Readonly<{
|
|
11
|
+
label: string;
|
|
12
|
+
message: string;
|
|
13
|
+
}>;
|
|
14
|
+
/** Draws the states of the phrase card. The surface owns the recording and the choices. */
|
|
15
|
+
export declare class PhraseCard {
|
|
16
|
+
private readonly prompt;
|
|
17
|
+
private readonly copy;
|
|
18
|
+
private readonly line;
|
|
19
|
+
private readonly segments;
|
|
20
|
+
private readonly bar;
|
|
21
|
+
private readonly ring;
|
|
22
|
+
private readonly seal;
|
|
23
|
+
private readonly check;
|
|
24
|
+
private readonly motion;
|
|
25
|
+
private rows;
|
|
26
|
+
private states;
|
|
27
|
+
private rendering;
|
|
28
|
+
private morph;
|
|
29
|
+
private rotation;
|
|
30
|
+
private version;
|
|
31
|
+
private disposed;
|
|
32
|
+
constructor(prompt: VoicePrompt, copy: PromptCopy, handedOver?: HandedOverRows | null);
|
|
33
|
+
/** The sentence with Record, after a check or a result. The wave comes back. */
|
|
34
|
+
showPrompt(): Promise<void>;
|
|
35
|
+
/** The end of speech: the wave flattens and opens into segments, the first one filled. */
|
|
36
|
+
beginCheck(): void;
|
|
37
|
+
show(view: PhraseView): Promise<void>;
|
|
38
|
+
/** The line stays in the card, which fades out as a whole when its surface closes. */
|
|
39
|
+
dispose(): void;
|
|
40
|
+
/** A newer state supersedes any state still on its way in. */
|
|
41
|
+
private begin;
|
|
42
|
+
private stale;
|
|
43
|
+
private queue;
|
|
44
|
+
private promptRows;
|
|
45
|
+
/**
|
|
46
|
+
* Rows whose content changes fade out together; `apply` then writes the new state, the card
|
|
47
|
+
* takes its new height, the ribbon row glides to its new place, and the new rows fade in one
|
|
48
|
+
* after another.
|
|
49
|
+
*/
|
|
50
|
+
private transition;
|
|
51
|
+
private markedText;
|
|
52
|
+
private detailNodes;
|
|
53
|
+
/** One line for the prompt; a caption and a detail line for the check and the result. */
|
|
54
|
+
private writeCaption;
|
|
55
|
+
private writeAction;
|
|
56
|
+
/** Changes the second caption line every 1.3 s and stays on the last line. */
|
|
57
|
+
private rotateDetails;
|
|
58
|
+
private swapDetail;
|
|
59
|
+
/** The flat line opens into six segments; the first stays filled, the rest turn to track. */
|
|
60
|
+
private openSegments;
|
|
61
|
+
/**
|
|
62
|
+
* Segments that finish fill one after another, 70 ms apart, and stop at a failed or held
|
|
63
|
+
* segment, which then shows its colour. Resolves when the result may follow.
|
|
64
|
+
*/
|
|
65
|
+
private setSegments;
|
|
66
|
+
/** The segments fill, the gaps close, the line shrinks as the ring draws, then seal and check. */
|
|
67
|
+
private sealLine;
|
|
68
|
+
private resetLine;
|
|
69
|
+
private paintStates;
|
|
70
|
+
private layoutSegments;
|
|
71
|
+
private tween;
|
|
72
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import type { VoiceCaptureEvent } from '../microphone';
|
|
2
|
+
import type { VoiceCheckingStyle } from './ribbon';
|
|
3
|
+
/** How the card shows a phrase: by default as it is, with a capital letter and a full stop. */
|
|
4
|
+
export type PhraseDisplay = (phrase: string) => string;
|
|
5
|
+
/** A new state of the card's rows. Rows that it leaves out keep their content. */
|
|
6
|
+
export type PromptChange = Readonly<{
|
|
7
|
+
title?: string;
|
|
8
|
+
phrase?: string;
|
|
9
|
+
status?: string;
|
|
10
|
+
/** The record button's label, and whether it shows. */
|
|
11
|
+
action?: Readonly<{
|
|
12
|
+
label?: string;
|
|
13
|
+
hidden?: boolean;
|
|
14
|
+
}>;
|
|
15
|
+
/** Runs while the changing rows are out of sight, for state that goes with them. */
|
|
16
|
+
apply?: () => void;
|
|
17
|
+
}>;
|
|
18
|
+
export declare class VoicePrompt {
|
|
19
|
+
/** The text to show for a phrase, such as the phrase with its commas. */
|
|
20
|
+
private readonly display;
|
|
21
|
+
readonly element: HTMLElement;
|
|
22
|
+
readonly title: HTMLHeadingElement;
|
|
23
|
+
readonly phrase: HTMLParagraphElement;
|
|
24
|
+
readonly status: HTMLParagraphElement;
|
|
25
|
+
readonly record: HTMLButtonElement;
|
|
26
|
+
readonly cancel: HTMLButtonElement;
|
|
27
|
+
readonly progress: HTMLElement;
|
|
28
|
+
/** The wave under the phrase. */
|
|
29
|
+
readonly ribbon: SVGSVGElement;
|
|
30
|
+
/** The row of the record button, which a phrase card reuses for its action. */
|
|
31
|
+
readonly actionRow: HTMLElement;
|
|
32
|
+
private readonly microphone;
|
|
33
|
+
private microphoneRequested;
|
|
34
|
+
private readonly visual;
|
|
35
|
+
private disposed;
|
|
36
|
+
private pendingChange;
|
|
37
|
+
private changing;
|
|
38
|
+
/** The card has been laid out once, and has since been hidden (a Wallet confirm took its place). */
|
|
39
|
+
private laidOut;
|
|
40
|
+
private hidden;
|
|
41
|
+
private readonly sizes;
|
|
42
|
+
/** A newer resize stops the one in progress, and takes over what waits for it to end. */
|
|
43
|
+
private frameFill;
|
|
44
|
+
private readonly onFrameFitted;
|
|
45
|
+
constructor(document: Document, label: string, phrase: string, checking?: VoiceCheckingStyle,
|
|
46
|
+
/** The text to show for a phrase, such as the phrase with its commas. */
|
|
47
|
+
display?: PhraseDisplay);
|
|
48
|
+
/**
|
|
49
|
+
* In an app's dialog with a step before this card, the corner control returns to that step.
|
|
50
|
+
* It still ends the card as cancelled; the Wallet tells the app that the person went back.
|
|
51
|
+
*/
|
|
52
|
+
showBack(): void;
|
|
53
|
+
/**
|
|
54
|
+
* Moves the rows to a new state. The rows whose content changes fade out together, take their
|
|
55
|
+
* new content and fade in one after another; the other rows stay still. A change that comes
|
|
56
|
+
* during a fade joins it, so no state in between is drawn.
|
|
57
|
+
*/
|
|
58
|
+
change(next: PromptChange): Promise<void>;
|
|
59
|
+
private readonly changesDrained;
|
|
60
|
+
private drainChanges;
|
|
61
|
+
/** The rows whose visible content `next` changes, top to bottom. */
|
|
62
|
+
private changingRows;
|
|
63
|
+
private rowOrder;
|
|
64
|
+
private writeChange;
|
|
65
|
+
/** The card is ready to record again, through a change of its rows. */
|
|
66
|
+
readyChange(message: string, next?: Omit<PromptChange, 'status'>): Promise<void>;
|
|
67
|
+
/** Starts without an entrance, because it takes another card's place. */
|
|
68
|
+
skipEntrance(): void;
|
|
69
|
+
/**
|
|
70
|
+
* In the Wallet iframe the parent eases the frame to every new card size. While the frame is
|
|
71
|
+
* taller than the card, the card fills it, so its edge follows the frame and never jumps; the
|
|
72
|
+
* surface around it keeps the card's own height, which is what the parent hears.
|
|
73
|
+
*/
|
|
74
|
+
fillFrame(fromCssPx?: number, rows?: 'top' | 'center', onFitted?: () => void): void;
|
|
75
|
+
private readonly releaseRows;
|
|
76
|
+
private frameFitted;
|
|
77
|
+
/** The card's rows, top to bottom: everything but its glow, its close button and its download. */
|
|
78
|
+
private rows;
|
|
79
|
+
/** A Wallet confirm hid the card (wallet-service.css); when it shows again, its rows come back. */
|
|
80
|
+
private observeSize;
|
|
81
|
+
setPhrase(phrase: string): void;
|
|
82
|
+
readonly capture: (event: VoiceCaptureEvent) => void;
|
|
83
|
+
/**
|
|
84
|
+
* This card replaces `previous` in the same surface. It skips the entrance and starts with the
|
|
85
|
+
* rows that `previous` shows, so the swap does not show; the caller then changes the rows.
|
|
86
|
+
*/
|
|
87
|
+
continueFrom(previous: Element): void;
|
|
88
|
+
/** Settle the visual when an enrollment sample is accepted. */
|
|
89
|
+
accepted(): void;
|
|
90
|
+
/** The wave rests flat while the glow keeps working. */
|
|
91
|
+
flattenLine(): void;
|
|
92
|
+
/** The glow settles. */
|
|
93
|
+
quietLine(): void;
|
|
94
|
+
/**
|
|
95
|
+
* Plays the unlock success in place and resolves when the check is drawn. Once
|
|
96
|
+
* the ring has formed it glides to the middle of `area` (the card by default).
|
|
97
|
+
*/
|
|
98
|
+
succeed(headline: string, message: string, area?: SuccessArea): Promise<void>;
|
|
99
|
+
ready(message: string): void;
|
|
100
|
+
private readonly changeMicrophone;
|
|
101
|
+
private useSystemMicrophone;
|
|
102
|
+
dispose(): void;
|
|
103
|
+
/** The ribbon turns into the ring and the copy may change, so the card's rows hold their size. */
|
|
104
|
+
private lockLayout;
|
|
105
|
+
/**
|
|
106
|
+
* The ring has formed: the reading copy fades where it is, only the ring glides
|
|
107
|
+
* to the middle of the free space, and the success copy arrives in place around
|
|
108
|
+
* it, headline first. The rows keep their locked size throughout.
|
|
109
|
+
*/
|
|
110
|
+
private beginSuccess;
|
|
111
|
+
private unlockLayout;
|
|
112
|
+
}
|
|
113
|
+
type SuccessArea = Readonly<{
|
|
114
|
+
top: number;
|
|
115
|
+
bottom: number;
|
|
116
|
+
}>;
|
|
117
|
+
/**
|
|
118
|
+
* A closed card fades out where it is, then goes. It leaves the layout at once, so a card or a
|
|
119
|
+
* confirm that comes next takes its place, and the frame never shows empty before it closes.
|
|
120
|
+
*/
|
|
121
|
+
export declare function leaveSurface(element: HTMLElement): void;
|
|
122
|
+
export {};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { VoiceActivityEvent } from '../microphone';
|
|
2
|
+
type RibbonPhase = 'idle' | 'listening' | 'processing';
|
|
3
|
+
/** How the line shows that the voice is being checked. */
|
|
4
|
+
export type VoiceCheckingStyle = 'ring' | 'segments';
|
|
5
|
+
/** Visual feedback from recorder activity. Never opens or retains an audio stream. */
|
|
6
|
+
export declare class VoiceRibbon {
|
|
7
|
+
private readonly checking;
|
|
8
|
+
readonly element: SVGSVGElement;
|
|
9
|
+
private readonly lines;
|
|
10
|
+
private readonly ripple;
|
|
11
|
+
private readonly track;
|
|
12
|
+
private readonly middle;
|
|
13
|
+
private readonly top;
|
|
14
|
+
private readonly bottom;
|
|
15
|
+
private readonly seal;
|
|
16
|
+
private readonly check;
|
|
17
|
+
private readonly motion;
|
|
18
|
+
private phase;
|
|
19
|
+
private stage;
|
|
20
|
+
private settleSuccess;
|
|
21
|
+
private successBegun;
|
|
22
|
+
private frame;
|
|
23
|
+
private previousTime;
|
|
24
|
+
private position;
|
|
25
|
+
private target;
|
|
26
|
+
private level;
|
|
27
|
+
private stopped;
|
|
28
|
+
constructor(document: Document, checking?: VoiceCheckingStyle);
|
|
29
|
+
readonly readLevel: () => number;
|
|
30
|
+
setPhase(phase: RibbonPhase): void;
|
|
31
|
+
/**
|
|
32
|
+
* Closes the ring into the success seal; resolves when the check is drawn.
|
|
33
|
+
* `begin` runs once the formed ring starts to close, after any capture in flight.
|
|
34
|
+
*/
|
|
35
|
+
succeed(begin: () => void): Promise<void>;
|
|
36
|
+
add(event: VoiceActivityEvent): void;
|
|
37
|
+
dispose(): void;
|
|
38
|
+
private readonly awaitSuccess;
|
|
39
|
+
private settle;
|
|
40
|
+
private append;
|
|
41
|
+
private wake;
|
|
42
|
+
private readonly draw;
|
|
43
|
+
private idleAmplitude;
|
|
44
|
+
private paint;
|
|
45
|
+
private paintLine;
|
|
46
|
+
private paintSegments;
|
|
47
|
+
/** Each segment brightens one beat after the one before it, from the resting line's opacity. */
|
|
48
|
+
private segmentOpacity;
|
|
49
|
+
private paintCapture;
|
|
50
|
+
private paintJoin;
|
|
51
|
+
/** The auth menu ring's arc: it grows and shrinks on its own clock while the ring turns. */
|
|
52
|
+
private spinnerArc;
|
|
53
|
+
private beginSuccess;
|
|
54
|
+
private paintSuccess;
|
|
55
|
+
private paintSeal;
|
|
56
|
+
private paintArc;
|
|
57
|
+
private displacement;
|
|
58
|
+
private linePath;
|
|
59
|
+
/** The stroke between two distances on a half's capture track. */
|
|
60
|
+
private halfPath;
|
|
61
|
+
}
|
|
62
|
+
export {};
|