@fusion.dev/helix-speech 0.0.0-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +104 -0
- package/dist/audio.d.ts +21 -0
- package/dist/audio.d.ts.map +1 -0
- package/dist/audio.js +42 -0
- package/dist/audio.js.map +1 -0
- package/dist/fallback-speech.d.ts +17 -0
- package/dist/fallback-speech.d.ts.map +1 -0
- package/dist/fallback-speech.js +50 -0
- package/dist/fallback-speech.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/moonshine-speech.d.ts +51 -0
- package/dist/moonshine-speech.d.ts.map +1 -0
- package/dist/moonshine-speech.js +189 -0
- package/dist/moonshine-speech.js.map +1 -0
- package/dist/protocol.d.ts +45 -0
- package/dist/protocol.d.ts.map +1 -0
- package/dist/protocol.js +2 -0
- package/dist/protocol.js.map +1 -0
- package/dist/worker/moonshine-worker.d.ts +2 -0
- package/dist/worker/moonshine-worker.d.ts.map +1 -0
- package/dist/worker/moonshine-worker.js +1 -0
- package/dist/worker/moonshine-worker.js.map +1 -0
- package/dist/worker/single-thread/moonshine.mjs +14 -0
- package/dist/worker/single-thread/moonshine.wasm +0 -0
- package/dist/worker/threaded/moonshine.mjs +20 -0
- package/dist/worker/threaded/moonshine.wasm +0 -0
- package/package.json +55 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Fusion Development Group
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# @fusion.dev/helix-speech
|
|
2
|
+
|
|
3
|
+
Speech engines for Helix dictation that recognize in the page, on the device.
|
|
4
|
+
The first is Moonshine, an open speech model that runs in WebAssembly.
|
|
5
|
+
|
|
6
|
+
```sh
|
|
7
|
+
bun add @fusion.dev/helix @fusion.dev/helix-speech
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
A field with `voice` listens through the browser's recognizer by default. Some
|
|
11
|
+
browsers have none that works. Brave has no speech service, and its on-device
|
|
12
|
+
install never finishes. Register this engine and every field on the page
|
|
13
|
+
listens through Moonshine instead:
|
|
14
|
+
|
|
15
|
+
```js
|
|
16
|
+
import { registerSpeechEngine } from '@fusion.dev/helix-core'
|
|
17
|
+
import { moonshineSpeech } from '@fusion.dev/helix-speech'
|
|
18
|
+
|
|
19
|
+
registerSpeechEngine(moonshineSpeech())
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Audio stays on the device, and no service is called. The first press
|
|
23
|
+
downloads the model, 51 MB for `tiny`, and the browser caches it. Call
|
|
24
|
+
`preload()` to download it before anyone presses.
|
|
25
|
+
|
|
26
|
+
To keep the browser's recognizer where it works, combine the two. A field
|
|
27
|
+
listens through the browser's recognizer, and through Moonshine where that
|
|
28
|
+
recognizer is missing or ends with an error that means it cannot recognize
|
|
29
|
+
here, such as Brave's `network`. The press that fails listens through
|
|
30
|
+
Moonshine, and so does every press after it.
|
|
31
|
+
|
|
32
|
+
```js
|
|
33
|
+
import { browserSpeech, registerSpeechEngine } from '@fusion.dev/helix-core'
|
|
34
|
+
import { fallbackSpeech, moonshineSpeech } from '@fusion.dev/helix-speech'
|
|
35
|
+
|
|
36
|
+
registerSpeechEngine(fallbackSpeech(browserSpeech, moonshineSpeech()))
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
A Helix sandbox preview listens through the engine its page registered, so
|
|
40
|
+
its examples dictate the same way.
|
|
41
|
+
|
|
42
|
+
## What it exports
|
|
43
|
+
|
|
44
|
+
| Import | What |
|
|
45
|
+
|---|---|
|
|
46
|
+
| `@fusion.dev/helix-speech` | `fallbackSpeech(first, second)`, `moonshineSpeech(options)`, and the types `MoonshineModel`, `MoonshineSpeech`, and `MoonshineSpeechOptions` |
|
|
47
|
+
| `@fusion.dev/helix-speech/worker/moonshine-worker.js` | The worker, whose folder an app serves itself |
|
|
48
|
+
|
|
49
|
+
| `moonshineSpeech` option | Default | What else it takes |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| `model` | `'tiny'` | `'small'`, a 165 MB download, or `'medium'`, 305 MB. Each is more accurate than the one before |
|
|
52
|
+
| `models` | The Moonshine CDN | A base URL, for an app that hosts the model files itself |
|
|
53
|
+
| `progress` | None | A function called with the download's progress, from 0 to 1 |
|
|
54
|
+
| `worker` | The `worker` folder beside the engine's module | The URL an app serves `dist/worker` from |
|
|
55
|
+
|
|
56
|
+
**It recognizes English.** Moonshine's streaming models are English only. A
|
|
57
|
+
field in another language ends listening with an error.
|
|
58
|
+
|
|
59
|
+
## How it runs
|
|
60
|
+
|
|
61
|
+
The engine captures the microphone in an audio worklet and sends the audio to
|
|
62
|
+
a worker, which runs the model and sends back each guess and each settled
|
|
63
|
+
phrase.
|
|
64
|
+
|
|
65
|
+
**The worker picks one of two builds.** A cross-origin isolated page can give
|
|
66
|
+
the worker threads, and the worker takes the threaded build that
|
|
67
|
+
`@moonshine-ai/moonshine-wasm` publishes. Any other page gets the
|
|
68
|
+
single-thread build. Moonshine does not publish that build.
|
|
69
|
+
`scripts/build-single-thread.sh` builds it from Moonshine's and ONNX
|
|
70
|
+
Runtime's sources, and the files are assets of a release of this repository.
|
|
71
|
+
`single-thread.json` names the release and pins the versions and the digest of
|
|
72
|
+
each file.
|
|
73
|
+
|
|
74
|
+
**The worker runs from a folder, `dist/worker`, as it is served.** The build
|
|
75
|
+
bundles the worker with Moonshine's JavaScript and puts both builds of the
|
|
76
|
+
Emscripten module beside it. The engine starts it from a URL a bundler does
|
|
77
|
+
not follow, so a bundler never meets the module's Node branches. Where the
|
|
78
|
+
package's files are served as they are, the engine finds the folder by
|
|
79
|
+
itself. A bundled app copies the folder to its static files and passes its URL
|
|
80
|
+
as `worker`:
|
|
81
|
+
|
|
82
|
+
```js
|
|
83
|
+
registerSpeechEngine(moonshineSpeech({ worker: '/helix-speech/' }))
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
fusion.dev does this with its `speech-copy` task, beside the MapLibre worker.
|
|
87
|
+
|
|
88
|
+
## Working on it
|
|
89
|
+
|
|
90
|
+
```sh
|
|
91
|
+
bun run --filter @fusion.dev/helix-speech build
|
|
92
|
+
bun run --filter @fusion.dev/helix-speech test
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The build downloads the single-thread files from the release into
|
|
96
|
+
`vendor/single-thread`, checks each against its digest, and copies it into
|
|
97
|
+
`dist`. While the repository is private, the download needs a token: the build
|
|
98
|
+
reads `GH_TOKEN`, or asks `gh` for yours.
|
|
99
|
+
|
|
100
|
+
To change the build, run `scripts/build-single-thread.sh`, publish its two
|
|
101
|
+
files as a new release, and put the release's tag and the digests the script
|
|
102
|
+
prints in `single-thread.json`.
|
|
103
|
+
|
|
104
|
+
MIT
|
package/dist/audio.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** The rate Moonshine's models take audio at. */
|
|
2
|
+
export declare const MODEL_RATE = 16000;
|
|
3
|
+
/**
|
|
4
|
+
* Resamples mono audio to the model's rate by linear interpolation. A
|
|
5
|
+
* microphone captures at the device's rate, usually 44.1 or 48 kHz, and
|
|
6
|
+
* Firefox refuses to connect it to an `AudioContext` running at another.
|
|
7
|
+
*/
|
|
8
|
+
export declare const resample: (input: Float32Array, rate: number) => Float32Array;
|
|
9
|
+
/**
|
|
10
|
+
* How loud a chunk is, from 0 to 1: the root mean square, scaled by the same
|
|
11
|
+
* factor as the browser engine's meter, so a voice at speaking level reads
|
|
12
|
+
* near the top of the ring either way.
|
|
13
|
+
*/
|
|
14
|
+
export declare const loudness: (samples: Float32Array) => number;
|
|
15
|
+
/**
|
|
16
|
+
* The audio worklet that hands the page each block the microphone delivers,
|
|
17
|
+
* averaged to mono. Averaging matters: a headset can open as stereo with the
|
|
18
|
+
* voice on one channel and silence on the other.
|
|
19
|
+
*/
|
|
20
|
+
export declare const CAPTURE_WORKLET = "registerProcessor('helix-speech-capture', class extends AudioWorkletProcessor {\n process(inputs) {\n const channels = inputs[0]\n if (channels && channels.length > 0) {\n const mono = new Float32Array(channels[0].length)\n for (const channel of channels) {\n for (let at = 0; at < mono.length; at += 1) {\n mono[at] += channel[at] / channels.length\n }\n }\n this.port.postMessage(mono, [mono.buffer])\n }\n return true\n }\n})";
|
|
21
|
+
//# sourceMappingURL=audio.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,eAAO,MAAM,UAAU,QAAS,CAAA;AAEhC;;;;GAIG;AACH,eAAO,MAAM,QAAQ,UAAW,YAAY,QAAQ,MAAM,KAAG,YAc5D,CAAA;AAED;;;;GAIG;AACH,eAAO,MAAM,QAAQ,YAAa,YAAY,KAAG,MAShD,CAAA;AAED;;;;GAIG;AACH,eAAO,MAAM,eAAe,0eAczB,CAAA"}
|
package/dist/audio.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export const MODEL_RATE = 16_000;
|
|
2
|
+
export const resample = (input, rate) => {
|
|
3
|
+
if (rate === MODEL_RATE) {
|
|
4
|
+
return input;
|
|
5
|
+
}
|
|
6
|
+
const step = rate / MODEL_RATE;
|
|
7
|
+
const output = new Float32Array(Math.floor(input.length / step));
|
|
8
|
+
for (let at = 0; at < output.length; at += 1) {
|
|
9
|
+
const position = at * step;
|
|
10
|
+
const before = Math.floor(position);
|
|
11
|
+
const after = Math.min(before + 1, input.length - 1);
|
|
12
|
+
const weight = position - before;
|
|
13
|
+
output[at] = (input[before] ?? 0) * (1 - weight) + (input[after] ?? 0) * weight;
|
|
14
|
+
}
|
|
15
|
+
return output;
|
|
16
|
+
};
|
|
17
|
+
export const loudness = (samples) => {
|
|
18
|
+
if (samples.length === 0) {
|
|
19
|
+
return 0;
|
|
20
|
+
}
|
|
21
|
+
let sum = 0;
|
|
22
|
+
for (const sample of samples) {
|
|
23
|
+
sum += sample * sample;
|
|
24
|
+
}
|
|
25
|
+
return Math.min(1, Math.sqrt(sum / samples.length) * 4);
|
|
26
|
+
};
|
|
27
|
+
export const CAPTURE_WORKLET = `registerProcessor('helix-speech-capture', class extends AudioWorkletProcessor {
|
|
28
|
+
process(inputs) {
|
|
29
|
+
const channels = inputs[0]
|
|
30
|
+
if (channels && channels.length > 0) {
|
|
31
|
+
const mono = new Float32Array(channels[0].length)
|
|
32
|
+
for (const channel of channels) {
|
|
33
|
+
for (let at = 0; at < mono.length; at += 1) {
|
|
34
|
+
mono[at] += channel[at] / channels.length
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
this.port.postMessage(mono, [mono.buffer])
|
|
38
|
+
}
|
|
39
|
+
return true
|
|
40
|
+
}
|
|
41
|
+
})`;
|
|
42
|
+
//# sourceMappingURL=audio.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audio.js","sourceRoot":"","sources":["../src/audio.ts"],"names":[],"mappings":"AACA,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAA;AAOhC,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,KAAmB,EAAE,IAAY,EAAgB,EAAE;IAC1E,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,OAAO,KAAK,CAAA;IACd,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,UAAU,CAAA;IAC9B,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAA;IAChE,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAA;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QACpD,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAA;QAChC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAA;IACjF,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AAOD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,OAAqB,EAAU,EAAE;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,CAAA;IACV,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,GAAG,IAAI,MAAM,GAAG,MAAM,CAAA;IACxB,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;AACzD,CAAC,CAAA;AAOD,MAAM,CAAC,MAAM,eAAe,GAAG;;;;;;;;;;;;;;GAc5B,CAAA"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { SpeechEngine } from '@fusion.dev/helix-core';
|
|
2
|
+
/**
|
|
3
|
+
* Combines two engines: a field listens through `first`, and through
|
|
4
|
+
* `second` where `first` is not available or cannot recognize here. The first
|
|
5
|
+
* time `first` ends with an error that means it cannot, before it heard
|
|
6
|
+
* anything, the same press listens through `second`, and so does every press
|
|
7
|
+
* after it on the page.
|
|
8
|
+
*
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { browserSpeech, registerSpeechEngine } from '@fusion.dev/helix-core'
|
|
11
|
+
* import { fallbackSpeech, moonshineSpeech } from '@fusion.dev/helix-speech'
|
|
12
|
+
*
|
|
13
|
+
* registerSpeechEngine(fallbackSpeech(browserSpeech, moonshineSpeech()))
|
|
14
|
+
* ```
|
|
15
|
+
*/
|
|
16
|
+
export declare const fallbackSpeech: (first: SpeechEngine, second: SpeechEngine) => SpeechEngine;
|
|
17
|
+
//# sourceMappingURL=fallback-speech.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fallback-speech.d.ts","sourceRoot":"","sources":["../src/fallback-speech.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAiC,MAAM,wBAAwB,CAAA;AAczF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,cAAc,UAAW,YAAY,UAAU,YAAY,KAAG,YAkD1E,CAAA"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
const CANNOT = new Set([
|
|
2
|
+
'language-not-supported',
|
|
3
|
+
'network',
|
|
4
|
+
'service-not-allowed',
|
|
5
|
+
'This browser has no speech recognition.',
|
|
6
|
+
]);
|
|
7
|
+
export const fallbackSpeech = (first, second) => {
|
|
8
|
+
let failed = false;
|
|
9
|
+
const listen = ({ lang, listener, }) => {
|
|
10
|
+
if (failed || !first.available()) {
|
|
11
|
+
return second.listen({ lang, listener });
|
|
12
|
+
}
|
|
13
|
+
let heard = false;
|
|
14
|
+
let stopped = false;
|
|
15
|
+
let current;
|
|
16
|
+
current = first.listen({
|
|
17
|
+
lang,
|
|
18
|
+
listener: {
|
|
19
|
+
end: (reason, message) => {
|
|
20
|
+
if (reason === 'error' && !heard && !stopped && message && CANNOT.has(message)) {
|
|
21
|
+
failed = true;
|
|
22
|
+
current = second.listen({ lang, listener });
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
listener.end(reason, message);
|
|
26
|
+
},
|
|
27
|
+
final: (text) => {
|
|
28
|
+
heard = true;
|
|
29
|
+
listener.final(text);
|
|
30
|
+
},
|
|
31
|
+
level: (level, bands) => listener.level?.(level, bands),
|
|
32
|
+
partial: (text) => {
|
|
33
|
+
heard = true;
|
|
34
|
+
listener.partial(text);
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
return {
|
|
39
|
+
stop: () => {
|
|
40
|
+
stopped = true;
|
|
41
|
+
current?.stop();
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
};
|
|
45
|
+
return {
|
|
46
|
+
available: () => first.available() || second.available(),
|
|
47
|
+
listen,
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
//# sourceMappingURL=fallback-speech.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fallback-speech.js","sourceRoot":"","sources":["../src/fallback-speech.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC;IACrB,wBAAwB;IACxB,SAAS;IACT,qBAAqB;IACrB,yCAAyC;CAC1C,CAAC,CAAA;AAgBF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,KAAmB,EAAE,MAAoB,EAAgB,EAAE;IACxF,IAAI,MAAM,GAAG,KAAK,CAAA;IAElB,MAAM,MAAM,GAAG,CAAC,EACd,IAAI,EACJ,QAAQ,GAIT,EAAiB,EAAE;QAClB,IAAI,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;YACjC,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;QAC1C,CAAC;QACD,IAAI,KAAK,GAAG,KAAK,CAAA;QACjB,IAAI,OAAO,GAAG,KAAK,CAAA;QACnB,IAAI,OAAkC,CAAA;QACtC,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC;YACrB,IAAI;YACJ,QAAQ,EAAE;gBACR,GAAG,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;oBACvB,IAAI,MAAM,KAAK,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,IAAI,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC/E,MAAM,GAAG,IAAI,CAAA;wBACb,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;wBAC3C,OAAM;oBACR,CAAC;oBACD,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;gBAC/B,CAAC;gBACD,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE;oBACd,KAAK,GAAG,IAAI,CAAA;oBACZ,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBACtB,CAAC;gBACD,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC;gBACvD,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;oBAChB,KAAK,GAAG,IAAI,CAAA;oBACZ,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;gBACxB,CAAC;aACF;SACF,CAAC,CAAA;QACF,OAAO;YACL,IAAI,EAAE,GAAG,EAAE;gBACT,OAAO,GAAG,IAAI,CAAA;gBACd,OAAO,EAAE,IAAI,EAAE,CAAA;YACjB,CAAC;SACF,CAAA;IACH,CAAC,CAAA;IAED,OAAO;QACL,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE;QACxD,MAAM;KACP,CAAA;AACH,CAAC,CAAA"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,sBAAsB,EAC3B,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAC9B,YAAY,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAGL,eAAe,GAChB,MAAM,uBAAuB,CAAA"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { SpeechEngine } from '@fusion.dev/helix-core';
|
|
2
|
+
import type { MoonshineModel } from './protocol.js';
|
|
3
|
+
/** How a Moonshine engine loads its model. */
|
|
4
|
+
export interface MoonshineSpeechOptions {
|
|
5
|
+
/**
|
|
6
|
+
* The model to recognize with. `tiny` is a 51 MB download, `small` 165 MB
|
|
7
|
+
* and `medium` 305 MB, each more accurate than the one before.
|
|
8
|
+
*/
|
|
9
|
+
model?: MoonshineModel;
|
|
10
|
+
/**
|
|
11
|
+
* A base URL to load the model files from, for an app that hosts them
|
|
12
|
+
* itself. The engine loads them from the Moonshine CDN without one.
|
|
13
|
+
*/
|
|
14
|
+
models?: string;
|
|
15
|
+
/** Called as the model downloads, from 0 to 1 across all its files. */
|
|
16
|
+
progress?(fraction: number): void;
|
|
17
|
+
/**
|
|
18
|
+
* The URL of the folder this package's `dist/worker` is served from. The
|
|
19
|
+
* engine uses the copy beside it without one, which works where the
|
|
20
|
+
* package's files are served as they are. A bundled app copies the folder
|
|
21
|
+
* to its static files and names it here.
|
|
22
|
+
*/
|
|
23
|
+
worker?: string | URL;
|
|
24
|
+
}
|
|
25
|
+
/** A speech engine that recognizes English on the device with Moonshine. */
|
|
26
|
+
export interface MoonshineSpeech extends SpeechEngine {
|
|
27
|
+
/**
|
|
28
|
+
* Downloads the model and starts the worker that runs it, so the first
|
|
29
|
+
* press listens at once. Listening loads the model too, where this was not
|
|
30
|
+
* called first.
|
|
31
|
+
*/
|
|
32
|
+
preload(): Promise<void>;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Creates an engine that recognizes speech in the page with Moonshine, an
|
|
36
|
+
* open speech model that runs in WebAssembly. Audio stays on the device, and
|
|
37
|
+
* no service is called. Only the model files are downloaded, once, and the
|
|
38
|
+
* browser caches them.
|
|
39
|
+
*
|
|
40
|
+
* **It recognizes English.** A field in another language ends listening with
|
|
41
|
+
* an error.
|
|
42
|
+
*
|
|
43
|
+
* ```ts
|
|
44
|
+
* import { registerSpeechEngine } from '@fusion.dev/helix-core'
|
|
45
|
+
* import { moonshineSpeech } from '@fusion.dev/helix-speech'
|
|
46
|
+
*
|
|
47
|
+
* registerSpeechEngine(moonshineSpeech())
|
|
48
|
+
* ```
|
|
49
|
+
*/
|
|
50
|
+
export declare const moonshineSpeech: (options?: MoonshineSpeechOptions) => MoonshineSpeech;
|
|
51
|
+
//# sourceMappingURL=moonshine-speech.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moonshine-speech.d.ts","sourceRoot":"","sources":["../src/moonshine-speech.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAiC,MAAM,wBAAwB,CAAA;AAEzF,OAAO,KAAK,EAAc,cAAc,EAAY,MAAM,eAAe,CAAA;AAEzE,8CAA8C;AAC9C,MAAM,WAAW,sBAAsB;IACrC;;;OAGG;IACH,KAAK,CAAC,EAAE,cAAc,CAAA;IACtB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,uEAAuE;IACvE,QAAQ,CAAC,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,GAAG,CAAA;CACtB;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACzB;AAWD;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,eAAe,aAAa,sBAAsB,KAAQ,eA+MtE,CAAA"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { CAPTURE_WORKLET, loudness, MODEL_RATE, resample } from './audio.js';
|
|
2
|
+
const CHUNK = 0.05;
|
|
3
|
+
const LEVEL_INTERVAL = 50;
|
|
4
|
+
const ENGLISH = /^en(-|$)/i;
|
|
5
|
+
export const moonshineSpeech = (options = {}) => {
|
|
6
|
+
let worker;
|
|
7
|
+
let loaded;
|
|
8
|
+
let sessions = 0;
|
|
9
|
+
const listeners = new Map();
|
|
10
|
+
const spawn = () => {
|
|
11
|
+
if (worker) {
|
|
12
|
+
return worker;
|
|
13
|
+
}
|
|
14
|
+
const here = import.meta.url;
|
|
15
|
+
const folder = new URL(options.worker ?? `${here.slice(0, here.lastIndexOf('/') + 1)}worker/`, location.href);
|
|
16
|
+
const spawned = new Worker(new URL('moonshine-worker.js', folder), { type: 'module' });
|
|
17
|
+
spawned.onmessage = (event) => {
|
|
18
|
+
const message = event.data;
|
|
19
|
+
switch (message.type) {
|
|
20
|
+
case 'progress':
|
|
21
|
+
options.progress?.(message.fraction);
|
|
22
|
+
return;
|
|
23
|
+
case 'partial':
|
|
24
|
+
listeners.get(message.id)?.partial(message.text);
|
|
25
|
+
return;
|
|
26
|
+
case 'final':
|
|
27
|
+
listeners.get(message.id)?.final(message.text);
|
|
28
|
+
return;
|
|
29
|
+
case 'error': {
|
|
30
|
+
const listener = listeners.get(message.id);
|
|
31
|
+
listeners.delete(message.id);
|
|
32
|
+
listener?.end('error', message.message);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
case 'stopped': {
|
|
36
|
+
const listener = listeners.get(message.id);
|
|
37
|
+
listeners.delete(message.id);
|
|
38
|
+
listener?.end('stopped');
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
default:
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
worker = spawned;
|
|
46
|
+
return spawned;
|
|
47
|
+
};
|
|
48
|
+
const send = (message, transfer = []) => {
|
|
49
|
+
spawn().postMessage(message, transfer);
|
|
50
|
+
};
|
|
51
|
+
const preload = () => {
|
|
52
|
+
loaded ??= new Promise((resolve, reject) => {
|
|
53
|
+
const target = spawn();
|
|
54
|
+
const settle = (event) => {
|
|
55
|
+
if (event.data.type === 'loaded') {
|
|
56
|
+
target.removeEventListener('message', settle);
|
|
57
|
+
resolve();
|
|
58
|
+
}
|
|
59
|
+
else if (event.data.type === 'failed') {
|
|
60
|
+
target.removeEventListener('message', settle);
|
|
61
|
+
loaded = undefined;
|
|
62
|
+
reject(new Error(event.data.message));
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
target.addEventListener('message', settle);
|
|
66
|
+
send({
|
|
67
|
+
model: options.model ?? 'tiny',
|
|
68
|
+
type: 'load',
|
|
69
|
+
...(options.models ? { models: options.models } : {}),
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
return loaded;
|
|
73
|
+
};
|
|
74
|
+
const listen = ({ lang, listener, }) => {
|
|
75
|
+
if (!ENGLISH.test(lang)) {
|
|
76
|
+
queueMicrotask(() => listener.end('error', `Moonshine recognizes English only, and the field is in ${lang}.`));
|
|
77
|
+
return { stop: () => { } };
|
|
78
|
+
}
|
|
79
|
+
sessions += 1;
|
|
80
|
+
const id = sessions;
|
|
81
|
+
listeners.set(id, listener);
|
|
82
|
+
let stopped = false;
|
|
83
|
+
let release = () => { };
|
|
84
|
+
const fail = (error) => {
|
|
85
|
+
release();
|
|
86
|
+
if (listeners.delete(id)) {
|
|
87
|
+
listener.end('error', error instanceof Error ? error.message : String(error));
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
preload().catch(fail);
|
|
91
|
+
send({ id, type: 'start' });
|
|
92
|
+
void (async () => {
|
|
93
|
+
const stream = await navigator.mediaDevices.getUserMedia({
|
|
94
|
+
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
|
|
95
|
+
});
|
|
96
|
+
if (stopped || !listeners.has(id)) {
|
|
97
|
+
for (const track of stream.getTracks()) {
|
|
98
|
+
track.stop();
|
|
99
|
+
}
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
const context = new AudioContext();
|
|
103
|
+
if (context.state === 'suspended') {
|
|
104
|
+
await context.resume();
|
|
105
|
+
}
|
|
106
|
+
const url = URL.createObjectURL(new Blob([CAPTURE_WORKLET], { type: 'text/javascript' }));
|
|
107
|
+
try {
|
|
108
|
+
await context.audioWorklet.addModule(url);
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
URL.revokeObjectURL(url);
|
|
112
|
+
}
|
|
113
|
+
const source = context.createMediaStreamSource(stream);
|
|
114
|
+
const node = new AudioWorkletNode(context, 'helix-speech-capture');
|
|
115
|
+
const size = Math.round(CHUNK * MODEL_RATE);
|
|
116
|
+
let chunk = new Float32Array(size);
|
|
117
|
+
let filled = 0;
|
|
118
|
+
let loudest = 0;
|
|
119
|
+
let reported = 0;
|
|
120
|
+
node.port.onmessage = (event) => {
|
|
121
|
+
if (stopped) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
const block = resample(event.data, context.sampleRate);
|
|
125
|
+
loudest = Math.max(loudest, loudness(block));
|
|
126
|
+
if (event.timeStamp - reported >= LEVEL_INTERVAL) {
|
|
127
|
+
listener.level?.(loudest);
|
|
128
|
+
loudest = 0;
|
|
129
|
+
reported = event.timeStamp;
|
|
130
|
+
}
|
|
131
|
+
let from = 0;
|
|
132
|
+
while (from < block.length) {
|
|
133
|
+
const taken = Math.min(size - filled, block.length - from);
|
|
134
|
+
chunk.set(block.subarray(from, from + taken), filled);
|
|
135
|
+
filled += taken;
|
|
136
|
+
from += taken;
|
|
137
|
+
if (filled === size) {
|
|
138
|
+
send({ id, samples: chunk, type: 'audio' }, [chunk.buffer]);
|
|
139
|
+
chunk = new Float32Array(size);
|
|
140
|
+
filled = 0;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
source.connect(node);
|
|
145
|
+
release = () => {
|
|
146
|
+
node.port.onmessage = null;
|
|
147
|
+
source.disconnect();
|
|
148
|
+
node.disconnect();
|
|
149
|
+
for (const track of stream.getTracks()) {
|
|
150
|
+
track.stop();
|
|
151
|
+
}
|
|
152
|
+
void context.close();
|
|
153
|
+
if (filled > 0) {
|
|
154
|
+
send({ id, samples: chunk.slice(0, filled), type: 'audio' });
|
|
155
|
+
filled = 0;
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
if (stopped || !listeners.has(id)) {
|
|
159
|
+
release();
|
|
160
|
+
}
|
|
161
|
+
})().catch(fail);
|
|
162
|
+
return {
|
|
163
|
+
stop: () => {
|
|
164
|
+
if (stopped) {
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
stopped = true;
|
|
168
|
+
release();
|
|
169
|
+
if (worker) {
|
|
170
|
+
send({ id, type: 'stop' });
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
listeners.delete(id);
|
|
174
|
+
listener.end('stopped');
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
};
|
|
179
|
+
return {
|
|
180
|
+
available: () => typeof WebAssembly === 'object' &&
|
|
181
|
+
typeof Worker === 'function' &&
|
|
182
|
+
typeof AudioWorkletNode === 'function' &&
|
|
183
|
+
typeof navigator === 'object' &&
|
|
184
|
+
typeof navigator.mediaDevices?.getUserMedia === 'function',
|
|
185
|
+
listen,
|
|
186
|
+
preload,
|
|
187
|
+
};
|
|
188
|
+
};
|
|
189
|
+
//# sourceMappingURL=moonshine-speech.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moonshine-speech.js","sourceRoot":"","sources":["../src/moonshine-speech.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAqC5E,MAAM,KAAK,GAAG,IAAI,CAAA;AAGlB,MAAM,cAAc,GAAG,EAAE,CAAA;AAGzB,MAAM,OAAO,GAAG,WAAW,CAAA;AAkB3B,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,OAAO,GAA2B,EAAE,EAAmB,EAAE;IACvF,IAAI,MAA0B,CAAA;IAC9B,IAAI,MAAiC,CAAA;IACrC,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,SAAS,GAAG,IAAI,GAAG,EAA0B,CAAA;IAEnD,MAAM,KAAK,GAAG,GAAW,EAAE;QACzB,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAA;QACf,CAAC;QAID,MAAM,IAAI,GAAG,OAAO,IAAI,CAAC,GAAG,CAAA;QAC5B,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,OAAO,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,EACtE,QAAQ,CAAC,IAAI,CACd,CAAA;QACD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;QACtF,OAAO,CAAC,SAAS,GAAG,CAAC,KAA+B,EAAE,EAAE;YACtD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAA;YAC1B,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;gBACrB,KAAK,UAAU;oBACb,OAAO,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;oBACpC,OAAM;gBACR,KAAK,SAAS;oBACZ,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;oBAChD,OAAM;gBACR,KAAK,OAAO;oBACV,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;oBAC9C,OAAM;gBACR,KAAK,OAAO,EAAE,CAAC;oBACb,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oBAC1C,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oBAC5B,QAAQ,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;oBACvC,OAAM;gBACR,CAAC;gBACD,KAAK,SAAS,EAAE,CAAC;oBACf,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oBAC1C,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;oBAC5B,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAA;oBACxB,OAAM;gBACR,CAAC;gBACD;oBACE,OAAM;YACV,CAAC;QACH,CAAC,CAAA;QACD,MAAM,GAAG,OAAO,CAAA;QAChB,OAAO,OAAO,CAAA;IAChB,CAAC,CAAA;IAED,MAAM,IAAI,GAAG,CAAC,OAAiB,EAAE,QAAQ,GAAmB,EAAE,EAAQ,EAAE;QACtE,KAAK,EAAE,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IACxC,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,GAAkB,EAAE;QAClC,MAAM,KAAK,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC/C,MAAM,MAAM,GAAG,KAAK,EAAE,CAAA;YACtB,MAAM,MAAM,GAAG,CAAC,KAA+B,EAAQ,EAAE;gBACvD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACjC,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;oBAC7C,OAAO,EAAE,CAAA;gBACX,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACxC,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;oBAC7C,MAAM,GAAG,SAAS,CAAA;oBAClB,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAA;gBACvC,CAAC;YACH,CAAC,CAAA;YACD,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;YAC1C,IAAI,CAAC;gBACH,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM;gBAC9B,IAAI,EAAE,MAAM;gBACZ,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QACF,OAAO,MAAM,CAAA;IACf,CAAC,CAAA;IAED,MAAM,MAAM,GAAG,CAAC,EACd,IAAI,EACJ,QAAQ,GAIT,EAAiB,EAAE;QAClB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,cAAc,CAAC,GAAG,EAAE,CAClB,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,0DAA0D,IAAI,GAAG,CAAC,CACzF,CAAA;YACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAA;QAC3B,CAAC;QACD,QAAQ,IAAI,CAAC,CAAA;QACb,MAAM,EAAE,GAAG,QAAQ,CAAA;QACnB,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QAC3B,IAAI,OAAO,GAAG,KAAK,CAAA;QACnB,IAAI,OAAO,GAAG,GAAS,EAAE,GAAE,CAAC,CAAA;QAE5B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAQ,EAAE;YACpC,OAAO,EAAE,CAAA;YACT,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzB,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;YAC/E,CAAC;QACH,CAAC,CAAA;QAKD,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACrB,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;QAC3B,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC;gBACvD,KAAK,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE;aAC3E,CAAC,CAAA;YAEF,IAAI,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAClC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;oBACvC,KAAK,CAAC,IAAI,EAAE,CAAA;gBACd,CAAC;gBACD,OAAM;YACR,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAA;YAClC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;gBAClC,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACxB,CAAC;YACD,MAAM,GAAG,GAAG,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,eAAe,CAAC,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAA;YACzF,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;YAC3C,CAAC;oBAAS,CAAC;gBACT,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;YAC1B,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAA;YACtD,MAAM,IAAI,GAAG,IAAI,gBAAgB,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAA;YAClE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,CAAA;YAC3C,IAAI,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,CAAA;YAClC,IAAI,MAAM,GAAG,CAAC,CAAA;YACd,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,QAAQ,GAAG,CAAC,CAAA;YAChB,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,KAAiC,EAAE,EAAE;gBAC1D,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAM;gBACR,CAAC;gBACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;gBACtD,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;gBAC5C,IAAI,KAAK,CAAC,SAAS,GAAG,QAAQ,IAAI,cAAc,EAAE,CAAC;oBACjD,QAAQ,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,CAAA;oBACzB,OAAO,GAAG,CAAC,CAAA;oBACX,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAA;gBAC5B,CAAC;gBACD,IAAI,IAAI,GAAG,CAAC,CAAA;gBACZ,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;oBAC1D,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,KAAK,CAAC,EAAE,MAAM,CAAC,CAAA;oBACrD,MAAM,IAAI,KAAK,CAAA;oBACf,IAAI,IAAI,KAAK,CAAA;oBACb,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;wBACpB,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;wBAC3D,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,CAAA;wBAC9B,MAAM,GAAG,CAAC,CAAA;oBACZ,CAAC;gBACH,CAAC;YACH,CAAC,CAAA;YACD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YACpB,OAAO,GAAG,GAAG,EAAE;gBACb,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;gBAC1B,MAAM,CAAC,UAAU,EAAE,CAAA;gBACnB,IAAI,CAAC,UAAU,EAAE,CAAA;gBACjB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;oBACvC,KAAK,CAAC,IAAI,EAAE,CAAA;gBACd,CAAC;gBACD,KAAK,OAAO,CAAC,KAAK,EAAE,CAAA;gBACpB,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;oBAC5D,MAAM,GAAG,CAAC,CAAA;gBACZ,CAAC;YACH,CAAC,CAAA;YACD,IAAI,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAClC,OAAO,EAAE,CAAA;YACX,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAEhB,OAAO;YACL,IAAI,EAAE,GAAG,EAAE;gBACT,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAM;gBACR,CAAC;gBACD,OAAO,GAAG,IAAI,CAAA;gBACd,OAAO,EAAE,CAAA;gBACT,IAAI,MAAM,EAAE,CAAC;oBACX,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;gBAC5B,CAAC;qBAAM,CAAC;oBACN,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;oBACpB,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBACzB,CAAC;YACH,CAAC;SACF,CAAA;IACH,CAAC,CAAA;IAED,OAAO;QACL,SAAS,EAAE,GAAG,EAAE,CACd,OAAO,WAAW,KAAK,QAAQ;YAC/B,OAAO,MAAM,KAAK,UAAU;YAC5B,OAAO,gBAAgB,KAAK,UAAU;YACtC,OAAO,SAAS,KAAK,QAAQ;YAC7B,OAAO,SAAS,CAAC,YAAY,EAAE,YAAY,KAAK,UAAU;QAC5D,MAAM;QACN,OAAO;KACR,CAAA;AACH,CAAC,CAAA"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** The model sizes an engine can load, smallest first. */
|
|
2
|
+
export type MoonshineModel = 'medium' | 'small' | 'tiny';
|
|
3
|
+
/** What the page tells the worker. */
|
|
4
|
+
export type ToWorker = {
|
|
5
|
+
/** Where the model files load from, or the Moonshine CDN when absent. */
|
|
6
|
+
models?: string;
|
|
7
|
+
model: MoonshineModel;
|
|
8
|
+
type: 'load';
|
|
9
|
+
} | {
|
|
10
|
+
id: number;
|
|
11
|
+
type: 'start';
|
|
12
|
+
} | {
|
|
13
|
+
id: number;
|
|
14
|
+
samples: Float32Array;
|
|
15
|
+
type: 'audio';
|
|
16
|
+
} | {
|
|
17
|
+
id: number;
|
|
18
|
+
type: 'stop';
|
|
19
|
+
};
|
|
20
|
+
/** What the worker tells the page. */
|
|
21
|
+
export type FromWorker = {
|
|
22
|
+
fraction: number;
|
|
23
|
+
type: 'progress';
|
|
24
|
+
} | {
|
|
25
|
+
type: 'loaded';
|
|
26
|
+
} | {
|
|
27
|
+
message: string;
|
|
28
|
+
type: 'failed';
|
|
29
|
+
} | {
|
|
30
|
+
id: number;
|
|
31
|
+
text: string;
|
|
32
|
+
type: 'partial';
|
|
33
|
+
} | {
|
|
34
|
+
id: number;
|
|
35
|
+
text: string;
|
|
36
|
+
type: 'final';
|
|
37
|
+
} | {
|
|
38
|
+
id: number;
|
|
39
|
+
type: 'stopped';
|
|
40
|
+
} | {
|
|
41
|
+
id: number;
|
|
42
|
+
message: string;
|
|
43
|
+
type: 'error';
|
|
44
|
+
};
|
|
45
|
+
//# sourceMappingURL=protocol.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":"AAAA,0DAA0D;AAC1D,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAA;AAExD,sCAAsC;AACtC,MAAM,MAAM,QAAQ,GAChB;IACE,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,cAAc,CAAA;IACrB,IAAI,EAAE,MAAM,CAAA;CACb,GACD;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAC7B;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAEhC,sCAAsC;AACtC,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,GACnC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAA;CAAE,GAC7C;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAC3C;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,SAAS,CAAA;CAAE,GAC/B;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,CAAA"}
|
package/dist/protocol.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"moonshine-worker.d.ts","sourceRoot":"","sources":["../../src/worker/moonshine-worker.ts"],"names":[],"mappings":""}
|