@elizaos/capacitor-swabble 1.0.0 → 2.0.3-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +90 -0
- package/android/build.gradle +17 -3
- package/dist/esm/web.d.ts.map +1 -1
- package/dist/esm/web.js +71 -17
- package/dist/esm/web.test.d.ts +2 -0
- package/dist/esm/web.test.d.ts.map +1 -0
- package/dist/esm/web.test.js +170 -0
- package/dist/plugin.cjs.js +71 -17
- package/dist/plugin.cjs.js.map +1 -1
- package/dist/plugin.js +71 -17
- package/dist/plugin.js.map +1 -1
- package/package.json +17 -15
- package/electrobun/src/global.d.ts +0 -1
- package/electrobun/src/index.ts +0 -786
- package/electrobun/tsconfig.json +0 -16
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shaw Walters and elizaOS Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# @elizaos/capacitor-swabble
|
|
2
|
+
|
|
3
|
+
Capacitor plugin for wake-word detection and live speech transcription. Integrates with Eliza agent UIs to give users a hands-free voice interface across iOS, Android, browser, and desktop (Electrobun + Whisper.cpp).
|
|
4
|
+
|
|
5
|
+
## What it does
|
|
6
|
+
|
|
7
|
+
- Listens for configurable trigger phrases ("eliza", "hey assistant", etc.) and emits a `wakeWord` event carrying the detected command text.
|
|
8
|
+
- Streams interim and final speech transcripts via a `transcript` event.
|
|
9
|
+
- Exposes microphone state changes, audio level data (for VU-meter visualizations), and errors as typed events.
|
|
10
|
+
- Manages microphone permissions across platforms.
|
|
11
|
+
|
|
12
|
+
## Platforms
|
|
13
|
+
|
|
14
|
+
| Platform | STT backend | Timing data |
|
|
15
|
+
|----------|-------------|-------------|
|
|
16
|
+
| iOS / macOS | Apple Speech framework | Yes |
|
|
17
|
+
| Android | SpeechRecognizer API | Partial |
|
|
18
|
+
| Browser | Web Speech API | No (postGap = -1) |
|
|
19
|
+
| Desktop (Electrobun) | Whisper.cpp via IPC bridge | Yes |
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
npm install @elizaos/capacitor-swabble @capacitor/core
|
|
25
|
+
npx cap sync
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
iOS requires the `Speech` and `AVFoundation` frameworks (linked automatically via the podspec). Android requires `RECORD_AUDIO` permission in your manifest.
|
|
29
|
+
|
|
30
|
+
## Usage
|
|
31
|
+
|
|
32
|
+
```typescript
|
|
33
|
+
import { Swabble } from "@elizaos/capacitor-swabble";
|
|
34
|
+
|
|
35
|
+
// Request microphone permission
|
|
36
|
+
await Swabble.requestPermissions();
|
|
37
|
+
|
|
38
|
+
// Listen for wake word + command
|
|
39
|
+
const handle = await Swabble.addListener("wakeWord", (event) => {
|
|
40
|
+
console.log("Wake word:", event.wakeWord);
|
|
41
|
+
console.log("Command:", event.command);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
// Start detection
|
|
45
|
+
await Swabble.start({
|
|
46
|
+
config: {
|
|
47
|
+
triggers: ["eliza"],
|
|
48
|
+
minCommandLength: 3,
|
|
49
|
+
locale: "en-US",
|
|
50
|
+
modelSize: "small", // Whisper model for desktop
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
// Stop later
|
|
55
|
+
await Swabble.stop();
|
|
56
|
+
handle.remove();
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Configuration
|
|
60
|
+
|
|
61
|
+
| Option | Type | Default | Description |
|
|
62
|
+
|--------|------|---------|-------------|
|
|
63
|
+
| `triggers` | `string[]` | required | Wake phrases to detect |
|
|
64
|
+
| `minPostTriggerGap` | `number` | — | Silence (seconds) required after trigger before command (native only) |
|
|
65
|
+
| `minCommandLength` | `number` | `1` | Minimum command length in characters |
|
|
66
|
+
| `locale` | `string` | `"en-US"` | Speech recognition locale |
|
|
67
|
+
| `sampleRate` | `number` | `16000` | Audio sample rate (Hz) |
|
|
68
|
+
| `modelSize` | `"tiny"\|"base"\|"small"\|"medium"\|"large"` | — | Whisper.cpp model size (desktop only) |
|
|
69
|
+
|
|
70
|
+
## Events
|
|
71
|
+
|
|
72
|
+
| Event | Description |
|
|
73
|
+
|-------|-------------|
|
|
74
|
+
| `wakeWord` | Trigger phrase detected; carries `wakeWord`, `command`, `transcript`, `postGap`, `confidence` |
|
|
75
|
+
| `transcript` | Speech transcript update (interim and final); carries segments with timing |
|
|
76
|
+
| `stateChange` | Microphone state: `idle`, `listening`, `processing`, `error` |
|
|
77
|
+
| `audioLevel` | RMS level + peak (~10 Hz); useful for microphone visualizations |
|
|
78
|
+
| `error` | Error with `code`, `message`, and `recoverable` flag |
|
|
79
|
+
|
|
80
|
+
## Known limitations
|
|
81
|
+
|
|
82
|
+
- **Web Speech API:** `postGap`, `start`, and `duration` in transcript segments are `-1` (timing unavailable). `setAudioDevice` throws on web.
|
|
83
|
+
- **Device selection:** Only supported on native platforms; ignored or rejected on browser.
|
|
84
|
+
|
|
85
|
+
## Building
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
bun run build # tsc then rollup — produces dist/esm/, dist/plugin.js, dist/plugin.cjs.js
|
|
89
|
+
bun run watch # tsc --watch (no rollup)
|
|
90
|
+
```
|
package/android/build.gradle
CHANGED
|
@@ -7,6 +7,16 @@ ext {
|
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
apply plugin: 'com.android.library'
|
|
10
|
+
// Explicitly apply the Kotlin Android plugin. The kotlin-gradle-plugin is on
|
|
11
|
+
// the root buildscript classpath, but without applying it here AGP 8.13 falls
|
|
12
|
+
// back to its "built-in Kotlin" compile path (build/intermediates/
|
|
13
|
+
// built_in_kotlinc), which compiles the .kt sources but does NOT bundle the
|
|
14
|
+
// resulting .class files into the *release* library jar. The app's
|
|
15
|
+
// :app:assembleRelease then links a library AAR with zero plugin classes, so
|
|
16
|
+
// the Capacitor plugin (and any manifest-declared component) is absent from
|
|
17
|
+
// the release dex. Applying the standard Kotlin plugin wires Kotlin
|
|
18
|
+
// compilation into both the debug and release jar-bundling tasks.
|
|
19
|
+
apply plugin: 'org.jetbrains.kotlin.android'
|
|
10
20
|
android {
|
|
11
21
|
namespace = "ai.eliza.plugins.swabble"
|
|
12
22
|
compileSdk project.hasProperty('compileSdkVersion') ? rootProject.ext.compileSdkVersion : 34
|
|
@@ -25,8 +35,12 @@ android {
|
|
|
25
35
|
}
|
|
26
36
|
|
|
27
37
|
compileOptions {
|
|
28
|
-
sourceCompatibility JavaVersion.
|
|
29
|
-
targetCompatibility JavaVersion.
|
|
38
|
+
sourceCompatibility JavaVersion.VERSION_21
|
|
39
|
+
targetCompatibility JavaVersion.VERSION_21
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
kotlinOptions {
|
|
43
|
+
jvmTarget = "21"
|
|
30
44
|
}
|
|
31
45
|
|
|
32
46
|
}
|
|
@@ -34,7 +48,7 @@ android {
|
|
|
34
48
|
repositories {
|
|
35
49
|
google()
|
|
36
50
|
maven {
|
|
37
|
-
url = uri(rootProject.ext.mavenCentralMirrorUrl)
|
|
51
|
+
url = uri(rootProject.ext.has('mavenCentralMirrorUrl') ? rootProject.ext.mavenCentralMirrorUrl : 'https://repo.maven.apache.org/maven2')
|
|
38
52
|
}
|
|
39
53
|
mavenCentral()
|
|
40
54
|
}
|
package/dist/esm/web.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAQ5C,OAAO,KAAK,EACV,aAAa,EACb,uBAAuB,EAEvB,mBAAmB,EACnB,kBAAkB,EACnB,MAAM,eAAe,CAAC;AA6JvB,qBAAa,UAAW,SAAQ,SAAS;IACvC,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAA8B;IAC9C,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,aAAa,CAA+C;IAGpE,OAAO,CAAC,aAAa,CAA4B;IACjD,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,gBAAgB,CAAoC;IAC5D,OAAO,CAAC,mBAAmB,CAAyB;IACpD,OAAO,CAAC,cAAc,CAAS;IAE/B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,qBAAqB;YAQf,oBAAoB;IAQlC,OAAO,CAAC,oBAAoB;IAuC5B,OAAO,CAAC,qBAAqB;YAOf,uBAAuB;IA+CrC,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,sBAAsB;IAWxB,KAAK,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAkFtE,OAAO,CAAC,kBAAkB;YAyCZ,yBAAyB;IAyBvC,OAAO,CAAC,wBAAwB;IAY1B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBrB,WAAW,IAAI,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,CAAC;IAI9C,SAAS,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAAA;KAAE,CAAC;IAItD,YAAY,CAAC,OAAO,EAAE;QAC1B,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;KAChC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBX,gBAAgB,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAkCpD,kBAAkB,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAkBtD,eAAe,IAAI,OAAO,CAAC;QAC/B,OAAO,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,OAAO,CAAA;SAAE,CAAC,CAAC;KAClE,CAAC;IAiBI,cAAc,CAAC,QAAQ,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAOpE"}
|
package/dist/esm/web.js
CHANGED
|
@@ -30,6 +30,34 @@ function subscribeDesktopBridgeEvent(options) {
|
|
|
30
30
|
const getSpeechRecognition = () => window.SpeechRecognition ||
|
|
31
31
|
window.webkitSpeechRecognition ||
|
|
32
32
|
null;
|
|
33
|
+
function normalizeConfig(config) {
|
|
34
|
+
if (!config || !Array.isArray(config.triggers)) {
|
|
35
|
+
throw new Error("Swabble config requires a triggers array");
|
|
36
|
+
}
|
|
37
|
+
const triggers = config.triggers
|
|
38
|
+
.filter((trigger) => typeof trigger === "string")
|
|
39
|
+
.map((trigger) => trigger.trim())
|
|
40
|
+
.filter(Boolean);
|
|
41
|
+
if (triggers.length === 0) {
|
|
42
|
+
throw new Error("Swabble config requires at least one non-empty trigger");
|
|
43
|
+
}
|
|
44
|
+
const minCommandLength = typeof config.minCommandLength === "number" &&
|
|
45
|
+
Number.isFinite(config.minCommandLength) &&
|
|
46
|
+
config.minCommandLength > 0
|
|
47
|
+
? Math.floor(config.minCommandLength)
|
|
48
|
+
: 1;
|
|
49
|
+
const sampleRate = typeof config.sampleRate === "number" &&
|
|
50
|
+
Number.isFinite(config.sampleRate) &&
|
|
51
|
+
config.sampleRate > 0
|
|
52
|
+
? Math.floor(config.sampleRate)
|
|
53
|
+
: 16000;
|
|
54
|
+
return {
|
|
55
|
+
...config,
|
|
56
|
+
triggers,
|
|
57
|
+
minCommandLength,
|
|
58
|
+
sampleRate,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
33
61
|
/**
|
|
34
62
|
* WakeWordGate detects trigger phrases in transcripts.
|
|
35
63
|
*
|
|
@@ -40,15 +68,22 @@ const getSpeechRecognition = () => window.SpeechRecognition ||
|
|
|
40
68
|
*/
|
|
41
69
|
class WakeWordGate {
|
|
42
70
|
constructor(config) {
|
|
43
|
-
|
|
71
|
+
const normalized = normalizeConfig(config);
|
|
72
|
+
this.triggers = normalized.triggers.map((t) => t.toLowerCase());
|
|
44
73
|
this.minCommandLength = config.minCommandLength ?? 1;
|
|
45
74
|
// Note: minPostTriggerGap cannot be enforced - Web Speech API lacks timing data
|
|
46
75
|
}
|
|
47
76
|
updateConfig(config) {
|
|
48
|
-
if (config.triggers)
|
|
49
|
-
this.triggers =
|
|
50
|
-
|
|
51
|
-
|
|
77
|
+
if (config.triggers) {
|
|
78
|
+
this.triggers = normalizeConfig({
|
|
79
|
+
triggers: config.triggers,
|
|
80
|
+
}).triggers.map((t) => t.toLowerCase());
|
|
81
|
+
}
|
|
82
|
+
if (typeof config.minCommandLength === "number" &&
|
|
83
|
+
Number.isFinite(config.minCommandLength) &&
|
|
84
|
+
config.minCommandLength > 0) {
|
|
85
|
+
this.minCommandLength = Math.floor(config.minCommandLength);
|
|
86
|
+
}
|
|
52
87
|
}
|
|
53
88
|
/**
|
|
54
89
|
* Match wake word in transcript using text-only detection.
|
|
@@ -190,6 +225,8 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
190
225
|
sink.connect(this.captureContext.destination);
|
|
191
226
|
}
|
|
192
227
|
computeRms(samples) {
|
|
228
|
+
if (samples.length === 0)
|
|
229
|
+
return 0;
|
|
193
230
|
let sum = 0;
|
|
194
231
|
for (let i = 0; i < samples.length; i++) {
|
|
195
232
|
sum += samples[i] * samples[i];
|
|
@@ -218,6 +255,7 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
218
255
|
async start(options) {
|
|
219
256
|
if (this.isActive)
|
|
220
257
|
return { started: true };
|
|
258
|
+
const config = normalizeConfig(options.config);
|
|
221
259
|
// Delegate to the native desktop bridge when available.
|
|
222
260
|
const rpc = this.getRendererRpc();
|
|
223
261
|
if (rpc) {
|
|
@@ -225,14 +263,14 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
225
263
|
const result = await this.invokeDesktopRequest({
|
|
226
264
|
rpcMethod: "swabbleStart",
|
|
227
265
|
ipcChannel: "swabble:start",
|
|
228
|
-
params: options,
|
|
266
|
+
params: { ...options, config },
|
|
229
267
|
});
|
|
230
268
|
if (result?.started) {
|
|
231
269
|
this.isActive = true;
|
|
232
270
|
this.usingNativeIpc = true;
|
|
233
|
-
this.config =
|
|
271
|
+
this.config = config;
|
|
234
272
|
this.setupNativeListeners();
|
|
235
|
-
await this.startNativeAudioCapture(
|
|
273
|
+
await this.startNativeAudioCapture(config.sampleRate ?? 16000);
|
|
236
274
|
return result;
|
|
237
275
|
}
|
|
238
276
|
}
|
|
@@ -247,13 +285,13 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
247
285
|
error: "Speech recognition not supported in this browser",
|
|
248
286
|
};
|
|
249
287
|
}
|
|
250
|
-
this.config =
|
|
251
|
-
this.wakeGate = new WakeWordGate(
|
|
288
|
+
this.config = config;
|
|
289
|
+
this.wakeGate = new WakeWordGate(config);
|
|
252
290
|
this.segments = [];
|
|
253
291
|
const recognition = new SpeechRecognitionAPI();
|
|
254
292
|
recognition.continuous = true;
|
|
255
293
|
recognition.interimResults = true;
|
|
256
|
-
recognition.lang =
|
|
294
|
+
recognition.lang = config.locale || "en-US";
|
|
257
295
|
recognition.onstart = () => {
|
|
258
296
|
this.isActive = true;
|
|
259
297
|
this.notifyListeners("stateChange", { state: "listening" });
|
|
@@ -291,10 +329,16 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
291
329
|
let transcript = "";
|
|
292
330
|
let isFinal = false;
|
|
293
331
|
for (let i = 0; i < event.results.length; i++) {
|
|
294
|
-
|
|
295
|
-
|
|
332
|
+
const result = event.results[i];
|
|
333
|
+
const first = result?.[0];
|
|
334
|
+
if (!first || typeof first.transcript !== "string")
|
|
335
|
+
continue;
|
|
336
|
+
transcript += first.transcript;
|
|
337
|
+
if (result.isFinal)
|
|
296
338
|
isFinal = true;
|
|
297
339
|
}
|
|
340
|
+
if (!transcript.trim())
|
|
341
|
+
return;
|
|
298
342
|
// Web Speech API does not provide word-level timing.
|
|
299
343
|
// Segments are provided for API compatibility but timing values are approximations.
|
|
300
344
|
const words = transcript.split(/\s+/).filter(Boolean);
|
|
@@ -401,10 +445,14 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
401
445
|
async checkPermissions() {
|
|
402
446
|
let microphone = "prompt";
|
|
403
447
|
try {
|
|
404
|
-
const result = await navigator.permissions
|
|
448
|
+
const result = await navigator.permissions?.query?.({
|
|
405
449
|
name: "microphone",
|
|
406
450
|
});
|
|
407
|
-
|
|
451
|
+
if (result?.state === "granted" ||
|
|
452
|
+
result?.state === "denied" ||
|
|
453
|
+
result?.state === "prompt") {
|
|
454
|
+
microphone = result.state;
|
|
455
|
+
}
|
|
408
456
|
}
|
|
409
457
|
catch {
|
|
410
458
|
/* permissions.query not supported for microphone in some browsers */
|
|
@@ -424,7 +472,11 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
424
472
|
}
|
|
425
473
|
async requestPermissions() {
|
|
426
474
|
try {
|
|
427
|
-
const stream = await navigator.mediaDevices
|
|
475
|
+
const stream = await navigator.mediaDevices?.getUserMedia?.({
|
|
476
|
+
audio: true,
|
|
477
|
+
});
|
|
478
|
+
if (!stream)
|
|
479
|
+
throw new Error("mediaDevices.getUserMedia unavailable");
|
|
428
480
|
stream.getTracks().forEach((track) => {
|
|
429
481
|
track.stop();
|
|
430
482
|
});
|
|
@@ -439,7 +491,9 @@ export class SwabbleWeb extends WebPlugin {
|
|
|
439
491
|
}
|
|
440
492
|
async getAudioDevices() {
|
|
441
493
|
try {
|
|
442
|
-
const devices = await navigator.mediaDevices
|
|
494
|
+
const devices = await navigator.mediaDevices?.enumerateDevices?.();
|
|
495
|
+
if (!devices)
|
|
496
|
+
return { devices: [] };
|
|
443
497
|
const audioInputs = devices
|
|
444
498
|
.filter((d) => d.kind === "audioinput")
|
|
445
499
|
.map((d, i) => ({
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web.test.d.ts","sourceRoot":"","sources":["../../src/web.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { SwabbleWeb } from "./web";
|
|
3
|
+
class FakeRecognition extends EventTarget {
|
|
4
|
+
constructor() {
|
|
5
|
+
super();
|
|
6
|
+
this.continuous = false;
|
|
7
|
+
this.interimResults = false;
|
|
8
|
+
this.lang = "";
|
|
9
|
+
this.onstart = null;
|
|
10
|
+
this.onend = null;
|
|
11
|
+
this.onerror = null;
|
|
12
|
+
this.onresult = null;
|
|
13
|
+
this.start = vi.fn(() => {
|
|
14
|
+
this.onstart?.();
|
|
15
|
+
});
|
|
16
|
+
this.stop = vi.fn(() => {
|
|
17
|
+
this.onend?.();
|
|
18
|
+
});
|
|
19
|
+
this.abort = vi.fn();
|
|
20
|
+
FakeRecognition.latest = this;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
FakeRecognition.latest = null;
|
|
24
|
+
function setWindow(overrides = {}) {
|
|
25
|
+
Object.defineProperty(globalThis, "window", {
|
|
26
|
+
configurable: true,
|
|
27
|
+
value: overrides,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
function setNavigator(value) {
|
|
31
|
+
Object.defineProperty(globalThis, "navigator", {
|
|
32
|
+
configurable: true,
|
|
33
|
+
value,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
function speechEvent(transcript, isFinal = true, confidence = 0.8) {
|
|
37
|
+
return {
|
|
38
|
+
results: [
|
|
39
|
+
{
|
|
40
|
+
isFinal,
|
|
41
|
+
0: { transcript, confidence },
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
resultIndex: 0,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
describe("SwabbleWeb fallback", () => {
|
|
48
|
+
afterEach(() => {
|
|
49
|
+
vi.restoreAllMocks();
|
|
50
|
+
vi.unstubAllGlobals();
|
|
51
|
+
FakeRecognition.latest = null;
|
|
52
|
+
});
|
|
53
|
+
it("reports unsupported speech recognition without microphone APIs", async () => {
|
|
54
|
+
setWindow();
|
|
55
|
+
setNavigator({});
|
|
56
|
+
await expect(new SwabbleWeb().checkPermissions()).resolves.toEqual({
|
|
57
|
+
microphone: "prompt",
|
|
58
|
+
speechRecognition: "not_supported",
|
|
59
|
+
});
|
|
60
|
+
await expect(new SwabbleWeb().requestPermissions()).resolves.toEqual({
|
|
61
|
+
microphone: "denied",
|
|
62
|
+
speechRecognition: "denied",
|
|
63
|
+
});
|
|
64
|
+
await expect(new SwabbleWeb().getAudioDevices()).resolves.toEqual({
|
|
65
|
+
devices: [],
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
it.each([
|
|
69
|
+
{ triggers: [] },
|
|
70
|
+
{ triggers: ["", " "] },
|
|
71
|
+
{ triggers: [123] },
|
|
72
|
+
])("rejects malformed start config %#", async (config) => {
|
|
73
|
+
setWindow({ SpeechRecognition: FakeRecognition });
|
|
74
|
+
setNavigator({});
|
|
75
|
+
await expect(new SwabbleWeb().start({ config })).rejects.toThrow("Swabble config requires");
|
|
76
|
+
expect(FakeRecognition.latest).toBeNull();
|
|
77
|
+
});
|
|
78
|
+
it("emits transcript and wake-word events from valid final speech results", async () => {
|
|
79
|
+
setWindow({ SpeechRecognition: FakeRecognition });
|
|
80
|
+
setNavigator({
|
|
81
|
+
mediaDevices: {
|
|
82
|
+
getUserMedia: vi.fn(async () => null),
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
const plugin = new SwabbleWeb();
|
|
86
|
+
const states = vi.fn();
|
|
87
|
+
const transcripts = vi.fn();
|
|
88
|
+
const wakeWords = vi.fn();
|
|
89
|
+
await plugin.addListener("stateChange", states);
|
|
90
|
+
await plugin.addListener("transcript", transcripts);
|
|
91
|
+
await plugin.addListener("wakeWord", wakeWords);
|
|
92
|
+
await expect(plugin.start({
|
|
93
|
+
config: {
|
|
94
|
+
triggers: [" Eliza "],
|
|
95
|
+
minCommandLength: Number.NaN,
|
|
96
|
+
locale: "en-US",
|
|
97
|
+
},
|
|
98
|
+
})).resolves.toEqual({ started: true });
|
|
99
|
+
FakeRecognition.latest?.onresult?.(speechEvent("Eliza open calendar"));
|
|
100
|
+
expect(states).toHaveBeenCalledWith({ state: "listening" });
|
|
101
|
+
expect(transcripts).toHaveBeenCalledWith(expect.objectContaining({
|
|
102
|
+
transcript: "Eliza open calendar",
|
|
103
|
+
isFinal: true,
|
|
104
|
+
}));
|
|
105
|
+
expect(wakeWords).toHaveBeenCalledWith(expect.objectContaining({
|
|
106
|
+
wakeWord: "eliza",
|
|
107
|
+
command: "open calendar",
|
|
108
|
+
postGap: -1,
|
|
109
|
+
}));
|
|
110
|
+
});
|
|
111
|
+
it("ignores malformed speech result payloads without emitting transcripts", async () => {
|
|
112
|
+
setWindow({ SpeechRecognition: FakeRecognition });
|
|
113
|
+
setNavigator({
|
|
114
|
+
mediaDevices: {
|
|
115
|
+
getUserMedia: vi.fn(async () => null),
|
|
116
|
+
},
|
|
117
|
+
});
|
|
118
|
+
const plugin = new SwabbleWeb();
|
|
119
|
+
const transcripts = vi.fn();
|
|
120
|
+
await plugin.addListener("transcript", transcripts);
|
|
121
|
+
await plugin.start({ config: { triggers: ["eliza"] } });
|
|
122
|
+
FakeRecognition.latest?.onresult?.({
|
|
123
|
+
results: [{ isFinal: true, 0: { transcript: 42 } }],
|
|
124
|
+
resultIndex: 0,
|
|
125
|
+
});
|
|
126
|
+
expect(transcripts).not.toHaveBeenCalled();
|
|
127
|
+
});
|
|
128
|
+
it("uses desktop bridge state changes and removes subscriptions on stop", async () => {
|
|
129
|
+
const listeners = new Map();
|
|
130
|
+
const swabbleStart = vi.fn(async () => ({ started: true }));
|
|
131
|
+
const swabbleStop = vi.fn(async () => undefined);
|
|
132
|
+
const onMessage = vi.fn((name, listener) => {
|
|
133
|
+
listeners.set(name, listener);
|
|
134
|
+
});
|
|
135
|
+
const offMessage = vi.fn((name) => {
|
|
136
|
+
listeners.delete(name);
|
|
137
|
+
});
|
|
138
|
+
setWindow({
|
|
139
|
+
__ELIZA_ELECTROBUN_RPC__: {
|
|
140
|
+
request: {
|
|
141
|
+
swabbleStart,
|
|
142
|
+
swabbleStop,
|
|
143
|
+
swabbleAudioChunk: vi.fn(async () => undefined),
|
|
144
|
+
},
|
|
145
|
+
onMessage,
|
|
146
|
+
offMessage,
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
setNavigator({
|
|
150
|
+
mediaDevices: {
|
|
151
|
+
getUserMedia: vi.fn(async () => null),
|
|
152
|
+
},
|
|
153
|
+
});
|
|
154
|
+
const plugin = new SwabbleWeb();
|
|
155
|
+
const states = vi.fn();
|
|
156
|
+
await plugin.addListener("stateChange", states);
|
|
157
|
+
await plugin.start({
|
|
158
|
+
config: { triggers: ["eliza"], sampleRate: Infinity },
|
|
159
|
+
});
|
|
160
|
+
expect(swabbleStart).toHaveBeenCalledWith({
|
|
161
|
+
config: { triggers: ["eliza"], minCommandLength: 1, sampleRate: 16000 },
|
|
162
|
+
});
|
|
163
|
+
listeners.get("swabbleStateChanged")?.({ listening: true });
|
|
164
|
+
await expect(plugin.isListening()).resolves.toEqual({ listening: true });
|
|
165
|
+
await plugin.stop();
|
|
166
|
+
expect(swabbleStop).toHaveBeenCalled();
|
|
167
|
+
expect(offMessage).toHaveBeenCalled();
|
|
168
|
+
expect(states).toHaveBeenLastCalledWith({ state: "idle" });
|
|
169
|
+
});
|
|
170
|
+
});
|