@elizaos/capacitor-talkmode 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ElizaosCapacitorTalkmode.podspec +18 -0
- package/android/build.gradle +46 -0
- package/android/src/main/AndroidManifest.xml +7 -0
- package/android/src/main/java/ai/eliza/plugins/talkmode/TalkModePlugin.kt +1202 -0
- package/dist/esm/definitions.d.ts +277 -0
- package/dist/esm/definitions.d.ts.map +1 -0
- package/dist/esm/definitions.js +1 -0
- package/dist/esm/index.d.ts +4 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +6 -0
- package/dist/esm/web.d.ts +46 -0
- package/dist/esm/web.d.ts.map +1 -0
- package/dist/esm/web.js +201 -0
- package/dist/plugin.cjs.js +214 -0
- package/dist/plugin.cjs.js.map +1 -0
- package/dist/plugin.js +217 -0
- package/dist/plugin.js.map +1 -0
- package/ios/Sources/TalkModePlugin/TalkModePlugin.swift +1121 -0
- package/package.json +83 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import type { PluginListenerHandle } from "@capacitor/core";
|
|
2
|
+
/**
|
|
3
|
+
* TTS voice directive from assistant response
|
|
4
|
+
*/
|
|
5
|
+
export interface TTSDirective {
|
|
6
|
+
/** Voice ID to use (ElevenLabs voice ID or alias) */
|
|
7
|
+
voiceId?: string;
|
|
8
|
+
/** Model ID for ElevenLabs */
|
|
9
|
+
modelId?: string;
|
|
10
|
+
/** Output format (e.g., "pcm_24000", "mp3_44100") */
|
|
11
|
+
outputFormat?: string;
|
|
12
|
+
/** Speech rate multiplier (0.5-2.0) */
|
|
13
|
+
speed?: number;
|
|
14
|
+
/** Words per minute rate */
|
|
15
|
+
rateWpm?: number;
|
|
16
|
+
/** Voice stability (0-1) */
|
|
17
|
+
stability?: number;
|
|
18
|
+
/** Voice similarity boost (0-1) */
|
|
19
|
+
similarity?: number;
|
|
20
|
+
/** Style exaggeration (0-1) */
|
|
21
|
+
style?: number;
|
|
22
|
+
/** Enable speaker boost */
|
|
23
|
+
speakerBoost?: boolean;
|
|
24
|
+
/** Seed for reproducible output */
|
|
25
|
+
seed?: number;
|
|
26
|
+
/** Normalize audio levels */
|
|
27
|
+
normalize?: boolean;
|
|
28
|
+
/** Language code (e.g., "en", "es") */
|
|
29
|
+
language?: string;
|
|
30
|
+
/** Latency optimization tier (1-4) */
|
|
31
|
+
latencyTier?: number;
|
|
32
|
+
/** Apply only to this utterance */
|
|
33
|
+
once?: boolean;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* TTS configuration
|
|
37
|
+
*/
|
|
38
|
+
export interface TTSConfig {
|
|
39
|
+
/** Default ElevenLabs voice ID */
|
|
40
|
+
voiceId?: string;
|
|
41
|
+
/** Default ElevenLabs model ID */
|
|
42
|
+
modelId?: string;
|
|
43
|
+
/** Default output format */
|
|
44
|
+
outputFormat?: string;
|
|
45
|
+
/** ElevenLabs API key */
|
|
46
|
+
apiKey?: string;
|
|
47
|
+
/** Voice aliases mapping (name -> voiceId) */
|
|
48
|
+
voiceAliases?: Record<string, string>;
|
|
49
|
+
/** Whether to interrupt playback when user speaks */
|
|
50
|
+
interruptOnSpeech?: boolean;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Options for speaking text
|
|
54
|
+
*/
|
|
55
|
+
export interface SpeakOptions {
|
|
56
|
+
/** Text to speak */
|
|
57
|
+
text: string;
|
|
58
|
+
/** Optional directive overrides */
|
|
59
|
+
directive?: TTSDirective;
|
|
60
|
+
/** Force use of system TTS */
|
|
61
|
+
useSystemTts?: boolean;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Result of speak operation
|
|
65
|
+
*/
|
|
66
|
+
export interface SpeakResult {
|
|
67
|
+
/** Whether speech completed successfully */
|
|
68
|
+
completed: boolean;
|
|
69
|
+
/** Whether playback was interrupted */
|
|
70
|
+
interrupted: boolean;
|
|
71
|
+
/** Time at which playback was interrupted (seconds from start) */
|
|
72
|
+
interruptedAt?: number;
|
|
73
|
+
/** Whether system TTS was used as fallback */
|
|
74
|
+
usedSystemTts: boolean;
|
|
75
|
+
/** Error message if failed */
|
|
76
|
+
error?: string;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Talk mode configuration
|
|
80
|
+
*/
|
|
81
|
+
export interface TalkModeConfig {
|
|
82
|
+
/** Session key for chat */
|
|
83
|
+
sessionKey?: string;
|
|
84
|
+
/** TTS configuration */
|
|
85
|
+
tts?: TTSConfig;
|
|
86
|
+
/** STT configuration (desktop whisper/web) */
|
|
87
|
+
stt?: {
|
|
88
|
+
/** STT engine preference */
|
|
89
|
+
engine?: "whisper" | "web";
|
|
90
|
+
/** Whisper model size */
|
|
91
|
+
modelSize?: "tiny" | "base" | "small" | "medium" | "large";
|
|
92
|
+
/** Language code (e.g., "en", "es") */
|
|
93
|
+
language?: string;
|
|
94
|
+
/** Audio sample rate in Hz (default: 16000) */
|
|
95
|
+
sampleRate?: number;
|
|
96
|
+
};
|
|
97
|
+
/** Silence window before finalizing transcript (ms) */
|
|
98
|
+
silenceWindowMs?: number;
|
|
99
|
+
/** Whether to use interrupt-on-speech */
|
|
100
|
+
interruptOnSpeech?: boolean;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Talk mode state
|
|
104
|
+
*/
|
|
105
|
+
export type TalkModeState = "idle" | "listening" | "processing" | "speaking" | "error";
|
|
106
|
+
/**
|
|
107
|
+
* Talk mode state event
|
|
108
|
+
*/
|
|
109
|
+
export interface TalkModeStateEvent {
|
|
110
|
+
/** Current state */
|
|
111
|
+
state: TalkModeState;
|
|
112
|
+
/** Previous state */
|
|
113
|
+
previousState: TalkModeState;
|
|
114
|
+
/** Status message */
|
|
115
|
+
statusText: string;
|
|
116
|
+
/** Whether system TTS is being used */
|
|
117
|
+
usingSystemTts?: boolean;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Transcript event during talk mode
|
|
121
|
+
*/
|
|
122
|
+
export interface TalkModeTranscriptEvent {
|
|
123
|
+
/** Transcript text */
|
|
124
|
+
transcript: string;
|
|
125
|
+
/** Whether this is final */
|
|
126
|
+
isFinal: boolean;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* TTS start event
|
|
130
|
+
*/
|
|
131
|
+
export interface TTSSpeakingEvent {
|
|
132
|
+
/** Text being spoken */
|
|
133
|
+
text: string;
|
|
134
|
+
/** Whether using system TTS */
|
|
135
|
+
isSystemTts: boolean;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* TTS completion event
|
|
139
|
+
*/
|
|
140
|
+
export interface TTSCompleteEvent {
|
|
141
|
+
/** Whether completed without interruption */
|
|
142
|
+
completed: boolean;
|
|
143
|
+
/** Interrupted at time (seconds) if interrupted */
|
|
144
|
+
interruptedAt?: number;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Talk mode error event
|
|
148
|
+
*/
|
|
149
|
+
export interface TalkModeErrorEvent {
|
|
150
|
+
/** Error code */
|
|
151
|
+
code: string;
|
|
152
|
+
/** Error message */
|
|
153
|
+
message: string;
|
|
154
|
+
/** Whether recoverable */
|
|
155
|
+
recoverable: boolean;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Permission status for talk mode
|
|
159
|
+
*/
|
|
160
|
+
export interface TalkModePermissionStatus {
|
|
161
|
+
/** Microphone permission */
|
|
162
|
+
microphone: "granted" | "denied" | "prompt";
|
|
163
|
+
/** Speech recognition permission */
|
|
164
|
+
speechRecognition: "granted" | "denied" | "prompt" | "not_supported";
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* TalkMode Plugin Interface
|
|
168
|
+
*
|
|
169
|
+
* Provides full conversation mode with STT → chat → TTS flow.
|
|
170
|
+
* Uses ElevenLabs for high-quality TTS with system TTS fallback.
|
|
171
|
+
*/
|
|
172
|
+
export interface TalkModePlugin {
|
|
173
|
+
/**
|
|
174
|
+
* Start talk mode
|
|
175
|
+
*
|
|
176
|
+
* @param options - Configuration options
|
|
177
|
+
* @returns Promise resolving when started
|
|
178
|
+
*/
|
|
179
|
+
start(options?: {
|
|
180
|
+
config?: TalkModeConfig;
|
|
181
|
+
}): Promise<{
|
|
182
|
+
started: boolean;
|
|
183
|
+
error?: string;
|
|
184
|
+
}>;
|
|
185
|
+
/**
|
|
186
|
+
* Stop talk mode
|
|
187
|
+
*
|
|
188
|
+
* @returns Promise that resolves when stopped
|
|
189
|
+
*/
|
|
190
|
+
stop(): Promise<void>;
|
|
191
|
+
/**
|
|
192
|
+
* Check if talk mode is enabled
|
|
193
|
+
*
|
|
194
|
+
* @returns Promise resolving to enabled status
|
|
195
|
+
*/
|
|
196
|
+
isEnabled(): Promise<{
|
|
197
|
+
enabled: boolean;
|
|
198
|
+
}>;
|
|
199
|
+
/**
|
|
200
|
+
* Get current state
|
|
201
|
+
*
|
|
202
|
+
* @returns Promise resolving to current state
|
|
203
|
+
*/
|
|
204
|
+
getState(): Promise<{
|
|
205
|
+
state: TalkModeState;
|
|
206
|
+
statusText: string;
|
|
207
|
+
}>;
|
|
208
|
+
/**
|
|
209
|
+
* Update configuration
|
|
210
|
+
*
|
|
211
|
+
* @param options - New configuration
|
|
212
|
+
* @returns Promise that resolves when updated
|
|
213
|
+
*/
|
|
214
|
+
updateConfig(options: {
|
|
215
|
+
config: Partial<TalkModeConfig>;
|
|
216
|
+
}): Promise<void>;
|
|
217
|
+
/**
|
|
218
|
+
* Speak text using TTS
|
|
219
|
+
*
|
|
220
|
+
* @param options - Text and options
|
|
221
|
+
* @returns Promise resolving to speak result
|
|
222
|
+
*/
|
|
223
|
+
speak(options: SpeakOptions): Promise<SpeakResult>;
|
|
224
|
+
/**
|
|
225
|
+
* Stop current TTS playback
|
|
226
|
+
*
|
|
227
|
+
* @returns Promise that resolves when stopped
|
|
228
|
+
*/
|
|
229
|
+
stopSpeaking(): Promise<{
|
|
230
|
+
interruptedAt?: number;
|
|
231
|
+
}>;
|
|
232
|
+
/**
|
|
233
|
+
* Check if currently speaking
|
|
234
|
+
*
|
|
235
|
+
* @returns Promise resolving to speaking status
|
|
236
|
+
*/
|
|
237
|
+
isSpeaking(): Promise<{
|
|
238
|
+
speaking: boolean;
|
|
239
|
+
}>;
|
|
240
|
+
/**
|
|
241
|
+
* Check permissions
|
|
242
|
+
*
|
|
243
|
+
* @returns Promise resolving to permission status
|
|
244
|
+
*/
|
|
245
|
+
checkPermissions(): Promise<TalkModePermissionStatus>;
|
|
246
|
+
/**
|
|
247
|
+
* Request permissions
|
|
248
|
+
*
|
|
249
|
+
* @returns Promise resolving to permission status after request
|
|
250
|
+
*/
|
|
251
|
+
requestPermissions(): Promise<TalkModePermissionStatus>;
|
|
252
|
+
/**
|
|
253
|
+
* Add listener for state changes
|
|
254
|
+
*/
|
|
255
|
+
addListener(eventName: "stateChange", listenerFunc: (event: TalkModeStateEvent) => void): Promise<PluginListenerHandle>;
|
|
256
|
+
/**
|
|
257
|
+
* Add listener for transcript updates during listening
|
|
258
|
+
*/
|
|
259
|
+
addListener(eventName: "transcript", listenerFunc: (event: TalkModeTranscriptEvent) => void): Promise<PluginListenerHandle>;
|
|
260
|
+
/**
|
|
261
|
+
* Add listener for TTS start
|
|
262
|
+
*/
|
|
263
|
+
addListener(eventName: "speaking", listenerFunc: (event: TTSSpeakingEvent) => void): Promise<PluginListenerHandle>;
|
|
264
|
+
/**
|
|
265
|
+
* Add listener for TTS completion
|
|
266
|
+
*/
|
|
267
|
+
addListener(eventName: "speakComplete", listenerFunc: (event: TTSCompleteEvent) => void): Promise<PluginListenerHandle>;
|
|
268
|
+
/**
|
|
269
|
+
* Add listener for errors
|
|
270
|
+
*/
|
|
271
|
+
addListener(eventName: "error", listenerFunc: (event: TalkModeErrorEvent) => void): Promise<PluginListenerHandle>;
|
|
272
|
+
/**
|
|
273
|
+
* Remove all listeners
|
|
274
|
+
*/
|
|
275
|
+
removeAllListeners(): Promise<void>;
|
|
276
|
+
}
|
|
277
|
+
//# sourceMappingURL=definitions.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"definitions.d.ts","sourceRoot":"","sources":["../../src/definitions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAE5D;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,qDAAqD;IACrD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mCAAmC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6BAA6B;IAC7B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mCAAmC;IACnC,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yBAAyB;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8CAA8C;IAC9C,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,qDAAqD;IACrD,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,mCAAmC;IACnC,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,8BAA8B;IAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,4CAA4C;IAC5C,SAAS,EAAE,OAAO,CAAC;IACnB,uCAAuC;IACvC,WAAW,EAAE,OAAO,CAAC;IACrB,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8CAA8C;IAC9C,aAAa,EAAE,OAAO,CAAC;IACvB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wBAAwB;IACxB,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,8CAA8C;IAC9C,GAAG,CAAC,EAAE;QACJ,4BAA4B;QAC5B,MAAM,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC;QAC3B,yBAAyB;QACzB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAC;QAC3D,uCAAuC;QACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,+CAA+C;QAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,uDAAuD;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB,MAAM,GACN,WAAW,GACX,YAAY,GACZ,UAAU,GACV,OAAO,CAAC;AAEZ;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,oBAAoB;IACpB,KAAK,EAAE,aAAa,CAAC;IACrB,qBAAqB;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B,qBAAqB;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,sBAAsB;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,6CAA6C;IAC7C,SAAS,EAAE,OAAO,CAAC;IACnB,mDAAmD;IACnD,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,0BAA0B;IAC1B,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,4BAA4B;IAC5B,UAAU,EAAE,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC5C,oCAAoC;IACpC,iBAAiB,EAAE,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,eAAe,CAAC;CACtE;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;OAKG;IACH,KAAK,CAAC,OAAO,CAAC,EAAE;QACd,MAAM,CAAC,EAAE,cAAc,CAAC;KACzB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAElD;;;;OAIG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtB;;;;OAIG;IACH,SAAS,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAE3C;;;;OAIG;IACH,QAAQ,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAElE;;;;;OAKG;IACH,YAAY,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1E;;;;;OAKG;IACH,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAEnD;;;;OAIG;IACH,YAAY,IAAI,OAAO,CAAC;QAAE,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAEpD;;;;OAIG;IACH,UAAU,IAAI,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAE7C;;;;OAIG;IACH,gBAAgB,IAAI,OAAO,CAAC,wBAAwB,CAAC,CAAC;IAEtD;;;;OAIG;IACH,kBAAkB,IAAI,OAAO,CAAC,wBAAwB,CAAC,CAAC;IAExD;;OAEG;IACH,WAAW,CACT,SAAS,EAAE,aAAa,EACxB,YAAY,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,GAChD,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,WAAW,CACT,SAAS,EAAE,YAAY,EACvB,YAAY,EAAE,CAAC,KAAK,EAAE,uBAAuB,KAAK,IAAI,GACrD,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,WAAW,CACT,SAAS,EAAE,UAAU,EACrB,YAAY,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAC9C,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,WAAW,CACT,SAAS,EAAE,eAAe,EAC1B,YAAY,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAC9C,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,WAAW,CACT,SAAS,EAAE,OAAO,EAClB,YAAY,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,GAChD,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACrC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD,cAAc,eAAe,CAAC;AAI9B,eAAO,MAAM,QAAQ,gBAEnB,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { WebPlugin } from "@capacitor/core";
|
|
2
|
+
import type { SpeakOptions, SpeakResult, TalkModeConfig, TalkModePermissionStatus, TalkModeState } from "./definitions";
|
|
3
|
+
/**
|
|
4
|
+
* Web implementation of TalkMode plugin
|
|
5
|
+
*
|
|
6
|
+
* Uses Web Speech API for TTS with limited functionality compared to native.
|
|
7
|
+
* ElevenLabs streaming is not supported on web due to CORS limitations.
|
|
8
|
+
*/
|
|
9
|
+
export declare class TalkModeWeb extends WebPlugin {
|
|
10
|
+
private config;
|
|
11
|
+
private state;
|
|
12
|
+
private statusText;
|
|
13
|
+
private synthesis;
|
|
14
|
+
private currentUtterance;
|
|
15
|
+
private recognition;
|
|
16
|
+
private enabled;
|
|
17
|
+
constructor();
|
|
18
|
+
start(options?: {
|
|
19
|
+
config?: TalkModeConfig;
|
|
20
|
+
}): Promise<{
|
|
21
|
+
started: boolean;
|
|
22
|
+
error?: string;
|
|
23
|
+
}>;
|
|
24
|
+
stop(): Promise<void>;
|
|
25
|
+
isEnabled(): Promise<{
|
|
26
|
+
enabled: boolean;
|
|
27
|
+
}>;
|
|
28
|
+
getState(): Promise<{
|
|
29
|
+
state: TalkModeState;
|
|
30
|
+
statusText: string;
|
|
31
|
+
}>;
|
|
32
|
+
updateConfig(options: {
|
|
33
|
+
config: Partial<TalkModeConfig>;
|
|
34
|
+
}): Promise<void>;
|
|
35
|
+
speak(options: SpeakOptions): Promise<SpeakResult>;
|
|
36
|
+
stopSpeaking(): Promise<{
|
|
37
|
+
interruptedAt?: number;
|
|
38
|
+
}>;
|
|
39
|
+
isSpeaking(): Promise<{
|
|
40
|
+
speaking: boolean;
|
|
41
|
+
}>;
|
|
42
|
+
checkPermissions(): Promise<TalkModePermissionStatus>;
|
|
43
|
+
requestPermissions(): Promise<TalkModePermissionStatus>;
|
|
44
|
+
private setState;
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=web.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EACV,YAAY,EACZ,WAAW,EACX,cAAc,EACd,wBAAwB,EACxB,aAAa,EACd,MAAM,eAAe,CAAC;AA8CvB;;;;;GAKG;AACH,qBAAa,WAAY,SAAQ,SAAS;IACxC,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,gBAAgB,CAAyC;IACjE,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,OAAO,CAAS;;IASlB,KAAK,CAAC,OAAO,CAAC,EAAE;QACpB,MAAM,CAAC,EAAE,cAAc,CAAC;KACzB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IA8E3C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IASrB,SAAS,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;IAI1C,QAAQ,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAIjE,YAAY,CAAC,OAAO,EAAE;QAC1B,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;KACjC,GAAG,OAAO,CAAC,IAAI,CAAC;IAIX,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;IAwDlD,YAAY,IAAI,OAAO,CAAC;QAAE,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IASnD,UAAU,IAAI,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC;IAI5C,gBAAgB,IAAI,OAAO,CAAC,wBAAwB,CAAC;IA2BrD,kBAAkB,IAAI,OAAO,CAAC,wBAAwB,CAAC;IAc7D,OAAO,CAAC,QAAQ;CAWjB"}
|
package/dist/esm/web.js
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { WebPlugin } from "@capacitor/core";
|
|
2
|
+
/**
|
|
3
|
+
* Web implementation of TalkMode plugin
|
|
4
|
+
*
|
|
5
|
+
* Uses Web Speech API for TTS with limited functionality compared to native.
|
|
6
|
+
* ElevenLabs streaming is not supported on web due to CORS limitations.
|
|
7
|
+
*/
|
|
8
|
+
export class TalkModeWeb extends WebPlugin {
|
|
9
|
+
constructor() {
|
|
10
|
+
super();
|
|
11
|
+
this.config = {};
|
|
12
|
+
this.state = "idle";
|
|
13
|
+
this.statusText = "Off";
|
|
14
|
+
this.synthesis = null;
|
|
15
|
+
this.currentUtterance = null;
|
|
16
|
+
this.recognition = null;
|
|
17
|
+
this.enabled = false;
|
|
18
|
+
if (typeof window !== "undefined" && window.speechSynthesis) {
|
|
19
|
+
this.synthesis = window.speechSynthesis;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
async start(options) {
|
|
23
|
+
if (options?.config) {
|
|
24
|
+
this.config = { ...this.config, ...options.config };
|
|
25
|
+
}
|
|
26
|
+
// Check for Web Speech API support
|
|
27
|
+
const SpeechRecognitionAPI = window.SpeechRecognition ||
|
|
28
|
+
window.webkitSpeechRecognition;
|
|
29
|
+
if (!SpeechRecognitionAPI) {
|
|
30
|
+
return {
|
|
31
|
+
started: false,
|
|
32
|
+
error: "Speech recognition not supported on this browser",
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
if (!this.synthesis) {
|
|
36
|
+
console.warn("[TalkMode] Speech synthesis not available on web");
|
|
37
|
+
}
|
|
38
|
+
this.enabled = true;
|
|
39
|
+
this.setState("listening", "Listening");
|
|
40
|
+
// Initialize speech recognition
|
|
41
|
+
this.recognition = new SpeechRecognitionAPI();
|
|
42
|
+
this.recognition.continuous = true;
|
|
43
|
+
this.recognition.interimResults = true;
|
|
44
|
+
this.recognition.onresult = (event) => {
|
|
45
|
+
const result = event.results[event.results.length - 1];
|
|
46
|
+
const transcript = result[0].transcript;
|
|
47
|
+
const isFinal = result.isFinal;
|
|
48
|
+
this.notifyListeners("transcript", { transcript, isFinal });
|
|
49
|
+
if (isFinal && transcript.trim()) {
|
|
50
|
+
// Note: Full talk mode flow would need Gateway plugin integration
|
|
51
|
+
// For web, we just emit the transcript
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
this.recognition.onerror = (event) => {
|
|
55
|
+
this.notifyListeners("error", {
|
|
56
|
+
code: event.error,
|
|
57
|
+
message: event.message || event.error,
|
|
58
|
+
recoverable: event.error !== "not-allowed",
|
|
59
|
+
});
|
|
60
|
+
};
|
|
61
|
+
this.recognition.onend = () => {
|
|
62
|
+
if (this.enabled && this.state === "listening") {
|
|
63
|
+
// Restart recognition if still enabled
|
|
64
|
+
try {
|
|
65
|
+
this.recognition?.start();
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
69
|
+
if (!msg.includes("already started")) {
|
|
70
|
+
console.warn("[TalkMode] Failed to restart recognition:", msg);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
try {
|
|
76
|
+
this.recognition.start();
|
|
77
|
+
return { started: true };
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
const message = error instanceof Error ? error.message : "Failed to start";
|
|
81
|
+
return { started: false, error: message };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async stop() {
|
|
85
|
+
this.enabled = false;
|
|
86
|
+
this.recognition?.stop();
|
|
87
|
+
this.recognition = null;
|
|
88
|
+
this.synthesis?.cancel();
|
|
89
|
+
this.currentUtterance = null;
|
|
90
|
+
this.setState("idle", "Off");
|
|
91
|
+
}
|
|
92
|
+
async isEnabled() {
|
|
93
|
+
return { enabled: this.enabled };
|
|
94
|
+
}
|
|
95
|
+
async getState() {
|
|
96
|
+
return { state: this.state, statusText: this.statusText };
|
|
97
|
+
}
|
|
98
|
+
async updateConfig(options) {
|
|
99
|
+
this.config = { ...this.config, ...options.config };
|
|
100
|
+
}
|
|
101
|
+
async speak(options) {
|
|
102
|
+
if (!this.synthesis) {
|
|
103
|
+
return {
|
|
104
|
+
completed: false,
|
|
105
|
+
interrupted: false,
|
|
106
|
+
usedSystemTts: false,
|
|
107
|
+
error: "Speech synthesis not available",
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
// Web can only use system TTS (no ElevenLabs due to CORS)
|
|
111
|
+
const text = options.text.trim();
|
|
112
|
+
if (!text) {
|
|
113
|
+
return { completed: true, interrupted: false, usedSystemTts: true };
|
|
114
|
+
}
|
|
115
|
+
this.setState("speaking", "Speaking");
|
|
116
|
+
this.notifyListeners("speaking", { text, isSystemTts: true });
|
|
117
|
+
return new Promise((resolve) => {
|
|
118
|
+
const utterance = new SpeechSynthesisUtterance(text);
|
|
119
|
+
this.currentUtterance = utterance;
|
|
120
|
+
// Always set language — fallback to en-US if directive doesn't specify.
|
|
121
|
+
// Without this, the browser uses the system locale, which may read
|
|
122
|
+
// numbers in the wrong language (e.g., Chinese on a Chinese-locale system).
|
|
123
|
+
utterance.lang = options.directive?.language || "en-US";
|
|
124
|
+
// Apply directive settings if available
|
|
125
|
+
if (options.directive?.speed) {
|
|
126
|
+
utterance.rate = options.directive.speed;
|
|
127
|
+
}
|
|
128
|
+
utterance.onend = () => {
|
|
129
|
+
this.currentUtterance = null;
|
|
130
|
+
this.notifyListeners("speakComplete", { completed: true });
|
|
131
|
+
this.setState("listening", "Listening");
|
|
132
|
+
resolve({ completed: true, interrupted: false, usedSystemTts: true });
|
|
133
|
+
};
|
|
134
|
+
utterance.onerror = (event) => {
|
|
135
|
+
this.currentUtterance = null;
|
|
136
|
+
this.notifyListeners("speakComplete", { completed: false });
|
|
137
|
+
this.setState("idle", "Speech error");
|
|
138
|
+
resolve({
|
|
139
|
+
completed: false,
|
|
140
|
+
interrupted: event.error === "interrupted",
|
|
141
|
+
usedSystemTts: true,
|
|
142
|
+
error: event.error,
|
|
143
|
+
});
|
|
144
|
+
};
|
|
145
|
+
this.synthesis?.speak(utterance);
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
async stopSpeaking() {
|
|
149
|
+
if (this.synthesis && this.currentUtterance) {
|
|
150
|
+
this.synthesis.cancel();
|
|
151
|
+
this.currentUtterance = null;
|
|
152
|
+
return { interruptedAt: undefined };
|
|
153
|
+
}
|
|
154
|
+
return {};
|
|
155
|
+
}
|
|
156
|
+
async isSpeaking() {
|
|
157
|
+
return { speaking: this.synthesis?.speaking ?? false };
|
|
158
|
+
}
|
|
159
|
+
async checkPermissions() {
|
|
160
|
+
// Check microphone permission
|
|
161
|
+
let microphone = "prompt";
|
|
162
|
+
try {
|
|
163
|
+
const result = await navigator.permissions.query({
|
|
164
|
+
name: "microphone",
|
|
165
|
+
});
|
|
166
|
+
microphone = result.state;
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
// Permissions API may not support microphone query
|
|
170
|
+
}
|
|
171
|
+
// Check if speech recognition is supported
|
|
172
|
+
const SpeechRecognitionAPI = window.SpeechRecognition ||
|
|
173
|
+
window.webkitSpeechRecognition;
|
|
174
|
+
const speechRecognition = SpeechRecognitionAPI ? "prompt" : "not_supported";
|
|
175
|
+
return { microphone, speechRecognition };
|
|
176
|
+
}
|
|
177
|
+
async requestPermissions() {
|
|
178
|
+
// Request microphone permission by attempting to get user media
|
|
179
|
+
try {
|
|
180
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
181
|
+
stream.getTracks().forEach((track) => {
|
|
182
|
+
track.stop();
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
// Permission denied or error
|
|
187
|
+
}
|
|
188
|
+
return this.checkPermissions();
|
|
189
|
+
}
|
|
190
|
+
setState(state, statusText) {
|
|
191
|
+
const previousState = this.state;
|
|
192
|
+
this.state = state;
|
|
193
|
+
this.statusText = statusText;
|
|
194
|
+
this.notifyListeners("stateChange", {
|
|
195
|
+
state,
|
|
196
|
+
previousState,
|
|
197
|
+
statusText,
|
|
198
|
+
usingSystemTts: true,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
}
|