@myscheme/voice-navigation-sdk 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/microphone-handler.d.ts +2 -0
- package/dist/microphone-handler.d.ts.map +1 -1
- package/dist/microphone-handler.js +32 -2
- package/dist/navigation-controller.d.ts +8 -1
- package/dist/navigation-controller.d.ts.map +1 -1
- package/dist/navigation-controller.js +122 -6
- package/dist/services/bedrock.d.ts.map +1 -1
- package/dist/services/bedrock.js +13 -1
- package/dist/ui.d.ts.map +1 -1
- package/dist/ui.js +32 -7
- package/package.json +1 -1
|
@@ -26,6 +26,7 @@ export declare class MicrophoneHandler {
|
|
|
26
26
|
private lastRecognitionTime;
|
|
27
27
|
private recognitionHealthCheckInterval;
|
|
28
28
|
private isRecovering;
|
|
29
|
+
private isStoppingIntentionally;
|
|
29
30
|
constructor(config: MicrophoneConfig);
|
|
30
31
|
hasMicrophonePermission(): boolean;
|
|
31
32
|
hasUnlockedAudioContext(): boolean;
|
|
@@ -41,6 +42,7 @@ export declare class MicrophoneHandler {
|
|
|
41
42
|
getAggregatedText(): string;
|
|
42
43
|
scheduleSilenceCheck(callbacks?: NavigationCallbacks): void;
|
|
43
44
|
rescheduleSilenceCheck(): void;
|
|
45
|
+
private verifyRecognizerHealth;
|
|
44
46
|
private startHealthCheck;
|
|
45
47
|
private stopHealthCheck;
|
|
46
48
|
private recoverFromStalledRecognizer;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"microphone-handler.d.ts","sourceRoot":"","sources":["../src/microphone-handler.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,GAAG,MAAM,wCAAwC,CAAC;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,UAAU,gBAAgB;IACxB,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,mBAAmB,CAAgB;IAC3C,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,oBAAoB,CAAkB;IAC9C,OAAO,CAAC,eAAe,CAA2B;IAClD,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,8BAA8B,CAEtB;IAChB,OAAO,CAAC,YAAY,CAAkB;
|
|
1
|
+
{"version":3,"file":"microphone-handler.d.ts","sourceRoot":"","sources":["../src/microphone-handler.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,GAAG,MAAM,wCAAwC,CAAC;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,UAAU,gBAAgB;IACxB,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,mBAAmB,CAAgB;IAC3C,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,oBAAoB,CAAkB;IAC9C,OAAO,CAAC,eAAe,CAA2B;IAClD,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,8BAA8B,CAEtB;IAChB,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,uBAAuB,CAAkB;gBAErC,MAAM,EAAE,gBAAgB;IAUpC,uBAAuB,IAAI,OAAO;IAIlC,uBAAuB,IAAI,OAAO;IAO5B,kBAAkB,IAAI,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAoB1D,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgBzC,kBAAkB,CAAC,aAAa,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAwDzD,2BAA2B,IAAI,OAAO,CAAC,OAAO,CAAC;IA2BrD,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAYnC,wBAAwB,IAAI,IAAI;IAO1B,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC;IAgBnC,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC;IAkBzC,iBAAiB,IAAI,IAAI;IAUzB,iBAAiB,IAAI,MAAM;IAO3B,oBAAoB,CAAC,SAAS,GAAE,mBAAwB,GAAG,IAAI;IAiB/D,sBAAsB,IAAI,IAAI;IAmB9B,OAAO,CAAC,sBAAsB;IA4B9B,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,eAAe;YAUT,4BAA4B;IAoCpC,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAiBpE,iBAAiB,IAAI,IAAI;IAqCnB,gBAAgB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAmBjD,cAAc,CAAC,SAAS,GAAE,mBAAwB,GAAG,OAAO,CAAC,OAAO,CAAC;IA6JrE,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;WA6CzB,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC;CAYvD"}
|
|
@@ -14,6 +14,7 @@ export class MicrophoneHandler {
|
|
|
14
14
|
this.lastRecognitionTime = 0;
|
|
15
15
|
this.recognitionHealthCheckInterval = null;
|
|
16
16
|
this.isRecovering = false;
|
|
17
|
+
this.isStoppingIntentionally = false;
|
|
17
18
|
this.azureSpeechService = config.azureSpeechService;
|
|
18
19
|
this.bedrockService = config.bedrockService;
|
|
19
20
|
this.language = config.language || "en-IN";
|
|
@@ -179,13 +180,28 @@ export class MicrophoneHandler {
|
|
|
179
180
|
}
|
|
180
181
|
console.log("[MicrophoneHandler] Rescheduling silence check");
|
|
181
182
|
this.scheduleSilenceCheck(this.activeCallbacks);
|
|
183
|
+
this.verifyRecognizerHealth();
|
|
184
|
+
}
|
|
185
|
+
verifyRecognizerHealth() {
|
|
186
|
+
if (!this.isRecording || !this.recognizer) {
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
const now = Date.now();
|
|
190
|
+
const timeSinceLastRecognition = now - this.lastRecognitionTime;
|
|
191
|
+
if (timeSinceLastRecognition > 15000) {
|
|
192
|
+
console.warn("[MicrophoneHandler] Recognizer may be stalled - last event was", timeSinceLastRecognition / 1000, "seconds ago. Triggering recovery.");
|
|
193
|
+
void this.recoverFromStalledRecognizer();
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
console.log("[MicrophoneHandler] Recognizer health OK - last event", (timeSinceLastRecognition / 1000).toFixed(1), "seconds ago");
|
|
197
|
+
}
|
|
182
198
|
}
|
|
183
199
|
startHealthCheck() {
|
|
184
200
|
this.stopHealthCheck();
|
|
185
201
|
this.recognitionHealthCheckInterval = setInterval(() => {
|
|
186
202
|
const now = Date.now();
|
|
187
203
|
const timeSinceLastRecognition = now - this.lastRecognitionTime;
|
|
188
|
-
if (this.isRecording && timeSinceLastRecognition >
|
|
204
|
+
if (this.isRecording && timeSinceLastRecognition > 30000) {
|
|
189
205
|
console.warn("[MicrophoneHandler] Recognizer appears stalled. Last recognition:", timeSinceLastRecognition / 1000, "seconds ago");
|
|
190
206
|
this.recoverFromStalledRecognizer();
|
|
191
207
|
}
|
|
@@ -281,6 +297,7 @@ export class MicrophoneHandler {
|
|
|
281
297
|
console.warn("[MicrophoneHandler] Recovery in progress, cannot start");
|
|
282
298
|
return false;
|
|
283
299
|
}
|
|
300
|
+
this.isStoppingIntentionally = false;
|
|
284
301
|
this.activeCallbacks = callbacks;
|
|
285
302
|
if (!this.hasPermission) {
|
|
286
303
|
const granted = await this.requestMicrophonePermission();
|
|
@@ -341,6 +358,7 @@ export class MicrophoneHandler {
|
|
|
341
358
|
}
|
|
342
359
|
}
|
|
343
360
|
else if (e.result.reason === sdk.ResultReason.NoMatch) {
|
|
361
|
+
this.lastRecognitionTime = Date.now();
|
|
344
362
|
console.log("[MicrophoneHandler] No speech recognized (NoMatch)");
|
|
345
363
|
}
|
|
346
364
|
};
|
|
@@ -353,9 +371,16 @@ export class MicrophoneHandler {
|
|
|
353
371
|
}
|
|
354
372
|
};
|
|
355
373
|
this.recognizer.sessionStopped = (_s, _e) => {
|
|
356
|
-
|
|
374
|
+
if (this.isStoppingIntentionally) {
|
|
375
|
+
console.log("[MicrophoneHandler] Session stopped (user-initiated)");
|
|
376
|
+
this.isRecording = false;
|
|
377
|
+
this.stopHealthCheck();
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
console.warn("[MicrophoneHandler] Session stopped unexpectedly by SDK - attempting auto-restart");
|
|
357
381
|
this.isRecording = false;
|
|
358
382
|
this.stopHealthCheck();
|
|
383
|
+
void this.recoverFromStalledRecognizer();
|
|
359
384
|
};
|
|
360
385
|
this.recognizer.startContinuousRecognitionAsync(() => {
|
|
361
386
|
console.log("[MicrophoneHandler] Continuous recognition started");
|
|
@@ -376,24 +401,29 @@ export class MicrophoneHandler {
|
|
|
376
401
|
async stopRecording() {
|
|
377
402
|
this.clearSilenceTimer();
|
|
378
403
|
this.stopHealthCheck();
|
|
404
|
+
this.isStoppingIntentionally = true;
|
|
379
405
|
if (!this.isRecording || !this.recognizer) {
|
|
380
406
|
console.warn("[MicrophoneHandler] Not currently recording");
|
|
407
|
+
this.isStoppingIntentionally = false;
|
|
381
408
|
return this.getAggregatedText();
|
|
382
409
|
}
|
|
383
410
|
return new Promise((resolve) => {
|
|
384
411
|
if (!this.recognizer) {
|
|
412
|
+
this.isStoppingIntentionally = false;
|
|
385
413
|
resolve(this.getAggregatedText());
|
|
386
414
|
return;
|
|
387
415
|
}
|
|
388
416
|
this.recognizer.stopContinuousRecognitionAsync(() => {
|
|
389
417
|
console.log("[MicrophoneHandler] Recognition stopped successfully");
|
|
390
418
|
this.isRecording = false;
|
|
419
|
+
this.isStoppingIntentionally = false;
|
|
391
420
|
const finalText = this.getAggregatedText();
|
|
392
421
|
this.disposeRecognizer();
|
|
393
422
|
resolve(finalText);
|
|
394
423
|
}, (error) => {
|
|
395
424
|
console.error("[MicrophoneHandler] Error stopping recognition:", error);
|
|
396
425
|
this.isRecording = false;
|
|
426
|
+
this.isStoppingIntentionally = false;
|
|
397
427
|
const finalText = this.getAggregatedText();
|
|
398
428
|
this.disposeRecognizer();
|
|
399
429
|
resolve(finalText);
|
|
@@ -16,6 +16,9 @@ export declare class VoiceNavigationController {
|
|
|
16
16
|
private vectorSearchService;
|
|
17
17
|
private pagesInitialized;
|
|
18
18
|
private uiInitialized;
|
|
19
|
+
private isSyncingButtonState;
|
|
20
|
+
private hasNavigatedAway;
|
|
21
|
+
private removeNavigationListener;
|
|
19
22
|
constructor(config: NavigationConfig);
|
|
20
23
|
private initializeUIAsync;
|
|
21
24
|
private shouldAutoStart;
|
|
@@ -29,7 +32,11 @@ export declare class VoiceNavigationController {
|
|
|
29
32
|
start(options?: {
|
|
30
33
|
reason?: "auto" | "user";
|
|
31
34
|
}): Promise<void>;
|
|
32
|
-
stop(
|
|
35
|
+
stop(options?: {
|
|
36
|
+
triggeredByUser?: boolean;
|
|
37
|
+
}): Promise<void>;
|
|
38
|
+
private trackNavigationAway;
|
|
39
|
+
private syncCustomButtonState;
|
|
33
40
|
private handlePartial;
|
|
34
41
|
private handleSegment;
|
|
35
42
|
private handleSilence;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"navigation-controller.d.ts","sourceRoot":"","sources":["../src/navigation-controller.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAMjB,MAAM,YAAY,CAAC;AA4DpB,qBAAa,yBAAyB;IACpC,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,EAAE,CAAc;IACxB,OAAO,CAAC,cAAc,CAAc;IACpC,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,qBAAqB,CAAkB;IAC/C,OAAO,CAAC,eAAe,CAA2B;IAClD,OAAO,CAAC,+BAA+B,CAAkB;IACzD,OAAO,CAAC,wBAAwB,CAA6B;IAC7D,OAAO,CAAC,mBAAmB,CAAoC;IAC/D,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,aAAa,CAAgB;
|
|
1
|
+
{"version":3,"file":"navigation-controller.d.ts","sourceRoot":"","sources":["../src/navigation-controller.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAMjB,MAAM,YAAY,CAAC;AA4DpB,qBAAa,yBAAyB;IACpC,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,EAAE,CAAc;IACxB,OAAO,CAAC,cAAc,CAAc;IACpC,OAAO,CAAC,YAAY,CAAkB;IACtC,OAAO,CAAC,qBAAqB,CAAkB;IAC/C,OAAO,CAAC,eAAe,CAA2B;IAClD,OAAO,CAAC,+BAA+B,CAAkB;IACzD,OAAO,CAAC,wBAAwB,CAA6B;IAC7D,OAAO,CAAC,mBAAmB,CAAoC;IAC/D,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,oBAAoB,CAAkB;IAC9C,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,OAAO,CAAC,wBAAwB,CAA6B;gBAEjD,MAAM,EAAE,gBAAgB;YA0JtB,iBAAiB;IAiD/B,OAAO,CAAC,eAAe;IAoBhB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAepC,oBAAoB,CAAC,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,IAAI;YAOtD,eAAe;IAmE7B,OAAO,CAAC,kBAAkB;YASZ,eAAe;IA8BhB,KAAK,CAChB,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAO,GACzC,OAAO,CAAC,IAAI,CAAC;IAuFH,IAAI,CACf,OAAO,GAAE;QAAE,eAAe,CAAC,EAAE,OAAO,CAAA;KAAO,GAC1C,OAAO,CAAC,IAAI,CAAC;IAqEhB,OAAO,CAAC,mBAAmB;IAoD3B,OAAO,CAAC,qBAAqB;IA4C7B,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,aAAa;YAWP,aAAa;YAOb,wBAAwB;IA2CtC,OAAO,CAAC,WAAW;YAuBL,iBAAiB;YAwBjB,iBAAiB;IA6RxB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAQnC,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAOxC,sBAAsB,IAAI,OAAO;YAO1B,qBAAqB;YAgFrB,yBAAyB;IAmMvC,OAAO,CAAC,oBAAoB;IAqD5B,OAAO,CAAC,yBAAyB;IAoCjC,OAAO,CAAC,UAAU;IAQlB,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,4BAA4B;IAiB7B,OAAO,IAAI,IAAI;IAgBtB,OAAO,CAAC,kBAAkB;IA8B1B,OAAO,CAAC,kBAAkB;IAwD1B,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,uBAAuB;IAqD/B,OAAO,CAAC,sBAAsB;IAW9B,OAAO,CAAC,wBAAwB;CAmBjC"}
|
|
@@ -38,6 +38,9 @@ export class VoiceNavigationController {
|
|
|
38
38
|
this.autoStartPendingUserInteraction = false;
|
|
39
39
|
this.removeAutoStartListeners = null;
|
|
40
40
|
this.vectorSearchService = null;
|
|
41
|
+
this.isSyncingButtonState = false;
|
|
42
|
+
this.hasNavigatedAway = false;
|
|
43
|
+
this.removeNavigationListener = null;
|
|
41
44
|
const openSearchConfig = resolveOpenSearchConfig(config);
|
|
42
45
|
this.config = {
|
|
43
46
|
...config,
|
|
@@ -119,6 +122,9 @@ export class VoiceNavigationController {
|
|
|
119
122
|
}
|
|
120
123
|
}
|
|
121
124
|
this.resumeAfterNavigation = this.consumeResumeState();
|
|
125
|
+
if (this.resumeAfterNavigation) {
|
|
126
|
+
this.hasNavigatedAway = true;
|
|
127
|
+
}
|
|
122
128
|
this.uiInitialized = this.initializeUIAsync();
|
|
123
129
|
if (this.resumeAfterNavigation) {
|
|
124
130
|
void this.start({ reason: "auto" });
|
|
@@ -235,13 +241,17 @@ export class VoiceNavigationController {
|
|
|
235
241
|
});
|
|
236
242
|
}
|
|
237
243
|
async toggleRecording() {
|
|
244
|
+
if (this.isSyncingButtonState) {
|
|
245
|
+
console.log("[VoiceNavigation] Skipping toggle - syncing button state");
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
238
248
|
if (!this.ui || !this.ui.root) {
|
|
239
249
|
console.warn("UI not initialized, cannot toggle recording");
|
|
240
250
|
return;
|
|
241
251
|
}
|
|
242
252
|
const currentState = this.ui.root.dataset.state;
|
|
243
253
|
if (currentState === "listening" || currentState === "thinking") {
|
|
244
|
-
await this.stop();
|
|
254
|
+
await this.stop({ triggeredByUser: true });
|
|
245
255
|
return;
|
|
246
256
|
}
|
|
247
257
|
if (currentState === "loading") {
|
|
@@ -286,7 +296,9 @@ export class VoiceNavigationController {
|
|
|
286
296
|
}
|
|
287
297
|
catch (error) {
|
|
288
298
|
setState(this.ui, "idle");
|
|
289
|
-
|
|
299
|
+
if (!this.ui.isCustomButton) {
|
|
300
|
+
updateStatus(this.ui, "Tap to enable voice navigation");
|
|
301
|
+
}
|
|
290
302
|
return;
|
|
291
303
|
}
|
|
292
304
|
}
|
|
@@ -302,6 +314,7 @@ export class VoiceNavigationController {
|
|
|
302
314
|
});
|
|
303
315
|
if (started) {
|
|
304
316
|
this.clearAutoStartFallback();
|
|
317
|
+
this.trackNavigationAway();
|
|
305
318
|
setState(this.ui, "listening");
|
|
306
319
|
emitEvent("state-change", { state: "listening" });
|
|
307
320
|
}
|
|
@@ -318,7 +331,8 @@ export class VoiceNavigationController {
|
|
|
318
331
|
emitEvent("error", { error });
|
|
319
332
|
}
|
|
320
333
|
}
|
|
321
|
-
async stop() {
|
|
334
|
+
async stop(options = {}) {
|
|
335
|
+
const triggeredByUser = options.triggeredByUser ?? false;
|
|
322
336
|
if (!this.ui || !this.ui.root) {
|
|
323
337
|
console.warn("UI not initialized, cannot stop");
|
|
324
338
|
return;
|
|
@@ -335,12 +349,96 @@ export class VoiceNavigationController {
|
|
|
335
349
|
setState(this.ui, "idle");
|
|
336
350
|
emitEvent("state-change", { state: "idle" });
|
|
337
351
|
updateTranscript(this.ui, "");
|
|
352
|
+
if (!triggeredByUser && this.ui.isCustomButton) {
|
|
353
|
+
if (!this.hasNavigatedAway) {
|
|
354
|
+
console.log("[VoiceNavigation] Programmatically clicking custom button to sync external UI state");
|
|
355
|
+
this.syncCustomButtonState();
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
console.log("[VoiceNavigation] Skipping custom button click - user has navigated away from start page");
|
|
359
|
+
}
|
|
360
|
+
}
|
|
338
361
|
}
|
|
339
362
|
catch (error) {
|
|
340
363
|
console.error("Failed to stop recording:", error);
|
|
341
364
|
showError(this.ui, error);
|
|
342
365
|
setState(this.ui, "error");
|
|
343
366
|
emitEvent("error", { error });
|
|
367
|
+
if (!triggeredByUser && this.ui.isCustomButton) {
|
|
368
|
+
if (!this.hasNavigatedAway) {
|
|
369
|
+
console.log("[VoiceNavigation] Programmatically clicking custom button after error");
|
|
370
|
+
this.syncCustomButtonState();
|
|
371
|
+
}
|
|
372
|
+
else {
|
|
373
|
+
console.log("[VoiceNavigation] Skipping custom button click after error - user has navigated away from start page");
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
trackNavigationAway() {
|
|
379
|
+
if (typeof window === "undefined") {
|
|
380
|
+
return;
|
|
381
|
+
}
|
|
382
|
+
if (this.removeNavigationListener) {
|
|
383
|
+
this.removeNavigationListener();
|
|
384
|
+
this.removeNavigationListener = null;
|
|
385
|
+
}
|
|
386
|
+
const startUrl = window.location.href;
|
|
387
|
+
const markNavigatedAway = () => {
|
|
388
|
+
if (window.location.href !== startUrl) {
|
|
389
|
+
this.hasNavigatedAway = true;
|
|
390
|
+
console.log("[VoiceNavigation] User navigated away from start page - custom button sync disabled");
|
|
391
|
+
if (this.removeNavigationListener) {
|
|
392
|
+
this.removeNavigationListener();
|
|
393
|
+
this.removeNavigationListener = null;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
};
|
|
397
|
+
window.addEventListener("popstate", markNavigatedAway);
|
|
398
|
+
const originalPushState = history.pushState.bind(history);
|
|
399
|
+
const originalReplaceState = history.replaceState.bind(history);
|
|
400
|
+
history.pushState = (...args) => {
|
|
401
|
+
originalPushState(...args);
|
|
402
|
+
markNavigatedAway();
|
|
403
|
+
};
|
|
404
|
+
history.replaceState = (...args) => {
|
|
405
|
+
originalReplaceState(...args);
|
|
406
|
+
markNavigatedAway();
|
|
407
|
+
};
|
|
408
|
+
this.removeNavigationListener = () => {
|
|
409
|
+
window.removeEventListener("popstate", markNavigatedAway);
|
|
410
|
+
history.pushState = originalPushState;
|
|
411
|
+
history.replaceState = originalReplaceState;
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
syncCustomButtonState() {
|
|
415
|
+
if (!this.ui.isCustomButton || !this.ui.button) {
|
|
416
|
+
return;
|
|
417
|
+
}
|
|
418
|
+
console.log("[VoiceNavigation] Syncing custom button state");
|
|
419
|
+
try {
|
|
420
|
+
this.isSyncingButtonState = true;
|
|
421
|
+
this.ui.button.setAttribute("aria-pressed", "false");
|
|
422
|
+
this.ui.button.setAttribute("data-voice-state", "idle");
|
|
423
|
+
this.ui.button.classList.remove("active", "recording", "listening");
|
|
424
|
+
this.ui.button.classList.add("inactive", "stopped");
|
|
425
|
+
console.log("[VoiceNavigation] Updated button attributes");
|
|
426
|
+
const event = new CustomEvent("voiceNavigation:stopped", {
|
|
427
|
+
bubbles: true,
|
|
428
|
+
detail: { stopped: true },
|
|
429
|
+
});
|
|
430
|
+
this.ui.button.dispatchEvent(event);
|
|
431
|
+
console.log("[VoiceNavigation] Dispatched voiceNavigation:stopped event");
|
|
432
|
+
this.ui.button.click();
|
|
433
|
+
console.log("[VoiceNavigation] Clicked button once to sync state");
|
|
434
|
+
setTimeout(() => {
|
|
435
|
+
this.isSyncingButtonState = false;
|
|
436
|
+
console.log("[VoiceNavigation] Button state sync complete");
|
|
437
|
+
}, 500);
|
|
438
|
+
}
|
|
439
|
+
catch (error) {
|
|
440
|
+
console.error("[VoiceNavigation] Error syncing button state:", error);
|
|
441
|
+
this.isSyncingButtonState = false;
|
|
344
442
|
}
|
|
345
443
|
}
|
|
346
444
|
handlePartial(text) {
|
|
@@ -397,7 +495,9 @@ export class VoiceNavigationController {
|
|
|
397
495
|
this.isPermissionRelatedError(error)) {
|
|
398
496
|
setState(this.ui, "idle");
|
|
399
497
|
this.handleAutoStartFallback("auto");
|
|
400
|
-
|
|
498
|
+
if (!this.ui.isCustomButton) {
|
|
499
|
+
updateStatus(this.ui, "Tap to start voice control");
|
|
500
|
+
}
|
|
401
501
|
return;
|
|
402
502
|
}
|
|
403
503
|
showError(this.ui, error);
|
|
@@ -455,6 +555,8 @@ export class VoiceNavigationController {
|
|
|
455
555
|
await this.provideActionFeedback(action, actionResult, originalText);
|
|
456
556
|
console.log("[VoiceNavigation] Search feedback complete, now navigating to:", actionResult.info.navigationUrl);
|
|
457
557
|
if (actionResult.info.navigationUrl) {
|
|
558
|
+
if (this.ui.isCustomButton)
|
|
559
|
+
this.syncCustomButtonState();
|
|
458
560
|
window.location.href = actionResult.info.navigationUrl;
|
|
459
561
|
actionResult.info.navigated = true;
|
|
460
562
|
actionResult.info.pendingNavigation = false;
|
|
@@ -463,6 +565,8 @@ export class VoiceNavigationController {
|
|
|
463
565
|
catch (err) {
|
|
464
566
|
console.error("[VoiceNavigation] Voice feedback error (search):", err);
|
|
465
567
|
if (actionResult.info.navigationUrl) {
|
|
568
|
+
if (this.ui.isCustomButton)
|
|
569
|
+
this.syncCustomButtonState();
|
|
466
570
|
window.location.href = actionResult.info.navigationUrl;
|
|
467
571
|
}
|
|
468
572
|
}
|
|
@@ -502,7 +606,7 @@ export class VoiceNavigationController {
|
|
|
502
606
|
}
|
|
503
607
|
const actionResult = performAgentAction(action, {
|
|
504
608
|
handler: this,
|
|
505
|
-
onStop: () => this.stop(),
|
|
609
|
+
onStop: () => this.stop({ triggeredByUser: false }),
|
|
506
610
|
transcript: originalText,
|
|
507
611
|
parameters: {
|
|
508
612
|
text: actionData.text || query,
|
|
@@ -526,24 +630,32 @@ export class VoiceNavigationController {
|
|
|
526
630
|
if (actionResult.info?.pendingNavigation) {
|
|
527
631
|
if (actionResult.info.navigationUrl) {
|
|
528
632
|
console.log("[VoiceNavigation] Feedback complete, now navigating to:", actionResult.info.navigationUrl);
|
|
633
|
+
if (this.ui.isCustomButton)
|
|
634
|
+
this.syncCustomButtonState();
|
|
529
635
|
window.location.href = actionResult.info.navigationUrl;
|
|
530
636
|
actionResult.info.navigated = true;
|
|
531
637
|
actionResult.info.pendingNavigation = false;
|
|
532
638
|
}
|
|
533
639
|
else if (actionResult.info.navigationType === "back") {
|
|
534
640
|
console.log("[VoiceNavigation] Feedback complete, navigating back");
|
|
641
|
+
if (this.ui.isCustomButton)
|
|
642
|
+
this.syncCustomButtonState();
|
|
535
643
|
window.history.back();
|
|
536
644
|
actionResult.info.navigated = true;
|
|
537
645
|
actionResult.info.pendingNavigation = false;
|
|
538
646
|
}
|
|
539
647
|
else if (actionResult.info.navigationType === "forward") {
|
|
540
648
|
console.log("[VoiceNavigation] Feedback complete, navigating forward");
|
|
649
|
+
if (this.ui.isCustomButton)
|
|
650
|
+
this.syncCustomButtonState();
|
|
541
651
|
window.history.forward();
|
|
542
652
|
actionResult.info.navigated = true;
|
|
543
653
|
actionResult.info.pendingNavigation = false;
|
|
544
654
|
}
|
|
545
655
|
else if (actionResult.info.navigationType === "reload") {
|
|
546
656
|
console.log("[VoiceNavigation] Feedback complete, reloading page");
|
|
657
|
+
if (this.ui.isCustomButton)
|
|
658
|
+
this.syncCustomButtonState();
|
|
547
659
|
window.location.reload();
|
|
548
660
|
actionResult.info.reloaded = true;
|
|
549
661
|
actionResult.info.pendingNavigation = false;
|
|
@@ -553,6 +665,8 @@ export class VoiceNavigationController {
|
|
|
553
665
|
catch (err) {
|
|
554
666
|
console.error("[VoiceNavigation] Voice feedback error (critical action):", err);
|
|
555
667
|
if (actionResult.info?.pendingNavigation) {
|
|
668
|
+
if (this.ui.isCustomButton)
|
|
669
|
+
this.syncCustomButtonState();
|
|
556
670
|
if (actionResult.info.navigationUrl) {
|
|
557
671
|
console.log("[VoiceNavigation] Feedback failed, but proceeding with navigation:", actionResult.info.navigationUrl);
|
|
558
672
|
window.location.href = actionResult.info.navigationUrl;
|
|
@@ -1045,7 +1159,9 @@ export class VoiceNavigationController {
|
|
|
1045
1159
|
window.removeEventListener(event, resume);
|
|
1046
1160
|
});
|
|
1047
1161
|
};
|
|
1048
|
-
|
|
1162
|
+
if (!this.ui.isCustomButton) {
|
|
1163
|
+
updateStatus(this.ui, "Tap to enable voice navigation");
|
|
1164
|
+
}
|
|
1049
1165
|
}
|
|
1050
1166
|
clearAutoStartFallback() {
|
|
1051
1167
|
if (this.removeAutoStartListeners) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../src/services/bedrock.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,mBAAmB,EAAmB,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,aAAa;IAE5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;
|
|
1
|
+
{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../src/services/bedrock.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,mBAAmB,EAAmB,MAAM,aAAa,CAAC;AACxE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,aAAa;IAE5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAqDD,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAqC;IACnD,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,gBAAgB,CAAgB;IACxC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,QAAQ,CAAkB;gBAEtB,MAAM,EAAE,aAAa;IA6CjC,eAAe,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAO7C,OAAO,CAAC,iBAAiB;IA+BnB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;YAWtD,qBAAqB;YA2ErB,mBAAmB;IAyEjC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,cAAc;IAO9C,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAWlC,iBAAiB;YAkFjB,eAAe;IAwFvB,uBAAuB,CAC3B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,CAAC;YA0BJ,wBAAwB;YAwExB,sBAAsB;CA+DrC"}
|
package/dist/services/bedrock.js
CHANGED
|
@@ -33,7 +33,19 @@ EXAMPLES:
|
|
|
33
33
|
MATCHING RULES FOR PAGE NAVIGATION:
|
|
34
34
|
- Match user's words to page names and keywords
|
|
35
35
|
- Choose the navigate_ action that best matches
|
|
36
|
-
- If user says "go to X" or "open X" or "show me X", find the page with that name or keyword
|
|
36
|
+
- If user says "go to X" or "open X" or "show me X", find the page with that name or keyword
|
|
37
|
+
|
|
38
|
+
SPECIAL PRIORITY RULES FOR STATES AND MINISTRIES:
|
|
39
|
+
- If user mentions a state/UT name (like "Assam", "Delhi", "Tamil Nadu", "Maharashtra"), ALWAYS use the state navigation action (navigate_state_*)
|
|
40
|
+
- If user mentions a ministry name or ministry keywords (like "education", "health", "agriculture"), ALWAYS use the ministry navigation action (navigate_ministry_*)
|
|
41
|
+
- User does NOT need to say "state of" or "ministry of" - the name alone is enough
|
|
42
|
+
- Examples:
|
|
43
|
+
* "go to Assam" → {"action": "navigate_state_assam"}
|
|
44
|
+
* "open Delhi" → {"action": "navigate_state_delhi"}
|
|
45
|
+
* "show Kerala" → {"action": "navigate_state_kerala"}
|
|
46
|
+
* "go to education" → {"action": "navigate_ministry_education"}
|
|
47
|
+
* "open health ministry" → {"action": "navigate_ministry_health"}
|
|
48
|
+
* "ministry of agriculture" → {"action": "navigate_ministry_agriculture"}`;
|
|
37
49
|
const DEFAULT_EMBEDDING_MODEL_ID = "cohere.embed-multilingual-v3";
|
|
38
50
|
export class BedrockService {
|
|
39
51
|
constructor(config) {
|
package/dist/ui.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ui.d.ts","sourceRoot":"","sources":["../src/ui.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAuC/E,eAAO,MAAM,2BAA2B,GACtC,YAAW,eAAgC,KAC1C,
|
|
1
|
+
{"version":3,"file":"ui.d.ts","sourceRoot":"","sources":["../src/ui.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAuC/E,eAAO,MAAM,2BAA2B,GACtC,YAAW,eAAgC,KAC1C,IAkIF,CAAC;AAKF,eAAO,MAAM,qBAAqB,GAChC,YAAW,eAAgC,KAC1C,UAgEF,CAAC;AA+EF,eAAO,MAAM,oBAAoB,GAC/B,gBAAgB,MAAM,KACrB,OAAO,CAAC,UAAU,CAuEpB,CAAC;AAKF,eAAO,MAAM,YAAY,GAAI,IAAI,UAAU,EAAE,MAAM,MAAM,KAAG,IAE3D,CAAC;AAKF,eAAO,MAAM,QAAQ,GAAI,IAAI,UAAU,EAAE,OAAO,eAAe,KAAG,IA0DjE,CAAC;AAKF,eAAO,MAAM,gBAAgB,GAAI,IAAI,UAAU,EAAE,MAAM,MAAM,KAAG,IAE/D,CAAC;AAKF,eAAO,MAAM,qBAAqB,GAChC,IAAI,UAAU,EACd,YAAY,MAAM,EAClB,WAAW,OAAO,EAClB,OAAM,GAAQ,KACb,IAkBF,CAAC;AAKF,eAAO,MAAM,SAAS,GAAI,IAAI,UAAU,EAAE,OAAO,KAAK,KAAG,IAGxD,CAAC;AAKF,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,SAAQ,GAAQ,KAAG,IAU1D,CAAC"}
|
package/dist/ui.js
CHANGED
|
@@ -85,6 +85,10 @@ export const injectFloatingControlStyles = (placement = "bottom-right") => {
|
|
|
85
85
|
color: #bae6fd;
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
.navigate-control__status:empty {
|
|
89
|
+
display: none;
|
|
90
|
+
}
|
|
91
|
+
|
|
88
92
|
.navigate-control__transcript {
|
|
89
93
|
font-size: 0.95rem;
|
|
90
94
|
line-height: 1.4;
|
|
@@ -123,6 +127,15 @@ export const injectFloatingControlStyles = (placement = "bottom-right") => {
|
|
|
123
127
|
background: #7c3aed;
|
|
124
128
|
}
|
|
125
129
|
|
|
130
|
+
.navigate-control--hidden {
|
|
131
|
+
display: none !important;
|
|
132
|
+
visibility: hidden !important;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
.navigate-control--hidden .navigate-control__panel {
|
|
136
|
+
display: none !important;
|
|
137
|
+
}
|
|
138
|
+
|
|
126
139
|
@media (max-width: 640px) {
|
|
127
140
|
.navigate-control {
|
|
128
141
|
${mobilePlacementStyles}
|
|
@@ -238,13 +251,12 @@ export const attachToCustomButton = async (buttonSelector) => {
|
|
|
238
251
|
if (!root) {
|
|
239
252
|
root = document.createElement("div");
|
|
240
253
|
root.id = "navigate-control-panel";
|
|
241
|
-
root.className = "navigate-control";
|
|
242
|
-
root.style.display = "none";
|
|
254
|
+
root.className = "navigate-control navigate-control--hidden";
|
|
243
255
|
}
|
|
244
256
|
root.dataset.state = root.dataset.state || "idle";
|
|
245
257
|
root.innerHTML = `
|
|
246
258
|
<div class="navigate-control__panel">
|
|
247
|
-
<div class="navigate-control__status"
|
|
259
|
+
<div class="navigate-control__status"></div>
|
|
248
260
|
<div class="navigate-control__transcript" role="status" aria-live="polite"></div>
|
|
249
261
|
<div class="navigate-control__action" aria-live="polite"></div>
|
|
250
262
|
</div>
|
|
@@ -264,6 +276,7 @@ export const attachToCustomButton = async (buttonSelector) => {
|
|
|
264
276
|
});
|
|
265
277
|
}
|
|
266
278
|
customButton.setAttribute("aria-pressed", "false");
|
|
279
|
+
customButton.dataset.voiceState = "idle";
|
|
267
280
|
if (!customButton.getAttribute("aria-label")) {
|
|
268
281
|
customButton.setAttribute("aria-label", "Start voice control");
|
|
269
282
|
}
|
|
@@ -282,6 +295,7 @@ export const updateStatus = (ui, text) => {
|
|
|
282
295
|
};
|
|
283
296
|
export const setState = (ui, state) => {
|
|
284
297
|
ui.root.dataset.state = state;
|
|
298
|
+
ui.button.dataset.voiceState = state;
|
|
285
299
|
const updateLabel = (text) => {
|
|
286
300
|
if (ui.label) {
|
|
287
301
|
ui.label.textContent = text;
|
|
@@ -292,9 +306,14 @@ export const setState = (ui, state) => {
|
|
|
292
306
|
updateLabel("Start");
|
|
293
307
|
ui.button.setAttribute("aria-pressed", "false");
|
|
294
308
|
ui.button.setAttribute("aria-label", "Start voice control");
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
309
|
+
if (ui.isCustomButton) {
|
|
310
|
+
updateStatus(ui, "");
|
|
311
|
+
updateTranscript(ui, "");
|
|
312
|
+
ui.action.textContent = "";
|
|
313
|
+
}
|
|
314
|
+
else {
|
|
315
|
+
updateStatus(ui, "Tap to start voice control");
|
|
316
|
+
}
|
|
298
317
|
break;
|
|
299
318
|
case "loading":
|
|
300
319
|
updateLabel("Loading...");
|
|
@@ -314,11 +333,17 @@ export const setState = (ui, state) => {
|
|
|
314
333
|
break;
|
|
315
334
|
case "error":
|
|
316
335
|
updateLabel("Error");
|
|
336
|
+
ui.button.setAttribute("aria-pressed", "false");
|
|
317
337
|
ui.button.setAttribute("aria-label", "Error occurred");
|
|
318
338
|
break;
|
|
319
339
|
}
|
|
320
340
|
if (ui.isCustomButton) {
|
|
321
|
-
|
|
341
|
+
if (state === "idle" || state === "error") {
|
|
342
|
+
ui.root.classList.add("navigate-control--hidden");
|
|
343
|
+
}
|
|
344
|
+
else {
|
|
345
|
+
ui.root.classList.remove("navigate-control--hidden");
|
|
346
|
+
}
|
|
322
347
|
}
|
|
323
348
|
};
|
|
324
349
|
export const updateTranscript = (ui, text) => {
|