@myscheme/voice-navigation-sdk 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -5
- package/dist/actions.d.ts.map +1 -1
- package/dist/actions.js +184 -44
- package/dist/microphone-handler.d.ts +8 -0
- package/dist/microphone-handler.d.ts.map +1 -1
- package/dist/microphone-handler.js +98 -24
- package/dist/navigation-controller.d.ts +4 -0
- package/dist/navigation-controller.d.ts.map +1 -1
- package/dist/navigation-controller.js +201 -7
- package/dist/server/azure-speech-handler.d.ts +13 -0
- package/dist/server/azure-speech-handler.d.ts.map +1 -0
- package/dist/server/azure-speech-handler.js +79 -0
- package/dist/server/bedrock-embedding-handler.d.ts +16 -0
- package/dist/server/bedrock-embedding-handler.d.ts.map +1 -0
- package/dist/server/bedrock-embedding-handler.js +183 -0
- package/dist/server/bedrock-handler.d.ts +15 -0
- package/dist/server/bedrock-handler.d.ts.map +1 -0
- package/dist/server/bedrock-handler.js +171 -0
- package/dist/server/index.d.ts +7 -0
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +7 -0
- package/dist/server/opensearch-env-handler.d.ts +22 -0
- package/dist/server/opensearch-env-handler.d.ts.map +1 -0
- package/dist/server/opensearch-env-handler.js +221 -0
- package/dist/services/azure-speech.d.ts +10 -2
- package/dist/services/azure-speech.d.ts.map +1 -1
- package/dist/services/azure-speech.js +66 -3
- package/dist/services/bedrock.d.ts +16 -4
- package/dist/services/bedrock.d.ts.map +1 -1
- package/dist/services/bedrock.js +293 -17
- package/dist/services/vector-search.d.ts.map +1 -1
- package/dist/services/vector-search.js +15 -2
- package/dist/services/voice-feedback.d.ts +46 -0
- package/dist/services/voice-feedback.d.ts.map +1 -0
- package/dist/services/voice-feedback.js +332 -0
- package/dist/types.d.ts +17 -9
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -30,6 +30,19 @@ The following packages are automatically installed:
|
|
|
30
30
|
- `@aws-sdk/client-bedrock-runtime` - AWS Bedrock integration
|
|
31
31
|
- `microsoft-cognitiveservices-speech-sdk` - Azure Speech-to-Text
|
|
32
32
|
|
|
33
|
+
## 📚 Next.js Integration
|
|
34
|
+
|
|
35
|
+
For Next.js projects, we recommend using **server-side API routes** for secure credential management:
|
|
36
|
+
|
|
37
|
+
📖 **[Complete Next.js Setup Guide →](./NEXT_JS_SETUP.md)**
|
|
38
|
+
|
|
39
|
+
The guide covers:
|
|
40
|
+
|
|
41
|
+
- ✅ Pages Router (Next.js 12 and below)
|
|
42
|
+
- ✅ App Router (Next.js 13+)
|
|
43
|
+
- ✅ Secure server-side token handling
|
|
44
|
+
- ✅ Troubleshooting common errors
|
|
45
|
+
|
|
33
46
|
## 🎯 Quick Start
|
|
34
47
|
|
|
35
48
|
### Basic Setup
|
|
@@ -858,7 +871,6 @@ console.log("New zoom:", result.info.newZoom);
|
|
|
858
871
|
**Solutions:**
|
|
859
872
|
|
|
860
873
|
1. **Check XML file exists:**
|
|
861
|
-
|
|
862
874
|
- File should be in `public/` folder
|
|
863
875
|
- Verify URL: `http://localhost:3000/navigation-pages.xml`
|
|
864
876
|
|
|
@@ -874,7 +886,6 @@ console.log("New zoom:", result.info.newZoom);
|
|
|
874
886
|
```
|
|
875
887
|
|
|
876
888
|
3. **Check browser console:**
|
|
877
|
-
|
|
878
889
|
- Look for `[VoiceNavigation]` messages
|
|
879
890
|
- Should see: "✓ Page registry initialized with X pages"
|
|
880
891
|
|
|
@@ -946,7 +957,6 @@ console.log("New zoom:", result.info.newZoom);
|
|
|
946
957
|
```
|
|
947
958
|
|
|
948
959
|
2. **Use temporary credentials:**
|
|
949
|
-
|
|
950
960
|
- AWS Cognito for user-specific tokens
|
|
951
961
|
- STS for temporary access keys
|
|
952
962
|
- API Gateway with authorization
|
|
@@ -1001,14 +1011,12 @@ console.log("New zoom:", result.info.newZoom);
|
|
|
1001
1011
|
⚠️ **Important Security Notes:**
|
|
1002
1012
|
|
|
1003
1013
|
1. **Credential Protection:**
|
|
1004
|
-
|
|
1005
1014
|
- Never hardcode AWS/Azure credentials
|
|
1006
1015
|
- Use environment variables
|
|
1007
1016
|
- Rotate keys regularly
|
|
1008
1017
|
- Use least-privilege IAM policies
|
|
1009
1018
|
|
|
1010
1019
|
2. **Production Setup:**
|
|
1011
|
-
|
|
1012
1020
|
- Implement AWS Cognito for temporary credentials
|
|
1013
1021
|
- Use API Gateway for request authorization
|
|
1014
1022
|
- Proxy sensitive operations through backend
|
package/dist/actions.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAEhB,aAAa,EACb,YAAY,EACZ,mBAAmB,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAEhB,aAAa,EACb,YAAY,EACZ,mBAAmB,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AA4DhE,eAAO,MAAM,eAAe,GAAI,UAAU,YAAY,KAAG,IAOxD,CAAC;AAKF,eAAO,MAAM,eAAe,QAAO,YAAY,GAAG,IAEjD,CAAC;AAgGF,eAAO,MAAM,YAAY,GAAI,OAAO,MAAM,KAAG,MAS5C,CAAC;AAKF,eAAO,MAAM,UAAU,GAAI,OAAO,MAAM,KAAG,MAM1C,CAAC;AAKF,eAAO,MAAM,iBAAiB,GAAI,QAAQ,MAAM,KAAG,MAElD,CAAC;AAKF,eAAO,MAAM,kBAAkB,GAC7B,QAAQ,gBAAgB,EACxB,UAAS,aAAkB,KAC1B,YAg7BF,CAAC;AAKF,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,KAAG,mBAAmB,GAAG,IAiCtE,CAAC;AAKF,eAAO,MAAM,eAAe,GAAI,QAAQ,MAAM,KAAG,MAAM,IAAI,gBAY1D,CAAC"}
|
package/dist/actions.js
CHANGED
|
@@ -28,9 +28,12 @@ const CORE_ACTIONS = new Set([
|
|
|
28
28
|
"quit",
|
|
29
29
|
"tab_next",
|
|
30
30
|
"tab_back",
|
|
31
|
+
"focus_next",
|
|
32
|
+
"focus_previous",
|
|
31
33
|
"click",
|
|
32
34
|
"enter",
|
|
33
35
|
"submit",
|
|
36
|
+
"activate",
|
|
34
37
|
"move_left",
|
|
35
38
|
"move_right",
|
|
36
39
|
"move_up",
|
|
@@ -66,6 +69,46 @@ const notifyNavigationIntent = (context, target) => {
|
|
|
66
69
|
handler.prepareForNavigation({ target });
|
|
67
70
|
}
|
|
68
71
|
};
|
|
72
|
+
const getFullscreenElement = () => {
|
|
73
|
+
const doc = document;
|
|
74
|
+
return (doc.fullscreenElement ||
|
|
75
|
+
doc.webkitFullscreenElement ||
|
|
76
|
+
doc.mozFullScreenElement ||
|
|
77
|
+
doc.msFullscreenElement ||
|
|
78
|
+
null);
|
|
79
|
+
};
|
|
80
|
+
const requestFullscreen = (element) => {
|
|
81
|
+
const el = element;
|
|
82
|
+
if (el.requestFullscreen) {
|
|
83
|
+
return el.requestFullscreen();
|
|
84
|
+
}
|
|
85
|
+
else if (el.webkitRequestFullscreen) {
|
|
86
|
+
return el.webkitRequestFullscreen();
|
|
87
|
+
}
|
|
88
|
+
else if (el.mozRequestFullScreen) {
|
|
89
|
+
return el.mozRequestFullScreen();
|
|
90
|
+
}
|
|
91
|
+
else if (el.msRequestFullscreen) {
|
|
92
|
+
return el.msRequestFullscreen();
|
|
93
|
+
}
|
|
94
|
+
return Promise.reject(new Error("Fullscreen API not available"));
|
|
95
|
+
};
|
|
96
|
+
const exitFullscreen = () => {
|
|
97
|
+
const doc = document;
|
|
98
|
+
if (doc.exitFullscreen) {
|
|
99
|
+
return doc.exitFullscreen();
|
|
100
|
+
}
|
|
101
|
+
else if (doc.webkitExitFullscreen) {
|
|
102
|
+
return doc.webkitExitFullscreen();
|
|
103
|
+
}
|
|
104
|
+
else if (doc.mozCancelFullScreen) {
|
|
105
|
+
return doc.mozCancelFullScreen();
|
|
106
|
+
}
|
|
107
|
+
else if (doc.msExitFullscreen) {
|
|
108
|
+
return doc.msExitFullscreen();
|
|
109
|
+
}
|
|
110
|
+
return Promise.reject(new Error("Fullscreen API not available"));
|
|
111
|
+
};
|
|
69
112
|
const ensureZoomBaseline = () => {
|
|
70
113
|
const root = document.documentElement;
|
|
71
114
|
if (!root.dataset.navigateZoom) {
|
|
@@ -184,26 +227,29 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
184
227
|
case "go_back": {
|
|
185
228
|
if (typeof window !== "undefined" && window.history.length > 1) {
|
|
186
229
|
notifyNavigationIntent(context);
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
info.
|
|
230
|
+
console.log("Preparing to navigate back");
|
|
231
|
+
info.navigated = false;
|
|
232
|
+
info.pendingNavigation = true;
|
|
233
|
+
info.navigationType = "back";
|
|
190
234
|
performed = true;
|
|
191
235
|
}
|
|
192
236
|
break;
|
|
193
237
|
}
|
|
194
238
|
case "go_forward": {
|
|
195
239
|
notifyNavigationIntent(context);
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
info.
|
|
240
|
+
console.log("Preparing to navigate forward");
|
|
241
|
+
info.navigated = false;
|
|
242
|
+
info.pendingNavigation = true;
|
|
243
|
+
info.navigationType = "forward";
|
|
199
244
|
performed = true;
|
|
200
245
|
break;
|
|
201
246
|
}
|
|
202
247
|
case "reload_page": {
|
|
203
248
|
notifyNavigationIntent(context, window.location.href);
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
info.
|
|
249
|
+
console.log("Preparing to reload page");
|
|
250
|
+
info.reloaded = false;
|
|
251
|
+
info.pendingNavigation = true;
|
|
252
|
+
info.navigationType = "reload";
|
|
207
253
|
performed = true;
|
|
208
254
|
break;
|
|
209
255
|
}
|
|
@@ -216,14 +262,14 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
216
262
|
}
|
|
217
263
|
case "copy_url": {
|
|
218
264
|
if (navigator.clipboard && window.isSecureContext) {
|
|
265
|
+
info.copied = true;
|
|
266
|
+
performed = true;
|
|
219
267
|
navigator.clipboard
|
|
220
268
|
.writeText(window.location.href)
|
|
221
269
|
.then(() => console.log("Page URL copied to clipboard"))
|
|
222
270
|
.catch((error) => {
|
|
223
271
|
console.warn("Failed to copy URL via clipboard API", error);
|
|
224
272
|
});
|
|
225
|
-
info.copied = true;
|
|
226
|
-
performed = true;
|
|
227
273
|
}
|
|
228
274
|
else {
|
|
229
275
|
try {
|
|
@@ -280,31 +326,40 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
280
326
|
break;
|
|
281
327
|
}
|
|
282
328
|
case "toggle_fullscreen": {
|
|
283
|
-
const element =
|
|
329
|
+
const element = getFullscreenElement();
|
|
284
330
|
if (element) {
|
|
285
|
-
|
|
286
|
-
.
|
|
287
|
-
|
|
288
|
-
|
|
331
|
+
exitFullscreen()
|
|
332
|
+
.then(() => {
|
|
333
|
+
console.log("Exited fullscreen");
|
|
334
|
+
})
|
|
335
|
+
.catch((error) => {
|
|
336
|
+
console.warn("Failed to exit fullscreen", error);
|
|
337
|
+
});
|
|
289
338
|
info.fullscreen = false;
|
|
290
339
|
performed = true;
|
|
291
340
|
}
|
|
292
|
-
else
|
|
293
|
-
document.documentElement
|
|
294
|
-
.
|
|
295
|
-
|
|
296
|
-
|
|
341
|
+
else {
|
|
342
|
+
requestFullscreen(document.documentElement)
|
|
343
|
+
.then(() => {
|
|
344
|
+
console.log("Entered fullscreen");
|
|
345
|
+
})
|
|
346
|
+
.catch((error) => {
|
|
347
|
+
console.warn("Failed to enter fullscreen", error);
|
|
348
|
+
});
|
|
297
349
|
info.fullscreen = true;
|
|
298
350
|
performed = true;
|
|
299
351
|
}
|
|
300
352
|
break;
|
|
301
353
|
}
|
|
302
354
|
case "exit_fullscreen": {
|
|
303
|
-
if (
|
|
304
|
-
|
|
305
|
-
.
|
|
306
|
-
|
|
307
|
-
|
|
355
|
+
if (getFullscreenElement()) {
|
|
356
|
+
exitFullscreen()
|
|
357
|
+
.then(() => {
|
|
358
|
+
console.log("Exited fullscreen");
|
|
359
|
+
})
|
|
360
|
+
.catch((error) => {
|
|
361
|
+
console.warn("Failed to exit fullscreen", error);
|
|
362
|
+
});
|
|
308
363
|
info.fullscreen = false;
|
|
309
364
|
performed = true;
|
|
310
365
|
}
|
|
@@ -417,15 +472,99 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
417
472
|
}
|
|
418
473
|
break;
|
|
419
474
|
}
|
|
475
|
+
case "focus_next": {
|
|
476
|
+
const focusableElements = Array.from(document.querySelectorAll('a, button, input, textarea, select, [tabindex]:not([tabindex="-1"])')).filter((el) => {
|
|
477
|
+
const rect = el.getBoundingClientRect();
|
|
478
|
+
return (!el.hasAttribute("disabled") &&
|
|
479
|
+
rect.width > 0 &&
|
|
480
|
+
rect.height > 0 &&
|
|
481
|
+
window.getComputedStyle(el).visibility !== "hidden");
|
|
482
|
+
});
|
|
483
|
+
const currentIndex = focusableElements.indexOf(document.activeElement);
|
|
484
|
+
const nextIndex = (currentIndex + 1) % focusableElements.length;
|
|
485
|
+
if (focusableElements[nextIndex]) {
|
|
486
|
+
focusableElements[nextIndex].focus();
|
|
487
|
+
focusableElements[nextIndex].scrollIntoView({
|
|
488
|
+
behavior: "smooth",
|
|
489
|
+
block: "center",
|
|
490
|
+
});
|
|
491
|
+
console.log("Shifted focus to next element");
|
|
492
|
+
info.focused = true;
|
|
493
|
+
info.direction = "next";
|
|
494
|
+
info.elementTag = focusableElements[nextIndex].tagName;
|
|
495
|
+
performed = true;
|
|
496
|
+
}
|
|
497
|
+
break;
|
|
498
|
+
}
|
|
499
|
+
case "focus_previous": {
|
|
500
|
+
const focusableElements = Array.from(document.querySelectorAll('a, button, input, textarea, select, [tabindex]:not([tabindex="-1"])')).filter((el) => {
|
|
501
|
+
const rect = el.getBoundingClientRect();
|
|
502
|
+
return (!el.hasAttribute("disabled") &&
|
|
503
|
+
rect.width > 0 &&
|
|
504
|
+
rect.height > 0 &&
|
|
505
|
+
window.getComputedStyle(el).visibility !== "hidden");
|
|
506
|
+
});
|
|
507
|
+
const currentIndex = focusableElements.indexOf(document.activeElement);
|
|
508
|
+
const prevIndex = currentIndex <= 0 ? focusableElements.length - 1 : currentIndex - 1;
|
|
509
|
+
if (focusableElements[prevIndex]) {
|
|
510
|
+
focusableElements[prevIndex].focus();
|
|
511
|
+
focusableElements[prevIndex].scrollIntoView({
|
|
512
|
+
behavior: "smooth",
|
|
513
|
+
block: "center",
|
|
514
|
+
});
|
|
515
|
+
console.log("Shifted focus to previous element");
|
|
516
|
+
info.focused = true;
|
|
517
|
+
info.direction = "previous";
|
|
518
|
+
info.elementTag = focusableElements[prevIndex].tagName;
|
|
519
|
+
performed = true;
|
|
520
|
+
}
|
|
521
|
+
break;
|
|
522
|
+
}
|
|
420
523
|
case "click":
|
|
421
524
|
case "enter":
|
|
422
|
-
case "submit":
|
|
525
|
+
case "submit":
|
|
526
|
+
case "activate": {
|
|
423
527
|
const activeElement = document.activeElement;
|
|
424
528
|
if (activeElement && activeElement !== document.body) {
|
|
425
|
-
activeElement
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
529
|
+
if (activeElement instanceof HTMLInputElement ||
|
|
530
|
+
activeElement instanceof HTMLTextAreaElement) {
|
|
531
|
+
const form = activeElement.closest("form");
|
|
532
|
+
if (form) {
|
|
533
|
+
form.requestSubmit();
|
|
534
|
+
console.log(`Submitted form for ${activeElement.tagName}`);
|
|
535
|
+
info.submitted = true;
|
|
536
|
+
performed = true;
|
|
537
|
+
}
|
|
538
|
+
else {
|
|
539
|
+
const enterEvent = new KeyboardEvent("keydown", {
|
|
540
|
+
key: "Enter",
|
|
541
|
+
code: "Enter",
|
|
542
|
+
keyCode: 13,
|
|
543
|
+
which: 13,
|
|
544
|
+
bubbles: true,
|
|
545
|
+
cancelable: true,
|
|
546
|
+
});
|
|
547
|
+
activeElement.dispatchEvent(enterEvent);
|
|
548
|
+
const keypressEvent = new KeyboardEvent("keypress", {
|
|
549
|
+
key: "Enter",
|
|
550
|
+
code: "Enter",
|
|
551
|
+
keyCode: 13,
|
|
552
|
+
which: 13,
|
|
553
|
+
bubbles: true,
|
|
554
|
+
cancelable: true,
|
|
555
|
+
});
|
|
556
|
+
activeElement.dispatchEvent(keypressEvent);
|
|
557
|
+
console.log(`Triggered Enter key on ${activeElement.tagName} input`);
|
|
558
|
+
info.keyPressed = "Enter";
|
|
559
|
+
performed = true;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
else {
|
|
563
|
+
activeElement.click();
|
|
564
|
+
console.log(`Clicked on focused element: ${activeElement.tagName}`);
|
|
565
|
+
info.clicked = true;
|
|
566
|
+
performed = true;
|
|
567
|
+
}
|
|
429
568
|
}
|
|
430
569
|
else {
|
|
431
570
|
console.log("No element is currently focused");
|
|
@@ -620,6 +759,8 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
620
759
|
const end = activeElement.selectionEnd || 0;
|
|
621
760
|
const selectedText = activeElement.value.substring(start, end);
|
|
622
761
|
if (selectedText && navigator.clipboard) {
|
|
762
|
+
info.cut = true;
|
|
763
|
+
performed = true;
|
|
623
764
|
navigator.clipboard
|
|
624
765
|
.writeText(selectedText)
|
|
625
766
|
.then(() => {
|
|
@@ -629,8 +770,6 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
629
770
|
activeElement.setSelectionRange(start, start);
|
|
630
771
|
activeElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: true }));
|
|
631
772
|
console.log("Cut text to clipboard");
|
|
632
|
-
info.cut = true;
|
|
633
|
-
performed = true;
|
|
634
773
|
})
|
|
635
774
|
.catch((error) => {
|
|
636
775
|
console.warn("Failed to cut text", error);
|
|
@@ -651,12 +790,12 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
651
790
|
const end = activeElement.selectionEnd || 0;
|
|
652
791
|
const selectedText = activeElement.value.substring(start, end);
|
|
653
792
|
if (selectedText && navigator.clipboard) {
|
|
793
|
+
info.copied = true;
|
|
794
|
+
performed = true;
|
|
654
795
|
navigator.clipboard
|
|
655
796
|
.writeText(selectedText)
|
|
656
797
|
.then(() => {
|
|
657
798
|
console.log("Copied text to clipboard");
|
|
658
|
-
info.copied = true;
|
|
659
|
-
performed = true;
|
|
660
799
|
})
|
|
661
800
|
.catch((error) => {
|
|
662
801
|
console.warn("Failed to copy text", error);
|
|
@@ -666,12 +805,12 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
666
805
|
const selection = window.getSelection();
|
|
667
806
|
const selectionText = selection?.toString();
|
|
668
807
|
if (selectionText && navigator.clipboard) {
|
|
808
|
+
info.copied = true;
|
|
809
|
+
performed = true;
|
|
669
810
|
navigator.clipboard
|
|
670
811
|
.writeText(selectionText)
|
|
671
812
|
.then(() => {
|
|
672
813
|
console.log("Copied selected text to clipboard");
|
|
673
|
-
info.copied = true;
|
|
674
|
-
performed = true;
|
|
675
814
|
})
|
|
676
815
|
.catch((error) => {
|
|
677
816
|
console.warn("Failed to copy text", error);
|
|
@@ -686,12 +825,12 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
686
825
|
const selection = window.getSelection();
|
|
687
826
|
const selectionText = selection?.toString();
|
|
688
827
|
if (selectionText && navigator.clipboard) {
|
|
828
|
+
info.copied = true;
|
|
829
|
+
performed = true;
|
|
689
830
|
navigator.clipboard
|
|
690
831
|
.writeText(selectionText)
|
|
691
832
|
.then(() => {
|
|
692
833
|
console.log("Copied selected text to clipboard");
|
|
693
|
-
info.copied = true;
|
|
694
|
-
performed = true;
|
|
695
834
|
})
|
|
696
835
|
.catch((error) => {
|
|
697
836
|
console.warn("Failed to copy text", error);
|
|
@@ -709,6 +848,8 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
709
848
|
(activeElement instanceof HTMLInputElement ||
|
|
710
849
|
activeElement instanceof HTMLTextAreaElement) &&
|
|
711
850
|
navigator.clipboard) {
|
|
851
|
+
info.pasted = true;
|
|
852
|
+
performed = true;
|
|
712
853
|
navigator.clipboard
|
|
713
854
|
.readText()
|
|
714
855
|
.then((clipboardText) => {
|
|
@@ -721,8 +862,6 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
721
862
|
activeElement.setSelectionRange(newPos, newPos);
|
|
722
863
|
activeElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: true }));
|
|
723
864
|
console.log("Pasted text from clipboard");
|
|
724
|
-
info.pasted = true;
|
|
725
|
-
performed = true;
|
|
726
865
|
})
|
|
727
866
|
.catch((error) => {
|
|
728
867
|
console.warn("Failed to paste text", error);
|
|
@@ -753,9 +892,10 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
753
892
|
const page = globalPageRegistry.getPageById(pageId);
|
|
754
893
|
if (page) {
|
|
755
894
|
notifyNavigationIntent(context, page.path);
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
info.
|
|
895
|
+
console.log(`Preparing to navigate to ${page.name} (${page.path})`);
|
|
896
|
+
info.navigated = false;
|
|
897
|
+
info.pendingNavigation = true;
|
|
898
|
+
info.navigationUrl = page.path;
|
|
759
899
|
info.pageName = page.name;
|
|
760
900
|
info.pagePath = page.path;
|
|
761
901
|
performed = true;
|
|
@@ -22,6 +22,10 @@ export declare class MicrophoneHandler {
|
|
|
22
22
|
private hasPermission;
|
|
23
23
|
private audioContext;
|
|
24
24
|
private audioContextUnlocked;
|
|
25
|
+
private activeCallbacks;
|
|
26
|
+
private lastRecognitionTime;
|
|
27
|
+
private recognitionHealthCheckInterval;
|
|
28
|
+
private isRecovering;
|
|
25
29
|
constructor(config: MicrophoneConfig);
|
|
26
30
|
hasMicrophonePermission(): boolean;
|
|
27
31
|
hasUnlockedAudioContext(): boolean;
|
|
@@ -36,6 +40,10 @@ export declare class MicrophoneHandler {
|
|
|
36
40
|
clearSilenceTimer(): void;
|
|
37
41
|
getAggregatedText(): string;
|
|
38
42
|
scheduleSilenceCheck(callbacks?: NavigationCallbacks): void;
|
|
43
|
+
rescheduleSilenceCheck(): void;
|
|
44
|
+
private startHealthCheck;
|
|
45
|
+
private stopHealthCheck;
|
|
46
|
+
private recoverFromStalledRecognizer;
|
|
39
47
|
sendToBedrockAgent(text: string): Promise<AgentActionResponse>;
|
|
40
48
|
disposeRecognizer(): void;
|
|
41
49
|
createRecognizer(): Promise<sdk.SpeechRecognizer>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"microphone-handler.d.ts","sourceRoot":"","sources":["../src/microphone-handler.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,GAAG,MAAM,wCAAwC,CAAC;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,UAAU,gBAAgB;IACxB,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,mBAAmB,CAAgB;IAC3C,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,oBAAoB,CAAkB;
|
|
1
|
+
{"version":3,"file":"microphone-handler.d.ts","sourceRoot":"","sources":["../src/microphone-handler.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,GAAG,MAAM,wCAAwC,CAAC;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,UAAU,gBAAgB;IACxB,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,UAAU,CAAqC;IACvD,OAAO,CAAC,mBAAmB,CAAgB;IAC3C,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,oBAAoB,CAAkB;IAC9C,OAAO,CAAC,eAAe,CAA2B;IAClD,OAAO,CAAC,mBAAmB,CAAa;IACxC,OAAO,CAAC,8BAA8B,CAEtB;IAChB,OAAO,CAAC,YAAY,CAAkB;gBAE1B,MAAM,EAAE,gBAAgB;IAUpC,uBAAuB,IAAI,OAAO;IAIlC,uBAAuB,IAAI,OAAO;IAO5B,kBAAkB,IAAI,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAoB1D,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC;IAgBzC,kBAAkB,CAAC,aAAa,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAwDzD,2BAA2B,IAAI,OAAO,CAAC,OAAO,CAAC;IA2BrD,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAYnC,wBAAwB,IAAI,IAAI;IAO1B,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC;IAgBnC,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC;IAkBzC,iBAAiB,IAAI,IAAI;IAUzB,iBAAiB,IAAI,MAAM;IAO3B,oBAAoB,CAAC,SAAS,GAAE,mBAAwB,GAAG,IAAI;IAiB/D,sBAAsB,IAAI,IAAI;IAgB9B,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,eAAe;YAUT,4BAA4B;IAoCpC,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAiBpE,iBAAiB,IAAI,IAAI;IAqCnB,gBAAgB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAmBjD,cAAc,CAAC,SAAS,GAAE,mBAAwB,GAAG,OAAO,CAAC,OAAO,CAAC;IA4IrE,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;WAwCzB,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC;CAYvD"}
|