@myscheme/voice-navigation-sdk 0.1.3 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +250 -1
- package/dist/actions.d.ts.map +1 -1
- package/dist/actions.js +397 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -3
- package/dist/microphone-handler.d.ts +2 -0
- package/dist/microphone-handler.d.ts.map +1 -1
- package/dist/microphone-handler.js +11 -2
- package/dist/navigation-controller.d.ts +2 -0
- package/dist/navigation-controller.d.ts.map +1 -1
- package/dist/navigation-controller.js +89 -10
- package/dist/server/azure-speech-handler.d.ts +12 -0
- package/dist/server/azure-speech-handler.d.ts.map +1 -0
- package/dist/server/azure-speech-handler.js +40 -0
- package/dist/server/bedrock-embedding-handler.d.ts +15 -0
- package/dist/server/bedrock-embedding-handler.d.ts.map +1 -0
- package/dist/server/bedrock-embedding-handler.js +123 -0
- package/dist/server/bedrock-handler.d.ts +14 -0
- package/dist/server/bedrock-handler.d.ts.map +1 -0
- package/dist/server/bedrock-handler.js +114 -0
- package/dist/server/index.d.ts +4 -0
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +4 -0
- package/dist/server/opensearch-env-handler.d.ts +22 -0
- package/dist/server/opensearch-env-handler.d.ts.map +1 -0
- package/dist/server/opensearch-env-handler.js +221 -0
- package/dist/services/azure-speech.d.ts +10 -2
- package/dist/services/azure-speech.d.ts.map +1 -1
- package/dist/services/azure-speech.js +66 -3
- package/dist/services/bedrock.d.ts +13 -4
- package/dist/services/bedrock.d.ts.map +1 -1
- package/dist/services/bedrock.js +171 -17
- package/dist/services/vector-search.d.ts.map +1 -1
- package/dist/services/vector-search.js +15 -2
- package/dist/types.d.ts +24 -11
- package/dist/types.d.ts.map +1 -1
- package/dist/ui.d.ts +4 -3
- package/dist/ui.d.ts.map +1 -1
- package/dist/ui.js +150 -17
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -11,7 +11,9 @@ A TypeScript SDK for voice-controlled navigation using Azure Speech-to-Text and
|
|
|
11
11
|
- 🖱️ **Rich browser and media controls**
|
|
12
12
|
- ♿ **Accessibility-first design**
|
|
13
13
|
- 🎨 **Customizable floating UI** control
|
|
14
|
-
-
|
|
14
|
+
- � **Flexible button options** - use default or your own custom button
|
|
15
|
+
- 📍 **Configurable button placement** - position the default button anywhere
|
|
16
|
+
- �📦 **Full TypeScript support**
|
|
15
17
|
- 🔄 **Reusable across multiple websites**
|
|
16
18
|
- 🔍 **Vector search integration** with OpenSearch
|
|
17
19
|
|
|
@@ -94,6 +96,7 @@ const controller = initNavigationOnMicrophone({
|
|
|
94
96
|
| `actionHandlers` | `object` | ❌ | Custom action callbacks |
|
|
95
97
|
| `pages` | `object` | ❌ | Dynamic page navigation configuration |
|
|
96
98
|
| `opensearch` | `object` | ❌ | OpenSearch vector search configuration |
|
|
99
|
+
| `ui` | `object` | ❌ | UI customization (button placement & style) |
|
|
97
100
|
|
|
98
101
|
### OpenSearch Configuration
|
|
99
102
|
|
|
@@ -112,6 +115,252 @@ When providing the optional `opensearch` block:
|
|
|
112
115
|
| `sourceFields` | `string[]` | ❌ | Source fields to retrieve |
|
|
113
116
|
| `apiPath` | `string` | ❌ | Proxy endpoint (default: `/api/voice-navigation/vector-search`) |
|
|
114
117
|
|
|
118
|
+
### UI Configuration
|
|
119
|
+
|
|
120
|
+
The library provides flexible button options - use the default floating button or integrate with your own custom button.
|
|
121
|
+
|
|
122
|
+
#### Configuration Options
|
|
123
|
+
|
|
124
|
+
| Property | Type | Required | Description |
|
|
125
|
+
| ---------------------- | ----------------- | -------- | ------------------------------------------------------------- |
|
|
126
|
+
| `showDefaultButton` | `boolean` | ❌ | Show library's default button (default: `true`) |
|
|
127
|
+
| `customButtonSelector` | `string` | ❌ | CSS selector for your custom button (e.g., `"#my-voice-btn"`) |
|
|
128
|
+
| `buttonPlacement` | `ButtonPlacement` | ❌ | Position of default button (default: `"bottom-right"`) |
|
|
129
|
+
|
|
130
|
+
**ButtonPlacement** options: `"bottom-right"` \| `"bottom-left"` \| `"top-right"` \| `"top-left"` \| `"center-right"` \| `"center-left"`
|
|
131
|
+
|
|
132
|
+
#### Option 1: Default Button with Custom Placement
|
|
133
|
+
|
|
134
|
+
Use the library's pre-styled floating button at different positions:
|
|
135
|
+
|
|
136
|
+
```typescript
|
|
137
|
+
const controller = initNavigationOnMicrophone({
|
|
138
|
+
azure: {
|
|
139
|
+
/* ... */
|
|
140
|
+
},
|
|
141
|
+
aws: {
|
|
142
|
+
/* ... */
|
|
143
|
+
},
|
|
144
|
+
|
|
145
|
+
// Place button at top-left corner
|
|
146
|
+
ui: {
|
|
147
|
+
showDefaultButton: true,
|
|
148
|
+
buttonPlacement: "top-left",
|
|
149
|
+
},
|
|
150
|
+
});
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
**Available placements:**
|
|
154
|
+
|
|
155
|
+
- `"bottom-right"` (default) - Bottom right corner
|
|
156
|
+
- `"bottom-left"` - Bottom left corner
|
|
157
|
+
- `"top-right"` - Top right corner
|
|
158
|
+
- `"top-left"` - Top left corner
|
|
159
|
+
- `"center-right"` - Vertically centered on right
|
|
160
|
+
- `"center-left"` - Vertically centered on left
|
|
161
|
+
|
|
162
|
+
#### Option 2: Custom Button
|
|
163
|
+
|
|
164
|
+
Use your own button design and have the library attach voice control functionality to it:
|
|
165
|
+
|
|
166
|
+
**1. Create your button in HTML:**
|
|
167
|
+
|
|
168
|
+
```html
|
|
169
|
+
<button id="my-voice-button" class="my-custom-style">
|
|
170
|
+
<span>🎤 Voice Control</span>
|
|
171
|
+
</button>
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
**2. Configure the SDK to use your button:**
|
|
175
|
+
|
|
176
|
+
```typescript
|
|
177
|
+
const controller = initNavigationOnMicrophone({
|
|
178
|
+
azure: {
|
|
179
|
+
/* ... */
|
|
180
|
+
},
|
|
181
|
+
aws: {
|
|
182
|
+
/* ... */
|
|
183
|
+
},
|
|
184
|
+
|
|
185
|
+
ui: {
|
|
186
|
+
showDefaultButton: false,
|
|
187
|
+
customButtonSelector: "#my-voice-button",
|
|
188
|
+
},
|
|
189
|
+
});
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
**Important Notes for Custom Buttons:**
|
|
193
|
+
|
|
194
|
+
- **Button must exist in DOM before SDK initialization** - Add the SDK initialization code after your button is rendered, or use `DOMContentLoaded` event
|
|
195
|
+
- The library will automatically add `aria-pressed` and `aria-label` attributes for accessibility
|
|
196
|
+
- The button can be any clickable element (button, div, etc.)
|
|
197
|
+
- The library handles all click events and state management
|
|
198
|
+
- Visual feedback panel will display near the button when voice control is active
|
|
199
|
+
- You are responsible for styling the button (pressed, disabled, etc.) based on your design requirements
|
|
200
|
+
|
|
201
|
+
**React Example with Custom Button:**
|
|
202
|
+
|
|
203
|
+
```tsx
|
|
204
|
+
import { useEffect, useRef } from "react";
|
|
205
|
+
import { initNavigationOnMicrophone } from "@myscheme/voice-navigation-sdk";
|
|
206
|
+
|
|
207
|
+
export default function VoiceNavigationProvider() {
|
|
208
|
+
const controllerRef = useRef(null);
|
|
209
|
+
|
|
210
|
+
useEffect(() => {
|
|
211
|
+
// Initialize after button is rendered
|
|
212
|
+
controllerRef.current = initNavigationOnMicrophone({
|
|
213
|
+
azure: {
|
|
214
|
+
subscriptionKey: process.env.NEXT_PUBLIC_AZURE_SPEECH_KEY!,
|
|
215
|
+
region: process.env.NEXT_PUBLIC_AZURE_SPEECH_REGION!,
|
|
216
|
+
},
|
|
217
|
+
aws: {
|
|
218
|
+
accessKeyId: process.env.NEXT_PUBLIC_AWS_ACCESS_KEY_ID!,
|
|
219
|
+
secretAccessKey: process.env.NEXT_PUBLIC_AWS_SECRET_ACCESS_KEY!,
|
|
220
|
+
modelId: "anthropic.claude-3-sonnet-20240229-v1:0",
|
|
221
|
+
region: "ap-south-1",
|
|
222
|
+
},
|
|
223
|
+
ui: {
|
|
224
|
+
showDefaultButton: false,
|
|
225
|
+
customButtonSelector: "#voice-control-btn",
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
return () => controllerRef.current?.destroy();
|
|
230
|
+
}, []);
|
|
231
|
+
|
|
232
|
+
return (
|
|
233
|
+
<button
|
|
234
|
+
id="voice-control-btn"
|
|
235
|
+
className="px-4 py-2 bg-blue-600 text-white rounded-full hover:bg-blue-700"
|
|
236
|
+
>
|
|
237
|
+
🎤 Voice Control
|
|
238
|
+
</button>
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
**Next.js App Router Example:**
|
|
244
|
+
|
|
245
|
+
```tsx
|
|
246
|
+
// app/components/voice-button.tsx
|
|
247
|
+
"use client";
|
|
248
|
+
|
|
249
|
+
export default function VoiceButton() {
|
|
250
|
+
return (
|
|
251
|
+
<button
|
|
252
|
+
id="voice-nav-button"
|
|
253
|
+
className="fixed bottom-4 right-4 w-16 h-16 rounded-full bg-gradient-to-r from-purple-500 to-pink-500 text-white shadow-lg hover:scale-110 transition-transform"
|
|
254
|
+
aria-label="Activate voice navigation"
|
|
255
|
+
>
|
|
256
|
+
🎤
|
|
257
|
+
</button>
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// app/providers/voice-navigation.tsx
|
|
262
|
+
("use client");
|
|
263
|
+
|
|
264
|
+
import { useEffect } from "react";
|
|
265
|
+
import { initNavigationOnMicrophone } from "@myscheme/voice-navigation-sdk";
|
|
266
|
+
|
|
267
|
+
export default function VoiceNavigationProvider() {
|
|
268
|
+
useEffect(() => {
|
|
269
|
+
const controller = initNavigationOnMicrophone({
|
|
270
|
+
azure: {
|
|
271
|
+
/* ... */
|
|
272
|
+
},
|
|
273
|
+
aws: {
|
|
274
|
+
/* ... */
|
|
275
|
+
},
|
|
276
|
+
ui: {
|
|
277
|
+
showDefaultButton: false,
|
|
278
|
+
customButtonSelector: "#voice-nav-button",
|
|
279
|
+
},
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
return () => controller.destroy();
|
|
283
|
+
}, []);
|
|
284
|
+
|
|
285
|
+
return null;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// app/layout.tsx
|
|
289
|
+
import VoiceButton from "./components/voice-button";
|
|
290
|
+
import VoiceNavigationProvider from "./providers/voice-navigation";
|
|
291
|
+
|
|
292
|
+
export default function RootLayout({ children }) {
|
|
293
|
+
return (
|
|
294
|
+
<html>
|
|
295
|
+
<body>
|
|
296
|
+
<VoiceButton />
|
|
297
|
+
<VoiceNavigationProvider />
|
|
298
|
+
{children}
|
|
299
|
+
</body>
|
|
300
|
+
</html>
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
**Vanilla JavaScript Example:**
|
|
306
|
+
|
|
307
|
+
```html
|
|
308
|
+
<!DOCTYPE html>
|
|
309
|
+
<html>
|
|
310
|
+
<head>
|
|
311
|
+
<style>
|
|
312
|
+
.voice-btn {
|
|
313
|
+
position: fixed;
|
|
314
|
+
bottom: 20px;
|
|
315
|
+
right: 20px;
|
|
316
|
+
width: 60px;
|
|
317
|
+
height: 60px;
|
|
318
|
+
border-radius: 50%;
|
|
319
|
+
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
320
|
+
color: white;
|
|
321
|
+
border: none;
|
|
322
|
+
cursor: pointer;
|
|
323
|
+
font-size: 24px;
|
|
324
|
+
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
|
|
325
|
+
transition: transform 0.2s;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
.voice-btn:hover {
|
|
329
|
+
transform: scale(1.1);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
.voice-btn:active {
|
|
333
|
+
transform: scale(0.95);
|
|
334
|
+
}
|
|
335
|
+
</style>
|
|
336
|
+
</head>
|
|
337
|
+
<body>
|
|
338
|
+
<button id="voice-control" class="voice-btn">🎤</button>
|
|
339
|
+
|
|
340
|
+
<script type="module">
|
|
341
|
+
import { initNavigationOnMicrophone } from "@myscheme/voice-navigation-sdk";
|
|
342
|
+
|
|
343
|
+
const controller = initNavigationOnMicrophone({
|
|
344
|
+
azure: {
|
|
345
|
+
subscriptionKey: "your-key",
|
|
346
|
+
region: "your-region",
|
|
347
|
+
},
|
|
348
|
+
aws: {
|
|
349
|
+
accessKeyId: "your-key",
|
|
350
|
+
secretAccessKey: "your-secret",
|
|
351
|
+
modelId: "anthropic.claude-3-sonnet-20240229-v1:0",
|
|
352
|
+
region: "ap-south-1",
|
|
353
|
+
},
|
|
354
|
+
ui: {
|
|
355
|
+
showDefaultButton: false,
|
|
356
|
+
customButtonSelector: "#voice-control",
|
|
357
|
+
},
|
|
358
|
+
});
|
|
359
|
+
</script>
|
|
360
|
+
</body>
|
|
361
|
+
</html>
|
|
362
|
+
```
|
|
363
|
+
|
|
115
364
|
## 🌐 Dynamic Page Navigation
|
|
116
365
|
|
|
117
366
|
Instead of hardcoding page navigation, configure pages dynamically using XML or direct configuration.
|
package/dist/actions.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAEhB,aAAa,EACb,YAAY,EACZ,mBAAmB,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../src/actions.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAEhB,aAAa,EACb,YAAY,EACZ,mBAAmB,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAyDhE,eAAO,MAAM,eAAe,GAAI,UAAU,YAAY,KAAG,IAOxD,CAAC;AAKF,eAAO,MAAM,eAAe,QAAO,YAAY,GAAG,IAEjD,CAAC;AAkCF,eAAO,MAAM,YAAY,GAAI,OAAO,MAAM,KAAG,MAS5C,CAAC;AAKF,eAAO,MAAM,UAAU,GAAI,OAAO,MAAM,KAAG,MAM1C,CAAC;AAKF,eAAO,MAAM,iBAAiB,GAAI,QAAQ,MAAM,KAAG,MAElD,CAAC;AAKF,eAAO,MAAM,kBAAkB,GAC7B,QAAQ,gBAAgB,EACxB,UAAS,aAAkB,KAC1B,YA4yBF,CAAC;AAKF,eAAO,MAAM,kBAAkB,GAAI,QAAQ,GAAG,KAAG,mBAAmB,GAAG,IAiCtE,CAAC;AAKF,eAAO,MAAM,eAAe,GAAI,QAAQ,MAAM,KAAG,MAAM,IAAI,gBAY1D,CAAC"}
|
package/dist/actions.js
CHANGED
|
@@ -25,6 +25,25 @@ const CORE_ACTIONS = new Set([
|
|
|
25
25
|
"unmute_media",
|
|
26
26
|
"search_content",
|
|
27
27
|
"stop",
|
|
28
|
+
"quit",
|
|
29
|
+
"tab_next",
|
|
30
|
+
"tab_back",
|
|
31
|
+
"click",
|
|
32
|
+
"enter",
|
|
33
|
+
"submit",
|
|
34
|
+
"move_left",
|
|
35
|
+
"move_right",
|
|
36
|
+
"move_up",
|
|
37
|
+
"move_down",
|
|
38
|
+
"move_beginning",
|
|
39
|
+
"move_end",
|
|
40
|
+
"select_all",
|
|
41
|
+
"select_line",
|
|
42
|
+
"type_text",
|
|
43
|
+
"cut",
|
|
44
|
+
"copy",
|
|
45
|
+
"paste",
|
|
46
|
+
"clear",
|
|
28
47
|
"unknown",
|
|
29
48
|
]);
|
|
30
49
|
let globalPageRegistry = null;
|
|
@@ -197,14 +216,14 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
197
216
|
}
|
|
198
217
|
case "copy_url": {
|
|
199
218
|
if (navigator.clipboard && window.isSecureContext) {
|
|
219
|
+
info.copied = true;
|
|
220
|
+
performed = true;
|
|
200
221
|
navigator.clipboard
|
|
201
222
|
.writeText(window.location.href)
|
|
202
223
|
.then(() => console.log("Page URL copied to clipboard"))
|
|
203
224
|
.catch((error) => {
|
|
204
225
|
console.warn("Failed to copy URL via clipboard API", error);
|
|
205
226
|
});
|
|
206
|
-
info.copied = true;
|
|
207
|
-
performed = true;
|
|
208
227
|
}
|
|
209
228
|
else {
|
|
210
229
|
try {
|
|
@@ -351,6 +370,382 @@ export const performAgentAction = (action, context = {}) => {
|
|
|
351
370
|
performed = true;
|
|
352
371
|
break;
|
|
353
372
|
}
|
|
373
|
+
case "quit": {
|
|
374
|
+
console.log("Quit command received");
|
|
375
|
+
if (onStop) {
|
|
376
|
+
onStop();
|
|
377
|
+
}
|
|
378
|
+
info.quit = true;
|
|
379
|
+
performed = true;
|
|
380
|
+
break;
|
|
381
|
+
}
|
|
382
|
+
case "tab_next": {
|
|
383
|
+
const focusableElements = Array.from(document.querySelectorAll('a, button, input, textarea, select, [tabindex]:not([tabindex="-1"])')).filter((el) => {
|
|
384
|
+
const rect = el.getBoundingClientRect();
|
|
385
|
+
return (!el.hasAttribute("disabled") &&
|
|
386
|
+
rect.width > 0 &&
|
|
387
|
+
rect.height > 0 &&
|
|
388
|
+
window.getComputedStyle(el).visibility !== "hidden");
|
|
389
|
+
});
|
|
390
|
+
const currentIndex = focusableElements.indexOf(document.activeElement);
|
|
391
|
+
const nextIndex = (currentIndex + 1) % focusableElements.length;
|
|
392
|
+
if (focusableElements[nextIndex]) {
|
|
393
|
+
focusableElements[nextIndex].focus();
|
|
394
|
+
console.log("Moved focus to next element");
|
|
395
|
+
info.focused = true;
|
|
396
|
+
info.direction = "next";
|
|
397
|
+
performed = true;
|
|
398
|
+
}
|
|
399
|
+
break;
|
|
400
|
+
}
|
|
401
|
+
case "tab_back": {
|
|
402
|
+
const focusableElements = Array.from(document.querySelectorAll('a, button, input, textarea, select, [tabindex]:not([tabindex="-1"])')).filter((el) => {
|
|
403
|
+
const rect = el.getBoundingClientRect();
|
|
404
|
+
return (!el.hasAttribute("disabled") &&
|
|
405
|
+
rect.width > 0 &&
|
|
406
|
+
rect.height > 0 &&
|
|
407
|
+
window.getComputedStyle(el).visibility !== "hidden");
|
|
408
|
+
});
|
|
409
|
+
const currentIndex = focusableElements.indexOf(document.activeElement);
|
|
410
|
+
const prevIndex = currentIndex <= 0 ? focusableElements.length - 1 : currentIndex - 1;
|
|
411
|
+
if (focusableElements[prevIndex]) {
|
|
412
|
+
focusableElements[prevIndex].focus();
|
|
413
|
+
console.log("Moved focus to previous element");
|
|
414
|
+
info.focused = true;
|
|
415
|
+
info.direction = "back";
|
|
416
|
+
performed = true;
|
|
417
|
+
}
|
|
418
|
+
break;
|
|
419
|
+
}
|
|
420
|
+
case "click":
|
|
421
|
+
case "enter":
|
|
422
|
+
case "submit": {
|
|
423
|
+
const activeElement = document.activeElement;
|
|
424
|
+
if (activeElement && activeElement !== document.body) {
|
|
425
|
+
activeElement.click();
|
|
426
|
+
console.log(`Clicked on focused element: ${activeElement.tagName}`);
|
|
427
|
+
info.clicked = true;
|
|
428
|
+
performed = true;
|
|
429
|
+
}
|
|
430
|
+
else {
|
|
431
|
+
console.log("No element is currently focused");
|
|
432
|
+
}
|
|
433
|
+
break;
|
|
434
|
+
}
|
|
435
|
+
case "move_left": {
|
|
436
|
+
const activeElement = document.activeElement;
|
|
437
|
+
if (activeElement &&
|
|
438
|
+
(activeElement instanceof HTMLInputElement ||
|
|
439
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
440
|
+
const currentPos = activeElement.selectionStart || 0;
|
|
441
|
+
activeElement.setSelectionRange(currentPos - 1, currentPos - 1);
|
|
442
|
+
console.log("Moved cursor left");
|
|
443
|
+
info.cursorMoved = true;
|
|
444
|
+
info.direction = "left";
|
|
445
|
+
performed = true;
|
|
446
|
+
}
|
|
447
|
+
break;
|
|
448
|
+
}
|
|
449
|
+
case "move_right": {
|
|
450
|
+
const activeElement = document.activeElement;
|
|
451
|
+
if (activeElement &&
|
|
452
|
+
(activeElement instanceof HTMLInputElement ||
|
|
453
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
454
|
+
const currentPos = activeElement.selectionStart || 0;
|
|
455
|
+
activeElement.setSelectionRange(currentPos + 1, currentPos + 1);
|
|
456
|
+
console.log("Moved cursor right");
|
|
457
|
+
info.cursorMoved = true;
|
|
458
|
+
info.direction = "right";
|
|
459
|
+
performed = true;
|
|
460
|
+
}
|
|
461
|
+
break;
|
|
462
|
+
}
|
|
463
|
+
case "move_up": {
|
|
464
|
+
const activeElement = document.activeElement;
|
|
465
|
+
if (activeElement instanceof HTMLTextAreaElement) {
|
|
466
|
+
const lines = activeElement.value.split("\n");
|
|
467
|
+
const currentPos = activeElement.selectionStart || 0;
|
|
468
|
+
let lineStart = 0;
|
|
469
|
+
let currentLine = 0;
|
|
470
|
+
for (let i = 0; i < lines.length; i++) {
|
|
471
|
+
if (currentPos <= lineStart + lines[i].length) {
|
|
472
|
+
currentLine = i;
|
|
473
|
+
break;
|
|
474
|
+
}
|
|
475
|
+
lineStart += lines[i].length + 1;
|
|
476
|
+
}
|
|
477
|
+
if (currentLine > 0) {
|
|
478
|
+
const posInLine = currentPos - lineStart;
|
|
479
|
+
const prevLineStart = lineStart - lines[currentLine - 1].length - 1;
|
|
480
|
+
const newPos = Math.min(prevLineStart + posInLine, prevLineStart + lines[currentLine - 1].length);
|
|
481
|
+
activeElement.setSelectionRange(newPos, newPos);
|
|
482
|
+
console.log("Moved cursor up");
|
|
483
|
+
info.cursorMoved = true;
|
|
484
|
+
info.direction = "up";
|
|
485
|
+
performed = true;
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
break;
|
|
489
|
+
}
|
|
490
|
+
case "move_down": {
|
|
491
|
+
const activeElement = document.activeElement;
|
|
492
|
+
if (activeElement instanceof HTMLTextAreaElement) {
|
|
493
|
+
const lines = activeElement.value.split("\n");
|
|
494
|
+
const currentPos = activeElement.selectionStart || 0;
|
|
495
|
+
let lineStart = 0;
|
|
496
|
+
let currentLine = 0;
|
|
497
|
+
for (let i = 0; i < lines.length; i++) {
|
|
498
|
+
if (currentPos <= lineStart + lines[i].length) {
|
|
499
|
+
currentLine = i;
|
|
500
|
+
break;
|
|
501
|
+
}
|
|
502
|
+
lineStart += lines[i].length + 1;
|
|
503
|
+
}
|
|
504
|
+
if (currentLine < lines.length - 1) {
|
|
505
|
+
const posInLine = currentPos - lineStart;
|
|
506
|
+
const nextLineStart = lineStart + lines[currentLine].length + 1;
|
|
507
|
+
const newPos = Math.min(nextLineStart + posInLine, nextLineStart + lines[currentLine + 1].length);
|
|
508
|
+
activeElement.setSelectionRange(newPos, newPos);
|
|
509
|
+
console.log("Moved cursor down");
|
|
510
|
+
info.cursorMoved = true;
|
|
511
|
+
info.direction = "down";
|
|
512
|
+
performed = true;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
break;
|
|
516
|
+
}
|
|
517
|
+
case "move_beginning": {
|
|
518
|
+
const activeElement = document.activeElement;
|
|
519
|
+
if (activeElement &&
|
|
520
|
+
(activeElement instanceof HTMLInputElement ||
|
|
521
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
522
|
+
activeElement.setSelectionRange(0, 0);
|
|
523
|
+
console.log("Moved cursor to beginning");
|
|
524
|
+
info.cursorMoved = true;
|
|
525
|
+
info.position = "beginning";
|
|
526
|
+
performed = true;
|
|
527
|
+
}
|
|
528
|
+
break;
|
|
529
|
+
}
|
|
530
|
+
case "move_end": {
|
|
531
|
+
const activeElement = document.activeElement;
|
|
532
|
+
if (activeElement &&
|
|
533
|
+
(activeElement instanceof HTMLInputElement ||
|
|
534
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
535
|
+
const length = activeElement.value.length;
|
|
536
|
+
activeElement.setSelectionRange(length, length);
|
|
537
|
+
console.log("Moved cursor to end");
|
|
538
|
+
info.cursorMoved = true;
|
|
539
|
+
info.position = "end";
|
|
540
|
+
performed = true;
|
|
541
|
+
}
|
|
542
|
+
break;
|
|
543
|
+
}
|
|
544
|
+
case "select_all": {
|
|
545
|
+
const activeElement = document.activeElement;
|
|
546
|
+
if (activeElement &&
|
|
547
|
+
(activeElement instanceof HTMLInputElement ||
|
|
548
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
549
|
+
activeElement.select();
|
|
550
|
+
console.log("Selected all text");
|
|
551
|
+
info.selected = true;
|
|
552
|
+
performed = true;
|
|
553
|
+
}
|
|
554
|
+
else {
|
|
555
|
+
const selection = window.getSelection();
|
|
556
|
+
if (selection) {
|
|
557
|
+
selection.selectAllChildren(document.body);
|
|
558
|
+
console.log("Selected all content on page");
|
|
559
|
+
info.selected = true;
|
|
560
|
+
performed = true;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
break;
|
|
564
|
+
}
|
|
565
|
+
case "select_line": {
|
|
566
|
+
const activeElement = document.activeElement;
|
|
567
|
+
if (activeElement instanceof HTMLTextAreaElement) {
|
|
568
|
+
const currentPos = activeElement.selectionStart || 0;
|
|
569
|
+
const value = activeElement.value;
|
|
570
|
+
let lineStart = value.lastIndexOf("\n", currentPos - 1) + 1;
|
|
571
|
+
let lineEnd = value.indexOf("\n", currentPos);
|
|
572
|
+
if (lineEnd === -1) {
|
|
573
|
+
lineEnd = value.length;
|
|
574
|
+
}
|
|
575
|
+
activeElement.setSelectionRange(lineStart, lineEnd);
|
|
576
|
+
console.log("Selected current line");
|
|
577
|
+
info.selected = true;
|
|
578
|
+
info.scope = "line";
|
|
579
|
+
performed = true;
|
|
580
|
+
}
|
|
581
|
+
else if (activeElement instanceof HTMLInputElement) {
|
|
582
|
+
activeElement.select();
|
|
583
|
+
console.log("Selected all text in input");
|
|
584
|
+
info.selected = true;
|
|
585
|
+
performed = true;
|
|
586
|
+
}
|
|
587
|
+
break;
|
|
588
|
+
}
|
|
589
|
+
case "type_text": {
|
|
590
|
+
const activeElement = document.activeElement;
|
|
591
|
+
const typeText = context?.parameters?.text || "";
|
|
592
|
+
if (activeElement &&
|
|
593
|
+
(activeElement instanceof HTMLInputElement ||
|
|
594
|
+
activeElement instanceof HTMLTextAreaElement) &&
|
|
595
|
+
typeText) {
|
|
596
|
+
const start = activeElement.selectionStart || 0;
|
|
597
|
+
const end = activeElement.selectionEnd || 0;
|
|
598
|
+
const value = activeElement.value;
|
|
599
|
+
activeElement.value =
|
|
600
|
+
value.substring(0, start) + typeText + value.substring(end);
|
|
601
|
+
const newPos = start + typeText.length;
|
|
602
|
+
activeElement.setSelectionRange(newPos, newPos);
|
|
603
|
+
activeElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: true }));
|
|
604
|
+
console.log(`Typed text: "${typeText}"`);
|
|
605
|
+
info.typed = true;
|
|
606
|
+
info.text = typeText;
|
|
607
|
+
performed = true;
|
|
608
|
+
}
|
|
609
|
+
else {
|
|
610
|
+
console.log("No text field is focused or no text provided");
|
|
611
|
+
}
|
|
612
|
+
break;
|
|
613
|
+
}
|
|
614
|
+
case "cut": {
|
|
615
|
+
const activeElement = document.activeElement;
|
|
616
|
+
if (activeElement &&
|
|
617
|
+
(activeElement instanceof HTMLInputElement ||
|
|
618
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
619
|
+
const start = activeElement.selectionStart || 0;
|
|
620
|
+
const end = activeElement.selectionEnd || 0;
|
|
621
|
+
const selectedText = activeElement.value.substring(start, end);
|
|
622
|
+
if (selectedText && navigator.clipboard) {
|
|
623
|
+
info.cut = true;
|
|
624
|
+
performed = true;
|
|
625
|
+
navigator.clipboard
|
|
626
|
+
.writeText(selectedText)
|
|
627
|
+
.then(() => {
|
|
628
|
+
const value = activeElement.value;
|
|
629
|
+
activeElement.value =
|
|
630
|
+
value.substring(0, start) + value.substring(end);
|
|
631
|
+
activeElement.setSelectionRange(start, start);
|
|
632
|
+
activeElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: true }));
|
|
633
|
+
console.log("Cut text to clipboard");
|
|
634
|
+
})
|
|
635
|
+
.catch((error) => {
|
|
636
|
+
console.warn("Failed to cut text", error);
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
else {
|
|
640
|
+
console.log("No text selected or clipboard API unavailable");
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
break;
|
|
644
|
+
}
|
|
645
|
+
case "copy": {
|
|
646
|
+
const activeElement = document.activeElement;
|
|
647
|
+
if (activeElement &&
|
|
648
|
+
(activeElement instanceof HTMLInputElement ||
|
|
649
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
650
|
+
const start = activeElement.selectionStart || 0;
|
|
651
|
+
const end = activeElement.selectionEnd || 0;
|
|
652
|
+
const selectedText = activeElement.value.substring(start, end);
|
|
653
|
+
if (selectedText && navigator.clipboard) {
|
|
654
|
+
info.copied = true;
|
|
655
|
+
performed = true;
|
|
656
|
+
navigator.clipboard
|
|
657
|
+
.writeText(selectedText)
|
|
658
|
+
.then(() => {
|
|
659
|
+
console.log("Copied text to clipboard");
|
|
660
|
+
})
|
|
661
|
+
.catch((error) => {
|
|
662
|
+
console.warn("Failed to copy text", error);
|
|
663
|
+
});
|
|
664
|
+
}
|
|
665
|
+
else {
|
|
666
|
+
const selection = window.getSelection();
|
|
667
|
+
const selectionText = selection?.toString();
|
|
668
|
+
if (selectionText && navigator.clipboard) {
|
|
669
|
+
info.copied = true;
|
|
670
|
+
performed = true;
|
|
671
|
+
navigator.clipboard
|
|
672
|
+
.writeText(selectionText)
|
|
673
|
+
.then(() => {
|
|
674
|
+
console.log("Copied selected text to clipboard");
|
|
675
|
+
})
|
|
676
|
+
.catch((error) => {
|
|
677
|
+
console.warn("Failed to copy text", error);
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
else {
|
|
681
|
+
console.log("No text selected");
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
else {
|
|
686
|
+
const selection = window.getSelection();
|
|
687
|
+
const selectionText = selection?.toString();
|
|
688
|
+
if (selectionText && navigator.clipboard) {
|
|
689
|
+
info.copied = true;
|
|
690
|
+
performed = true;
|
|
691
|
+
navigator.clipboard
|
|
692
|
+
.writeText(selectionText)
|
|
693
|
+
.then(() => {
|
|
694
|
+
console.log("Copied selected text to clipboard");
|
|
695
|
+
})
|
|
696
|
+
.catch((error) => {
|
|
697
|
+
console.warn("Failed to copy text", error);
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
else {
|
|
701
|
+
console.log("No text selected");
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
break;
|
|
705
|
+
}
|
|
706
|
+
case "paste": {
|
|
707
|
+
const activeElement = document.activeElement;
|
|
708
|
+
if (activeElement &&
|
|
709
|
+
(activeElement instanceof HTMLInputElement ||
|
|
710
|
+
activeElement instanceof HTMLTextAreaElement) &&
|
|
711
|
+
navigator.clipboard) {
|
|
712
|
+
info.pasted = true;
|
|
713
|
+
performed = true;
|
|
714
|
+
navigator.clipboard
|
|
715
|
+
.readText()
|
|
716
|
+
.then((clipboardText) => {
|
|
717
|
+
const start = activeElement.selectionStart || 0;
|
|
718
|
+
const end = activeElement.selectionEnd || 0;
|
|
719
|
+
const value = activeElement.value;
|
|
720
|
+
activeElement.value =
|
|
721
|
+
value.substring(0, start) + clipboardText + value.substring(end);
|
|
722
|
+
const newPos = start + clipboardText.length;
|
|
723
|
+
activeElement.setSelectionRange(newPos, newPos);
|
|
724
|
+
activeElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: true }));
|
|
725
|
+
console.log("Pasted text from clipboard");
|
|
726
|
+
})
|
|
727
|
+
.catch((error) => {
|
|
728
|
+
console.warn("Failed to paste text", error);
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
else {
|
|
732
|
+
console.log("No text field is focused or clipboard API unavailable");
|
|
733
|
+
}
|
|
734
|
+
break;
|
|
735
|
+
}
|
|
736
|
+
case "clear": {
|
|
737
|
+
const activeElement = document.activeElement;
|
|
738
|
+
if (activeElement &&
|
|
739
|
+
(activeElement instanceof HTMLInputElement ||
|
|
740
|
+
activeElement instanceof HTMLTextAreaElement)) {
|
|
741
|
+
activeElement.value = "";
|
|
742
|
+
activeElement.dispatchEvent(new Event("input", { bubbles: true, cancelable: true }));
|
|
743
|
+
console.log("Cleared text field");
|
|
744
|
+
info.cleared = true;
|
|
745
|
+
performed = true;
|
|
746
|
+
}
|
|
747
|
+
break;
|
|
748
|
+
}
|
|
354
749
|
default: {
|
|
355
750
|
if (action.startsWith("navigate_") && globalPageRegistry) {
|
|
356
751
|
const pageId = globalPageRegistry.getPageIdFromAction(action);
|