@cuekit-ai/react 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-UD3LZUJ2.mjs → chunk-PLOG3DEN.mjs} +494 -24
- package/dist/index.d.mts +6 -100
- package/dist/index.d.ts +6 -100
- package/dist/index.js +635 -832
- package/dist/index.mjs +120 -807
- package/dist/{webrtc-service-UYILN4PB.mjs → webrtc-service-BHI4M7YJ.mjs} +3 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ConnectionState,
|
|
3
|
+
GlobalStore,
|
|
3
4
|
Participant,
|
|
4
5
|
ParticipantEvent,
|
|
5
6
|
RoomEvent,
|
|
@@ -11,23 +12,29 @@ import {
|
|
|
11
12
|
_apiKey,
|
|
12
13
|
_appId,
|
|
13
14
|
authenticate,
|
|
15
|
+
captureFullDOMStructure,
|
|
14
16
|
connectToRoom,
|
|
15
17
|
createAudioAnalyser,
|
|
16
18
|
disconnectFromRoom,
|
|
19
|
+
executeAction,
|
|
20
|
+
getFullDOMStructure,
|
|
17
21
|
getParticipants,
|
|
18
22
|
getRoom,
|
|
19
23
|
getRoomName,
|
|
24
|
+
onStateChange,
|
|
20
25
|
sendData,
|
|
26
|
+
sendRuntimeData,
|
|
21
27
|
sendScreenStatus,
|
|
22
28
|
sendStaticData,
|
|
23
29
|
sendUserCommand,
|
|
24
30
|
setApiKey,
|
|
25
31
|
setAppId,
|
|
26
32
|
setAudioContainer,
|
|
33
|
+
setNavigationHandler,
|
|
27
34
|
setServerUrl,
|
|
28
35
|
setWebRTCCallbacks,
|
|
29
36
|
setWebRTCConfig
|
|
30
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-PLOG3DEN.mjs";
|
|
31
38
|
|
|
32
39
|
// node_modules/inline-style-parser/index.js
|
|
33
40
|
var require_inline_style_parser = __commonJS({
|
|
@@ -377,205 +384,6 @@ function InitCuekit(config) {
|
|
|
377
384
|
setWebRTCConfig(webRTCConfig);
|
|
378
385
|
}
|
|
379
386
|
|
|
380
|
-
// src/core/intent-store.ts
|
|
381
|
-
var store = {
|
|
382
|
-
screenMetadata: {},
|
|
383
|
-
allElementsData: []
|
|
384
|
-
};
|
|
385
|
-
var GlobalStore = {
|
|
386
|
-
// 🔹 Screen Metadata Methods
|
|
387
|
-
setMetadata(screen, metadata) {
|
|
388
|
-
store.screenMetadata[screen] = metadata;
|
|
389
|
-
},
|
|
390
|
-
getMetadata(screen) {
|
|
391
|
-
return store.screenMetadata[screen];
|
|
392
|
-
},
|
|
393
|
-
clearMetadata(screen) {
|
|
394
|
-
delete store.screenMetadata[screen];
|
|
395
|
-
},
|
|
396
|
-
// 🔹 Generic Store Access Methods
|
|
397
|
-
setData(key, value) {
|
|
398
|
-
store[key] = value;
|
|
399
|
-
},
|
|
400
|
-
getData(key) {
|
|
401
|
-
return store[key];
|
|
402
|
-
},
|
|
403
|
-
clearData(key) {
|
|
404
|
-
delete store[key];
|
|
405
|
-
},
|
|
406
|
-
// 🔹 Element Data Management
|
|
407
|
-
setElement(elementData) {
|
|
408
|
-
const index2 = store.allElementsData.findIndex((e2) => e2.elementId === elementData.elementId);
|
|
409
|
-
if (index2 >= 0) {
|
|
410
|
-
console.log("Updating existing element");
|
|
411
|
-
store.allElementsData[index2] = elementData;
|
|
412
|
-
} else {
|
|
413
|
-
console.log("Adding new element");
|
|
414
|
-
store.allElementsData.push(elementData);
|
|
415
|
-
}
|
|
416
|
-
},
|
|
417
|
-
getElementById(elementId) {
|
|
418
|
-
const match = store.allElementsData.find((e2) => e2.elementId === elementId);
|
|
419
|
-
if (!match) {
|
|
420
|
-
console.warn(`[GlobalStore] No element found for ID: ${elementId}`);
|
|
421
|
-
console.log("All elements in store:", store.allElementsData);
|
|
422
|
-
}
|
|
423
|
-
return match;
|
|
424
|
-
},
|
|
425
|
-
deleteElementById(id) {
|
|
426
|
-
store.allElementsData = store.allElementsData.filter((e2) => e2.elementId !== id);
|
|
427
|
-
},
|
|
428
|
-
clearAllElements() {
|
|
429
|
-
store.allElementsData = [];
|
|
430
|
-
}
|
|
431
|
-
};
|
|
432
|
-
|
|
433
|
-
// src/core/navigation.ts
|
|
434
|
-
var navigation;
|
|
435
|
-
var navigationHandler = null;
|
|
436
|
-
function setNavigationHandler(handler) {
|
|
437
|
-
navigationHandler = handler;
|
|
438
|
-
}
|
|
439
|
-
function navigate(path2, params) {
|
|
440
|
-
const safeParams = params || {};
|
|
441
|
-
const absolutePath = path2.startsWith("/") ? path2 : `/${path2}`;
|
|
442
|
-
if (navigationHandler) {
|
|
443
|
-
try {
|
|
444
|
-
navigationHandler(absolutePath, safeParams);
|
|
445
|
-
} catch (error) {
|
|
446
|
-
console.error("[CueKit] navigation handler failed, falling back to default:", error);
|
|
447
|
-
}
|
|
448
|
-
return;
|
|
449
|
-
}
|
|
450
|
-
let fullPath = absolutePath;
|
|
451
|
-
if (safeParams) {
|
|
452
|
-
const searchParams = new URLSearchParams(safeParams).toString();
|
|
453
|
-
if (searchParams) {
|
|
454
|
-
fullPath += `?${searchParams}`;
|
|
455
|
-
}
|
|
456
|
-
}
|
|
457
|
-
if (navigation) {
|
|
458
|
-
navigation.push(fullPath);
|
|
459
|
-
} else {
|
|
460
|
-
if (typeof window !== "undefined") {
|
|
461
|
-
window.location.href = fullPath;
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
}
|
|
465
|
-
var getCurrentPath = () => {
|
|
466
|
-
if (typeof window === "undefined") return "";
|
|
467
|
-
return window.location.pathname;
|
|
468
|
-
};
|
|
469
|
-
var getSearchParams = () => {
|
|
470
|
-
if (typeof window === "undefined") return new URLSearchParams();
|
|
471
|
-
return new URLSearchParams(window.location.search);
|
|
472
|
-
};
|
|
473
|
-
var safeNavigate = (name2, params = {}) => {
|
|
474
|
-
if (name2) {
|
|
475
|
-
navigate(name2, params);
|
|
476
|
-
} else {
|
|
477
|
-
console.warn("[CueKit] route name not provided");
|
|
478
|
-
}
|
|
479
|
-
};
|
|
480
|
-
function getCurrentScreenName() {
|
|
481
|
-
try {
|
|
482
|
-
const path2 = getCurrentPath();
|
|
483
|
-
return path2 || "UnknownScreen";
|
|
484
|
-
} catch (e2) {
|
|
485
|
-
return "UnknownScreen";
|
|
486
|
-
}
|
|
487
|
-
}
|
|
488
|
-
function getCurrentRouteParams() {
|
|
489
|
-
try {
|
|
490
|
-
const params = {};
|
|
491
|
-
const searchParams = getSearchParams();
|
|
492
|
-
if (searchParams instanceof URLSearchParams) {
|
|
493
|
-
searchParams.forEach((value, key) => {
|
|
494
|
-
params[key] = value;
|
|
495
|
-
});
|
|
496
|
-
} else {
|
|
497
|
-
return searchParams;
|
|
498
|
-
}
|
|
499
|
-
return params;
|
|
500
|
-
} catch (e2) {
|
|
501
|
-
return {};
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
function onStateChange() {
|
|
505
|
-
const routeName = getCurrentScreenName();
|
|
506
|
-
const params = getCurrentRouteParams();
|
|
507
|
-
if (params && params.metadata) {
|
|
508
|
-
try {
|
|
509
|
-
const metadata = JSON.parse(params.metadata);
|
|
510
|
-
GlobalStore.setMetadata(routeName, metadata);
|
|
511
|
-
} catch (error) {
|
|
512
|
-
console.error("Failed to parse metadata from URL:", error);
|
|
513
|
-
}
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
var handleNavigationAndClick = (routeName, elementHash) => {
|
|
517
|
-
safeNavigate(routeName);
|
|
518
|
-
if (typeof MutationObserver === "undefined" || typeof document === "undefined") return;
|
|
519
|
-
const observer = new MutationObserver((mutationsList, observer2) => {
|
|
520
|
-
setTimeout(() => {
|
|
521
|
-
const allElements = document.querySelectorAll("*");
|
|
522
|
-
let elementToClick = null;
|
|
523
|
-
for (const element3 of allElements) {
|
|
524
|
-
if (element3 instanceof HTMLElement) {
|
|
525
|
-
const tagName = element3.tagName.toLowerCase();
|
|
526
|
-
const text7 = (element3.textContent || "").trim().substring(0, 50);
|
|
527
|
-
let sibling = element3.previousElementSibling;
|
|
528
|
-
let position3 = 1;
|
|
529
|
-
while (sibling) {
|
|
530
|
-
if (sibling.tagName === element3.tagName) {
|
|
531
|
-
position3++;
|
|
532
|
-
}
|
|
533
|
-
sibling = sibling.previousElementSibling;
|
|
534
|
-
}
|
|
535
|
-
const path2 = getElementPath(element3);
|
|
536
|
-
const idString = `${tagName}[${position3}]_(${text7})_${path2}`;
|
|
537
|
-
let hash = 0;
|
|
538
|
-
for (let i2 = 0; i2 < idString.length; i2++) {
|
|
539
|
-
const char = idString.charCodeAt(i2);
|
|
540
|
-
hash = (hash << 5) - hash + char;
|
|
541
|
-
hash |= 0;
|
|
542
|
-
}
|
|
543
|
-
const elementHashValue = hash.toString(36);
|
|
544
|
-
if (elementHashValue === elementHash) {
|
|
545
|
-
elementToClick = element3;
|
|
546
|
-
break;
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
}
|
|
550
|
-
if (elementToClick) {
|
|
551
|
-
elementToClick.click();
|
|
552
|
-
observer2.disconnect();
|
|
553
|
-
}
|
|
554
|
-
}, 100);
|
|
555
|
-
});
|
|
556
|
-
observer.observe(document.body, { childList: true, subtree: true });
|
|
557
|
-
};
|
|
558
|
-
function getElementPath(element3) {
|
|
559
|
-
if (element3.id) {
|
|
560
|
-
return `id(${element3.id})`;
|
|
561
|
-
}
|
|
562
|
-
const path2 = [];
|
|
563
|
-
let current = element3;
|
|
564
|
-
while (current && current !== document.body) {
|
|
565
|
-
let index2 = 1;
|
|
566
|
-
let sibling = current.previousElementSibling;
|
|
567
|
-
while (sibling) {
|
|
568
|
-
if (sibling.tagName === current.tagName) {
|
|
569
|
-
index2++;
|
|
570
|
-
}
|
|
571
|
-
sibling = sibling.previousElementSibling;
|
|
572
|
-
}
|
|
573
|
-
path2.unshift(`${current.tagName.toLowerCase()}[${index2}]`);
|
|
574
|
-
current = current.parentElement;
|
|
575
|
-
}
|
|
576
|
-
return path2.join("/");
|
|
577
|
-
}
|
|
578
|
-
|
|
579
387
|
// src/utils/webrtc-config.ts
|
|
580
388
|
var configureWebRTCServer = (config) => {
|
|
581
389
|
if (!config.apiKey) {
|
|
@@ -654,7 +462,7 @@ var CuekitProvider = ({
|
|
|
654
462
|
deviceId = "",
|
|
655
463
|
appId,
|
|
656
464
|
children,
|
|
657
|
-
navigationHandler
|
|
465
|
+
navigationHandler,
|
|
658
466
|
onConnectionStateChange,
|
|
659
467
|
onParticipantUpdate
|
|
660
468
|
}) => {
|
|
@@ -678,13 +486,13 @@ var CuekitProvider = ({
|
|
|
678
486
|
}
|
|
679
487
|
}, [apiKey]);
|
|
680
488
|
useEffect(() => {
|
|
681
|
-
setNavigationHandler(
|
|
489
|
+
setNavigationHandler(navigationHandler ?? null);
|
|
682
490
|
return () => {
|
|
683
491
|
setNavigationHandler(null);
|
|
684
492
|
};
|
|
685
|
-
}, [
|
|
493
|
+
}, [navigationHandler]);
|
|
686
494
|
useEffect(() => {
|
|
687
|
-
import("./webrtc-service-
|
|
495
|
+
import("./webrtc-service-BHI4M7YJ.mjs").then(({ setWebRTCCallbacks: setWebRTCCallbacks2 }) => {
|
|
688
496
|
setWebRTCCallbacks2({
|
|
689
497
|
onNavigationCommand: (command) => {
|
|
690
498
|
console.log("\u{1F9ED} Processing navigation command:", command);
|
|
@@ -696,8 +504,8 @@ var CuekitProvider = ({
|
|
|
696
504
|
console.log("\u{1F3AF} AI Intent:", command.text, "->", command.actionType);
|
|
697
505
|
if (command.actionType === "navigate" && command.current_page) {
|
|
698
506
|
console.log(`\u{1F9ED} Navigating to: ${command.current_page}`);
|
|
699
|
-
if (
|
|
700
|
-
|
|
507
|
+
if (navigationHandler) {
|
|
508
|
+
navigationHandler(command.current_page, {
|
|
701
509
|
intent: command.intent,
|
|
702
510
|
text: command.text,
|
|
703
511
|
confidence: command.confidence
|
|
@@ -727,7 +535,7 @@ var CuekitProvider = ({
|
|
|
727
535
|
}
|
|
728
536
|
});
|
|
729
537
|
});
|
|
730
|
-
}, [onConnectionStateChange, onParticipantUpdate,
|
|
538
|
+
}, [onConnectionStateChange, onParticipantUpdate, navigationHandler]);
|
|
731
539
|
useEffect(() => {
|
|
732
540
|
const updateGlobalStore = (id) => {
|
|
733
541
|
if (typeof window !== "undefined" && globalThis.CuekitStore) {
|
|
@@ -796,12 +604,12 @@ var CuekitProvider = ({
|
|
|
796
604
|
import React11, { useState as useState10, useEffect as useEffect10, useCallback as useCallback5, useRef as useRef7 } from "react";
|
|
797
605
|
|
|
798
606
|
// src/hooks/use-cuekit.ts
|
|
799
|
-
import { useState as useState3, useCallback as useCallback2,
|
|
607
|
+
import { useState as useState3, useCallback as useCallback2, useRef as useRef2 } from "react";
|
|
800
608
|
|
|
801
609
|
// src/hooks/use-webrtc.ts
|
|
802
610
|
import { useState as useState2, useEffect as useEffect2, useCallback, useRef, useMemo } from "react";
|
|
803
611
|
var useWebRTC = (options) => {
|
|
804
|
-
const [
|
|
612
|
+
const [isConnected, setIsConnected] = useState2(false);
|
|
805
613
|
const [isConnecting, setIsConnecting] = useState2(false);
|
|
806
614
|
const [connectionState, setConnectionState] = useState2(null);
|
|
807
615
|
const [participants, setParticipants] = useState2([]);
|
|
@@ -865,7 +673,7 @@ var useWebRTC = (options) => {
|
|
|
865
673
|
return getParticipants().map((p) => p.identity);
|
|
866
674
|
}, [participants]);
|
|
867
675
|
return {
|
|
868
|
-
isConnected
|
|
676
|
+
isConnected,
|
|
869
677
|
isConnecting,
|
|
870
678
|
connectionState,
|
|
871
679
|
room,
|
|
@@ -882,464 +690,6 @@ var useWebRTC = (options) => {
|
|
|
882
690
|
};
|
|
883
691
|
};
|
|
884
692
|
|
|
885
|
-
// src/utils/jsx-encoder.ts
|
|
886
|
-
function generateStableDOMId(element3) {
|
|
887
|
-
const tagName = element3.tagName.toLowerCase();
|
|
888
|
-
const text7 = (element3.textContent || "").trim().substring(0, 50);
|
|
889
|
-
let sibling = element3.previousElementSibling;
|
|
890
|
-
let position3 = 1;
|
|
891
|
-
while (sibling) {
|
|
892
|
-
if (sibling.tagName === element3.tagName) {
|
|
893
|
-
position3++;
|
|
894
|
-
}
|
|
895
|
-
sibling = sibling.previousElementSibling;
|
|
896
|
-
}
|
|
897
|
-
const path2 = getElementPath2(element3);
|
|
898
|
-
const idString = `${tagName}[${position3}]_(${text7})_${path2}`;
|
|
899
|
-
let hash = 0;
|
|
900
|
-
for (let i2 = 0; i2 < idString.length; i2++) {
|
|
901
|
-
const char = idString.charCodeAt(i2);
|
|
902
|
-
hash = (hash << 5) - hash + char;
|
|
903
|
-
hash |= 0;
|
|
904
|
-
}
|
|
905
|
-
return hash.toString(36);
|
|
906
|
-
}
|
|
907
|
-
function getElementPath2(element3) {
|
|
908
|
-
if (element3.id) {
|
|
909
|
-
return `id(${element3.id})`;
|
|
910
|
-
}
|
|
911
|
-
if (element3.tagName.toLowerCase() === "body") {
|
|
912
|
-
return "/body";
|
|
913
|
-
}
|
|
914
|
-
let ix = 0;
|
|
915
|
-
const siblings = element3.parentNode?.children || new HTMLCollection();
|
|
916
|
-
for (let i2 = 0; i2 < siblings.length; i2++) {
|
|
917
|
-
const sibling = siblings[i2];
|
|
918
|
-
if (sibling === element3) {
|
|
919
|
-
return `${getElementPath2(element3.parentNode)}/${element3.tagName}[${ix + 1}]`;
|
|
920
|
-
}
|
|
921
|
-
if (sibling.nodeType === 1 && sibling.tagName === element3.tagName) {
|
|
922
|
-
ix++;
|
|
923
|
-
}
|
|
924
|
-
}
|
|
925
|
-
return "not_found";
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
// src/utils/patch-react.ts
|
|
929
|
-
function getImmediateText(element3) {
|
|
930
|
-
let text7 = "";
|
|
931
|
-
if (element3.childNodes) {
|
|
932
|
-
for (const node2 of Array.from(element3.childNodes)) {
|
|
933
|
-
if (node2.nodeType === 3) {
|
|
934
|
-
text7 += node2.textContent || "";
|
|
935
|
-
}
|
|
936
|
-
}
|
|
937
|
-
}
|
|
938
|
-
return text7.trim();
|
|
939
|
-
}
|
|
940
|
-
function captureFullDOMStructure() {
|
|
941
|
-
console.log("\u{1F333} Capturing full DOM structure...");
|
|
942
|
-
const components = [];
|
|
943
|
-
const interactiveElements = document.querySelectorAll(
|
|
944
|
-
'a, button, input, textarea, select, [role="button"], [onclick]'
|
|
945
|
-
);
|
|
946
|
-
interactiveElements.forEach((element3) => {
|
|
947
|
-
if (element3 instanceof HTMLElement && !element3.closest("[data-cuekit-ignore]")) {
|
|
948
|
-
const nodeData = buildFlatDOMNode(element3);
|
|
949
|
-
if (nodeData) {
|
|
950
|
-
components.push(nodeData);
|
|
951
|
-
}
|
|
952
|
-
}
|
|
953
|
-
});
|
|
954
|
-
const result = { components };
|
|
955
|
-
console.log("\u{1F333} Full DOM structure captured:", result);
|
|
956
|
-
return result;
|
|
957
|
-
}
|
|
958
|
-
function buildFlatDOMNode(element3) {
|
|
959
|
-
if (element3.tagName.toLowerCase() === "script" || element3.hasAttribute("data-cuekit-ignore") || element3.style.display === "none" || element3.style.visibility === "hidden") {
|
|
960
|
-
return null;
|
|
961
|
-
}
|
|
962
|
-
const hash = generateStableDOMId(element3);
|
|
963
|
-
const text7 = getImmediateText(element3).substring(0, 100);
|
|
964
|
-
const isClickable = isElementClickable(element3);
|
|
965
|
-
const componentType = element3.tagName.toLowerCase();
|
|
966
|
-
return {
|
|
967
|
-
hash,
|
|
968
|
-
text: text7,
|
|
969
|
-
isClickable,
|
|
970
|
-
componentType,
|
|
971
|
-
children: []
|
|
972
|
-
// No children in a flat structure
|
|
973
|
-
};
|
|
974
|
-
}
|
|
975
|
-
function isElementClickable(element3) {
|
|
976
|
-
const interactiveSelectors = [
|
|
977
|
-
"button",
|
|
978
|
-
"a",
|
|
979
|
-
"input",
|
|
980
|
-
"select",
|
|
981
|
-
"textarea",
|
|
982
|
-
'[role="button"]',
|
|
983
|
-
'[role="link"]',
|
|
984
|
-
'[role="tab"]',
|
|
985
|
-
"[data-onclick-id]",
|
|
986
|
-
"[data-on-press-id]",
|
|
987
|
-
"[onclick]",
|
|
988
|
-
"[onmousedown]",
|
|
989
|
-
"[onmouseup]",
|
|
990
|
-
"[ontouchstart]",
|
|
991
|
-
"[ontouchend]",
|
|
992
|
-
"[onkeydown]",
|
|
993
|
-
"[onkeyup]",
|
|
994
|
-
"[onkeypress]"
|
|
995
|
-
];
|
|
996
|
-
for (const selector of interactiveSelectors) {
|
|
997
|
-
if (element3.matches(selector)) {
|
|
998
|
-
return true;
|
|
999
|
-
}
|
|
1000
|
-
}
|
|
1001
|
-
const hasClickEvents = element3.onclick !== null || element3.getAttribute("onclick") !== null;
|
|
1002
|
-
const hasInteractiveEvents = element3.ontouchstart !== null || element3.getAttribute("ontouchstart") !== null || element3.ontouchend !== null || element3.getAttribute("ontouchend") !== null || element3.onkeydown !== null || element3.getAttribute("onkeydown") !== null || element3.onkeyup !== null || element3.getAttribute("onkeyup") !== null || element3.onkeypress !== null || element3.getAttribute("onkeypress") !== null;
|
|
1003
|
-
const hasPointerCursor = element3.style.cursor === "pointer" || getComputedStyle(element3).cursor === "pointer";
|
|
1004
|
-
const hasTabIndex = element3.hasAttribute("tabindex") && parseInt(element3.getAttribute("tabindex") || "0") >= 0;
|
|
1005
|
-
const hasInteractiveDataAttrs = element3.hasAttribute("data-clickable") || element3.hasAttribute("data-interactive") || element3.hasAttribute("data-action") || element3.hasAttribute("data-handler");
|
|
1006
|
-
const hasInteractiveAria = element3.hasAttribute("aria-pressed") || element3.hasAttribute("aria-expanded") || element3.hasAttribute("aria-selected") || element3.hasAttribute("aria-checked");
|
|
1007
|
-
return hasClickEvents || hasInteractiveEvents || hasPointerCursor || hasTabIndex || hasInteractiveDataAttrs || hasInteractiveAria;
|
|
1008
|
-
}
|
|
1009
|
-
|
|
1010
|
-
// src/utils/sse-service.ts
|
|
1011
|
-
var eventSource = null;
|
|
1012
|
-
var isConnected2 = false;
|
|
1013
|
-
var serverUrl = WEBRTC_BACKEND_SERVER_URL || "https://bdd4c945f073.ngrok-free.app";
|
|
1014
|
-
var callbacks = {};
|
|
1015
|
-
var sessionId = null;
|
|
1016
|
-
function setSSECallbacks(newCallbacks) {
|
|
1017
|
-
callbacks = newCallbacks;
|
|
1018
|
-
}
|
|
1019
|
-
async function connectSSE(newSessionId) {
|
|
1020
|
-
if (eventSource) {
|
|
1021
|
-
eventSource.close();
|
|
1022
|
-
}
|
|
1023
|
-
if (typeof EventSource === "undefined") {
|
|
1024
|
-
console.warn("\u{1F527} EventSource not available, SSE functionality disabled");
|
|
1025
|
-
callbacks.onError?.("EventSource not available");
|
|
1026
|
-
return;
|
|
1027
|
-
}
|
|
1028
|
-
try {
|
|
1029
|
-
const url = `${serverUrl}/navigation/stream?session_id=${newSessionId}`;
|
|
1030
|
-
eventSource = new EventSource(url);
|
|
1031
|
-
sessionId = newSessionId;
|
|
1032
|
-
eventSource.onopen = () => {
|
|
1033
|
-
console.log("\u{1F517} SSE connection opened");
|
|
1034
|
-
isConnected2 = true;
|
|
1035
|
-
callbacks.onConnectionChange?.(true);
|
|
1036
|
-
};
|
|
1037
|
-
eventSource.onmessage = (event) => {
|
|
1038
|
-
console.log("\u{1F4E1} SSE MESSAGE RECEIVED:", event.data);
|
|
1039
|
-
try {
|
|
1040
|
-
const data = JSON.parse(event.data);
|
|
1041
|
-
console.log("\u{1F4E1} SSE MESSAGE PARSED:", data);
|
|
1042
|
-
handleSSEMessage(data);
|
|
1043
|
-
} catch (error) {
|
|
1044
|
-
console.error("\u274C Failed to parse SSE message:", error);
|
|
1045
|
-
}
|
|
1046
|
-
};
|
|
1047
|
-
eventSource.onerror = (error) => {
|
|
1048
|
-
console.error("\u274C SSE connection error:", error);
|
|
1049
|
-
isConnected2 = false;
|
|
1050
|
-
callbacks.onConnectionChange?.(false);
|
|
1051
|
-
callbacks.onError?.("SSE connection failed");
|
|
1052
|
-
};
|
|
1053
|
-
} catch (error) {
|
|
1054
|
-
console.error("\u274C Failed to create SSE connection:", error);
|
|
1055
|
-
callbacks.onError?.("Failed to create SSE connection");
|
|
1056
|
-
}
|
|
1057
|
-
}
|
|
1058
|
-
function handleSSEMessage(data) {
|
|
1059
|
-
console.log("\u{1F50D} Processing SSE message type:", data.type);
|
|
1060
|
-
console.log("\u{1F50D} Full SSE data:", data);
|
|
1061
|
-
console.log("\u{1F50D} SSE data.data:", data.data);
|
|
1062
|
-
const actionEvent = data;
|
|
1063
|
-
console.log("\u{1F50D} Created action event:", actionEvent);
|
|
1064
|
-
callbacks.onActionEvent?.(actionEvent);
|
|
1065
|
-
switch (actionEvent.type) {
|
|
1066
|
-
case "static_data_ready":
|
|
1067
|
-
console.log("\u{1F4E6} Handling static_data_ready event");
|
|
1068
|
-
callbacks.onStaticDataUpdate?.(actionEvent.data);
|
|
1069
|
-
break;
|
|
1070
|
-
case "ai_intent":
|
|
1071
|
-
console.log("\u{1F3AF} Handling ai_intent event");
|
|
1072
|
-
callbacks.onIntentUpdate?.(actionEvent.data);
|
|
1073
|
-
break;
|
|
1074
|
-
case "request_runtime_data":
|
|
1075
|
-
console.log("\u{1F4E6} Handling request_runtime_data event");
|
|
1076
|
-
sendRuntimeData();
|
|
1077
|
-
break;
|
|
1078
|
-
case "user_speech_chunk":
|
|
1079
|
-
console.log("\u{1F464} Handling user_speech_chunk event");
|
|
1080
|
-
const userSpeechEntry = {
|
|
1081
|
-
speaker: "user",
|
|
1082
|
-
text: actionEvent.data?.text_chunk,
|
|
1083
|
-
is_final: actionEvent.data?.is_final
|
|
1084
|
-
};
|
|
1085
|
-
callbacks.onConversationUpdate?.(userSpeechEntry);
|
|
1086
|
-
break;
|
|
1087
|
-
case "ai_speech_chunk":
|
|
1088
|
-
console.log("\u{1F916} Handling ai_speech_chunk event");
|
|
1089
|
-
callbacks.onConversationUpdate?.({
|
|
1090
|
-
speaker: "ai",
|
|
1091
|
-
text: actionEvent.data?.text_chunk,
|
|
1092
|
-
is_final: actionEvent.data?.is_final
|
|
1093
|
-
});
|
|
1094
|
-
break;
|
|
1095
|
-
case "ai_interrupted":
|
|
1096
|
-
console.log("\u{1F507} Handling ai_interrupted event");
|
|
1097
|
-
break;
|
|
1098
|
-
case "tool_log":
|
|
1099
|
-
console.log("\u{1F527} Handling tool_log event");
|
|
1100
|
-
callbacks.onToolStatusUpdate?.(actionEvent.data);
|
|
1101
|
-
break;
|
|
1102
|
-
case "connection":
|
|
1103
|
-
console.log("\u{1F517} Handling connection event");
|
|
1104
|
-
break;
|
|
1105
|
-
case "keepalive":
|
|
1106
|
-
console.log("\u{1F493} Handling keepalive event");
|
|
1107
|
-
break;
|
|
1108
|
-
default:
|
|
1109
|
-
console.log("\u{1F50D} Unknown SSE message type:", data.type);
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
async function sendRuntimeData() {
|
|
1113
|
-
if (!sessionId) {
|
|
1114
|
-
console.error("\u274C Cannot send runtime data without a session ID");
|
|
1115
|
-
return;
|
|
1116
|
-
}
|
|
1117
|
-
try {
|
|
1118
|
-
const domStructure = captureFullDOMStructure();
|
|
1119
|
-
const screenName = getCurrentScreenName();
|
|
1120
|
-
const response = await fetch(`${serverUrl}/api/runtime/data/${sessionId}`, {
|
|
1121
|
-
method: "POST",
|
|
1122
|
-
headers: {
|
|
1123
|
-
"Content-Type": "application/json",
|
|
1124
|
-
Authorization: `Bearer ${_apiKey}`
|
|
1125
|
-
},
|
|
1126
|
-
body: JSON.stringify({
|
|
1127
|
-
session_id: sessionId,
|
|
1128
|
-
components: domStructure.components,
|
|
1129
|
-
screen: screenName
|
|
1130
|
-
})
|
|
1131
|
-
});
|
|
1132
|
-
if (!response.ok) {
|
|
1133
|
-
const errorData = await response.json();
|
|
1134
|
-
throw new Error(errorData.detail || "Failed to send runtime data");
|
|
1135
|
-
}
|
|
1136
|
-
console.log("\u{1F4E6} Runtime data sent successfully");
|
|
1137
|
-
} catch (error) {
|
|
1138
|
-
console.error("\u274C Failed to send runtime data:", error);
|
|
1139
|
-
callbacks.onError?.("Failed to send runtime data");
|
|
1140
|
-
}
|
|
1141
|
-
}
|
|
1142
|
-
async function sendDashboardData(dashboardData) {
|
|
1143
|
-
try {
|
|
1144
|
-
console.log("\u{1F4E4} SSEService: Sending dashboard data...");
|
|
1145
|
-
const response = await fetch(`${serverUrl}/ai/data`, {
|
|
1146
|
-
method: "POST",
|
|
1147
|
-
headers: {
|
|
1148
|
-
"Content-Type": "application/json"
|
|
1149
|
-
},
|
|
1150
|
-
body: JSON.stringify({
|
|
1151
|
-
type: "dashboard_data",
|
|
1152
|
-
data: dashboardData
|
|
1153
|
-
})
|
|
1154
|
-
});
|
|
1155
|
-
if (!response.ok) {
|
|
1156
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
1157
|
-
}
|
|
1158
|
-
console.log("\u2705 SSEService: Dashboard data sent successfully");
|
|
1159
|
-
} catch (error) {
|
|
1160
|
-
console.error("\u274C SSEService: Failed to send dashboard data:", error);
|
|
1161
|
-
}
|
|
1162
|
-
}
|
|
1163
|
-
async function sendElementData(appId = "default") {
|
|
1164
|
-
try {
|
|
1165
|
-
console.log("\u{1F4E4} SSEService: Sending element data...");
|
|
1166
|
-
const domStructure = captureFullDOMStructure();
|
|
1167
|
-
const currentPage = getCurrentScreenName();
|
|
1168
|
-
const elementData = {
|
|
1169
|
-
app_id: appId,
|
|
1170
|
-
current_page: currentPage,
|
|
1171
|
-
dom_structure: domStructure,
|
|
1172
|
-
timestamp: Date.now()
|
|
1173
|
-
};
|
|
1174
|
-
const response = await fetch(`${serverUrl}/ai/data`, {
|
|
1175
|
-
method: "POST",
|
|
1176
|
-
headers: {
|
|
1177
|
-
"Content-Type": "application/json"
|
|
1178
|
-
},
|
|
1179
|
-
body: JSON.stringify(elementData)
|
|
1180
|
-
});
|
|
1181
|
-
if (!response.ok) {
|
|
1182
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
1183
|
-
}
|
|
1184
|
-
console.log("\u2705 SSEService: Element data sent successfully");
|
|
1185
|
-
return elementData;
|
|
1186
|
-
} catch (error) {
|
|
1187
|
-
console.error("\u274C SSEService: Failed to send element data:", error);
|
|
1188
|
-
throw error;
|
|
1189
|
-
}
|
|
1190
|
-
}
|
|
1191
|
-
function disconnectSSE() {
|
|
1192
|
-
console.log("\u{1F50C} SSEService: Disconnecting SSE...");
|
|
1193
|
-
if (eventSource) {
|
|
1194
|
-
eventSource.close();
|
|
1195
|
-
eventSource = null;
|
|
1196
|
-
}
|
|
1197
|
-
isConnected2 = false;
|
|
1198
|
-
callbacks.onConnectionChange?.(false);
|
|
1199
|
-
}
|
|
1200
|
-
function getSSEConnectionStatus() {
|
|
1201
|
-
return isConnected2 && eventSource?.readyState === EventSource.OPEN;
|
|
1202
|
-
}
|
|
1203
|
-
function getSSEConnectionState() {
|
|
1204
|
-
if (isConnected2 && eventSource?.readyState === EventSource.OPEN) return "connected";
|
|
1205
|
-
if (eventSource?.readyState === EventSource.CONNECTING) return "connecting";
|
|
1206
|
-
return "disconnected";
|
|
1207
|
-
}
|
|
1208
|
-
|
|
1209
|
-
// src/utils/element-service.ts
|
|
1210
|
-
function executeAction(action) {
|
|
1211
|
-
console.log("\u{1F3AF} Executing element action:", action);
|
|
1212
|
-
const { action_type, target_element, target } = action;
|
|
1213
|
-
switch (action_type) {
|
|
1214
|
-
case "click":
|
|
1215
|
-
return clickElement(target_element);
|
|
1216
|
-
case "navigate":
|
|
1217
|
-
return navigateToElement(target_element || target);
|
|
1218
|
-
case "input":
|
|
1219
|
-
case "focus":
|
|
1220
|
-
return focusElement(target_element);
|
|
1221
|
-
case "toggle":
|
|
1222
|
-
return toggleElement(target_element);
|
|
1223
|
-
default:
|
|
1224
|
-
console.warn(`\u26A0\uFE0F Unknown action type: ${action_type}`);
|
|
1225
|
-
return false;
|
|
1226
|
-
}
|
|
1227
|
-
}
|
|
1228
|
-
function getFullDOMStructure() {
|
|
1229
|
-
console.log("\u{1F333} ElementService: Getting full DOM structure...");
|
|
1230
|
-
return captureFullDOMStructure();
|
|
1231
|
-
}
|
|
1232
|
-
function clickElement(elementId) {
|
|
1233
|
-
if (!elementId) {
|
|
1234
|
-
console.warn("\u26A0\uFE0F No element ID provided for click action");
|
|
1235
|
-
return false;
|
|
1236
|
-
}
|
|
1237
|
-
const domStructure = getFullDOMStructure();
|
|
1238
|
-
const elementToClick = findElementById(domStructure, elementId);
|
|
1239
|
-
if (elementToClick) {
|
|
1240
|
-
console.log(`\u{1F3AF} Clicking element: ${elementId}`);
|
|
1241
|
-
const domElement = findDOMElementById(elementId);
|
|
1242
|
-
if (domElement) {
|
|
1243
|
-
domElement.click();
|
|
1244
|
-
return true;
|
|
1245
|
-
}
|
|
1246
|
-
} else {
|
|
1247
|
-
console.warn(`\u26A0\uFE0F Element not found: ${elementId}`);
|
|
1248
|
-
}
|
|
1249
|
-
return false;
|
|
1250
|
-
}
|
|
1251
|
-
function navigateToElement(target) {
|
|
1252
|
-
if (!target) {
|
|
1253
|
-
console.warn("\u26A0\uFE0F No target provided for navigation action");
|
|
1254
|
-
return false;
|
|
1255
|
-
}
|
|
1256
|
-
console.log(`\u{1F9ED} Navigating to: ${target}`);
|
|
1257
|
-
if (target.includes("/") || target.startsWith("http")) {
|
|
1258
|
-
safeNavigate(target, {});
|
|
1259
|
-
} else {
|
|
1260
|
-
handleNavigationAndClick(target, target);
|
|
1261
|
-
}
|
|
1262
|
-
return true;
|
|
1263
|
-
}
|
|
1264
|
-
function focusElement(elementId) {
|
|
1265
|
-
if (!elementId) {
|
|
1266
|
-
console.warn("\u26A0\uFE0F No element ID provided for focus action");
|
|
1267
|
-
return false;
|
|
1268
|
-
}
|
|
1269
|
-
const domElement = findDOMElementById(elementId);
|
|
1270
|
-
if (domElement instanceof HTMLInputElement || domElement instanceof HTMLTextAreaElement || domElement instanceof HTMLSelectElement) {
|
|
1271
|
-
console.log(`\u{1F4DD} Focusing element: ${elementId}`);
|
|
1272
|
-
domElement.focus();
|
|
1273
|
-
return true;
|
|
1274
|
-
} else {
|
|
1275
|
-
console.warn(`\u26A0\uFE0F Focusable element not found: ${elementId}`);
|
|
1276
|
-
return false;
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1279
|
-
function toggleElement(elementId) {
|
|
1280
|
-
if (!elementId) {
|
|
1281
|
-
console.warn("\u26A0\uFE0F No element ID provided for toggle action");
|
|
1282
|
-
return false;
|
|
1283
|
-
}
|
|
1284
|
-
const domElement = findDOMElementById(elementId);
|
|
1285
|
-
if (domElement instanceof HTMLElement) {
|
|
1286
|
-
console.log(`\u{1F504} Toggling element: ${elementId}`);
|
|
1287
|
-
if (domElement instanceof HTMLInputElement) {
|
|
1288
|
-
if (domElement.type === "checkbox") {
|
|
1289
|
-
domElement.checked = !domElement.checked;
|
|
1290
|
-
} else if (domElement.type === "radio") {
|
|
1291
|
-
domElement.checked = true;
|
|
1292
|
-
}
|
|
1293
|
-
domElement.dispatchEvent(new Event("change", { bubbles: true }));
|
|
1294
|
-
} else {
|
|
1295
|
-
domElement.click();
|
|
1296
|
-
}
|
|
1297
|
-
return true;
|
|
1298
|
-
} else {
|
|
1299
|
-
console.warn(`\u26A0\uFE0F Toggleable element not found: ${elementId}`);
|
|
1300
|
-
return false;
|
|
1301
|
-
}
|
|
1302
|
-
}
|
|
1303
|
-
function findElementById(domStructure, elementId) {
|
|
1304
|
-
const searchInComponents = (components) => {
|
|
1305
|
-
for (const component of components) {
|
|
1306
|
-
if (component.hash === elementId) {
|
|
1307
|
-
return component;
|
|
1308
|
-
}
|
|
1309
|
-
if (component.children.length > 0) {
|
|
1310
|
-
const found = searchInComponents(component.children);
|
|
1311
|
-
if (found) return found;
|
|
1312
|
-
}
|
|
1313
|
-
}
|
|
1314
|
-
return null;
|
|
1315
|
-
};
|
|
1316
|
-
return searchInComponents(domStructure.components);
|
|
1317
|
-
}
|
|
1318
|
-
function findDOMElementById(elementId) {
|
|
1319
|
-
const allElements = document.querySelectorAll("*");
|
|
1320
|
-
for (const element3 of allElements) {
|
|
1321
|
-
if (element3 instanceof HTMLElement) {
|
|
1322
|
-
const hash = generateHash(element3);
|
|
1323
|
-
if (hash === elementId) {
|
|
1324
|
-
return element3;
|
|
1325
|
-
}
|
|
1326
|
-
}
|
|
1327
|
-
}
|
|
1328
|
-
return null;
|
|
1329
|
-
}
|
|
1330
|
-
function generateHash(element3) {
|
|
1331
|
-
const tagName = element3.tagName.toLowerCase();
|
|
1332
|
-
const text7 = (element3.textContent || "").trim().substring(0, 50);
|
|
1333
|
-
const idString = `${tagName}_${text7}_${element3.tagName}`;
|
|
1334
|
-
let hash = 0;
|
|
1335
|
-
for (let i2 = 0; i2 < idString.length; i2++) {
|
|
1336
|
-
const char = idString.charCodeAt(i2);
|
|
1337
|
-
hash = (hash << 5) - hash + char;
|
|
1338
|
-
hash |= 0;
|
|
1339
|
-
}
|
|
1340
|
-
return hash.toString(36);
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
693
|
// src/hooks/use-cuekit.ts
|
|
1344
694
|
var useCuekit = (options) => {
|
|
1345
695
|
const [messages, setMessages] = useState3([]);
|
|
@@ -1391,89 +741,85 @@ var useCuekit = (options) => {
|
|
|
1391
741
|
}, []);
|
|
1392
742
|
const [micState, setMicState] = useState3("idle");
|
|
1393
743
|
const [status, setStatus] = useState3("");
|
|
1394
|
-
const
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
}
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
if (entry.speaker === "user") {
|
|
1427
|
-
setMicState("thinking");
|
|
1428
|
-
} else if (entry.speaker === "ai") {
|
|
1429
|
-
setTimeout(() => setMicState("listening"), 1e3);
|
|
1430
|
-
}
|
|
1431
|
-
}
|
|
1432
|
-
break;
|
|
1433
|
-
}
|
|
1434
|
-
case "ai_intent": {
|
|
1435
|
-
const intent = event.data;
|
|
1436
|
-
if (intent.actionType === "click" && intent.actionMetadata.elementId) {
|
|
1437
|
-
executeAction({
|
|
1438
|
-
action_type: "click",
|
|
1439
|
-
target_element: intent.actionMetadata.elementId
|
|
1440
|
-
});
|
|
1441
|
-
} else if (intent.actionType === "navigate" && intent.actionMetadata.routeName) {
|
|
1442
|
-
executeAction({
|
|
1443
|
-
action_type: "navigate",
|
|
1444
|
-
target_element: intent.actionMetadata.routeName
|
|
1445
|
-
});
|
|
1446
|
-
}
|
|
1447
|
-
break;
|
|
1448
|
-
}
|
|
1449
|
-
case "ai_interrupted": {
|
|
1450
|
-
handleAIInterruption();
|
|
1451
|
-
break;
|
|
1452
|
-
}
|
|
1453
|
-
case "keepalive": {
|
|
1454
|
-
if (currentUserMessageRef.current) {
|
|
1455
|
-
const finalMessageId = currentUserMessageRef.current.id;
|
|
1456
|
-
setMessages(
|
|
1457
|
-
(prev) => prev.map((m) => m.id === finalMessageId ? { ...m, isFinal: true } : m)
|
|
1458
|
-
);
|
|
1459
|
-
currentUserMessageRef.current = null;
|
|
1460
|
-
}
|
|
1461
|
-
if (currentAIMessageRef.current) {
|
|
1462
|
-
const finalMessageId = currentAIMessageRef.current.id;
|
|
1463
|
-
setMessages(
|
|
1464
|
-
(prev) => prev.map((m) => m.id === finalMessageId ? { ...m, isFinal: true } : m)
|
|
1465
|
-
);
|
|
1466
|
-
currentAIMessageRef.current = null;
|
|
1467
|
-
}
|
|
1468
|
-
break;
|
|
744
|
+
const handleNavigationCommand = (event) => {
|
|
745
|
+
console.log(`\u{1F9E0} Processing event in useCuekit: ${event.type}`, event);
|
|
746
|
+
switch (event.type) {
|
|
747
|
+
case "user_speech_chunk":
|
|
748
|
+
case "ai_speech_chunk": {
|
|
749
|
+
const role = event.type === "user_speech_chunk" ? "user" : "ai";
|
|
750
|
+
if (role === "user" && currentAIMessageRef.current) {
|
|
751
|
+
const finalMessageId = currentAIMessageRef.current.id;
|
|
752
|
+
setMessages(
|
|
753
|
+
(prev) => prev.map((m) => m.id === finalMessageId ? { ...m, isFinal: true } : m)
|
|
754
|
+
);
|
|
755
|
+
currentAIMessageRef.current = null;
|
|
756
|
+
}
|
|
757
|
+
if (role === "ai" && currentUserMessageRef.current) {
|
|
758
|
+
const finalMessageId = currentUserMessageRef.current.id;
|
|
759
|
+
setMessages(
|
|
760
|
+
(prev) => prev.map((m) => m.id === finalMessageId ? { ...m, isFinal: true } : m)
|
|
761
|
+
);
|
|
762
|
+
currentUserMessageRef.current = null;
|
|
763
|
+
}
|
|
764
|
+
const entry = {
|
|
765
|
+
speaker: role,
|
|
766
|
+
text: event.data.text_chunk,
|
|
767
|
+
is_final: event.data.is_final,
|
|
768
|
+
timestamp: new Date(event.timestamp || Date.now()).toISOString()
|
|
769
|
+
};
|
|
770
|
+
handleMessageChunk(entry.text, entry.speaker, entry.is_final);
|
|
771
|
+
if (entry.is_final) {
|
|
772
|
+
if (entry.speaker === "user") {
|
|
773
|
+
setMicState("thinking");
|
|
774
|
+
} else if (entry.speaker === "ai") {
|
|
775
|
+
setTimeout(() => setMicState("listening"), 1e3);
|
|
1469
776
|
}
|
|
1470
777
|
}
|
|
1471
|
-
|
|
1472
|
-
onConnectionChange: (isConnected3) => {
|
|
1473
|
-
setIsSseConnected(isConnected3);
|
|
778
|
+
break;
|
|
1474
779
|
}
|
|
1475
|
-
|
|
1476
|
-
|
|
780
|
+
case "ai_intent": {
|
|
781
|
+
const intent = event.data;
|
|
782
|
+
if (intent.actionType === "click" && intent.actionMetadata.elementId) {
|
|
783
|
+
executeAction({
|
|
784
|
+
action_type: "click",
|
|
785
|
+
target_element: intent.actionMetadata.elementId
|
|
786
|
+
});
|
|
787
|
+
} else if (intent.actionType === "navigate" && intent.actionMetadata.routeName) {
|
|
788
|
+
executeAction({
|
|
789
|
+
action_type: "navigate",
|
|
790
|
+
target_element: intent.actionMetadata.routeName
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
break;
|
|
794
|
+
}
|
|
795
|
+
case "request_runtime_data": {
|
|
796
|
+
console.log("\u{1F9E0} Requesting runtime data");
|
|
797
|
+
sendRuntimeData();
|
|
798
|
+
break;
|
|
799
|
+
}
|
|
800
|
+
case "ai_interrupted": {
|
|
801
|
+
handleAIInterruption();
|
|
802
|
+
break;
|
|
803
|
+
}
|
|
804
|
+
case "keepalive": {
|
|
805
|
+
if (currentUserMessageRef.current) {
|
|
806
|
+
const finalMessageId = currentUserMessageRef.current.id;
|
|
807
|
+
setMessages(
|
|
808
|
+
(prev) => prev.map((m) => m.id === finalMessageId ? { ...m, isFinal: true } : m)
|
|
809
|
+
);
|
|
810
|
+
currentUserMessageRef.current = null;
|
|
811
|
+
}
|
|
812
|
+
if (currentAIMessageRef.current) {
|
|
813
|
+
const finalMessageId = currentAIMessageRef.current.id;
|
|
814
|
+
setMessages(
|
|
815
|
+
(prev) => prev.map((m) => m.id === finalMessageId ? { ...m, isFinal: true } : m)
|
|
816
|
+
);
|
|
817
|
+
currentAIMessageRef.current = null;
|
|
818
|
+
}
|
|
819
|
+
break;
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
};
|
|
1477
823
|
const handleConnectionStateChange = (state) => {
|
|
1478
824
|
switch (state) {
|
|
1479
825
|
case "connecting":
|
|
@@ -1497,32 +843,22 @@ var useCuekit = (options) => {
|
|
|
1497
843
|
};
|
|
1498
844
|
const webrtc = useWebRTC({
|
|
1499
845
|
...options,
|
|
1500
|
-
onConnectionStateChange: handleConnectionStateChange
|
|
846
|
+
onConnectionStateChange: handleConnectionStateChange,
|
|
847
|
+
onNavigationCommand: handleNavigationCommand
|
|
1501
848
|
});
|
|
1502
849
|
const connect = useCallback2(
|
|
1503
850
|
async (identity, apiKey, appId) => {
|
|
1504
|
-
|
|
1505
|
-
if (authData?.session_id) {
|
|
1506
|
-
connectSSE(authData.session_id);
|
|
1507
|
-
}
|
|
851
|
+
await webrtc.connect(identity, apiKey, appId);
|
|
1508
852
|
},
|
|
1509
853
|
[webrtc]
|
|
1510
854
|
);
|
|
1511
855
|
const disconnect = useCallback2(async () => {
|
|
1512
856
|
await webrtc.disconnect();
|
|
1513
|
-
disconnectSSE();
|
|
1514
857
|
clearMessages();
|
|
1515
858
|
setMicState("idle");
|
|
1516
859
|
}, [webrtc, clearMessages]);
|
|
1517
|
-
useEffect3(() => {
|
|
1518
|
-
if (lastActionEvent?.type === "ai_interrupted") {
|
|
1519
|
-
handleAIInterruption();
|
|
1520
|
-
}
|
|
1521
|
-
}, [lastActionEvent, handleAIInterruption]);
|
|
1522
860
|
return {
|
|
1523
861
|
...webrtc,
|
|
1524
|
-
isSseConnected,
|
|
1525
|
-
lastActionEvent,
|
|
1526
862
|
messages,
|
|
1527
863
|
micState,
|
|
1528
864
|
setMicState,
|
|
@@ -7263,7 +6599,7 @@ function createTokenizer(parser, initialize, from) {
|
|
|
7263
6599
|
function handleConstruct(construct) {
|
|
7264
6600
|
return start2;
|
|
7265
6601
|
function start2(code4) {
|
|
7266
|
-
info =
|
|
6602
|
+
info = store();
|
|
7267
6603
|
currentConstruct = construct;
|
|
7268
6604
|
if (!construct.partial) {
|
|
7269
6605
|
context.currentConstruct = construct;
|
|
@@ -7308,7 +6644,7 @@ function createTokenizer(parser, initialize, from) {
|
|
|
7308
6644
|
context.events = construct.resolveTo(context.events, context);
|
|
7309
6645
|
}
|
|
7310
6646
|
}
|
|
7311
|
-
function
|
|
6647
|
+
function store() {
|
|
7312
6648
|
const startPoint = now();
|
|
7313
6649
|
const startPrevious = context.previous;
|
|
7314
6650
|
const startCurrentConstruct = context.currentConstruct;
|
|
@@ -14288,7 +13624,7 @@ var ChatPopup = ({
|
|
|
14288
13624
|
onSendText,
|
|
14289
13625
|
onEndCall,
|
|
14290
13626
|
messages,
|
|
14291
|
-
isConnected
|
|
13627
|
+
isConnected,
|
|
14292
13628
|
micState,
|
|
14293
13629
|
error,
|
|
14294
13630
|
currentTheme = "dark",
|
|
@@ -14875,7 +14211,7 @@ var ChatPopup = ({
|
|
|
14875
14211
|
}
|
|
14876
14212
|
}
|
|
14877
14213
|
),
|
|
14878
|
-
|
|
14214
|
+
isConnected && onEndCall && /* @__PURE__ */ React6.createElement(
|
|
14879
14215
|
"button",
|
|
14880
14216
|
{
|
|
14881
14217
|
type: "submit",
|
|
@@ -17065,13 +16401,12 @@ var MicButton = ({
|
|
|
17065
16401
|
const aiSpeechTimeoutRef = useRef7(null);
|
|
17066
16402
|
const activeAITracksRef = useRef7(/* @__PURE__ */ new Set());
|
|
17067
16403
|
const {
|
|
17068
|
-
isConnected
|
|
16404
|
+
isConnected,
|
|
17069
16405
|
isConnecting,
|
|
17070
16406
|
error: voiceError,
|
|
17071
16407
|
connect: voiceConnect,
|
|
17072
16408
|
disconnect: voiceDisconnect,
|
|
17073
16409
|
sendUserCommand: sendUserCommand2,
|
|
17074
|
-
lastActionEvent,
|
|
17075
16410
|
messages: messageManagerMessages,
|
|
17076
16411
|
micState,
|
|
17077
16412
|
setMicState,
|
|
@@ -17151,8 +16486,8 @@ var MicButton = ({
|
|
|
17151
16486
|
console.log("\u{1F3A4} MicButton: Current active AI tracks:", Array.from(activeAITracksRef.current));
|
|
17152
16487
|
console.log("\u{1F3A4} MicButton: Current status:", status);
|
|
17153
16488
|
console.log("\u{1F3A4} MicButton: Current mic state:", micState);
|
|
17154
|
-
console.log("\u{1F3A4} MicButton: Is listening:",
|
|
17155
|
-
console.log("\u{1F3A4} MicButton: Is connected:",
|
|
16489
|
+
console.log("\u{1F3A4} MicButton: Is listening:", isConnected);
|
|
16490
|
+
console.log("\u{1F3A4} MicButton: Is connected:", isConnected);
|
|
17156
16491
|
if (isSpeaking && trackId) {
|
|
17157
16492
|
console.log("\u{1F3A4} MicButton: ===== AI SPEECH START =====");
|
|
17158
16493
|
console.log("\u{1F3A4} MicButton: Adding track to active set:", trackId);
|
|
@@ -17207,7 +16542,7 @@ var MicButton = ({
|
|
|
17207
16542
|
console.log("\u{1F3A4} MicButton: - Status:", status);
|
|
17208
16543
|
console.log("\u{1F3A4} MicButton: ================================");
|
|
17209
16544
|
},
|
|
17210
|
-
[status, micState,
|
|
16545
|
+
[status, micState, isConnected]
|
|
17211
16546
|
);
|
|
17212
16547
|
useEffect10(() => {
|
|
17213
16548
|
if (audioContainerRef.current) {
|
|
@@ -17220,10 +16555,9 @@ var MicButton = ({
|
|
|
17220
16555
|
}
|
|
17221
16556
|
};
|
|
17222
16557
|
}, [handleAISpeech]);
|
|
17223
|
-
const isListening =
|
|
17224
|
-
const
|
|
17225
|
-
|
|
17226
|
-
if (!isConnected4) {
|
|
16558
|
+
const isListening = isConnected;
|
|
16559
|
+
const getUserFriendlyStatus = (micState2, isConnected2) => {
|
|
16560
|
+
if (!isConnected2) {
|
|
17227
16561
|
return "Connecting...";
|
|
17228
16562
|
}
|
|
17229
16563
|
if (status && !status.includes("error") && !status.includes("failed") && !status.includes("Connection error") && !status.includes("Unable to")) {
|
|
@@ -17233,28 +16567,28 @@ var MicButton = ({
|
|
|
17233
16567
|
if (micState2 === "thinking") return "Thinking...";
|
|
17234
16568
|
if (micState2 === "replying") return "Responding...";
|
|
17235
16569
|
if (micState2 === "idle") {
|
|
17236
|
-
if (
|
|
16570
|
+
if (isConnected2) return "Listening...";
|
|
17237
16571
|
return "Connecting...";
|
|
17238
16572
|
}
|
|
17239
|
-
return
|
|
16573
|
+
return isConnected2 ? "Ready" : "Connecting...";
|
|
17240
16574
|
};
|
|
17241
16575
|
useEffect10(() => {
|
|
17242
|
-
if (
|
|
16576
|
+
if (isConnected) {
|
|
17243
16577
|
console.log("\u{1F3A4} MicButton: WebRTC and SSE connections established - ready for commands");
|
|
17244
16578
|
} else {
|
|
17245
16579
|
console.log("\u{1F3A4} MicButton: WebRTC not yet connected - ignoring speech");
|
|
17246
16580
|
}
|
|
17247
|
-
}, [
|
|
16581
|
+
}, [isConnected]);
|
|
17248
16582
|
useEffect10(() => {
|
|
17249
16583
|
console.log("\u{1F3A4} MicButton: Auto-open check:", {
|
|
17250
|
-
isConnected
|
|
16584
|
+
isConnected,
|
|
17251
16585
|
chatIsOpen: isChatOpen
|
|
17252
16586
|
});
|
|
17253
|
-
if (
|
|
16587
|
+
if (isConnected && !isChatOpen) {
|
|
17254
16588
|
console.log("\u{1F3A4} MicButton: Auto-opening chat popup");
|
|
17255
16589
|
openChat();
|
|
17256
16590
|
}
|
|
17257
|
-
}, [
|
|
16591
|
+
}, [isConnected, isChatOpen, openChat]);
|
|
17258
16592
|
useEffect10(() => {
|
|
17259
16593
|
if (messageManagerMessages.length > 0 && !isChatOpen) {
|
|
17260
16594
|
console.log("\u{1F3A4} MicButton: Auto-opening chat popup due to messages");
|
|
@@ -17262,7 +16596,7 @@ var MicButton = ({
|
|
|
17262
16596
|
}
|
|
17263
16597
|
}, [messageManagerMessages.length, isChatOpen, openChat]);
|
|
17264
16598
|
const handleMicClick = useCallback5(() => {
|
|
17265
|
-
const shouldStop = micState === "listening" &&
|
|
16599
|
+
const shouldStop = micState === "listening" && isConnected;
|
|
17266
16600
|
if (shouldStop) {
|
|
17267
16601
|
console.log("\u{1F3A4} MicButton: User wants to stop - closing everything");
|
|
17268
16602
|
voiceDisconnect().then(() => {
|
|
@@ -17287,7 +16621,7 @@ var MicButton = ({
|
|
|
17287
16621
|
}
|
|
17288
16622
|
}, [
|
|
17289
16623
|
micState,
|
|
17290
|
-
|
|
16624
|
+
isConnected,
|
|
17291
16625
|
voiceDisconnect,
|
|
17292
16626
|
voiceConnect,
|
|
17293
16627
|
apiKey,
|
|
@@ -17295,19 +16629,6 @@ var MicButton = ({
|
|
|
17295
16629
|
openChat,
|
|
17296
16630
|
showBorderGlow
|
|
17297
16631
|
]);
|
|
17298
|
-
useEffect10(() => {
|
|
17299
|
-
if (!isConnected3) {
|
|
17300
|
-
return;
|
|
17301
|
-
}
|
|
17302
|
-
if (transcript && transcript.trim() && !aiSpeakingRef.current) {
|
|
17303
|
-
console.log("\u{1F3A4} MicButton: Processing new transcript:", transcript);
|
|
17304
|
-
sendUserCommand2(transcript).then(() => {
|
|
17305
|
-
console.log("\u{1F3A4} MicButton: User command sent successfully");
|
|
17306
|
-
}).catch((error) => {
|
|
17307
|
-
console.error("\u{1F3A4} MicButton: Failed to send user command:", error);
|
|
17308
|
-
});
|
|
17309
|
-
}
|
|
17310
|
-
}, [transcript, isConnected3, sendUserCommand2]);
|
|
17311
16632
|
const handleSendText = async (textToSend) => {
|
|
17312
16633
|
console.log("\u{1F3A4} MicButton: handleSendText called with:", textToSend);
|
|
17313
16634
|
setMicState("thinking");
|
|
@@ -17315,7 +16636,7 @@ var MicButton = ({
|
|
|
17315
16636
|
if (!isChatOpen) {
|
|
17316
16637
|
openChat();
|
|
17317
16638
|
}
|
|
17318
|
-
if (
|
|
16639
|
+
if (isConnected) {
|
|
17319
16640
|
console.log("\u{1F3A4} MicButton: Sending via WebRTC");
|
|
17320
16641
|
try {
|
|
17321
16642
|
await sendUserCommand2(textToSend);
|
|
@@ -17512,13 +16833,13 @@ var MicButton = ({
|
|
|
17512
16833
|
text: msg.text,
|
|
17513
16834
|
sender: msg.role === "ai" ? "assistant" : "user"
|
|
17514
16835
|
})),
|
|
17515
|
-
isConnected:
|
|
16836
|
+
isConnected: isConnected ?? false,
|
|
17516
16837
|
micState,
|
|
17517
16838
|
participants,
|
|
17518
16839
|
error: voiceError,
|
|
17519
16840
|
currentTheme,
|
|
17520
16841
|
onThemeToggle: setCurrentTheme,
|
|
17521
|
-
status: getUserFriendlyStatus(micState,
|
|
16842
|
+
status: getUserFriendlyStatus(micState, isConnected ?? false),
|
|
17522
16843
|
anchor: { position: screenPosition, bottom: bottomSpace, size: buttonSize }
|
|
17523
16844
|
}
|
|
17524
16845
|
), isChatOpen && isChatMinimized && /* @__PURE__ */ React11.createElement(
|
|
@@ -17554,19 +16875,11 @@ export {
|
|
|
17554
16875
|
VoiceIntensityVisualizer,
|
|
17555
16876
|
captureFullDOMStructure,
|
|
17556
16877
|
configureWebRTCServer,
|
|
17557
|
-
connectSSE,
|
|
17558
|
-
disconnectSSE,
|
|
17559
16878
|
executeAction,
|
|
17560
16879
|
getFullDOMStructure,
|
|
17561
|
-
getSSEConnectionState,
|
|
17562
|
-
getSSEConnectionStatus,
|
|
17563
16880
|
getWebRTCServerConfig,
|
|
17564
16881
|
initWebRTC,
|
|
17565
16882
|
initWebRTCWithDeployedBackend,
|
|
17566
|
-
sendDashboardData,
|
|
17567
|
-
sendElementData,
|
|
17568
|
-
sendRuntimeData,
|
|
17569
|
-
setSSECallbacks,
|
|
17570
16883
|
useCuekit,
|
|
17571
16884
|
useQubeContext,
|
|
17572
16885
|
useWebRTC
|