@cuekit-ai/react 1.3.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-PLOG3DEN.mjs → chunk-ZGHSIEXZ.mjs} +132 -162
- package/dist/index.d.mts +22 -13
- package/dist/index.d.ts +22 -13
- package/dist/index.js +909 -246
- package/dist/index.mjs +829 -132
- package/dist/{webrtc-service-BHI4M7YJ.mjs → webrtc-service-SDVOO4LS.mjs} +1 -1
- package/package.json +4 -2
|
@@ -4,7 +4,13 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
|
4
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
5
|
var __getProtoOf = Object.getPrototypeOf;
|
|
6
6
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
-
var
|
|
7
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
8
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
9
|
+
}) : x)(function(x) {
|
|
10
|
+
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
11
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
12
|
+
});
|
|
13
|
+
var __commonJS = (cb, mod) => function __require2() {
|
|
8
14
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
9
15
|
};
|
|
10
16
|
var __export = (target, all) => {
|
|
@@ -22670,49 +22676,6 @@ var setWebRTCConnectionState = (state) => {
|
|
|
22670
22676
|
// src/constants/index.ts
|
|
22671
22677
|
var WEBRTC_BACKEND_SERVER_URL = "https://api-webrtc-dev.ansyr.ai";
|
|
22672
22678
|
|
|
22673
|
-
// src/utils/jsx-encoder.ts
|
|
22674
|
-
function generateStableDOMId(element) {
|
|
22675
|
-
const tagName = element.tagName.toLowerCase();
|
|
22676
|
-
const text = (element.textContent || "").trim().substring(0, 50);
|
|
22677
|
-
let sibling = element.previousElementSibling;
|
|
22678
|
-
let position = 1;
|
|
22679
|
-
while (sibling) {
|
|
22680
|
-
if (sibling.tagName === element.tagName) {
|
|
22681
|
-
position++;
|
|
22682
|
-
}
|
|
22683
|
-
sibling = sibling.previousElementSibling;
|
|
22684
|
-
}
|
|
22685
|
-
const path = getElementPath(element);
|
|
22686
|
-
const idString = `${tagName}[${position}]_(${text})_${path}`;
|
|
22687
|
-
let hash = 0;
|
|
22688
|
-
for (let i = 0; i < idString.length; i++) {
|
|
22689
|
-
const char = idString.charCodeAt(i);
|
|
22690
|
-
hash = (hash << 5) - hash + char;
|
|
22691
|
-
hash |= 0;
|
|
22692
|
-
}
|
|
22693
|
-
return hash.toString(36);
|
|
22694
|
-
}
|
|
22695
|
-
function getElementPath(element) {
|
|
22696
|
-
if (element.id) {
|
|
22697
|
-
return `id(${element.id})`;
|
|
22698
|
-
}
|
|
22699
|
-
if (element.tagName.toLowerCase() === "body") {
|
|
22700
|
-
return "/body";
|
|
22701
|
-
}
|
|
22702
|
-
let ix = 0;
|
|
22703
|
-
const siblings = element.parentNode?.children || new HTMLCollection();
|
|
22704
|
-
for (let i = 0; i < siblings.length; i++) {
|
|
22705
|
-
const sibling = siblings[i];
|
|
22706
|
-
if (sibling === element) {
|
|
22707
|
-
return `${getElementPath(element.parentNode)}/${element.tagName}[${ix + 1}]`;
|
|
22708
|
-
}
|
|
22709
|
-
if (sibling.nodeType === 1 && sibling.tagName === element.tagName) {
|
|
22710
|
-
ix++;
|
|
22711
|
-
}
|
|
22712
|
-
}
|
|
22713
|
-
return "not_found";
|
|
22714
|
-
}
|
|
22715
|
-
|
|
22716
22679
|
// src/core/intent-store.ts
|
|
22717
22680
|
var store = {
|
|
22718
22681
|
screenMetadata: {},
|
|
@@ -22743,18 +22706,14 @@ var GlobalStore = {
|
|
|
22743
22706
|
setElement(elementData) {
|
|
22744
22707
|
const index = store.allElementsData.findIndex((e2) => e2.elementId === elementData.elementId);
|
|
22745
22708
|
if (index >= 0) {
|
|
22746
|
-
console.log("Updating existing element");
|
|
22747
22709
|
store.allElementsData[index] = elementData;
|
|
22748
22710
|
} else {
|
|
22749
|
-
console.log("Adding new element");
|
|
22750
22711
|
store.allElementsData.push(elementData);
|
|
22751
22712
|
}
|
|
22752
22713
|
},
|
|
22753
22714
|
getElementById(elementId) {
|
|
22754
22715
|
const match = store.allElementsData.find((e2) => e2.elementId === elementId);
|
|
22755
22716
|
if (!match) {
|
|
22756
|
-
console.warn(`[GlobalStore] No element found for ID: ${elementId}`);
|
|
22757
|
-
console.log("All elements in store:", store.allElementsData);
|
|
22758
22717
|
}
|
|
22759
22718
|
return match;
|
|
22760
22719
|
},
|
|
@@ -22779,7 +22738,6 @@ function navigate(path, params) {
|
|
|
22779
22738
|
try {
|
|
22780
22739
|
navigationHandler(absolutePath, safeParams);
|
|
22781
22740
|
} catch (error) {
|
|
22782
|
-
console.error("[CueKit] navigation handler failed, falling back to default:", error);
|
|
22783
22741
|
}
|
|
22784
22742
|
return;
|
|
22785
22743
|
}
|
|
@@ -22810,7 +22768,6 @@ var safeNavigate = (name, params = {}) => {
|
|
|
22810
22768
|
if (name) {
|
|
22811
22769
|
navigate(name, params);
|
|
22812
22770
|
} else {
|
|
22813
|
-
console.warn("[CueKit] route name not provided");
|
|
22814
22771
|
}
|
|
22815
22772
|
};
|
|
22816
22773
|
function getCurrentScreenName() {
|
|
@@ -22845,7 +22802,6 @@ function onStateChange() {
|
|
|
22845
22802
|
const metadata = JSON.parse(params.metadata);
|
|
22846
22803
|
GlobalStore.setMetadata(routeName, metadata);
|
|
22847
22804
|
} catch (error) {
|
|
22848
|
-
console.error("Failed to parse metadata from URL:", error);
|
|
22849
22805
|
}
|
|
22850
22806
|
}
|
|
22851
22807
|
}
|
|
@@ -22868,7 +22824,7 @@ var handleNavigationAndClick = (routeName, elementHash) => {
|
|
|
22868
22824
|
}
|
|
22869
22825
|
sibling = sibling.previousElementSibling;
|
|
22870
22826
|
}
|
|
22871
|
-
const path =
|
|
22827
|
+
const path = getElementPath(element);
|
|
22872
22828
|
const idString = `${tagName}[${position}]_(${text})_${path}`;
|
|
22873
22829
|
let hash = 0;
|
|
22874
22830
|
for (let i = 0; i < idString.length; i++) {
|
|
@@ -22891,7 +22847,7 @@ var handleNavigationAndClick = (routeName, elementHash) => {
|
|
|
22891
22847
|
});
|
|
22892
22848
|
observer.observe(document.body, { childList: true, subtree: true });
|
|
22893
22849
|
};
|
|
22894
|
-
function
|
|
22850
|
+
function getElementPath(element) {
|
|
22895
22851
|
if (element.id) {
|
|
22896
22852
|
return `id(${element.id})`;
|
|
22897
22853
|
}
|
|
@@ -22912,10 +22868,54 @@ function getElementPath2(element) {
|
|
|
22912
22868
|
return path.join("/");
|
|
22913
22869
|
}
|
|
22914
22870
|
|
|
22871
|
+
// src/utils/jsx-encoder.ts
|
|
22872
|
+
function generateStableDOMId(element) {
|
|
22873
|
+
const tagName = element.tagName.toLowerCase();
|
|
22874
|
+
const text = (element.textContent || "").trim().substring(0, 50);
|
|
22875
|
+
let sibling = element.previousElementSibling;
|
|
22876
|
+
let position = 1;
|
|
22877
|
+
while (sibling) {
|
|
22878
|
+
if (sibling.tagName === element.tagName) {
|
|
22879
|
+
position++;
|
|
22880
|
+
}
|
|
22881
|
+
sibling = sibling.previousElementSibling;
|
|
22882
|
+
}
|
|
22883
|
+
const path = getElementPath2(element);
|
|
22884
|
+
const idString = `${tagName}[${position}]_(${text})_${path}`;
|
|
22885
|
+
let hash = 0;
|
|
22886
|
+
for (let i = 0; i < idString.length; i++) {
|
|
22887
|
+
const char = idString.charCodeAt(i);
|
|
22888
|
+
hash = (hash << 5) - hash + char;
|
|
22889
|
+
hash |= 0;
|
|
22890
|
+
}
|
|
22891
|
+
return hash.toString(36);
|
|
22892
|
+
}
|
|
22893
|
+
function getElementPath2(element) {
|
|
22894
|
+
if (element.id) {
|
|
22895
|
+
return `id(${element.id})`;
|
|
22896
|
+
}
|
|
22897
|
+
if (element.tagName.toLowerCase() === "body") {
|
|
22898
|
+
return "/body";
|
|
22899
|
+
}
|
|
22900
|
+
let ix = 0;
|
|
22901
|
+
const siblings = element.parentNode?.children || new HTMLCollection();
|
|
22902
|
+
for (let i = 0; i < siblings.length; i++) {
|
|
22903
|
+
const sibling = siblings[i];
|
|
22904
|
+
if (sibling === element) {
|
|
22905
|
+
return `${getElementPath2(element.parentNode)}/${element.tagName}[${ix + 1}]`;
|
|
22906
|
+
}
|
|
22907
|
+
if (sibling.nodeType === 1 && sibling.tagName === element.tagName) {
|
|
22908
|
+
ix++;
|
|
22909
|
+
}
|
|
22910
|
+
}
|
|
22911
|
+
return "not_found";
|
|
22912
|
+
}
|
|
22913
|
+
|
|
22915
22914
|
// src/utils/element-service.ts
|
|
22916
|
-
var INTERACTIVE_ELEMENT_SELECTOR = 'a, button, input, textarea, select, [role="button"], [onclick]';
|
|
22915
|
+
var INTERACTIVE_ELEMENT_SELECTOR = 'a, button, input, textarea, select, [role="button"], [onclick], [data-ansyr-static], [data-ansyr-dynamic]';
|
|
22917
22916
|
function getInteractiveElements() {
|
|
22918
|
-
|
|
22917
|
+
const elements = document.querySelectorAll(INTERACTIVE_ELEMENT_SELECTOR);
|
|
22918
|
+
return Array.from(new Set(Array.from(elements)));
|
|
22919
22919
|
}
|
|
22920
22920
|
function getImmediateText(element) {
|
|
22921
22921
|
let text = "";
|
|
@@ -22929,38 +22929,56 @@ function getImmediateText(element) {
|
|
|
22929
22929
|
return text.trim();
|
|
22930
22930
|
}
|
|
22931
22931
|
function captureFullDOMStructure() {
|
|
22932
|
-
console.log("\u{1F333} Capturing full DOM structure...");
|
|
22933
|
-
const
|
|
22934
|
-
const
|
|
22935
|
-
|
|
22936
|
-
|
|
22937
|
-
|
|
22938
|
-
|
|
22939
|
-
|
|
22940
|
-
|
|
22932
|
+
console.log("\u{1F333} Capturing full DOM structure for Cuekit...");
|
|
22933
|
+
const interactiveElements = [];
|
|
22934
|
+
const descriptionsMap = /* @__PURE__ */ new Map();
|
|
22935
|
+
document.querySelectorAll("[data-for]").forEach((el) => {
|
|
22936
|
+
const targetId = el.getAttribute("data-for");
|
|
22937
|
+
if (!targetId) return;
|
|
22938
|
+
const tags = [];
|
|
22939
|
+
const description = el.getAttribute("data-ansyr-description");
|
|
22940
|
+
if (description) {
|
|
22941
|
+
tags.push(description);
|
|
22942
|
+
}
|
|
22943
|
+
if (el.textContent) {
|
|
22944
|
+
tags.push(el.textContent.trim());
|
|
22945
|
+
}
|
|
22946
|
+
if (tags.length > 0) {
|
|
22947
|
+
const existingTags = descriptionsMap.get(targetId) || [];
|
|
22948
|
+
descriptionsMap.set(targetId, [...existingTags, ...tags]);
|
|
22941
22949
|
}
|
|
22942
22950
|
});
|
|
22943
|
-
const
|
|
22951
|
+
const allElements = getInteractiveElements();
|
|
22952
|
+
allElements.forEach((element) => {
|
|
22953
|
+
const style = getComputedStyle(element);
|
|
22954
|
+
if (element.closest("[data-cuekit-ignore]") || !isElementClickable(element) || element.tagName.toLowerCase() === "script" || style.display === "none" || style.visibility === "hidden") {
|
|
22955
|
+
return;
|
|
22956
|
+
}
|
|
22957
|
+
const staticId = element.getAttribute("data-ansyr-static");
|
|
22958
|
+
const dynamicId = element.getAttribute("data-ansyr-dynamic");
|
|
22959
|
+
const id = staticId || dynamicId;
|
|
22960
|
+
const tags = [];
|
|
22961
|
+
const directDescription = element.getAttribute("data-ansyr-description");
|
|
22962
|
+
if (directDescription) {
|
|
22963
|
+
tags.push(directDescription);
|
|
22964
|
+
}
|
|
22965
|
+
if (id && descriptionsMap.has(id)) {
|
|
22966
|
+
tags.push(...descriptionsMap.get(id) || []);
|
|
22967
|
+
}
|
|
22968
|
+
const dto = {
|
|
22969
|
+
testID: id || generateStableDOMId(element),
|
|
22970
|
+
type: element.tagName.toLowerCase(),
|
|
22971
|
+
textContent: getImmediateText(element) || element.textContent?.trim() || "",
|
|
22972
|
+
tags
|
|
22973
|
+
};
|
|
22974
|
+
interactiveElements.push(dto);
|
|
22975
|
+
});
|
|
22976
|
+
const result = {
|
|
22977
|
+
components: interactiveElements
|
|
22978
|
+
};
|
|
22944
22979
|
console.log("\u{1F333} Full DOM structure captured:", result);
|
|
22945
22980
|
return result;
|
|
22946
22981
|
}
|
|
22947
|
-
function buildFlatDOMNode(element) {
|
|
22948
|
-
if (element.tagName.toLowerCase() === "script" || element.hasAttribute("data-cuekit-ignore") || element.style.display === "none" || element.style.visibility === "hidden") {
|
|
22949
|
-
return null;
|
|
22950
|
-
}
|
|
22951
|
-
const hash = generateStableDOMId(element);
|
|
22952
|
-
const text = getImmediateText(element).substring(0, 100);
|
|
22953
|
-
const isClickable = isElementClickable(element);
|
|
22954
|
-
const componentType = element.tagName.toLowerCase();
|
|
22955
|
-
return {
|
|
22956
|
-
hash,
|
|
22957
|
-
text,
|
|
22958
|
-
isClickable,
|
|
22959
|
-
componentType,
|
|
22960
|
-
children: []
|
|
22961
|
-
// No children in a flat structure
|
|
22962
|
-
};
|
|
22963
|
-
}
|
|
22964
22982
|
function isElementClickable(element) {
|
|
22965
22983
|
const interactiveSelectors = [
|
|
22966
22984
|
"button",
|
|
@@ -23014,34 +23032,23 @@ function executeAction(action) {
|
|
|
23014
23032
|
}
|
|
23015
23033
|
}
|
|
23016
23034
|
function getFullDOMStructure() {
|
|
23017
|
-
console.log("\u{1F333} ElementService: Getting full DOM structure...");
|
|
23018
23035
|
return captureFullDOMStructure();
|
|
23019
23036
|
}
|
|
23020
23037
|
function clickElement(elementId) {
|
|
23021
23038
|
if (!elementId) {
|
|
23022
|
-
console.warn("\u26A0\uFE0F No element ID provided for click action");
|
|
23023
23039
|
return false;
|
|
23024
23040
|
}
|
|
23025
|
-
const
|
|
23026
|
-
|
|
23027
|
-
|
|
23028
|
-
|
|
23029
|
-
const domElement = findDOMElementById(elementId);
|
|
23030
|
-
if (domElement) {
|
|
23031
|
-
domElement.click();
|
|
23032
|
-
return true;
|
|
23033
|
-
}
|
|
23034
|
-
} else {
|
|
23035
|
-
console.warn(`\u26A0\uFE0F Element not found: ${elementId}`);
|
|
23041
|
+
const domElement = findDOMElementById(elementId);
|
|
23042
|
+
if (domElement) {
|
|
23043
|
+
domElement.click();
|
|
23044
|
+
return true;
|
|
23036
23045
|
}
|
|
23037
23046
|
return false;
|
|
23038
23047
|
}
|
|
23039
23048
|
function navigateToElement(target) {
|
|
23040
23049
|
if (!target) {
|
|
23041
|
-
console.warn("\u26A0\uFE0F No target provided for navigation action");
|
|
23042
23050
|
return false;
|
|
23043
23051
|
}
|
|
23044
|
-
console.log(`\u{1F9ED} Navigating to: ${target}`);
|
|
23045
23052
|
if (target.includes("/") || target.startsWith("http")) {
|
|
23046
23053
|
safeNavigate(target, {});
|
|
23047
23054
|
} else {
|
|
@@ -23051,27 +23058,22 @@ function navigateToElement(target) {
|
|
|
23051
23058
|
}
|
|
23052
23059
|
function focusElement(elementId) {
|
|
23053
23060
|
if (!elementId) {
|
|
23054
|
-
console.warn("\u26A0\uFE0F No element ID provided for focus action");
|
|
23055
23061
|
return false;
|
|
23056
23062
|
}
|
|
23057
23063
|
const domElement = findDOMElementById(elementId);
|
|
23058
23064
|
if (domElement instanceof HTMLInputElement || domElement instanceof HTMLTextAreaElement || domElement instanceof HTMLSelectElement) {
|
|
23059
|
-
console.log(`\u{1F4DD} Focusing element: ${elementId}`);
|
|
23060
23065
|
domElement.focus();
|
|
23061
23066
|
return true;
|
|
23062
23067
|
} else {
|
|
23063
|
-
console.warn(`\u26A0\uFE0F Focusable element not found: ${elementId}`);
|
|
23064
23068
|
return false;
|
|
23065
23069
|
}
|
|
23066
23070
|
}
|
|
23067
23071
|
function toggleElement(elementId) {
|
|
23068
23072
|
if (!elementId) {
|
|
23069
|
-
console.warn("\u26A0\uFE0F No element ID provided for toggle action");
|
|
23070
23073
|
return false;
|
|
23071
23074
|
}
|
|
23072
23075
|
const domElement = findDOMElementById(elementId);
|
|
23073
23076
|
if (domElement instanceof HTMLElement) {
|
|
23074
|
-
console.log(`\u{1F504} Toggling element: ${elementId}`);
|
|
23075
23077
|
if (domElement instanceof HTMLInputElement) {
|
|
23076
23078
|
if (domElement.type === "checkbox") {
|
|
23077
23079
|
domElement.checked = !domElement.checked;
|
|
@@ -23084,34 +23086,20 @@ function toggleElement(elementId) {
|
|
|
23084
23086
|
}
|
|
23085
23087
|
return true;
|
|
23086
23088
|
} else {
|
|
23087
|
-
console.warn(`\u26A0\uFE0F Toggleable element not found: ${elementId}`);
|
|
23088
23089
|
return false;
|
|
23089
23090
|
}
|
|
23090
23091
|
}
|
|
23091
|
-
function findElementById(domStructure, elementId) {
|
|
23092
|
-
const searchInComponents = (components) => {
|
|
23093
|
-
for (const component of components) {
|
|
23094
|
-
if (component.hash === elementId) {
|
|
23095
|
-
return component;
|
|
23096
|
-
}
|
|
23097
|
-
if (component.children.length > 0) {
|
|
23098
|
-
const found = searchInComponents(component.children);
|
|
23099
|
-
if (found) return found;
|
|
23100
|
-
}
|
|
23101
|
-
}
|
|
23102
|
-
return null;
|
|
23103
|
-
};
|
|
23104
|
-
return searchInComponents(domStructure.components);
|
|
23105
|
-
}
|
|
23106
23092
|
function findDOMElementById(elementId) {
|
|
23093
|
+
if (!elementId) {
|
|
23094
|
+
return null;
|
|
23095
|
+
}
|
|
23107
23096
|
const interactiveElements = getInteractiveElements();
|
|
23108
23097
|
for (const element of interactiveElements) {
|
|
23109
23098
|
if (element instanceof HTMLElement) {
|
|
23110
|
-
|
|
23111
|
-
const
|
|
23112
|
-
|
|
23113
|
-
if (
|
|
23114
|
-
console.log("\u{1F50D} Found element:", element);
|
|
23099
|
+
const staticId = element.getAttribute("data-ansyr-static");
|
|
23100
|
+
const dynamicId = element.getAttribute("data-ansyr-dynamic");
|
|
23101
|
+
const currentElementId = staticId || dynamicId || generateStableDOMId(element);
|
|
23102
|
+
if (currentElementId === elementId) {
|
|
23115
23103
|
return element;
|
|
23116
23104
|
}
|
|
23117
23105
|
}
|
|
@@ -23139,31 +23127,29 @@ function setWebRTCCallbacks(newCallbacks) {
|
|
|
23139
23127
|
}
|
|
23140
23128
|
async function authenticate(userIdentity, apiKey, appId) {
|
|
23141
23129
|
try {
|
|
23142
|
-
|
|
23130
|
+
const authPayload = {
|
|
23131
|
+
user_identity: userIdentity,
|
|
23132
|
+
app_id: appId || _appId
|
|
23133
|
+
};
|
|
23143
23134
|
const response = await fetch(`${serverUrl}/auth/login`, {
|
|
23144
23135
|
method: "POST",
|
|
23145
23136
|
headers: {
|
|
23146
23137
|
"Content-Type": "application/json",
|
|
23147
23138
|
"X-API-Key": apiKey
|
|
23148
23139
|
},
|
|
23149
|
-
body: JSON.stringify(
|
|
23150
|
-
user_identity: userIdentity,
|
|
23151
|
-
app_id: appId || _appId
|
|
23152
|
-
})
|
|
23140
|
+
body: JSON.stringify(authPayload)
|
|
23153
23141
|
});
|
|
23154
23142
|
if (!response.ok) {
|
|
23155
23143
|
const errorData = await response.json();
|
|
23156
23144
|
throw new Error(errorData.detail || "Authentication failed");
|
|
23157
23145
|
}
|
|
23158
23146
|
const authData = await response.json();
|
|
23159
|
-
console.log("\u{1F3A4} WebRTCService: Authentication successful:", authData);
|
|
23160
23147
|
livekitUrl = authData.livekit_url;
|
|
23161
23148
|
token = authData.livekit_token;
|
|
23162
23149
|
roomName = authData.room_name;
|
|
23163
23150
|
userIdentity = authData.user_identity;
|
|
23164
23151
|
return authData;
|
|
23165
23152
|
} catch (error) {
|
|
23166
|
-
console.error("\u{1F3A4} WebRTCService: Authentication failed:", error);
|
|
23167
23153
|
throw error;
|
|
23168
23154
|
}
|
|
23169
23155
|
}
|
|
@@ -23173,7 +23159,6 @@ async function connectToRoom(newLivekitUrl, newToken) {
|
|
|
23173
23159
|
if (!url || !authToken) {
|
|
23174
23160
|
throw new Error("Missing LiveKit URL or token. Please authenticate first.");
|
|
23175
23161
|
}
|
|
23176
|
-
console.log("\u{1F3A4} WebRTCService: Connecting to room...", { url, hasToken: !!authToken });
|
|
23177
23162
|
try {
|
|
23178
23163
|
setWebRTCConnectionState({ isConnected: false, isConnecting: true });
|
|
23179
23164
|
room = new Room({
|
|
@@ -23182,17 +23167,13 @@ async function connectToRoom(newLivekitUrl, newToken) {
|
|
|
23182
23167
|
});
|
|
23183
23168
|
setupEventListeners();
|
|
23184
23169
|
await room.connect(url, authToken);
|
|
23185
|
-
console.log("\u{1F3A4} WebRTCService: Successfully connected to room:", room.name);
|
|
23186
23170
|
setWebRTCConnectionState({ isConnected: true, isConnecting: false });
|
|
23187
23171
|
try {
|
|
23188
23172
|
await room.localParticipant.setMicrophoneEnabled(true);
|
|
23189
|
-
console.log("\u{1F3A4} WebRTCService: Microphone enabled");
|
|
23190
23173
|
} catch (micError) {
|
|
23191
|
-
console.warn("\u{1F3A4} WebRTCService: Failed to enable microphone:", micError);
|
|
23192
23174
|
}
|
|
23193
23175
|
return { success: true };
|
|
23194
23176
|
} catch (error) {
|
|
23195
|
-
console.error("\u{1F3A4} WebRTCService: Failed to connect to room:", error);
|
|
23196
23177
|
setWebRTCConnectionState({ isConnected: false, isConnecting: false });
|
|
23197
23178
|
throw error;
|
|
23198
23179
|
}
|
|
@@ -23200,23 +23181,17 @@ async function connectToRoom(newLivekitUrl, newToken) {
|
|
|
23200
23181
|
function setupEventListeners() {
|
|
23201
23182
|
if (!room) return;
|
|
23202
23183
|
room.on(RoomEvent.ConnectionStateChanged, (state) => {
|
|
23203
|
-
console.log("\u{1F3A4} WebRTCService: Connection state changed:", state);
|
|
23204
23184
|
callbacks.onConnectionStateChange?.(state);
|
|
23205
23185
|
if (state === ConnectionState.Connected) {
|
|
23206
|
-
console.log("\u{1F3A4} WebRTCService: Successfully connected to room");
|
|
23207
23186
|
setWebRTCConnectionState({ isConnected: true, isConnecting: false });
|
|
23208
23187
|
} else if (state === ConnectionState.Disconnected) {
|
|
23209
|
-
console.log("\u{1F3A4} WebRTCService: Disconnected from room");
|
|
23210
23188
|
setWebRTCConnectionState({ isConnected: false, isConnecting: false });
|
|
23211
23189
|
} else if (state === ConnectionState.Connecting) {
|
|
23212
|
-
console.log("\u{1F3A4} WebRTCService: Connecting to room...");
|
|
23213
23190
|
setWebRTCConnectionState({ isConnected: false, isConnecting: true });
|
|
23214
23191
|
}
|
|
23215
23192
|
}).on(RoomEvent.ParticipantConnected, (participant) => {
|
|
23216
|
-
console.log("\u{1F3A4} WebRTCService: Participant connected:", participant.identity);
|
|
23217
23193
|
updateParticipantsList();
|
|
23218
23194
|
}).on(RoomEvent.ParticipantDisconnected, (participant) => {
|
|
23219
|
-
console.log("\u{1F3A4} WebRTCService: Participant disconnected:", participant.identity);
|
|
23220
23195
|
updateParticipantsList();
|
|
23221
23196
|
}).on(RoomEvent.TrackSubscribed, (track, publication, participant) => {
|
|
23222
23197
|
if (track.kind === Track.Kind.Audio && !participant.isLocal) {
|
|
@@ -23228,9 +23203,8 @@ function setupEventListeners() {
|
|
|
23228
23203
|
if (element instanceof HTMLAudioElement) {
|
|
23229
23204
|
const trackId = track.sid || `track_${Date.now()}_${Math.random()}`;
|
|
23230
23205
|
callbacks.onAISpeechStart?.(trackId);
|
|
23231
|
-
element.play().catch(
|
|
23232
|
-
|
|
23233
|
-
);
|
|
23206
|
+
element.play().catch((error) => {
|
|
23207
|
+
});
|
|
23234
23208
|
element.addEventListener("ended", () => callbacks.onAISpeechEnd?.(trackId));
|
|
23235
23209
|
element.addEventListener("pause", () => callbacks.onAISpeechEnd?.(trackId));
|
|
23236
23210
|
}
|
|
@@ -23240,13 +23214,12 @@ function setupEventListeners() {
|
|
|
23240
23214
|
}).on(RoomEvent.TrackUnsubscribed, (track) => {
|
|
23241
23215
|
track.detach().forEach((element) => element.remove());
|
|
23242
23216
|
}).on(RoomEvent.DataReceived, (payload, participant) => {
|
|
23243
|
-
|
|
23217
|
+
const decodedPayload = new TextDecoder().decode(payload);
|
|
23244
23218
|
try {
|
|
23245
|
-
const message = JSON.parse(
|
|
23246
|
-
console.log("\u{1F4E1} LiveKit data received:", message);
|
|
23219
|
+
const message = JSON.parse(decodedPayload);
|
|
23247
23220
|
callbacks.onNavigationCommand?.(message);
|
|
23248
23221
|
} catch (error) {
|
|
23249
|
-
const message =
|
|
23222
|
+
const message = decodedPayload;
|
|
23250
23223
|
callbacks.onNavigationCommand?.({ type: "raw_text", data: message });
|
|
23251
23224
|
}
|
|
23252
23225
|
}).on(RoomEvent.Disconnected, () => {
|
|
@@ -23265,26 +23238,25 @@ function updateParticipantsList() {
|
|
|
23265
23238
|
async function sendData(data, reliable = true) {
|
|
23266
23239
|
if (!room) throw new Error("Not connected to room");
|
|
23267
23240
|
try {
|
|
23268
|
-
console.log("\
|
|
23241
|
+
console.log("\u2B06\uFE0F Sending data to backend [DataChannel]:", data);
|
|
23269
23242
|
const encoder = new TextEncoder();
|
|
23270
23243
|
const encodedData = encoder.encode(data);
|
|
23271
23244
|
await room.localParticipant.publishData(encodedData, {
|
|
23272
23245
|
reliable
|
|
23273
23246
|
});
|
|
23274
23247
|
} catch (error) {
|
|
23275
|
-
console.error("Error sending data:", error);
|
|
23276
23248
|
throw error;
|
|
23277
23249
|
}
|
|
23278
23250
|
}
|
|
23279
23251
|
async function sendScreenStatus(screenData) {
|
|
23280
23252
|
try {
|
|
23253
|
+
console.log("\u2B06\uFE0F Sending to backend [/ai/data]:", JSON.stringify(screenData, null, 2));
|
|
23281
23254
|
await fetch(`${serverUrl}/ai/data`, {
|
|
23282
23255
|
method: "POST",
|
|
23283
23256
|
headers: { "Content-Type": "application/json" },
|
|
23284
23257
|
body: JSON.stringify(screenData)
|
|
23285
23258
|
});
|
|
23286
23259
|
} catch (error) {
|
|
23287
|
-
console.error("Error sending screen status:", error);
|
|
23288
23260
|
throw error;
|
|
23289
23261
|
}
|
|
23290
23262
|
}
|
|
@@ -23299,12 +23271,10 @@ function getParticipants() {
|
|
|
23299
23271
|
}
|
|
23300
23272
|
async function sendUserCommand(command) {
|
|
23301
23273
|
if (!room) return;
|
|
23302
|
-
console.log(`\u{1F4AC} Sending user command: "${command}"`);
|
|
23303
23274
|
await sendData(command);
|
|
23304
23275
|
}
|
|
23305
23276
|
async function sendRuntimeData() {
|
|
23306
23277
|
if (!room) {
|
|
23307
|
-
console.error("\u274C Cannot send runtime data without a room connection");
|
|
23308
23278
|
return;
|
|
23309
23279
|
}
|
|
23310
23280
|
try {
|
|
@@ -23317,28 +23287,28 @@ async function sendRuntimeData() {
|
|
|
23317
23287
|
current_screen: screenName
|
|
23318
23288
|
}
|
|
23319
23289
|
};
|
|
23320
|
-
console.log("\u{1F4E6} Sending runtime data response");
|
|
23321
23290
|
await sendData(JSON.stringify(response));
|
|
23322
|
-
console.log("\u{1F4E6} Runtime data sent successfully");
|
|
23323
23291
|
} catch (error) {
|
|
23324
|
-
console.error("\u274C Failed to send runtime data:", error);
|
|
23325
23292
|
}
|
|
23326
23293
|
}
|
|
23327
23294
|
async function sendStaticData(componentData, appId = "default") {
|
|
23328
23295
|
try {
|
|
23296
|
+
const staticDataPayload = {
|
|
23297
|
+
type: "dashboard_data",
|
|
23298
|
+
app_id: appId,
|
|
23299
|
+
data: componentData
|
|
23300
|
+
};
|
|
23301
|
+
console.log("\u2B06\uFE0F Sending to backend [/ai/data]:", JSON.stringify(staticDataPayload, null, 2));
|
|
23329
23302
|
const response = await fetch(`${serverUrl}/ai/data`, {
|
|
23330
23303
|
method: "POST",
|
|
23331
23304
|
headers: {
|
|
23332
23305
|
"Content-Type": "application/json"
|
|
23333
23306
|
},
|
|
23334
|
-
body: JSON.stringify(
|
|
23335
|
-
type: "dashboard_data",
|
|
23336
|
-
app_id: appId,
|
|
23337
|
-
data: componentData
|
|
23338
|
-
})
|
|
23307
|
+
body: JSON.stringify(staticDataPayload)
|
|
23339
23308
|
});
|
|
23340
23309
|
if (response.ok) {
|
|
23341
23310
|
const result = await response.json();
|
|
23311
|
+
console.log("\u2B07\uFE0F Received from backend [/ai/data]:", JSON.stringify(result, null, 2));
|
|
23342
23312
|
return { success: true, data: result };
|
|
23343
23313
|
} else {
|
|
23344
23314
|
const errorText = await response.text();
|
|
@@ -23368,6 +23338,7 @@ function getRoom() {
|
|
|
23368
23338
|
}
|
|
23369
23339
|
|
|
23370
23340
|
export {
|
|
23341
|
+
__require,
|
|
23371
23342
|
__commonJS,
|
|
23372
23343
|
__export,
|
|
23373
23344
|
__toESM,
|
|
@@ -23384,7 +23355,6 @@ export {
|
|
|
23384
23355
|
ParticipantEvent,
|
|
23385
23356
|
Track,
|
|
23386
23357
|
createAudioAnalyser,
|
|
23387
|
-
Participant,
|
|
23388
23358
|
ConnectionState,
|
|
23389
23359
|
captureFullDOMStructure,
|
|
23390
23360
|
executeAction,
|
package/dist/index.d.mts
CHANGED
|
@@ -254,21 +254,20 @@ declare const initWebRTCWithDeployedBackend: (apiKey: string, customConfig?: Par
|
|
|
254
254
|
*/
|
|
255
255
|
declare const initWebRTC: (apiKey: string) => WebRTCServerConfig;
|
|
256
256
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
};
|
|
257
|
+
interface InteractiveElementDto {
|
|
258
|
+
testID: string;
|
|
259
|
+
type: string;
|
|
260
|
+
textContent: string;
|
|
261
|
+
tags: string[];
|
|
262
|
+
}
|
|
263
|
+
interface DOMStructurePayload {
|
|
264
|
+
components: InteractiveElementDto[];
|
|
265
|
+
}
|
|
267
266
|
|
|
268
267
|
/**
|
|
269
268
|
* Capture the full DOM structure as a flat list of interactive elements
|
|
270
269
|
*/
|
|
271
|
-
declare function captureFullDOMStructure():
|
|
270
|
+
declare function captureFullDOMStructure(): DOMStructurePayload;
|
|
272
271
|
interface ElementAction {
|
|
273
272
|
action_type: 'click' | 'navigate' | 'input' | 'focus' | 'toggle';
|
|
274
273
|
target_element?: string;
|
|
@@ -276,6 +275,16 @@ interface ElementAction {
|
|
|
276
275
|
instruction?: string;
|
|
277
276
|
}
|
|
278
277
|
declare function executeAction(action: ElementAction): boolean;
|
|
279
|
-
declare function getFullDOMStructure():
|
|
278
|
+
declare function getFullDOMStructure(): DOMStructurePayload;
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Generates a stable, unique 8-character ID for an interactive element.
|
|
282
|
+
*
|
|
283
|
+
* @param routePath The path of the current route (e.g., from useLocation().pathname).
|
|
284
|
+
* @param elementIdentifier A stable, developer-defined string that uniquely identifies
|
|
285
|
+
* the element within its dynamic context (e.g., `van-link-${van.id}`).
|
|
286
|
+
* @returns An 8-character hexadecimal string.
|
|
287
|
+
*/
|
|
288
|
+
declare function generateDynamicId(routePath: string, elementIdentifier: string): string;
|
|
280
289
|
|
|
281
|
-
export { BorderGlow, ChatPopup, type ChatPopupProps, CuekitProvider, type
|
|
290
|
+
export { BorderGlow, ChatPopup, type ChatPopupProps, CuekitProvider, type DOMStructurePayload, type ElementAction, InitCuekit, type InteractiveElementDto, MicButton, type MicButtonProps, type MicState$1 as MicState, type NavigationCommand, type ServerConfig, type TokenRequest, type TokenResponse, VoiceIntensityVisualizer, type WebRTCServerConfig, captureFullDOMStructure, configureWebRTCServer, executeAction, generateDynamicId, getFullDOMStructure, getWebRTCServerConfig, initWebRTC, initWebRTCWithDeployedBackend, useCuekit, useQubeContext, useWebRTC };
|
package/dist/index.d.ts
CHANGED
|
@@ -254,21 +254,20 @@ declare const initWebRTCWithDeployedBackend: (apiKey: string, customConfig?: Par
|
|
|
254
254
|
*/
|
|
255
255
|
declare const initWebRTC: (apiKey: string) => WebRTCServerConfig;
|
|
256
256
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
};
|
|
257
|
+
interface InteractiveElementDto {
|
|
258
|
+
testID: string;
|
|
259
|
+
type: string;
|
|
260
|
+
textContent: string;
|
|
261
|
+
tags: string[];
|
|
262
|
+
}
|
|
263
|
+
interface DOMStructurePayload {
|
|
264
|
+
components: InteractiveElementDto[];
|
|
265
|
+
}
|
|
267
266
|
|
|
268
267
|
/**
|
|
269
268
|
* Capture the full DOM structure as a flat list of interactive elements
|
|
270
269
|
*/
|
|
271
|
-
declare function captureFullDOMStructure():
|
|
270
|
+
declare function captureFullDOMStructure(): DOMStructurePayload;
|
|
272
271
|
interface ElementAction {
|
|
273
272
|
action_type: 'click' | 'navigate' | 'input' | 'focus' | 'toggle';
|
|
274
273
|
target_element?: string;
|
|
@@ -276,6 +275,16 @@ interface ElementAction {
|
|
|
276
275
|
instruction?: string;
|
|
277
276
|
}
|
|
278
277
|
declare function executeAction(action: ElementAction): boolean;
|
|
279
|
-
declare function getFullDOMStructure():
|
|
278
|
+
declare function getFullDOMStructure(): DOMStructurePayload;
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Generates a stable, unique 8-character ID for an interactive element.
|
|
282
|
+
*
|
|
283
|
+
* @param routePath The path of the current route (e.g., from useLocation().pathname).
|
|
284
|
+
* @param elementIdentifier A stable, developer-defined string that uniquely identifies
|
|
285
|
+
* the element within its dynamic context (e.g., `van-link-${van.id}`).
|
|
286
|
+
* @returns An 8-character hexadecimal string.
|
|
287
|
+
*/
|
|
288
|
+
declare function generateDynamicId(routePath: string, elementIdentifier: string): string;
|
|
280
289
|
|
|
281
|
-
export { BorderGlow, ChatPopup, type ChatPopupProps, CuekitProvider, type
|
|
290
|
+
export { BorderGlow, ChatPopup, type ChatPopupProps, CuekitProvider, type DOMStructurePayload, type ElementAction, InitCuekit, type InteractiveElementDto, MicButton, type MicButtonProps, type MicState$1 as MicState, type NavigationCommand, type ServerConfig, type TokenRequest, type TokenResponse, VoiceIntensityVisualizer, type WebRTCServerConfig, captureFullDOMStructure, configureWebRTCServer, executeAction, generateDynamicId, getFullDOMStructure, getWebRTCServerConfig, initWebRTC, initWebRTCWithDeployedBackend, useCuekit, useQubeContext, useWebRTC };
|