@midscene/web 0.8.8 → 0.8.9-beta-20241223095005.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -595,6 +595,14 @@ var midscene_element_inspector = (() => {
|
|
|
595
595
|
})(NodeType || {});
|
|
596
596
|
|
|
597
597
|
// src/extractor/dom-util.ts
|
|
598
|
+
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF = "midscene-description-ref";
|
|
599
|
+
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID = "midscene-description-id";
|
|
600
|
+
function isUserDescribedElement(node) {
|
|
601
|
+
if (node instanceof Element) {
|
|
602
|
+
return node.hasAttribute(USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF);
|
|
603
|
+
}
|
|
604
|
+
return false;
|
|
605
|
+
}
|
|
598
606
|
function isFormElement(node) {
|
|
599
607
|
return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
|
|
600
608
|
}
|
|
@@ -962,7 +970,7 @@ var midscene_element_inspector = (() => {
|
|
|
962
970
|
tagName = node.tagName.toLowerCase();
|
|
963
971
|
}
|
|
964
972
|
const parentElement = node.parentElement;
|
|
965
|
-
if (parentElement && parentElement instanceof HTMLElement) {
|
|
973
|
+
if (!tagName && parentElement && parentElement instanceof HTMLElement) {
|
|
966
974
|
tagName = parentElement.tagName.toLowerCase();
|
|
967
975
|
}
|
|
968
976
|
return tagName ? `<${tagName}>` : "";
|
|
@@ -973,6 +981,81 @@ var midscene_element_inspector = (() => {
|
|
|
973
981
|
if (!rect || rect.width < CONTAINER_MINI_WIDTH || rect.height < CONTAINER_MINI_HEIGHT) {
|
|
974
982
|
return null;
|
|
975
983
|
}
|
|
984
|
+
if (isUserDescribedElement(node)) {
|
|
985
|
+
console.log("isUserDescribedElement", node);
|
|
986
|
+
const element = node;
|
|
987
|
+
const descriptionId = element.getAttribute(
|
|
988
|
+
USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF
|
|
989
|
+
);
|
|
990
|
+
const targetSelector = `[${USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID}="${descriptionId}"]`;
|
|
991
|
+
const targetElement = document.querySelectorAll(targetSelector);
|
|
992
|
+
if (targetElement.length === 0) {
|
|
993
|
+
console.warn(
|
|
994
|
+
"cannot find element for Midscene description",
|
|
995
|
+
targetSelector
|
|
996
|
+
);
|
|
997
|
+
return null;
|
|
998
|
+
}
|
|
999
|
+
if (targetElement.length > 1) {
|
|
1000
|
+
console.warn(
|
|
1001
|
+
"multiple elements found for Midscene description",
|
|
1002
|
+
targetSelector
|
|
1003
|
+
);
|
|
1004
|
+
}
|
|
1005
|
+
const descriptionElement = targetElement[0];
|
|
1006
|
+
const description = descriptionElement.innerText;
|
|
1007
|
+
try {
|
|
1008
|
+
const descriptionJson = JSON.parse(description);
|
|
1009
|
+
if (!Array.isArray(descriptionJson)) {
|
|
1010
|
+
console.warn("description is not a valid JSON", description);
|
|
1011
|
+
return null;
|
|
1012
|
+
}
|
|
1013
|
+
const infoList = descriptionJson.map((item, index) => {
|
|
1014
|
+
const overallRect = {
|
|
1015
|
+
left: item.rect.x + rect.left,
|
|
1016
|
+
top: item.rect.y + rect.top,
|
|
1017
|
+
width: item.rect.width,
|
|
1018
|
+
height: item.rect.height
|
|
1019
|
+
};
|
|
1020
|
+
const center = [
|
|
1021
|
+
Math.round(overallRect.left + overallRect.width / 2),
|
|
1022
|
+
Math.round(overallRect.top + overallRect.height / 2)
|
|
1023
|
+
];
|
|
1024
|
+
const nodeType = item.text ? NodeType.TEXT : NodeType.CONTAINER;
|
|
1025
|
+
const nodeHashId = midsceneGenerateHash(
|
|
1026
|
+
null,
|
|
1027
|
+
`${index}-${item.text || ""}`,
|
|
1028
|
+
overallRect
|
|
1029
|
+
);
|
|
1030
|
+
return {
|
|
1031
|
+
id: nodeHashId,
|
|
1032
|
+
indexId: indexId++,
|
|
1033
|
+
zoom: 1,
|
|
1034
|
+
locator: "",
|
|
1035
|
+
nodeType,
|
|
1036
|
+
nodePath,
|
|
1037
|
+
nodeHashId,
|
|
1038
|
+
attributes: __spreadProps(__spreadValues({}, item.description ? { description: item.description } : {}), {
|
|
1039
|
+
nodeType
|
|
1040
|
+
}),
|
|
1041
|
+
content: item.text || "",
|
|
1042
|
+
rect: overallRect,
|
|
1043
|
+
center,
|
|
1044
|
+
screenWidth: window.innerWidth,
|
|
1045
|
+
screenHeight: window.innerHeight
|
|
1046
|
+
};
|
|
1047
|
+
});
|
|
1048
|
+
return infoList;
|
|
1049
|
+
} catch (e) {
|
|
1050
|
+
console.error(e);
|
|
1051
|
+
console.warn(
|
|
1052
|
+
"description is not a valid JSON",
|
|
1053
|
+
targetSelector,
|
|
1054
|
+
description
|
|
1055
|
+
);
|
|
1056
|
+
return null;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
976
1059
|
if (isFormElement(node)) {
|
|
977
1060
|
const attributes = getNodeAttributes(node);
|
|
978
1061
|
let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
|
|
@@ -1143,6 +1226,10 @@ var midscene_element_inspector = (() => {
|
|
|
1143
1226
|
return null;
|
|
1144
1227
|
}
|
|
1145
1228
|
const elementInfo = collectElementInfo(node, nodePath, baseZoom);
|
|
1229
|
+
if (Array.isArray(elementInfo)) {
|
|
1230
|
+
elementInfoArray.push(...elementInfo);
|
|
1231
|
+
return null;
|
|
1232
|
+
}
|
|
1146
1233
|
if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.TEXT || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.FORM_ITEM || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.CONTAINER) {
|
|
1147
1234
|
elementInfoArray.push(elementInfo);
|
|
1148
1235
|
return elementInfo;
|
|
@@ -583,6 +583,14 @@ var midscene_element_inspector = (() => {
|
|
|
583
583
|
})(NodeType || {});
|
|
584
584
|
|
|
585
585
|
// src/extractor/dom-util.ts
|
|
586
|
+
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF = "midscene-description-ref";
|
|
587
|
+
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID = "midscene-description-id";
|
|
588
|
+
function isUserDescribedElement(node) {
|
|
589
|
+
if (node instanceof Element) {
|
|
590
|
+
return node.hasAttribute(USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF);
|
|
591
|
+
}
|
|
592
|
+
return false;
|
|
593
|
+
}
|
|
586
594
|
function isFormElement(node) {
|
|
587
595
|
return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
|
|
588
596
|
}
|
|
@@ -960,7 +968,7 @@ var midscene_element_inspector = (() => {
|
|
|
960
968
|
tagName = node.tagName.toLowerCase();
|
|
961
969
|
}
|
|
962
970
|
const parentElement = node.parentElement;
|
|
963
|
-
if (parentElement && parentElement instanceof HTMLElement) {
|
|
971
|
+
if (!tagName && parentElement && parentElement instanceof HTMLElement) {
|
|
964
972
|
tagName = parentElement.tagName.toLowerCase();
|
|
965
973
|
}
|
|
966
974
|
return tagName ? `<${tagName}>` : "";
|
|
@@ -971,6 +979,81 @@ var midscene_element_inspector = (() => {
|
|
|
971
979
|
if (!rect || rect.width < CONTAINER_MINI_WIDTH || rect.height < CONTAINER_MINI_HEIGHT) {
|
|
972
980
|
return null;
|
|
973
981
|
}
|
|
982
|
+
if (isUserDescribedElement(node)) {
|
|
983
|
+
console.log("isUserDescribedElement", node);
|
|
984
|
+
const element = node;
|
|
985
|
+
const descriptionId = element.getAttribute(
|
|
986
|
+
USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF
|
|
987
|
+
);
|
|
988
|
+
const targetSelector = `[${USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID}="${descriptionId}"]`;
|
|
989
|
+
const targetElement = document.querySelectorAll(targetSelector);
|
|
990
|
+
if (targetElement.length === 0) {
|
|
991
|
+
console.warn(
|
|
992
|
+
"cannot find element for Midscene description",
|
|
993
|
+
targetSelector
|
|
994
|
+
);
|
|
995
|
+
return null;
|
|
996
|
+
}
|
|
997
|
+
if (targetElement.length > 1) {
|
|
998
|
+
console.warn(
|
|
999
|
+
"multiple elements found for Midscene description",
|
|
1000
|
+
targetSelector
|
|
1001
|
+
);
|
|
1002
|
+
}
|
|
1003
|
+
const descriptionElement = targetElement[0];
|
|
1004
|
+
const description = descriptionElement.innerText;
|
|
1005
|
+
try {
|
|
1006
|
+
const descriptionJson = JSON.parse(description);
|
|
1007
|
+
if (!Array.isArray(descriptionJson)) {
|
|
1008
|
+
console.warn("description is not a valid JSON", description);
|
|
1009
|
+
return null;
|
|
1010
|
+
}
|
|
1011
|
+
const infoList = descriptionJson.map((item, index) => {
|
|
1012
|
+
const overallRect = {
|
|
1013
|
+
left: item.rect.x + rect.left,
|
|
1014
|
+
top: item.rect.y + rect.top,
|
|
1015
|
+
width: item.rect.width,
|
|
1016
|
+
height: item.rect.height
|
|
1017
|
+
};
|
|
1018
|
+
const center = [
|
|
1019
|
+
Math.round(overallRect.left + overallRect.width / 2),
|
|
1020
|
+
Math.round(overallRect.top + overallRect.height / 2)
|
|
1021
|
+
];
|
|
1022
|
+
const nodeType = item.text ? NodeType.TEXT : NodeType.CONTAINER;
|
|
1023
|
+
const nodeHashId = midsceneGenerateHash(
|
|
1024
|
+
null,
|
|
1025
|
+
`${index}-${item.text || ""}`,
|
|
1026
|
+
overallRect
|
|
1027
|
+
);
|
|
1028
|
+
return {
|
|
1029
|
+
id: nodeHashId,
|
|
1030
|
+
indexId: indexId++,
|
|
1031
|
+
zoom: 1,
|
|
1032
|
+
locator: "",
|
|
1033
|
+
nodeType,
|
|
1034
|
+
nodePath,
|
|
1035
|
+
nodeHashId,
|
|
1036
|
+
attributes: __spreadProps(__spreadValues({}, item.description ? { description: item.description } : {}), {
|
|
1037
|
+
nodeType
|
|
1038
|
+
}),
|
|
1039
|
+
content: item.text || "",
|
|
1040
|
+
rect: overallRect,
|
|
1041
|
+
center,
|
|
1042
|
+
screenWidth: window.innerWidth,
|
|
1043
|
+
screenHeight: window.innerHeight
|
|
1044
|
+
};
|
|
1045
|
+
});
|
|
1046
|
+
return infoList;
|
|
1047
|
+
} catch (e) {
|
|
1048
|
+
console.error(e);
|
|
1049
|
+
console.warn(
|
|
1050
|
+
"description is not a valid JSON",
|
|
1051
|
+
targetSelector,
|
|
1052
|
+
description
|
|
1053
|
+
);
|
|
1054
|
+
return null;
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
974
1057
|
if (isFormElement(node)) {
|
|
975
1058
|
const attributes = getNodeAttributes(node);
|
|
976
1059
|
let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
|
|
@@ -1141,6 +1224,10 @@ var midscene_element_inspector = (() => {
|
|
|
1141
1224
|
return null;
|
|
1142
1225
|
}
|
|
1143
1226
|
const elementInfo = collectElementInfo(node, nodePath, baseZoom);
|
|
1227
|
+
if (Array.isArray(elementInfo)) {
|
|
1228
|
+
elementInfoArray.push(...elementInfo);
|
|
1229
|
+
return null;
|
|
1230
|
+
}
|
|
1144
1231
|
if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.TEXT || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.FORM_ITEM || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.CONTAINER) {
|
|
1145
1232
|
elementInfoArray.push(elementInfo);
|
|
1146
1233
|
return elementInfo;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/web",
|
|
3
3
|
"description": "An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language. See https://midscenejs.com/ for details.",
|
|
4
|
-
"version": "0.8.
|
|
4
|
+
"version": "0.8.9-beta-20241223095005.0",
|
|
5
5
|
"repository": "https://github.com/web-infra-dev/midscene",
|
|
6
6
|
"homepage": "https://midscenejs.com/",
|
|
7
7
|
"jsnext:source": "./src/index.ts",
|
|
@@ -72,8 +72,8 @@
|
|
|
72
72
|
"express": "4.21.1",
|
|
73
73
|
"inquirer": "10.1.5",
|
|
74
74
|
"openai": "4.57.1",
|
|
75
|
-
"@midscene/core": "0.8.
|
|
76
|
-
"@midscene/shared": "0.8.
|
|
75
|
+
"@midscene/core": "0.8.9-beta-20241223095005.0",
|
|
76
|
+
"@midscene/shared": "0.8.9-beta-20241223095005.0"
|
|
77
77
|
},
|
|
78
78
|
"devDependencies": {
|
|
79
79
|
"@modern-js/module-tools": "2.60.6",
|