@midscene/web 0.8.9-beta-20241223095005.0 → 0.8.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -595,14 +595,6 @@ var midscene_element_inspector = (() => {
|
|
|
595
595
|
})(NodeType || {});
|
|
596
596
|
|
|
597
597
|
// src/extractor/dom-util.ts
|
|
598
|
-
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF = "midscene-description-ref";
|
|
599
|
-
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID = "midscene-description-id";
|
|
600
|
-
function isUserDescribedElement(node) {
|
|
601
|
-
if (node instanceof Element) {
|
|
602
|
-
return node.hasAttribute(USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF);
|
|
603
|
-
}
|
|
604
|
-
return false;
|
|
605
|
-
}
|
|
606
598
|
function isFormElement(node) {
|
|
607
599
|
return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
|
|
608
600
|
}
|
|
@@ -970,7 +962,7 @@ var midscene_element_inspector = (() => {
|
|
|
970
962
|
tagName = node.tagName.toLowerCase();
|
|
971
963
|
}
|
|
972
964
|
const parentElement = node.parentElement;
|
|
973
|
-
if (
|
|
965
|
+
if (parentElement && parentElement instanceof HTMLElement) {
|
|
974
966
|
tagName = parentElement.tagName.toLowerCase();
|
|
975
967
|
}
|
|
976
968
|
return tagName ? `<${tagName}>` : "";
|
|
@@ -981,81 +973,6 @@ var midscene_element_inspector = (() => {
|
|
|
981
973
|
if (!rect || rect.width < CONTAINER_MINI_WIDTH || rect.height < CONTAINER_MINI_HEIGHT) {
|
|
982
974
|
return null;
|
|
983
975
|
}
|
|
984
|
-
if (isUserDescribedElement(node)) {
|
|
985
|
-
console.log("isUserDescribedElement", node);
|
|
986
|
-
const element = node;
|
|
987
|
-
const descriptionId = element.getAttribute(
|
|
988
|
-
USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF
|
|
989
|
-
);
|
|
990
|
-
const targetSelector = `[${USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID}="${descriptionId}"]`;
|
|
991
|
-
const targetElement = document.querySelectorAll(targetSelector);
|
|
992
|
-
if (targetElement.length === 0) {
|
|
993
|
-
console.warn(
|
|
994
|
-
"cannot find element for Midscene description",
|
|
995
|
-
targetSelector
|
|
996
|
-
);
|
|
997
|
-
return null;
|
|
998
|
-
}
|
|
999
|
-
if (targetElement.length > 1) {
|
|
1000
|
-
console.warn(
|
|
1001
|
-
"multiple elements found for Midscene description",
|
|
1002
|
-
targetSelector
|
|
1003
|
-
);
|
|
1004
|
-
}
|
|
1005
|
-
const descriptionElement = targetElement[0];
|
|
1006
|
-
const description = descriptionElement.innerText;
|
|
1007
|
-
try {
|
|
1008
|
-
const descriptionJson = JSON.parse(description);
|
|
1009
|
-
if (!Array.isArray(descriptionJson)) {
|
|
1010
|
-
console.warn("description is not a valid JSON", description);
|
|
1011
|
-
return null;
|
|
1012
|
-
}
|
|
1013
|
-
const infoList = descriptionJson.map((item, index) => {
|
|
1014
|
-
const overallRect = {
|
|
1015
|
-
left: item.rect.x + rect.left,
|
|
1016
|
-
top: item.rect.y + rect.top,
|
|
1017
|
-
width: item.rect.width,
|
|
1018
|
-
height: item.rect.height
|
|
1019
|
-
};
|
|
1020
|
-
const center = [
|
|
1021
|
-
Math.round(overallRect.left + overallRect.width / 2),
|
|
1022
|
-
Math.round(overallRect.top + overallRect.height / 2)
|
|
1023
|
-
];
|
|
1024
|
-
const nodeType = item.text ? NodeType.TEXT : NodeType.CONTAINER;
|
|
1025
|
-
const nodeHashId = midsceneGenerateHash(
|
|
1026
|
-
null,
|
|
1027
|
-
`${index}-${item.text || ""}`,
|
|
1028
|
-
overallRect
|
|
1029
|
-
);
|
|
1030
|
-
return {
|
|
1031
|
-
id: nodeHashId,
|
|
1032
|
-
indexId: indexId++,
|
|
1033
|
-
zoom: 1,
|
|
1034
|
-
locator: "",
|
|
1035
|
-
nodeType,
|
|
1036
|
-
nodePath,
|
|
1037
|
-
nodeHashId,
|
|
1038
|
-
attributes: __spreadProps(__spreadValues({}, item.description ? { description: item.description } : {}), {
|
|
1039
|
-
nodeType
|
|
1040
|
-
}),
|
|
1041
|
-
content: item.text || "",
|
|
1042
|
-
rect: overallRect,
|
|
1043
|
-
center,
|
|
1044
|
-
screenWidth: window.innerWidth,
|
|
1045
|
-
screenHeight: window.innerHeight
|
|
1046
|
-
};
|
|
1047
|
-
});
|
|
1048
|
-
return infoList;
|
|
1049
|
-
} catch (e) {
|
|
1050
|
-
console.error(e);
|
|
1051
|
-
console.warn(
|
|
1052
|
-
"description is not a valid JSON",
|
|
1053
|
-
targetSelector,
|
|
1054
|
-
description
|
|
1055
|
-
);
|
|
1056
|
-
return null;
|
|
1057
|
-
}
|
|
1058
|
-
}
|
|
1059
976
|
if (isFormElement(node)) {
|
|
1060
977
|
const attributes = getNodeAttributes(node);
|
|
1061
978
|
let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
|
|
@@ -1226,10 +1143,6 @@ var midscene_element_inspector = (() => {
|
|
|
1226
1143
|
return null;
|
|
1227
1144
|
}
|
|
1228
1145
|
const elementInfo = collectElementInfo(node, nodePath, baseZoom);
|
|
1229
|
-
if (Array.isArray(elementInfo)) {
|
|
1230
|
-
elementInfoArray.push(...elementInfo);
|
|
1231
|
-
return null;
|
|
1232
|
-
}
|
|
1233
1146
|
if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.TEXT || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.FORM_ITEM || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.CONTAINER) {
|
|
1234
1147
|
elementInfoArray.push(elementInfo);
|
|
1235
1148
|
return elementInfo;
|
|
@@ -583,14 +583,6 @@ var midscene_element_inspector = (() => {
|
|
|
583
583
|
})(NodeType || {});
|
|
584
584
|
|
|
585
585
|
// src/extractor/dom-util.ts
|
|
586
|
-
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF = "midscene-description-ref";
|
|
587
|
-
var USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID = "midscene-description-id";
|
|
588
|
-
function isUserDescribedElement(node) {
|
|
589
|
-
if (node instanceof Element) {
|
|
590
|
-
return node.hasAttribute(USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF);
|
|
591
|
-
}
|
|
592
|
-
return false;
|
|
593
|
-
}
|
|
594
586
|
function isFormElement(node) {
|
|
595
587
|
return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
|
|
596
588
|
}
|
|
@@ -968,7 +960,7 @@ var midscene_element_inspector = (() => {
|
|
|
968
960
|
tagName = node.tagName.toLowerCase();
|
|
969
961
|
}
|
|
970
962
|
const parentElement = node.parentElement;
|
|
971
|
-
if (
|
|
963
|
+
if (parentElement && parentElement instanceof HTMLElement) {
|
|
972
964
|
tagName = parentElement.tagName.toLowerCase();
|
|
973
965
|
}
|
|
974
966
|
return tagName ? `<${tagName}>` : "";
|
|
@@ -979,81 +971,6 @@ var midscene_element_inspector = (() => {
|
|
|
979
971
|
if (!rect || rect.width < CONTAINER_MINI_WIDTH || rect.height < CONTAINER_MINI_HEIGHT) {
|
|
980
972
|
return null;
|
|
981
973
|
}
|
|
982
|
-
if (isUserDescribedElement(node)) {
|
|
983
|
-
console.log("isUserDescribedElement", node);
|
|
984
|
-
const element = node;
|
|
985
|
-
const descriptionId = element.getAttribute(
|
|
986
|
-
USER_DESCRIBED_ELEMENT_ATTRIBUTE_REF
|
|
987
|
-
);
|
|
988
|
-
const targetSelector = `[${USER_DESCRIBED_ELEMENT_ATTRIBUTE_ID}="${descriptionId}"]`;
|
|
989
|
-
const targetElement = document.querySelectorAll(targetSelector);
|
|
990
|
-
if (targetElement.length === 0) {
|
|
991
|
-
console.warn(
|
|
992
|
-
"cannot find element for Midscene description",
|
|
993
|
-
targetSelector
|
|
994
|
-
);
|
|
995
|
-
return null;
|
|
996
|
-
}
|
|
997
|
-
if (targetElement.length > 1) {
|
|
998
|
-
console.warn(
|
|
999
|
-
"multiple elements found for Midscene description",
|
|
1000
|
-
targetSelector
|
|
1001
|
-
);
|
|
1002
|
-
}
|
|
1003
|
-
const descriptionElement = targetElement[0];
|
|
1004
|
-
const description = descriptionElement.innerText;
|
|
1005
|
-
try {
|
|
1006
|
-
const descriptionJson = JSON.parse(description);
|
|
1007
|
-
if (!Array.isArray(descriptionJson)) {
|
|
1008
|
-
console.warn("description is not a valid JSON", description);
|
|
1009
|
-
return null;
|
|
1010
|
-
}
|
|
1011
|
-
const infoList = descriptionJson.map((item, index) => {
|
|
1012
|
-
const overallRect = {
|
|
1013
|
-
left: item.rect.x + rect.left,
|
|
1014
|
-
top: item.rect.y + rect.top,
|
|
1015
|
-
width: item.rect.width,
|
|
1016
|
-
height: item.rect.height
|
|
1017
|
-
};
|
|
1018
|
-
const center = [
|
|
1019
|
-
Math.round(overallRect.left + overallRect.width / 2),
|
|
1020
|
-
Math.round(overallRect.top + overallRect.height / 2)
|
|
1021
|
-
];
|
|
1022
|
-
const nodeType = item.text ? NodeType.TEXT : NodeType.CONTAINER;
|
|
1023
|
-
const nodeHashId = midsceneGenerateHash(
|
|
1024
|
-
null,
|
|
1025
|
-
`${index}-${item.text || ""}`,
|
|
1026
|
-
overallRect
|
|
1027
|
-
);
|
|
1028
|
-
return {
|
|
1029
|
-
id: nodeHashId,
|
|
1030
|
-
indexId: indexId++,
|
|
1031
|
-
zoom: 1,
|
|
1032
|
-
locator: "",
|
|
1033
|
-
nodeType,
|
|
1034
|
-
nodePath,
|
|
1035
|
-
nodeHashId,
|
|
1036
|
-
attributes: __spreadProps(__spreadValues({}, item.description ? { description: item.description } : {}), {
|
|
1037
|
-
nodeType
|
|
1038
|
-
}),
|
|
1039
|
-
content: item.text || "",
|
|
1040
|
-
rect: overallRect,
|
|
1041
|
-
center,
|
|
1042
|
-
screenWidth: window.innerWidth,
|
|
1043
|
-
screenHeight: window.innerHeight
|
|
1044
|
-
};
|
|
1045
|
-
});
|
|
1046
|
-
return infoList;
|
|
1047
|
-
} catch (e) {
|
|
1048
|
-
console.error(e);
|
|
1049
|
-
console.warn(
|
|
1050
|
-
"description is not a valid JSON",
|
|
1051
|
-
targetSelector,
|
|
1052
|
-
description
|
|
1053
|
-
);
|
|
1054
|
-
return null;
|
|
1055
|
-
}
|
|
1056
|
-
}
|
|
1057
974
|
if (isFormElement(node)) {
|
|
1058
975
|
const attributes = getNodeAttributes(node);
|
|
1059
976
|
let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
|
|
@@ -1224,10 +1141,6 @@ var midscene_element_inspector = (() => {
|
|
|
1224
1141
|
return null;
|
|
1225
1142
|
}
|
|
1226
1143
|
const elementInfo = collectElementInfo(node, nodePath, baseZoom);
|
|
1227
|
-
if (Array.isArray(elementInfo)) {
|
|
1228
|
-
elementInfoArray.push(...elementInfo);
|
|
1229
|
-
return null;
|
|
1230
|
-
}
|
|
1231
1144
|
if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.TEXT || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.FORM_ITEM || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.CONTAINER) {
|
|
1232
1145
|
elementInfoArray.push(elementInfo);
|
|
1233
1146
|
return elementInfo;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/web",
|
|
3
3
|
"description": "An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language. See https://midscenejs.com/ for details.",
|
|
4
|
-
"version": "0.8.9
|
|
4
|
+
"version": "0.8.9",
|
|
5
5
|
"repository": "https://github.com/web-infra-dev/midscene",
|
|
6
6
|
"homepage": "https://midscenejs.com/",
|
|
7
7
|
"jsnext:source": "./src/index.ts",
|
|
@@ -72,8 +72,8 @@
|
|
|
72
72
|
"express": "4.21.1",
|
|
73
73
|
"inquirer": "10.1.5",
|
|
74
74
|
"openai": "4.57.1",
|
|
75
|
-
"@midscene/
|
|
76
|
-
"@midscene/
|
|
75
|
+
"@midscene/shared": "0.8.9",
|
|
76
|
+
"@midscene/core": "0.8.9"
|
|
77
77
|
},
|
|
78
78
|
"devDependencies": {
|
|
79
79
|
"@modern-js/module-tools": "2.60.6",
|