@midscene/shared 0.16.9-beta-20250508125518.0 → 0.16.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/es/common.js +2 -0
  2. package/dist/es/env.d.ts +3 -1
  3. package/dist/es/env.js +3 -0
  4. package/dist/es/extractor-debug.js +16 -21
  5. package/dist/es/extractor.d.ts +47 -2
  6. package/dist/es/extractor.js +168 -104
  7. package/dist/es/fs.d.ts +1 -2
  8. package/dist/es/fs.js +4 -9
  9. package/dist/es/img.d.ts +1 -1
  10. package/dist/es/index-305e7a7e.d.ts +32 -0
  11. package/dist/es/logger.js +2 -0
  12. package/dist/lib/common.js +2 -0
  13. package/dist/lib/env.d.ts +3 -1
  14. package/dist/lib/env.js +4 -0
  15. package/dist/lib/extractor-debug.js +16 -21
  16. package/dist/lib/extractor.d.ts +47 -2
  17. package/dist/lib/extractor.js +169 -107
  18. package/dist/lib/fs.d.ts +1 -2
  19. package/dist/lib/fs.js +4 -10
  20. package/dist/lib/img.d.ts +1 -1
  21. package/dist/lib/index-305e7a7e.d.ts +32 -0
  22. package/dist/lib/logger.js +2 -0
  23. package/dist/script/htmlElement.js +166 -101
  24. package/dist/script/htmlElementDebug.js +16 -21
  25. package/dist/types/env.d.ts +3 -1
  26. package/dist/types/extractor.d.ts +47 -2
  27. package/dist/types/fs.d.ts +1 -2
  28. package/dist/types/img.d.ts +1 -1
  29. package/dist/types/index-305e7a7e.d.ts +32 -0
  30. package/package.json +2 -6
  31. package/src/env.ts +3 -0
  32. package/src/extractor/client-extractor.ts +211 -0
  33. package/src/extractor/index.ts +3 -5
  34. package/src/extractor/util.ts +2 -31
  35. package/src/extractor/web-extractor.ts +19 -2
  36. package/src/node/fs.ts +4 -12
  37. package/src/types/index.ts +0 -5
  38. package/dist/es/index-df4e781b.d.ts +0 -77
  39. package/dist/es/types.d.ts +0 -2
  40. package/dist/es/types.js +0 -6
  41. package/dist/lib/index-df4e781b.d.ts +0 -77
  42. package/dist/lib/types.d.ts +0 -2
  43. package/dist/lib/types.js +0 -31
  44. package/dist/types/index-df4e781b.d.ts +0 -77
  45. package/dist/types/types.d.ts +0 -2
  46. package/src/extractor/locator.ts +0 -117
@@ -578,10 +578,8 @@ var midscene_element_inspector = (() => {
578
578
  // src/extractor/index.ts
579
579
  var extractor_exports = {};
580
580
  __export(extractor_exports, {
581
+ clientExtractTextWithPosition: () => extractTextWithPosition2,
581
582
  descriptionOfTree: () => descriptionOfTree,
582
- getElementInfoByNode: () => getElementInfoByNode,
583
- getXpathsById: () => getXpathsById,
584
- setNodeHashCacheListOnWindow: () => setNodeHashCacheListOnWindow,
585
583
  traverseTree: () => traverseTree,
586
584
  treeToList: () => treeToList,
587
585
  webExtractNodeTree: () => extractTreeNode,
@@ -1075,33 +1073,8 @@ ${indentStr}${after}`;
1075
1073
  }
1076
1074
  function midsceneGenerateHash(node, content, rect) {
1077
1075
  const slicedHash = generateHashId(rect, content);
1078
- if (node) {
1079
- if (!window.midsceneNodeHashCacheList) {
1080
- setNodeHashCacheListOnWindow();
1081
- }
1082
- setNodeToCacheList(node, slicedHash);
1083
- }
1084
1076
  return slicedHash;
1085
1077
  }
1086
- function setNodeHashCacheListOnWindow() {
1087
- if (typeof window !== "undefined") {
1088
- window.midsceneNodeHashCacheList = [];
1089
- }
1090
- }
1091
- function setNodeToCacheList(node, id) {
1092
- if (typeof window !== "undefined") {
1093
- window.midsceneNodeHashCacheList.push({ node, id });
1094
- }
1095
- }
1096
- function getNodeFromCacheList(id) {
1097
- var _a;
1098
- if (typeof window !== "undefined") {
1099
- return (_a = window.midsceneNodeHashCacheList.find(
1100
- (item) => item.id === id
1101
- )) == null ? void 0 : _a.node;
1102
- }
1103
- return null;
1104
- }
1105
1078
  function getTopDocument() {
1106
1079
  const container = document.body || document;
1107
1080
  return container;
@@ -1162,7 +1135,9 @@ ${indentStr}${after}`;
1162
1135
  Math.round(rect.left + rect.width / 2),
1163
1136
  Math.round(rect.top + rect.height / 2)
1164
1137
  ],
1165
- zoom: rect.zoom
1138
+ zoom: rect.zoom,
1139
+ screenWidth: currentWindow.innerWidth,
1140
+ screenHeight: currentWindow.innerHeight
1166
1141
  };
1167
1142
  return elementInfo;
1168
1143
  }
@@ -1188,7 +1163,9 @@ ${indentStr}${after}`;
1188
1163
  Math.round(rect.left + rect.width / 2),
1189
1164
  Math.round(rect.top + rect.height / 2)
1190
1165
  ],
1191
- zoom: rect.zoom
1166
+ zoom: rect.zoom,
1167
+ screenWidth: currentWindow.innerWidth,
1168
+ screenHeight: currentWindow.innerHeight
1192
1169
  };
1193
1170
  return elementInfo;
1194
1171
  }
@@ -1214,7 +1191,9 @@ ${indentStr}${after}`;
1214
1191
  Math.round(rect.left + rect.width / 2),
1215
1192
  Math.round(rect.top + rect.height / 2)
1216
1193
  ],
1217
- zoom: rect.zoom
1194
+ zoom: rect.zoom,
1195
+ screenWidth: currentWindow.innerWidth,
1196
+ screenHeight: currentWindow.innerHeight
1218
1197
  };
1219
1198
  return elementInfo;
1220
1199
  }
@@ -1244,9 +1223,12 @@ ${indentStr}${after}`;
1244
1223
  Math.round(rect.left + rect.width / 2),
1245
1224
  Math.round(rect.top + rect.height / 2)
1246
1225
  ],
1226
+ // attributes,
1247
1227
  content: text,
1248
1228
  rect,
1249
- zoom: rect.zoom
1229
+ zoom: rect.zoom,
1230
+ screenWidth: currentWindow.innerWidth,
1231
+ screenHeight: currentWindow.innerHeight
1250
1232
  };
1251
1233
  return elementInfo;
1252
1234
  }
@@ -1270,7 +1252,9 @@ ${indentStr}${after}`;
1270
1252
  Math.round(rect.left + rect.width / 2),
1271
1253
  Math.round(rect.top + rect.height / 2)
1272
1254
  ],
1273
- zoom: rect.zoom
1255
+ zoom: rect.zoom,
1256
+ screenWidth: currentWindow.innerWidth,
1257
+ screenHeight: currentWindow.innerHeight
1274
1258
  };
1275
1259
  return elementInfo;
1276
1260
  }
@@ -1379,83 +1363,164 @@ ${indentStr}${after}`;
1379
1363
  };
1380
1364
  }
1381
1365
 
1382
- // src/extractor/locator.ts
1383
- var getElementIndex = (element) => {
1384
- let index = 1;
1385
- let prev = element.previousElementSibling;
1386
- while (prev) {
1387
- if (prev.nodeName.toLowerCase() === element.nodeName.toLowerCase()) {
1388
- index++;
1366
+ // src/extractor/client-extractor.ts
1367
+ function getNodeAttributes2(node) {
1368
+ var _a;
1369
+ const attrs = {};
1370
+ if (node && node.nodeType === 1) {
1371
+ const element = node;
1372
+ for (let i = 0; i < element.attributes.length; i++) {
1373
+ const attr = element.attributes[i];
1374
+ attrs[attr.nodeName] = (_a = attr.nodeValue) != null ? _a : "";
1389
1375
  }
1390
- prev = prev.previousElementSibling;
1391
1376
  }
1392
- return index;
1393
- };
1394
- var findFirstAncestorWithId = (element) => {
1395
- let current = element;
1396
- while (current == null ? void 0 : current.parentElement) {
1397
- if (current.id) {
1398
- return current;
1399
- }
1400
- current = current.parentElement;
1377
+ return attrs;
1378
+ }
1379
+ function getRect2(attributes) {
1380
+ var _a, _b, _c, _d;
1381
+ const x = Math.round(Number.parseFloat((_a = attributes.x) != null ? _a : "0"));
1382
+ const y = Math.round(Number.parseFloat((_b = attributes.y) != null ? _b : "0"));
1383
+ const width = Math.round(Number.parseFloat((_c = attributes.width) != null ? _c : "0"));
1384
+ const height = Math.round(Number.parseFloat((_d = attributes.height) != null ? _d : "0"));
1385
+ return {
1386
+ left: Math.max(0, Math.floor(x)),
1387
+ top: Math.max(0, Math.floor(y)),
1388
+ width: Math.max(0, width),
1389
+ height: Math.max(0, height)
1390
+ };
1391
+ }
1392
+ function validTextNodeContent(node) {
1393
+ var _a;
1394
+ if (node.nodeType === 3) {
1395
+ return ((_a = node.nodeValue) == null ? void 0 : _a.trim()) || "";
1401
1396
  }
1402
- return null;
1403
- };
1404
- var getElementXPath = (element) => {
1405
- if (element.nodeType !== Node.ELEMENT_NODE)
1397
+ return "";
1398
+ }
1399
+ function getXPathForElement(element) {
1400
+ if (element.nodeType !== 1) {
1406
1401
  return "";
1407
- const el = element;
1408
- if (el === document.documentElement) {
1409
- return "/html";
1410
- }
1411
- if (el === document.body) {
1412
- return "/html/body";
1413
1402
  }
1414
- if (el.id) {
1415
- return `//*[@id="${el.id}"]`;
1416
- }
1417
- const ancestorWithId = findFirstAncestorWithId(el);
1418
- if (ancestorWithId) {
1419
- const ancestorPath = `//*[@id="${ancestorWithId.id}"]`;
1420
- let current = el;
1421
- const pathParts = [];
1422
- while (current && current !== ancestorWithId) {
1423
- const index2 = getElementIndex(current);
1424
- const tagName2 = current.nodeName.toLowerCase();
1425
- pathParts.unshift(`${tagName2}[${index2}]`);
1426
- current = current.parentElement;
1403
+ const getIndex = (sib, name) => {
1404
+ let count = 1;
1405
+ for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) {
1406
+ if (cur.nodeType === 1 && cur.nodeName === name) {
1407
+ count++;
1408
+ }
1427
1409
  }
1428
- return pathParts.length > 0 ? `${ancestorPath}/${pathParts.join("/")}` : ancestorPath;
1429
- }
1430
- if (!el.parentNode) {
1431
- return `/${el.nodeName.toLowerCase()}`;
1432
- }
1433
- const index = getElementIndex(el);
1434
- const tagName = el.nodeName.toLowerCase();
1435
- if (el.parentNode) {
1436
- const parentXPath = getElementXPath(el.parentNode);
1437
- return `${parentXPath}/${tagName}[${index}]`;
1438
- }
1439
- return `/${tagName}[${index}]`;
1440
- };
1441
- function generateXPaths(node) {
1442
- if (!node)
1443
- return [];
1444
- const xPath = getElementXPath(node);
1445
- return [xPath];
1410
+ return count;
1411
+ };
1412
+ const buildAttributePart = (elem) => {
1413
+ const attributes = ["id", "resource-id", "content-desc", "class"];
1414
+ for (const attr of attributes) {
1415
+ if (elem.hasAttribute(attr)) {
1416
+ const value = elem.getAttribute(attr);
1417
+ if (value && value.trim() !== "") {
1418
+ return `[@${attr}="${value}"]`;
1419
+ }
1420
+ }
1421
+ }
1422
+ return "";
1423
+ };
1424
+ const getPath = (node, path = "") => {
1425
+ if (node.parentNode) {
1426
+ path = getPath(node.parentNode, path);
1427
+ }
1428
+ if (node.nodeType === 1) {
1429
+ const elem = node;
1430
+ const tagName = elem.nodeName.toLowerCase();
1431
+ let part = `/${tagName}`;
1432
+ const attributePart = buildAttributePart(elem);
1433
+ if (attributePart) {
1434
+ part += attributePart;
1435
+ } else {
1436
+ const index = getIndex(node, node.nodeName);
1437
+ if (index > 1) {
1438
+ part += `[${index}]`;
1439
+ }
1440
+ }
1441
+ path += part;
1442
+ }
1443
+ return path;
1444
+ };
1445
+ return getPath(element);
1446
1446
  }
1447
- function getXpathsById(id) {
1448
- let node = getNodeFromCacheList(id);
1449
- if (!node) {
1450
- return null;
1447
+ function extractTextWithPosition2(initNode) {
1448
+ const elementInfoArray = [];
1449
+ let nodeIndex = 1;
1450
+ function dfs(node, parentNode = null) {
1451
+ if (!node) {
1452
+ return;
1453
+ }
1454
+ const currentNodeDes = { node, children: [] };
1455
+ if (parentNode) {
1456
+ parentNode.children.push(currentNodeDes);
1457
+ }
1458
+ collectElementInfo2(node);
1459
+ if (node.childNodes && node.childNodes.length > 0) {
1460
+ for (let i = 0; i < node.childNodes.length; i++) {
1461
+ dfs(node.childNodes[i], currentNodeDes);
1462
+ }
1463
+ }
1451
1464
  }
1452
- if (isTextElement(node)) {
1453
- node = node.parentElement;
1465
+ function collectElementInfo2(node) {
1466
+ const attributes = getNodeAttributes2(node);
1467
+ const rect = getRect2(attributes);
1468
+ const nodeHashId = midsceneGenerateHash(null, attributes.placeholder, rect);
1469
+ const text = validTextNodeContent(node);
1470
+ let nodeType;
1471
+ switch (node.nodeName.toUpperCase()) {
1472
+ case "TEXT":
1473
+ nodeType = "TEXT Node" /* TEXT */;
1474
+ break;
1475
+ case "IMAGE":
1476
+ nodeType = "IMG Node" /* IMG */;
1477
+ break;
1478
+ case "BUTTON":
1479
+ nodeType = "BUTTON Node" /* BUTTON */;
1480
+ break;
1481
+ case "SEARCHINPUT":
1482
+ case "TEXTINPUT":
1483
+ case "INPUT":
1484
+ nodeType = "FORM_ITEM Node" /* FORM_ITEM */;
1485
+ break;
1486
+ case "NAV":
1487
+ case "LIST":
1488
+ case "CELL":
1489
+ nodeType = "CONTAINER Node" /* CONTAINER */;
1490
+ break;
1491
+ default:
1492
+ if (attributes.id === "android:id/input" || attributes.id === "android:id/inputArea") {
1493
+ nodeType = "FORM_ITEM Node" /* FORM_ITEM */;
1494
+ } else {
1495
+ nodeType = "CONTAINER Node" /* CONTAINER */;
1496
+ }
1497
+ break;
1498
+ }
1499
+ const xpath = getXPathForElement(node);
1500
+ const elementInfo = {
1501
+ id: nodeHashId,
1502
+ indexId: nodeIndex++,
1503
+ nodeHashId,
1504
+ locator: xpath,
1505
+ attributes: __spreadValues({
1506
+ nodeType
1507
+ }, attributes),
1508
+ content: text,
1509
+ rect,
1510
+ center: [
1511
+ Math.round(rect.left + rect.width / 2),
1512
+ Math.round(rect.top + rect.height / 2)
1513
+ ],
1514
+ nodeType
1515
+ };
1516
+ if (elementInfo.nodeType !== "CONTAINER Node" /* CONTAINER */) {
1517
+ elementInfoArray.push(elementInfo);
1518
+ }
1454
1519
  }
1455
- return generateXPaths(node);
1456
- }
1457
- function getElementInfoByNode(node) {
1458
- return collectElementInfo(node, window, document);
1520
+ const rootNode = initNode;
1521
+ const rootDescriptor = { node: rootNode, children: [] };
1522
+ dfs(rootNode, rootDescriptor);
1523
+ return elementInfoArray;
1459
1524
  }
1460
1525
  return __toCommonJS(extractor_exports);
1461
1526
  })();
@@ -934,24 +934,8 @@ var midscene_element_inspector = (() => {
934
934
  }
935
935
  function midsceneGenerateHash(node, content, rect) {
936
936
  const slicedHash = generateHashId(rect, content);
937
- if (node) {
938
- if (!window.midsceneNodeHashCacheList) {
939
- setNodeHashCacheListOnWindow();
940
- }
941
- setNodeToCacheList(node, slicedHash);
942
- }
943
937
  return slicedHash;
944
938
  }
945
- function setNodeHashCacheListOnWindow() {
946
- if (typeof window !== "undefined") {
947
- window.midsceneNodeHashCacheList = [];
948
- }
949
- }
950
- function setNodeToCacheList(node, id) {
951
- if (typeof window !== "undefined") {
952
- window.midsceneNodeHashCacheList.push({ node, id });
953
- }
954
- }
955
939
  function setMidsceneVisibleRectOnWindow() {
956
940
  if (typeof window !== "undefined") {
957
941
  window.midsceneVisibleRect = visibleRect;
@@ -1022,7 +1006,9 @@ var midscene_element_inspector = (() => {
1022
1006
  Math.round(rect.left + rect.width / 2),
1023
1007
  Math.round(rect.top + rect.height / 2)
1024
1008
  ],
1025
- zoom: rect.zoom
1009
+ zoom: rect.zoom,
1010
+ screenWidth: currentWindow.innerWidth,
1011
+ screenHeight: currentWindow.innerHeight
1026
1012
  };
1027
1013
  return elementInfo;
1028
1014
  }
@@ -1048,7 +1034,9 @@ var midscene_element_inspector = (() => {
1048
1034
  Math.round(rect.left + rect.width / 2),
1049
1035
  Math.round(rect.top + rect.height / 2)
1050
1036
  ],
1051
- zoom: rect.zoom
1037
+ zoom: rect.zoom,
1038
+ screenWidth: currentWindow.innerWidth,
1039
+ screenHeight: currentWindow.innerHeight
1052
1040
  };
1053
1041
  return elementInfo;
1054
1042
  }
@@ -1074,7 +1062,9 @@ var midscene_element_inspector = (() => {
1074
1062
  Math.round(rect.left + rect.width / 2),
1075
1063
  Math.round(rect.top + rect.height / 2)
1076
1064
  ],
1077
- zoom: rect.zoom
1065
+ zoom: rect.zoom,
1066
+ screenWidth: currentWindow.innerWidth,
1067
+ screenHeight: currentWindow.innerHeight
1078
1068
  };
1079
1069
  return elementInfo;
1080
1070
  }
@@ -1104,9 +1094,12 @@ var midscene_element_inspector = (() => {
1104
1094
  Math.round(rect.left + rect.width / 2),
1105
1095
  Math.round(rect.top + rect.height / 2)
1106
1096
  ],
1097
+ // attributes,
1107
1098
  content: text,
1108
1099
  rect,
1109
- zoom: rect.zoom
1100
+ zoom: rect.zoom,
1101
+ screenWidth: currentWindow.innerWidth,
1102
+ screenHeight: currentWindow.innerHeight
1110
1103
  };
1111
1104
  return elementInfo;
1112
1105
  }
@@ -1130,7 +1123,9 @@ var midscene_element_inspector = (() => {
1130
1123
  Math.round(rect.left + rect.width / 2),
1131
1124
  Math.round(rect.top + rect.height / 2)
1132
1125
  ],
1133
- zoom: rect.zoom
1126
+ zoom: rect.zoom,
1127
+ screenWidth: currentWindow.innerWidth,
1128
+ screenHeight: currentWindow.innerHeight
1134
1129
  };
1135
1130
  return elementInfo;
1136
1131
  }
@@ -8,6 +8,7 @@ declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
8
8
  declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
9
9
  declare const MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
10
10
  declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
11
+ declare const MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
11
12
  declare const OPENAI_API_KEY = "OPENAI_API_KEY";
12
13
  declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
13
14
  declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -50,6 +51,7 @@ declare const allConfigFromEnv: () => {
50
51
  MATCH_BY_POSITION: string | undefined;
51
52
  MIDSCENE_REPORT_TAG_NAME: string | undefined;
52
53
  MIDSCENE_OPENAI_SOCKS_PROXY: string | undefined;
54
+ MIDSCENE_OPENAI_HTTP_PROXY: string | undefined;
53
55
  MIDSCENE_USE_AZURE_OPENAI: string | undefined;
54
56
  MIDSCENE_AZURE_OPENAI_SCOPE: string | undefined;
55
57
  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON: string | undefined;
@@ -80,4 +82,4 @@ declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfi
80
82
  declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
81
83
  declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
82
84
 
83
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
85
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
@@ -1,2 +1,47 @@
1
- import './constants.js';
2
- export { E as ElementInfo, a as ElementNode, d as descriptionOfTree, h as getElementInfoByNode, g as getXpathsById, s as setNodeHashCacheListOnWindow, t as traverseTree, b as treeToList, c as webExtractNodeTree, f as webExtractNodeTreeAsString, e as webExtractTextWithPosition } from './index-df4e781b.js';
1
+ import { NodeType } from './constants.js';
2
+ import { B as BaseElement, E as ElementTreeNode } from './index-305e7a7e.js';
3
+
4
+ declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean): string;
5
+ declare function treeToList<T extends BaseElement>(tree: ElementTreeNode<T>): T[];
6
+ declare function traverseTree<T extends BaseElement, ReturnNodeType extends BaseElement>(tree: ElementTreeNode<T>, onNode: (node: T) => ReturnNodeType): ElementTreeNode<ReturnNodeType>;
7
+
8
+ interface WebElementInfo extends ElementInfo {
9
+ zoom: number;
10
+ screenWidth?: number;
11
+ screenHeight?: number;
12
+ }
13
+ interface WebElementNode {
14
+ node: WebElementInfo | null;
15
+ children: WebElementNode[];
16
+ }
17
+ declare function extractTextWithPosition$1(initNode: globalThis.Node, debugMode?: boolean): WebElementInfo[];
18
+ declare function extractTreeNodeAsString(initNode: globalThis.Node, debugMode?: boolean): string;
19
+ declare function extractTreeNode(initNode: globalThis.Node, debugMode?: boolean): WebElementNode;
20
+
21
+ declare function extractTextWithPosition(initNode: globalThis.Document): ElementInfo[];
22
+
23
+ interface ElementInfo {
24
+ id: string;
25
+ indexId: number;
26
+ nodeHashId: string;
27
+ locator: string;
28
+ attributes: {
29
+ nodeType: NodeType;
30
+ [key: string]: string;
31
+ };
32
+ nodeType: NodeType;
33
+ content: string;
34
+ rect: {
35
+ left: number;
36
+ top: number;
37
+ width: number;
38
+ height: number;
39
+ };
40
+ center: [number, number];
41
+ }
42
+ interface ElementNode {
43
+ node: ElementInfo | null;
44
+ children: ElementNode[];
45
+ }
46
+
47
+ export { type ElementInfo, type ElementNode, extractTextWithPosition as clientExtractTextWithPosition, descriptionOfTree, traverseTree, treeToList, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition$1 as webExtractTextWithPosition };
@@ -10,7 +10,6 @@ declare function getRunningPkgInfo(dir?: string): PkgInfo | null;
10
10
  * @returns {string|null} - The most recent package.json file path or null
11
11
  */
12
12
  declare function findNearestPackageJson(dir: string): string | null;
13
- declare function getElementInfosScriptContent(): string;
14
13
  declare function getExtraReturnLogic(tree?: boolean): Promise<string | null>;
15
14
 
16
- export { findNearestPackageJson, getElementInfosScriptContent, getExtraReturnLogic, getRunningPkgInfo };
15
+ export { findNearestPackageJson, getExtraReturnLogic, getRunningPkgInfo };
@@ -1,6 +1,6 @@
1
1
  import { Buffer } from 'node:buffer';
2
2
  import Jimp from 'jimp';
3
- import { R as Rect, B as BaseElement } from './index-df4e781b.js';
3
+ import { R as Rect, B as BaseElement } from './index-305e7a7e.js';
4
4
  import './constants.js';
5
5
 
6
6
  interface Size {
@@ -0,0 +1,32 @@
1
+ import { NodeType } from './constants.js';
2
+
3
+ interface Point {
4
+ left: number;
5
+ top: number;
6
+ }
7
+ interface Size {
8
+ width: number;
9
+ height: number;
10
+ dpr?: number;
11
+ }
12
+ type Rect = Point & Size & {
13
+ zoom?: number;
14
+ };
15
+ declare abstract class BaseElement {
16
+ abstract id: string;
17
+ abstract indexId?: number;
18
+ abstract attributes: {
19
+ nodeType: NodeType;
20
+ [key: string]: string;
21
+ };
22
+ abstract content: string;
23
+ abstract rect: Rect;
24
+ abstract center: [number, number];
25
+ abstract locator?: string;
26
+ }
27
+ interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
28
+ node: ElementType | null;
29
+ children: ElementTreeNode<ElementType>[];
30
+ }
31
+
32
+ export { BaseElement as B, type ElementTreeNode as E, type Rect as R };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@midscene/shared",
3
- "version": "0.16.9-beta-20250508125518.0",
3
+ "version": "0.16.9",
4
4
  "repository": "https://github.com/web-infra-dev/midscene",
5
5
  "homepage": "https://midscenejs.com/",
6
6
  "types": "./dist/types/index.d.ts",
@@ -17,8 +17,7 @@
17
17
  "./keyboard-layout": "./dist/lib/us-keyboard-layout.js",
18
18
  "./logger": "./dist/lib/logger.js",
19
19
  "./common": "./dist/lib/common.js",
20
- "./env": "./dist/lib/env.js",
21
- "./types": "./dist/lib/types.js"
20
+ "./env": "./dist/lib/env.js"
22
21
  },
23
22
  "typesVersions": {
24
23
  "*": {
@@ -54,9 +53,6 @@
54
53
  ],
55
54
  "env": [
56
55
  "./dist/types/env.d.ts"
57
- ],
58
- "types": [
59
- "./dist/types/types.d.ts"
60
56
  ]
61
57
  }
62
58
  },
package/src/env.ts CHANGED
@@ -14,6 +14,7 @@ export const MIDSCENE_MCP_USE_PUPPETEER_MODE =
14
14
  export const MIDSCENE_FORCE_DEEP_THINK = 'MIDSCENE_FORCE_DEEP_THINK';
15
15
 
16
16
  export const MIDSCENE_OPENAI_SOCKS_PROXY = 'MIDSCENE_OPENAI_SOCKS_PROXY';
17
+ export const MIDSCENE_OPENAI_HTTP_PROXY = 'MIDSCENE_OPENAI_HTTP_PROXY';
17
18
  export const OPENAI_API_KEY = 'OPENAI_API_KEY';
18
19
  export const OPENAI_BASE_URL = 'OPENAI_BASE_URL';
19
20
  export const OPENAI_MAX_TOKENS = 'OPENAI_MAX_TOKENS';
@@ -75,6 +76,8 @@ export const allConfigFromEnv = () => {
75
76
  process.env[MIDSCENE_REPORT_TAG_NAME] || undefined,
76
77
  [MIDSCENE_OPENAI_SOCKS_PROXY]:
77
78
  process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || undefined,
79
+ [MIDSCENE_OPENAI_HTTP_PROXY]:
80
+ process.env[MIDSCENE_OPENAI_HTTP_PROXY] || undefined,
78
81
  [MIDSCENE_USE_AZURE_OPENAI]:
79
82
  process.env[MIDSCENE_USE_AZURE_OPENAI] || undefined,
80
83
  [MIDSCENE_AZURE_OPENAI_SCOPE]: