@midscene/shared 0.17.2-beta-20250523061809.0 → 0.17.2-beta-20250523075813.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/dist/es/constants.d.ts +1 -1
  2. package/dist/es/constants.js +1 -1
  3. package/dist/es/extractor-debug.js +62 -8
  4. package/dist/es/extractor.js +62 -8
  5. package/dist/lib/constants.d.ts +1 -1
  6. package/dist/lib/constants.js +1 -1
  7. package/dist/lib/extractor-debug.js +62 -8
  8. package/dist/lib/extractor.js +62 -8
  9. package/dist/script/htmlElement.js +61 -8
  10. package/dist/script/htmlElementDebug.js +61 -8
  11. package/dist/types/constants.d.ts +1 -1
  12. package/package.json +1 -1
  13. package/src/constants/index.ts +1 -1
  14. package/src/extractor/web-extractor.ts +63 -0
package/dist/es/constants.d.ts CHANGED
@@ -6,7 +6,7 @@ declare enum NodeType {
6
6
  CONTAINER = "CONTAINER Node",
7
7
  FORM_ITEM = "FORM_ITEM Node",
8
8
  BUTTON = "BUTTON Node",
9
- A = "A Node",
9
+ A = "Anchor Node",
10
10
  IMG = "IMG Node",
11
11
  TEXT = "TEXT Node",
12
12
  POSITION = "POSITION Node"
package/dist/es/constants.js CHANGED
@@ -7,7 +7,7 @@ var NodeType = /* @__PURE__ */ ((NodeType2) => {
7
7
  NodeType2["CONTAINER"] = "CONTAINER Node";
8
8
  NodeType2["FORM_ITEM"] = "FORM_ITEM Node";
9
9
  NodeType2["BUTTON"] = "BUTTON Node";
10
- NodeType2["A"] = "A Node";
10
+ NodeType2["A"] = "Anchor Node";
11
11
  NodeType2["IMG"] = "IMG Node";
12
12
  NodeType2["TEXT"] = "TEXT Node";
13
13
  NodeType2["POSITION"] = "POSITION Node";
package/dist/es/extractor-debug.js CHANGED
@@ -475,10 +475,20 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
475
475
  return elementInfo;
476
476
  }
477
477
  if (isButtonElement(node)) {
478
+ const rect2 = mergeElementAndChildrenRects(
479
+ node,
480
+ currentWindow,
481
+ currentDocument,
482
+ baseZoom,
483
+ visibleOnly
484
+ );
485
+ if (!rect2) {
486
+ return null;
487
+ }
478
488
  const attributes = getNodeAttributes(node, currentWindow);
479
489
  const pseudo = getPseudoElementContent(node, currentWindow);
480
490
  const content = node.innerText || pseudo.before || pseudo.after || "";
481
- const nodeHashId = midsceneGenerateHash(node, content, rect);
491
+ const nodeHashId = midsceneGenerateHash(node, content, rect2);
482
492
  const selector = setDataForNode(node, nodeHashId, false, currentWindow);
483
493
  const elementInfo = {
484
494
  id: nodeHashId,
@@ -492,12 +502,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
492
502
  nodeType: "BUTTON Node" /* BUTTON */
493
503
  },
494
504
  content,
495
- rect,
505
+ rect: rect2,
496
506
  center: [
497
- Math.round(rect.left + rect.width / 2),
498
- Math.round(rect.top + rect.height / 2)
507
+ Math.round(rect2.left + rect2.width / 2),
508
+ Math.round(rect2.top + rect2.height / 2)
499
509
  ],
500
- zoom: rect.zoom
510
+ zoom: rect2.zoom
501
511
  };
502
512
  return elementInfo;
503
513
  }
@@ -572,12 +582,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
572
582
  id: nodeHashId,
573
583
  indexId: indexId++,
574
584
  nodeHashId,
575
- nodeType: "A Node" /* A */,
585
+ nodeType: "Anchor Node" /* A */,
576
586
  locator: selector,
577
587
  attributes: {
578
588
  ...attributes,
579
589
  htmlTagName: tagNameOfNode(node),
580
- nodeType: "A Node" /* A */
590
+ nodeType: "Anchor Node" /* A */
581
591
  },
582
592
  content,
583
593
  rect,
@@ -659,7 +669,7 @@ function extractTreeNode(initNode, debugMode2 = false) {
659
669
  node: elementInfo,
660
670
  children: []
661
671
  };
662
- if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "A Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
672
+ if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "Anchor Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
663
673
  return nodeInfo;
664
674
  }
665
675
  const rect = getRect(node, baseZoom, currentWindow);
@@ -714,6 +724,50 @@ function extractTreeNode(initNode, debugMode2 = false) {
714
724
  children: topChildren
715
725
  };
716
726
  }
727
+ function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
728
+ const selfRect = elementRect(
729
+ node,
730
+ currentWindow,
731
+ currentDocument,
732
+ baseZoom,
733
+ visibleOnly
734
+ );
735
+ if (!selfRect)
736
+ return null;
737
+ let minLeft = selfRect.left;
738
+ let minTop = selfRect.top;
739
+ let maxRight = selfRect.left + selfRect.width;
740
+ let maxBottom = selfRect.top + selfRect.height;
741
+ function traverse(child) {
742
+ for (let i = 0; i < child.childNodes.length; i++) {
743
+ const sub = child.childNodes[i];
744
+ if (sub.nodeType === 1) {
745
+ const rect = elementRect(
746
+ sub,
747
+ currentWindow,
748
+ currentDocument,
749
+ baseZoom,
750
+ visibleOnly
751
+ );
752
+ if (rect) {
753
+ minLeft = Math.min(minLeft, rect.left);
754
+ minTop = Math.min(minTop, rect.top);
755
+ maxRight = Math.max(maxRight, rect.left + rect.width);
756
+ maxBottom = Math.max(maxBottom, rect.top + rect.height);
757
+ }
758
+ traverse(sub);
759
+ }
760
+ }
761
+ }
762
+ traverse(node);
763
+ return {
764
+ ...selfRect,
765
+ left: minLeft,
766
+ top: minTop,
767
+ width: maxRight - minLeft,
768
+ height: maxBottom - minTop
769
+ };
770
+ }
717
771
 
718
772
  // src/extractor/debug.ts
719
773
  console.log(extractTextWithPosition(document.body, true));
package/dist/es/extractor.js CHANGED
@@ -603,10 +603,20 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
603
603
  return elementInfo;
604
604
  }
605
605
  if (isButtonElement(node)) {
606
+ const rect2 = mergeElementAndChildrenRects(
607
+ node,
608
+ currentWindow,
609
+ currentDocument,
610
+ baseZoom,
611
+ visibleOnly
612
+ );
613
+ if (!rect2) {
614
+ return null;
615
+ }
606
616
  const attributes = getNodeAttributes(node, currentWindow);
607
617
  const pseudo = getPseudoElementContent(node, currentWindow);
608
618
  const content = node.innerText || pseudo.before || pseudo.after || "";
609
- const nodeHashId = midsceneGenerateHash(node, content, rect);
619
+ const nodeHashId = midsceneGenerateHash(node, content, rect2);
610
620
  const selector = setDataForNode(node, nodeHashId, false, currentWindow);
611
621
  const elementInfo = {
612
622
  id: nodeHashId,
@@ -620,12 +630,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
620
630
  nodeType: "BUTTON Node" /* BUTTON */
621
631
  },
622
632
  content,
623
- rect,
633
+ rect: rect2,
624
634
  center: [
625
- Math.round(rect.left + rect.width / 2),
626
- Math.round(rect.top + rect.height / 2)
635
+ Math.round(rect2.left + rect2.width / 2),
636
+ Math.round(rect2.top + rect2.height / 2)
627
637
  ],
628
- zoom: rect.zoom
638
+ zoom: rect2.zoom
629
639
  };
630
640
  return elementInfo;
631
641
  }
@@ -700,12 +710,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
700
710
  id: nodeHashId,
701
711
  indexId: indexId++,
702
712
  nodeHashId,
703
- nodeType: "A Node" /* A */,
713
+ nodeType: "Anchor Node" /* A */,
704
714
  locator: selector,
705
715
  attributes: {
706
716
  ...attributes,
707
717
  htmlTagName: tagNameOfNode(node),
708
- nodeType: "A Node" /* A */
718
+ nodeType: "Anchor Node" /* A */
709
719
  },
710
720
  content,
711
721
  rect,
@@ -791,7 +801,7 @@ function extractTreeNode(initNode, debugMode2 = false) {
791
801
  node: elementInfo,
792
802
  children: []
793
803
  };
794
- if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "A Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
804
+ if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "Anchor Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
795
805
  return nodeInfo;
796
806
  }
797
807
  const rect = getRect(node, baseZoom, currentWindow);
@@ -846,6 +856,50 @@ function extractTreeNode(initNode, debugMode2 = false) {
846
856
  children: topChildren
847
857
  };
848
858
  }
859
+ function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
860
+ const selfRect = elementRect(
861
+ node,
862
+ currentWindow,
863
+ currentDocument,
864
+ baseZoom,
865
+ visibleOnly
866
+ );
867
+ if (!selfRect)
868
+ return null;
869
+ let minLeft = selfRect.left;
870
+ let minTop = selfRect.top;
871
+ let maxRight = selfRect.left + selfRect.width;
872
+ let maxBottom = selfRect.top + selfRect.height;
873
+ function traverse(child) {
874
+ for (let i = 0; i < child.childNodes.length; i++) {
875
+ const sub = child.childNodes[i];
876
+ if (sub.nodeType === 1) {
877
+ const rect = elementRect(
878
+ sub,
879
+ currentWindow,
880
+ currentDocument,
881
+ baseZoom,
882
+ visibleOnly
883
+ );
884
+ if (rect) {
885
+ minLeft = Math.min(minLeft, rect.left);
886
+ minTop = Math.min(minTop, rect.top);
887
+ maxRight = Math.max(maxRight, rect.left + rect.width);
888
+ maxBottom = Math.max(maxBottom, rect.top + rect.height);
889
+ }
890
+ traverse(sub);
891
+ }
892
+ }
893
+ }
894
+ traverse(node);
895
+ return {
896
+ ...selfRect,
897
+ left: minLeft,
898
+ top: minTop,
899
+ width: maxRight - minLeft,
900
+ height: maxBottom - minTop
901
+ };
902
+ }
849
903
 
850
904
  // src/extractor/locator.ts
851
905
  var getElementIndex = (element) => {
package/dist/lib/constants.d.ts CHANGED
@@ -6,7 +6,7 @@ declare enum NodeType {
6
6
  CONTAINER = "CONTAINER Node",
7
7
  FORM_ITEM = "FORM_ITEM Node",
8
8
  BUTTON = "BUTTON Node",
9
- A = "A Node",
9
+ A = "Anchor Node",
10
10
  IMG = "IMG Node",
11
11
  TEXT = "TEXT Node",
12
12
  POSITION = "POSITION Node"
package/dist/lib/constants.js CHANGED
@@ -41,7 +41,7 @@ var NodeType = /* @__PURE__ */ ((NodeType2) => {
41
41
  NodeType2["CONTAINER"] = "CONTAINER Node";
42
42
  NodeType2["FORM_ITEM"] = "FORM_ITEM Node";
43
43
  NodeType2["BUTTON"] = "BUTTON Node";
44
- NodeType2["A"] = "A Node";
44
+ NodeType2["A"] = "Anchor Node";
45
45
  NodeType2["IMG"] = "IMG Node";
46
46
  NodeType2["TEXT"] = "TEXT Node";
47
47
  NodeType2["POSITION"] = "POSITION Node";
package/dist/lib/extractor-debug.js CHANGED
@@ -477,10 +477,20 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
477
477
  return elementInfo;
478
478
  }
479
479
  if (isButtonElement(node)) {
480
+ const rect2 = mergeElementAndChildrenRects(
481
+ node,
482
+ currentWindow,
483
+ currentDocument,
484
+ baseZoom,
485
+ visibleOnly
486
+ );
487
+ if (!rect2) {
488
+ return null;
489
+ }
480
490
  const attributes = getNodeAttributes(node, currentWindow);
481
491
  const pseudo = getPseudoElementContent(node, currentWindow);
482
492
  const content = node.innerText || pseudo.before || pseudo.after || "";
483
- const nodeHashId = midsceneGenerateHash(node, content, rect);
493
+ const nodeHashId = midsceneGenerateHash(node, content, rect2);
484
494
  const selector = setDataForNode(node, nodeHashId, false, currentWindow);
485
495
  const elementInfo = {
486
496
  id: nodeHashId,
@@ -494,12 +504,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
494
504
  nodeType: "BUTTON Node" /* BUTTON */
495
505
  },
496
506
  content,
497
- rect,
507
+ rect: rect2,
498
508
  center: [
499
- Math.round(rect.left + rect.width / 2),
500
- Math.round(rect.top + rect.height / 2)
509
+ Math.round(rect2.left + rect2.width / 2),
510
+ Math.round(rect2.top + rect2.height / 2)
501
511
  ],
502
- zoom: rect.zoom
512
+ zoom: rect2.zoom
503
513
  };
504
514
  return elementInfo;
505
515
  }
@@ -574,12 +584,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
574
584
  id: nodeHashId,
575
585
  indexId: indexId++,
576
586
  nodeHashId,
577
- nodeType: "A Node" /* A */,
587
+ nodeType: "Anchor Node" /* A */,
578
588
  locator: selector,
579
589
  attributes: {
580
590
  ...attributes,
581
591
  htmlTagName: tagNameOfNode(node),
582
- nodeType: "A Node" /* A */
592
+ nodeType: "Anchor Node" /* A */
583
593
  },
584
594
  content,
585
595
  rect,
@@ -661,7 +671,7 @@ function extractTreeNode(initNode, debugMode2 = false) {
661
671
  node: elementInfo,
662
672
  children: []
663
673
  };
664
- if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "A Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
674
+ if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "Anchor Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
665
675
  return nodeInfo;
666
676
  }
667
677
  const rect = getRect(node, baseZoom, currentWindow);
@@ -716,6 +726,50 @@ function extractTreeNode(initNode, debugMode2 = false) {
716
726
  children: topChildren
717
727
  };
718
728
  }
729
+ function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
730
+ const selfRect = elementRect(
731
+ node,
732
+ currentWindow,
733
+ currentDocument,
734
+ baseZoom,
735
+ visibleOnly
736
+ );
737
+ if (!selfRect)
738
+ return null;
739
+ let minLeft = selfRect.left;
740
+ let minTop = selfRect.top;
741
+ let maxRight = selfRect.left + selfRect.width;
742
+ let maxBottom = selfRect.top + selfRect.height;
743
+ function traverse(child) {
744
+ for (let i = 0; i < child.childNodes.length; i++) {
745
+ const sub = child.childNodes[i];
746
+ if (sub.nodeType === 1) {
747
+ const rect = elementRect(
748
+ sub,
749
+ currentWindow,
750
+ currentDocument,
751
+ baseZoom,
752
+ visibleOnly
753
+ );
754
+ if (rect) {
755
+ minLeft = Math.min(minLeft, rect.left);
756
+ minTop = Math.min(minTop, rect.top);
757
+ maxRight = Math.max(maxRight, rect.left + rect.width);
758
+ maxBottom = Math.max(maxBottom, rect.top + rect.height);
759
+ }
760
+ traverse(sub);
761
+ }
762
+ }
763
+ }
764
+ traverse(node);
765
+ return {
766
+ ...selfRect,
767
+ left: minLeft,
768
+ top: minTop,
769
+ width: maxRight - minLeft,
770
+ height: maxBottom - minTop
771
+ };
772
+ }
719
773
 
720
774
  // src/extractor/debug.ts
721
775
  console.log(extractTextWithPosition(document.body, true));
package/dist/lib/extractor.js CHANGED
@@ -640,10 +640,20 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
640
640
  return elementInfo;
641
641
  }
642
642
  if (isButtonElement(node)) {
643
+ const rect2 = mergeElementAndChildrenRects(
644
+ node,
645
+ currentWindow,
646
+ currentDocument,
647
+ baseZoom,
648
+ visibleOnly
649
+ );
650
+ if (!rect2) {
651
+ return null;
652
+ }
643
653
  const attributes = getNodeAttributes(node, currentWindow);
644
654
  const pseudo = getPseudoElementContent(node, currentWindow);
645
655
  const content = node.innerText || pseudo.before || pseudo.after || "";
646
- const nodeHashId = midsceneGenerateHash(node, content, rect);
656
+ const nodeHashId = midsceneGenerateHash(node, content, rect2);
647
657
  const selector = setDataForNode(node, nodeHashId, false, currentWindow);
648
658
  const elementInfo = {
649
659
  id: nodeHashId,
@@ -657,12 +667,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
657
667
  nodeType: "BUTTON Node" /* BUTTON */
658
668
  },
659
669
  content,
660
- rect,
670
+ rect: rect2,
661
671
  center: [
662
- Math.round(rect.left + rect.width / 2),
663
- Math.round(rect.top + rect.height / 2)
672
+ Math.round(rect2.left + rect2.width / 2),
673
+ Math.round(rect2.top + rect2.height / 2)
664
674
  ],
665
- zoom: rect.zoom
675
+ zoom: rect2.zoom
666
676
  };
667
677
  return elementInfo;
668
678
  }
@@ -737,12 +747,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
737
747
  id: nodeHashId,
738
748
  indexId: indexId++,
739
749
  nodeHashId,
740
- nodeType: "A Node" /* A */,
750
+ nodeType: "Anchor Node" /* A */,
741
751
  locator: selector,
742
752
  attributes: {
743
753
  ...attributes,
744
754
  htmlTagName: tagNameOfNode(node),
745
- nodeType: "A Node" /* A */
755
+ nodeType: "Anchor Node" /* A */
746
756
  },
747
757
  content,
748
758
  rect,
@@ -828,7 +838,7 @@ function extractTreeNode(initNode, debugMode2 = false) {
828
838
  node: elementInfo,
829
839
  children: []
830
840
  };
831
- if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "A Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
841
+ if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "Anchor Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
832
842
  return nodeInfo;
833
843
  }
834
844
  const rect = getRect(node, baseZoom, currentWindow);
@@ -883,6 +893,50 @@ function extractTreeNode(initNode, debugMode2 = false) {
883
893
  children: topChildren
884
894
  };
885
895
  }
896
+ function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
897
+ const selfRect = elementRect(
898
+ node,
899
+ currentWindow,
900
+ currentDocument,
901
+ baseZoom,
902
+ visibleOnly
903
+ );
904
+ if (!selfRect)
905
+ return null;
906
+ let minLeft = selfRect.left;
907
+ let minTop = selfRect.top;
908
+ let maxRight = selfRect.left + selfRect.width;
909
+ let maxBottom = selfRect.top + selfRect.height;
910
+ function traverse(child) {
911
+ for (let i = 0; i < child.childNodes.length; i++) {
912
+ const sub = child.childNodes[i];
913
+ if (sub.nodeType === 1) {
914
+ const rect = elementRect(
915
+ sub,
916
+ currentWindow,
917
+ currentDocument,
918
+ baseZoom,
919
+ visibleOnly
920
+ );
921
+ if (rect) {
922
+ minLeft = Math.min(minLeft, rect.left);
923
+ minTop = Math.min(minTop, rect.top);
924
+ maxRight = Math.max(maxRight, rect.left + rect.width);
925
+ maxBottom = Math.max(maxBottom, rect.top + rect.height);
926
+ }
927
+ traverse(sub);
928
+ }
929
+ }
930
+ }
931
+ traverse(node);
932
+ return {
933
+ ...selfRect,
934
+ left: minLeft,
935
+ top: minTop,
936
+ width: maxRight - minLeft,
937
+ height: maxBottom - minTop
938
+ };
939
+ }
886
940
 
887
941
  // src/extractor/locator.ts
888
942
  var getElementIndex = (element) => {
package/dist/script/htmlElement.js CHANGED
@@ -1200,10 +1200,20 @@ ${indentStr}${after}`;
1200
1200
  return elementInfo;
1201
1201
  }
1202
1202
  if (isButtonElement(node)) {
1203
+ const rect2 = mergeElementAndChildrenRects(
1204
+ node,
1205
+ currentWindow,
1206
+ currentDocument,
1207
+ baseZoom,
1208
+ visibleOnly
1209
+ );
1210
+ if (!rect2) {
1211
+ return null;
1212
+ }
1203
1213
  const attributes = getNodeAttributes(node, currentWindow);
1204
1214
  const pseudo = getPseudoElementContent(node, currentWindow);
1205
1215
  const content = node.innerText || pseudo.before || pseudo.after || "";
1206
- const nodeHashId = midsceneGenerateHash(node, content, rect);
1216
+ const nodeHashId = midsceneGenerateHash(node, content, rect2);
1207
1217
  const selector = setDataForNode(node, nodeHashId, false, currentWindow);
1208
1218
  const elementInfo = {
1209
1219
  id: nodeHashId,
@@ -1216,12 +1226,12 @@ ${indentStr}${after}`;
1216
1226
  nodeType: "BUTTON Node" /* BUTTON */
1217
1227
  }),
1218
1228
  content,
1219
- rect,
1229
+ rect: rect2,
1220
1230
  center: [
1221
- Math.round(rect.left + rect.width / 2),
1222
- Math.round(rect.top + rect.height / 2)
1231
+ Math.round(rect2.left + rect2.width / 2),
1232
+ Math.round(rect2.top + rect2.height / 2)
1223
1233
  ],
1224
- zoom: rect.zoom
1234
+ zoom: rect2.zoom
1225
1235
  };
1226
1236
  return elementInfo;
1227
1237
  }
@@ -1293,11 +1303,11 @@ ${indentStr}${after}`;
1293
1303
  id: nodeHashId,
1294
1304
  indexId: indexId++,
1295
1305
  nodeHashId,
1296
- nodeType: "A Node" /* A */,
1306
+ nodeType: "Anchor Node" /* A */,
1297
1307
  locator: selector,
1298
1308
  attributes: __spreadProps(__spreadValues({}, attributes), {
1299
1309
  htmlTagName: tagNameOfNode(node),
1300
- nodeType: "A Node" /* A */
1310
+ nodeType: "Anchor Node" /* A */
1301
1311
  }),
1302
1312
  content,
1303
1313
  rect,
@@ -1382,7 +1392,7 @@ ${indentStr}${after}`;
1382
1392
  node: elementInfo,
1383
1393
  children: []
1384
1394
  };
1385
- if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "A Node" /* A */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
1395
+ if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "Anchor Node" /* A */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
1386
1396
  return nodeInfo;
1387
1397
  }
1388
1398
  const rect = getRect(node, baseZoom, currentWindow);
@@ -1437,6 +1447,49 @@ ${indentStr}${after}`;
1437
1447
  children: topChildren
1438
1448
  };
1439
1449
  }
1450
+ function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
1451
+ const selfRect = elementRect(
1452
+ node,
1453
+ currentWindow,
1454
+ currentDocument,
1455
+ baseZoom,
1456
+ visibleOnly
1457
+ );
1458
+ if (!selfRect)
1459
+ return null;
1460
+ let minLeft = selfRect.left;
1461
+ let minTop = selfRect.top;
1462
+ let maxRight = selfRect.left + selfRect.width;
1463
+ let maxBottom = selfRect.top + selfRect.height;
1464
+ function traverse(child) {
1465
+ for (let i = 0; i < child.childNodes.length; i++) {
1466
+ const sub = child.childNodes[i];
1467
+ if (sub.nodeType === 1) {
1468
+ const rect = elementRect(
1469
+ sub,
1470
+ currentWindow,
1471
+ currentDocument,
1472
+ baseZoom,
1473
+ visibleOnly
1474
+ );
1475
+ if (rect) {
1476
+ minLeft = Math.min(minLeft, rect.left);
1477
+ minTop = Math.min(minTop, rect.top);
1478
+ maxRight = Math.max(maxRight, rect.left + rect.width);
1479
+ maxBottom = Math.max(maxBottom, rect.top + rect.height);
1480
+ }
1481
+ traverse(sub);
1482
+ }
1483
+ }
1484
+ }
1485
+ traverse(node);
1486
+ return __spreadProps(__spreadValues({}, selfRect), {
1487
+ left: minLeft,
1488
+ top: minTop,
1489
+ width: maxRight - minLeft,
1490
+ height: maxBottom - minTop
1491
+ });
1492
+ }
1440
1493
 
1441
1494
  // src/extractor/locator.ts
1442
1495
  var getElementIndex = (element) => {
package/dist/script/htmlElementDebug.js CHANGED
@@ -1049,10 +1049,20 @@ var midscene_element_inspector = (() => {
1049
1049
  return elementInfo;
1050
1050
  }
1051
1051
  if (isButtonElement(node)) {
1052
+ const rect2 = mergeElementAndChildrenRects(
1053
+ node,
1054
+ currentWindow,
1055
+ currentDocument,
1056
+ baseZoom,
1057
+ visibleOnly
1058
+ );
1059
+ if (!rect2) {
1060
+ return null;
1061
+ }
1052
1062
  const attributes = getNodeAttributes(node, currentWindow);
1053
1063
  const pseudo = getPseudoElementContent(node, currentWindow);
1054
1064
  const content = node.innerText || pseudo.before || pseudo.after || "";
1055
- const nodeHashId = midsceneGenerateHash(node, content, rect);
1065
+ const nodeHashId = midsceneGenerateHash(node, content, rect2);
1056
1066
  const selector = setDataForNode(node, nodeHashId, false, currentWindow);
1057
1067
  const elementInfo = {
1058
1068
  id: nodeHashId,
@@ -1065,12 +1075,12 @@ var midscene_element_inspector = (() => {
1065
1075
  nodeType: "BUTTON Node" /* BUTTON */
1066
1076
  }),
1067
1077
  content,
1068
- rect,
1078
+ rect: rect2,
1069
1079
  center: [
1070
- Math.round(rect.left + rect.width / 2),
1071
- Math.round(rect.top + rect.height / 2)
1080
+ Math.round(rect2.left + rect2.width / 2),
1081
+ Math.round(rect2.top + rect2.height / 2)
1072
1082
  ],
1073
- zoom: rect.zoom
1083
+ zoom: rect2.zoom
1074
1084
  };
1075
1085
  return elementInfo;
1076
1086
  }
@@ -1142,11 +1152,11 @@ var midscene_element_inspector = (() => {
1142
1152
  id: nodeHashId,
1143
1153
  indexId: indexId++,
1144
1154
  nodeHashId,
1145
- nodeType: "A Node" /* A */,
1155
+ nodeType: "Anchor Node" /* A */,
1146
1156
  locator: selector,
1147
1157
  attributes: __spreadProps(__spreadValues({}, attributes), {
1148
1158
  htmlTagName: tagNameOfNode(node),
1149
- nodeType: "A Node" /* A */
1159
+ nodeType: "Anchor Node" /* A */
1150
1160
  }),
1151
1161
  content,
1152
1162
  rect,
@@ -1227,7 +1237,7 @@ var midscene_element_inspector = (() => {
1227
1237
  node: elementInfo,
1228
1238
  children: []
1229
1239
  };
1230
- if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "A Node" /* A */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
1240
+ if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "Anchor Node" /* A */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
1231
1241
  return nodeInfo;
1232
1242
  }
1233
1243
  const rect = getRect(node, baseZoom, currentWindow);
@@ -1282,6 +1292,49 @@ var midscene_element_inspector = (() => {
1282
1292
  children: topChildren
1283
1293
  };
1284
1294
  }
1295
+ function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
1296
+ const selfRect = elementRect(
1297
+ node,
1298
+ currentWindow,
1299
+ currentDocument,
1300
+ baseZoom,
1301
+ visibleOnly
1302
+ );
1303
+ if (!selfRect)
1304
+ return null;
1305
+ let minLeft = selfRect.left;
1306
+ let minTop = selfRect.top;
1307
+ let maxRight = selfRect.left + selfRect.width;
1308
+ let maxBottom = selfRect.top + selfRect.height;
1309
+ function traverse(child) {
1310
+ for (let i = 0; i < child.childNodes.length; i++) {
1311
+ const sub = child.childNodes[i];
1312
+ if (sub.nodeType === 1) {
1313
+ const rect = elementRect(
1314
+ sub,
1315
+ currentWindow,
1316
+ currentDocument,
1317
+ baseZoom,
1318
+ visibleOnly
1319
+ );
1320
+ if (rect) {
1321
+ minLeft = Math.min(minLeft, rect.left);
1322
+ minTop = Math.min(minTop, rect.top);
1323
+ maxRight = Math.max(maxRight, rect.left + rect.width);
1324
+ maxBottom = Math.max(maxBottom, rect.top + rect.height);
1325
+ }
1326
+ traverse(sub);
1327
+ }
1328
+ }
1329
+ }
1330
+ traverse(node);
1331
+ return __spreadProps(__spreadValues({}, selfRect), {
1332
+ left: minLeft,
1333
+ top: minTop,
1334
+ width: maxRight - minLeft,
1335
+ height: maxBottom - minTop
1336
+ });
1337
+ }
1285
1338
 
1286
1339
  // src/extractor/debug.ts
1287
1340
  console.log(extractTextWithPosition(document.body, true));
package/dist/types/constants.d.ts CHANGED
@@ -6,7 +6,7 @@ declare enum NodeType {
6
6
  CONTAINER = "CONTAINER Node",
7
7
  FORM_ITEM = "FORM_ITEM Node",
8
8
  BUTTON = "BUTTON Node",
9
- A = "A Node",
9
+ A = "Anchor Node",
10
10
  IMG = "IMG Node",
11
11
  TEXT = "TEXT Node",
12
12
  POSITION = "POSITION Node"
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@midscene/shared",
3
- "version": "0.17.2-beta-20250523061809.0",
3
+ "version": "0.17.2-beta-20250523075813.0",
4
4
  "repository": "https://github.com/web-infra-dev/midscene",
5
5
  "homepage": "https://midscenejs.com/",
6
6
  "types": "./dist/types/index.d.ts",
package/src/constants/index.ts CHANGED
@@ -9,7 +9,7 @@ export enum NodeType {
9
9
  CONTAINER = 'CONTAINER Node',
10
10
  FORM_ITEM = 'FORM_ITEM Node',
11
11
  BUTTON = 'BUTTON Node',
12
- A = 'A Node',
12
+ A = 'Anchor Node',
13
13
  IMG = 'IMG Node',
14
14
  TEXT = 'TEXT Node',
15
15
  POSITION = 'POSITION Node',
package/src/extractor/web-extractor.ts CHANGED
@@ -127,6 +127,16 @@ export function collectElementInfo(
127
127
  }
128
128
 
129
129
  if (isButtonElement(node)) {
130
+ const rect = mergeElementAndChildrenRects(
131
+ node,
132
+ currentWindow,
133
+ currentDocument,
134
+ baseZoom,
135
+ visibleOnly,
136
+ );
137
+ if (!rect) {
138
+ return null;
139
+ }
130
140
  const attributes = getNodeAttributes(node, currentWindow);
131
141
  const pseudo = getPseudoElementContent(node, currentWindow);
132
142
  const content = node.innerText || pseudo.before || pseudo.after || '';
@@ -427,3 +437,56 @@ export function extractTreeNode(
427
437
  children: topChildren,
428
438
  };
429
439
  }
440
+
441
+ export function mergeElementAndChildrenRects(
442
+ node: Node,
443
+ currentWindow: typeof window,
444
+ currentDocument: typeof document,
445
+ baseZoom = 1,
446
+ visibleOnly = true,
447
+ ) {
448
+ const selfRect = elementRect(
449
+ node,
450
+ currentWindow,
451
+ currentDocument,
452
+ baseZoom,
453
+ visibleOnly,
454
+ );
455
+ if (!selfRect) return null;
456
+
457
+ let minLeft = selfRect.left;
458
+ let minTop = selfRect.top;
459
+ let maxRight = selfRect.left + selfRect.width;
460
+ let maxBottom = selfRect.top + selfRect.height;
461
+
462
+ function traverse(child: Node) {
463
+ for (let i = 0; i < child.childNodes.length; i++) {
464
+ const sub = child.childNodes[i];
465
+ if (sub.nodeType === 1) {
466
+ const rect = elementRect(
467
+ sub,
468
+ currentWindow,
469
+ currentDocument,
470
+ baseZoom,
471
+ visibleOnly,
472
+ );
473
+ if (rect) {
474
+ minLeft = Math.min(minLeft, rect.left);
475
+ minTop = Math.min(minTop, rect.top);
476
+ maxRight = Math.max(maxRight, rect.left + rect.width);
477
+ maxBottom = Math.max(maxBottom, rect.top + rect.height);
478
+ }
479
+ traverse(sub);
480
+ }
481
+ }
482
+ }
483
+ traverse(node);
484
+
485
+ return {
486
+ ...selfRect,
487
+ left: minLeft,
488
+ top: minTop,
489
+ width: maxRight - minLeft,
490
+ height: maxBottom - minTop,
491
+ };
492
+ }