@one2x/playwright 1.57.0-alpha.15 → 1.57.0-alpha.17

package/lib/mcp/browser/tools/mouse.js

@@ -133,21 +133,46 @@ const mouseDragToXY = (0, import_tool.defineTabTool)({
     response.setIncludeAutoScreenshot();
   }
 });
+function generateSamplingPoints(x, y, radius) {
+  if (radius === 0)
+    return [{ x, y }];
+  const circumference = 2 * Math.PI * radius;
+  const numDirections = Math.floor(circumference / 2);
+  const points = [{ x, y }];
+  for (let i = 0; i < numDirections; i++) {
+    const angle = i * 2 * Math.PI / numDirections;
+    points.push({
+      x: x + radius * Math.cos(angle),
+      y: y + radius * Math.sin(angle)
+    });
+  }
+  return points;
+}
 const inspectAtXY = (0, import_tool.defineTabTool)({
   capability: "core",
   schema: {
     name: "browser_inspect_element_at_xy",
     title: "Inspect elements at coordinates",
-    description: "Inspect all meaningful elements at x,y coordinates (excluding parent containers). Returns element details for generating test assertions including suggested selector, HTML, and computed styles",
+    description: "Inspect all meaningful elements at x,y coordinates with optional radius (default 16px). Returns element details for generating test assertions including suggested selector, HTML, and computed styles",
     inputSchema: elementSchema.extend({
       x: import_bundle.z.number().describe("X coordinate"),
-      y: import_bundle.z.number().describe("Y coordinate")
+      y: import_bundle.z.number().describe("Y coordinate"),
+      radius: import_bundle.z.number().min(0).optional().default(16).describe("Search radius (default: 16). Use radius=0 for exact coordinate only.")
     }),
     type: "readOnly"
   },
   handle: async (tab, params, response) => {
-    const elementHandlesData = await tab.page.evaluateHandle(({ x, y }) => {
-      const allElements = document.elementsFromPoint(x, y);
+    const radius = params.radius ?? 16;
+    if (radius < 0)
+      throw new Error("radius must be non-negative");
+    const samplingPoints = generateSamplingPoints(params.x, params.y, radius);
+    const elementHandlesData = await tab.page.evaluateHandle(({ points }) => {
+      const elementSet = /* @__PURE__ */ new Set();
+      for (const point of points) {
+        const elements = document.elementsFromPoint(point.x, point.y);
+        elements.forEach((el) => elementSet.add(el));
+      }
+      const allElements = Array.from(elementSet);
       const meaningfulElements = allElements.filter((element, index) => {
         for (let i = 0; i < allElements.length; i++) {
           if (i !== index && element.contains(allElements[i]))
@@ -155,8 +180,52 @@ const inspectAtXY = (0, import_tool.defineTabTool)({
         }
         return true;
       });
-      return meaningfulElements;
-    }, params);
+      const consolidatedElements = [];
+      const svgSet = /* @__PURE__ */ new Set();
+      for (const element of meaningfulElements) {
+        let current = element;
+        let parentSVG = null;
+        while (current) {
+          if (current.tagName === "svg") {
+            parentSVG = current;
+            break;
+          }
+          current = current.parentElement;
+        }
+        if (parentSVG) {
+          if (!svgSet.has(parentSVG)) {
+            svgSet.add(parentSVG);
+            consolidatedElements.push(parentSVG);
+          }
+        } else {
+          consolidatedElements.push(element);
+        }
+      }
+      const sortedElements = [];
+      const processed = /* @__PURE__ */ new Set();
+      const consolidatedSet = new Set(consolidatedElements);
+      for (const point of points) {
+        const elements = document.elementsFromPoint(point.x, point.y);
+        for (const el of elements) {
+          let elementToCheck = el;
+          if (el.tagName !== "svg") {
+            let current = el;
+            while (current) {
+              if (current.tagName === "svg") {
+                elementToCheck = current;
+                break;
+              }
+              current = current.parentElement;
+            }
+          }
+          if (consolidatedSet.has(elementToCheck) && !processed.has(elementToCheck)) {
+            sortedElements.push(elementToCheck);
+            processed.add(elementToCheck);
+          }
+        }
+      }
+      return sortedElements;
+    }, { points: samplingPoints });
     const elementHandlesArray = await elementHandlesData.evaluateHandle((arr) => arr);
     const elementHandles = [];
     const length = await elementHandlesArray.evaluate((arr) => arr.length);
@@ -169,7 +238,7 @@ const inspectAtXY = (0, import_tool.defineTabTool)({
     const elementsData = [];
     for (const elementHandle of elementHandles) {
       const pathToRoot = await elementHandle.getPathWithRefs().catch(() => "");
-      const elementData = await elementHandle.evaluate((element) => {
+      const elementData = await elementHandle.evaluate((element, centerPoint) => {
         const computedStyle = window.getComputedStyle(element);
         function serializeAttributes(el) {
           const essentialAttrs = ["id", "class", "type", "name", "role", "aria-label"];
@@ -250,13 +319,47 @@ const inspectAtXY = (0, import_tool.defineTabTool)({
           return currentHtml;
         }
         const ariaRef = element._ariaRef;
+        const bounds = element.getBoundingClientRect();
+        const containsCenterPoint = bounds.left <= centerPoint.x && centerPoint.x <= bounds.right && bounds.top <= centerPoint.y && centerPoint.y <= bounds.bottom;
+        const elementCenterX = bounds.left + bounds.width / 2;
+        const elementCenterY = bounds.top + bounds.height / 2;
+        const distanceToCenter = Math.sqrt(
+          Math.pow(elementCenterX - centerPoint.x, 2) + Math.pow(elementCenterY - centerPoint.y, 2)
+        );
+        let svgColors = null;
+        if (element.tagName.toLowerCase() === "svg") {
+          const colorSet = /* @__PURE__ */ new Set();
+          const fillSet = /* @__PURE__ */ new Set();
+          const strokeSet = /* @__PURE__ */ new Set();
+          const allDescendants = element.querySelectorAll("*");
+          for (const descendant of Array.from(allDescendants)) {
+            const style = window.getComputedStyle(descendant);
+            const fill = descendant.getAttribute("fill") || style.fill;
+            if (fill && fill !== "none" && fill !== "transparent" && !fill.startsWith("url("))
+              fillSet.add(fill);
+            const stroke = descendant.getAttribute("stroke") || style.stroke;
+            if (stroke && stroke !== "none" && stroke !== "transparent" && !stroke.startsWith("url("))
+              strokeSet.add(stroke);
+            const color = style.color;
+            if (color && color !== "transparent")
+              colorSet.add(color);
+          }
+          svgColors = {
+            colors: Array.from(colorSet),
+            fills: Array.from(fillSet),
+            strokes: Array.from(strokeSet)
+          };
+        }
         return {
           ref: ariaRef?.ref || null,
           role: ariaRef?.role || null,
           element: ariaRef?.name || element.tagName.toLowerCase(),
           truncatedHTML: truncateIterativeDeepening(element, 300),
           tagName: element.tagName.toLowerCase(),
-          bounds: element.getBoundingClientRect(),
+          bounds,
+          containsCenterPoint,
+          distanceToCenter,
+          svgColors,
           computedStyles: {
             display: computedStyle.display,
             visibility: computedStyle.visibility,
@@ -277,16 +380,26 @@ const inspectAtXY = (0, import_tool.defineTabTool)({
           disabled: element.disabled || void 0,
           readonly: element.readOnly || void 0
         };
-      });
+      }, { x: params.x, y: params.y });
       elementsData.push({
         ...elementData,
         pathToRoot
       });
     }
+    const elementsContainingCenter = [];
+    const elementsNotContainingCenter = [];
+    for (const el of elementsData) {
+      if (el.containsCenterPoint)
+        elementsContainingCenter.push(el);
+      else
+        elementsNotContainingCenter.push(el);
+    }
+    elementsNotContainingCenter.sort((a, b) => a.distanceToCenter - b.distanceToCenter);
+    const sortedElementsData = [...elementsContainingCenter, ...elementsNotContainingCenter];
     let suggestedSelectors = "";
-    if (elementsData.length > 0 && elementsData[0].ref) {
+    if (sortedElementsData.length > 0 && sortedElementsData[0].ref) {
       try {
-        const topElement = elementsData[0];
+        const topElement = sortedElementsData[0];
         const { locator } = await tab.refLocator({
           element: topElement.element,
           ref: topElement.ref
@@ -296,13 +409,71 @@ const inspectAtXY = (0, import_tool.defineTabTool)({
       }
     }
     let result = "";
-    if (elementsData.length > 1)
-      result += `\u26A0\uFE0F Warning: Found ${elementsData.length} elements at (${params.x}, ${params.y})
+    if (sortedElementsData.length === 0) {
+      const viewport = await tab.page.viewportSize();
+      const searchArea = radius === 0 ? `at coordinates (${params.x}, ${params.y})` : `within ${radius}px radius of (${params.x}, ${params.y})`;
+      result += `\u26A0\uFE0F No elements found ${searchArea}
 
 `;
-    elementsData.forEach((elementInfo, index) => {
+      result += `**Possible reasons:**
+`;
+      result += `- The coordinates may be outside the viewport
+`;
+      if (viewport) {
+        result += `  - Current viewport size: ${viewport.width}x${viewport.height}
+`;
+        if (params.x < 0 || params.y < 0 || params.x >= viewport.width || params.y >= viewport.height)
+          result += `  - \u26A0\uFE0F Coordinates are outside viewport bounds!
+`;
+      }
+      result += `- The page may still be loading or animating
+`;
+      result += `- The search area may be on empty space (no rendered elements)
+`;
+      result += `- Elements in this area may have been removed from the DOM
+`;
+      if (radius > 0)
+        result += `- Try increasing the radius to expand the search area
+`;
+      result += `
+**Suggestions:**
+`;
+      result += `- Take a screenshot with \`browser_take_screenshot\` to see the current page state
+`;
+      result += `- Wait for the page to stabilize with \`browser_wait_for\`
+`;
+      result += `- Try different coordinates closer to visible elements
+`;
+      if (viewport)
+        result += `- Ensure coordinates are within viewport bounds (0-${viewport.width}, 0-${viewport.height})
+`;
+      response.addResult(result);
+      return;
+    }
+    if (sortedElementsData.length > 1) {
+      const numAtCenter = elementsContainingCenter.length;
+      const numNearby = elementsNotContainingCenter.length;
+      const searchArea = radius === 0 ? `at (${params.x}, ${params.y})` : `within ${radius}px radius of (${params.x}, ${params.y})`;
+      let warning = `\u26A0\uFE0F Warning: Found ${sortedElementsData.length} elements ${searchArea}`;
+      if (numAtCenter > 0 && numNearby > 0)
+        warning += ` (${numAtCenter} at center, ${numNearby} nearby)`;
+      else if (numAtCenter > 0)
+        warning += ` (all at center)`;
+      else if (numNearby > 0)
+        warning += ` (all nearby)`;
+      result += warning + "\n\n";
+    }
+    sortedElementsData.forEach((elementInfo, index) => {
       const isTopmost = index === 0;
-      const header = isTopmost ? `**Element at (${params.x}, ${params.y}) - Topmost (will receive clicks):**` : `**Element ${index + 1} at (${params.x}, ${params.y}) - Below topmost:**`;
+      const locationDesc = radius === 0 ? `at (${params.x}, ${params.y})` : `within ${radius}px of (${params.x}, ${params.y})`;
+      let category = "";
+      if (elementInfo.containsCenterPoint && isTopmost)
+        category = " - Topmost at center (will receive clicks)";
+      else if (elementInfo.containsCenterPoint)
+        category = " - At center (below topmost)";
+      else
+        category = ` - Nearby (${Math.round(elementInfo.distanceToCenter)}px from center)`;
+      const header = `**Element ${index + 1} ${locationDesc}${category}:**`;
       result += header + "\n";
       result += `- RefId: ${elementInfo.ref || "N/A"}
 `;
@@ -329,6 +500,34 @@ ${elementInfo.truncatedHTML}
 `;
       result += `- height: ${elementInfo.bounds.height}
 `;
+      result += `- Contains center point (${params.x}, ${params.y}): ${elementInfo.containsCenterPoint ? "\u2713 Yes" : "\u2717 No"}
+`;
+      if (!elementInfo.containsCenterPoint)
+        result += `- Distance to center: ${Math.round(elementInfo.distanceToCenter)}px
+`;
+      if (elementInfo.svgColors) {
+        result += `
+**SVG Colors (from descendants):**
+`;
+        if (elementInfo.svgColors.fills.length > 0)
+          result += `- Fill colors: ${elementInfo.svgColors.fills.join(", ")}
+`;
+        else
+          result += `- Fill colors: none
+`;
+        if (elementInfo.svgColors.strokes.length > 0)
+          result += `- Stroke colors: ${elementInfo.svgColors.strokes.join(", ")}
+`;
+        else
+          result += `- Stroke colors: none
+`;
+        if (elementInfo.svgColors.colors.length > 0)
+          result += `- Text colors: ${elementInfo.svgColors.colors.join(", ")}
+`;
+        else
+          result += `- Text colors: none
+`;
+      }
       result += `
 **Key Computed Styles:**
 `;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@one2x/playwright",
-  "version": "1.57.0-alpha.15",
+  "version": "1.57.0-alpha.17",
   "description": "A high-level API to automate web browsers",
   "repository": {
     "type": "git",
@@ -65,7 +65,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "content-type": "^1.0.5",
-    "playwright-core": "npm:@one2x/playwright-core@1.57.0-alpha.15",
+    "playwright-core": "npm:@one2x/playwright-core@1.57.0-alpha.17",
     "raw-body": "^2.5.2"
   },
   "optionalDependencies": {