cdp-skill 1.0.8 → 1.0.14

package/src/dom/click-executor.js

@@ -20,7 +20,8 @@ import {
   getCurrentUrl,
   getElementAtPoint,
   detectNavigation,
-  releaseObject
+  releaseObject,
+  isContextDestroyed
 } from '../utils.js';
 
 /**
@@ -36,9 +37,20 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
   if (!elementLocator) throw new Error('Element locator is required');
   if (!inputEmulator) throw new Error('Input emulator is required');
 
+  const getFrameContext = elementLocator.getFrameContext || null;
   const actionabilityChecker = createActionabilityChecker(session);
   const elementValidator = createElementValidator(session);
 
+  /** Build Runtime.evaluate params with frame context when in an iframe. */
+  function frameEvalParams(expression, returnByValue = true) {
+    const params = { expression, returnByValue };
+    if (getFrameContext) {
+      const contextId = getFrameContext();
+      if (contextId) params.contextId = contextId;
+    }
+    return params;
+  }
+
   function calculateVisibleCenter(box, viewport = null) {
     let visibleBox = { ...box };
 
@@ -58,13 +70,10 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
   }
 
   async function getViewportBounds() {
-    const result = await session.send('Runtime.evaluate', {
-      expression: `({
+    const result = await session.send('Runtime.evaluate', frameEvalParams(`({
         width: window.innerWidth || document.documentElement.clientWidth,
         height: window.innerHeight || document.documentElement.clientHeight
-      })`,
-      returnByValue: true
-    });
+      })`, true));
     return result.result.value;
   }
 
@@ -85,8 +94,7 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
 
     const urlBefore = checkNavigation ? await getCurrentUrl(session) : null;
 
-    const result = await session.send('Runtime.evaluate', {
-      expression: `
+    const detectExpr = `
         (function() {
           return new Promise((resolve) => {
             const timeout = ${timeout};
@@ -142,10 +150,10 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
             }, stableTime);
           });
         })()
-      `,
-      returnByValue: true,
-      awaitPromise: true
-    });
+      `;
+    const detectParams = frameEvalParams(detectExpr, true);
+    detectParams.awaitPromise = true;
+    const result = await session.send('Runtime.evaluate', detectParams);
 
     const changeResult = result.result.value || { type: 'none', changeCount: 0 };
 
@@ -245,10 +253,7 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
       })()
     `;
 
-    const result = await session.send('Runtime.evaluate', {
-      expression,
-      returnByValue: true
-    });
+    const result = await session.send('Runtime.evaluate', frameEvalParams(expression, true));
 
     if (result.exceptionDetails || !result.result.value) {
       return null;
@@ -302,10 +307,24 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
   }
 
   async function executeJsClickOnRef(ref) {
-    const result = await session.send('Runtime.evaluate', {
-      expression: `
+    const result = await session.send('Runtime.evaluate', frameEvalParams(`
         (function() {
-          const el = window.__ariaRefs && window.__ariaRefs.get(${JSON.stringify(ref)});
+          let el = window.__ariaRefs && window.__ariaRefs.get(${JSON.stringify(ref)});
+
+          // Re-resolve if element is missing or stale
+          if (!el || !el.isConnected) {
+            const meta = window.__ariaRefMeta && window.__ariaRefMeta.get(${JSON.stringify(ref)});
+            if (meta && meta.selector) {
+              try {
+                const candidate = document.querySelector(meta.selector);
+                if (candidate && candidate.isConnected) {
+                  el = candidate;
+                  if (window.__ariaRefs) window.__ariaRefs.set(${JSON.stringify(ref)}, el);
+                }
+              } catch (e) {}
+            }
+          }
+
           if (!el) {
             return { success: false, reason: 'ref not found in __ariaRefs' };
           }
@@ -319,9 +338,7 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
           el.click();
           return { success: true };
         })()
-      `,
-      returnByValue: true
-    });
+      `, true));
 
     const value = result.result.value || {};
     if (!value.success) {
@@ -330,42 +347,87 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
   }
 
   async function clickWithVerification(x, y, targetObjectId) {
+    // Use pointerdown for verification instead of click.
+    // React (and similar frameworks) re-render elements between mousedown and click,
+    // destroying the original DOM node and its event listeners. pointerdown fires
+    // synchronously at the start of the interaction, before any re-render.
+    // Also listen on document (capture phase) as a fallback — if the click target
+    // is the element or a descendant, count it as received.
     await session.send('Runtime.callFunctionOn', {
       objectId: targetObjectId,
       functionDeclaration: `function() {
         this.__clickReceived = false;
-        this.__clickHandler = () => { this.__clickReceived = true; };
-        this.addEventListener('click', this.__clickHandler, { once: true });
+        const self = this;
+        this.__ptrHandler = (e) => { self.__clickReceived = true; };
+        this.addEventListener('pointerdown', this.__ptrHandler, { once: true });
+        // Document-level capture fallback: catch clicks that bubble from descendants
+        this.__docHandler = (e) => {
+          if (self.contains(e.target) || e.target === self) {
+            self.__clickReceived = true;
+          }
+        };
+        document.addEventListener('pointerdown', this.__docHandler, { capture: true, once: true });
       }`
     });
 
-    await inputEmulator.click(x, y);
-    await sleep(50);
+    try {
+      await inputEmulator.click(x, y);
+      await sleep(50);
 
-    const verifyResult = await session.send('Runtime.callFunctionOn', {
-      objectId: targetObjectId,
-      functionDeclaration: `function() {
-        this.removeEventListener('click', this.__clickHandler);
-        const received = this.__clickReceived;
-        delete this.__clickReceived;
-        delete this.__clickHandler;
-        return received;
-      }`,
-      returnByValue: true
-    });
+      let verifyResult;
+      try {
+        verifyResult = await session.send('Runtime.callFunctionOn', {
+          objectId: targetObjectId,
+          functionDeclaration: `function() {
+            this.removeEventListener('pointerdown', this.__ptrHandler);
+            document.removeEventListener('pointerdown', this.__docHandler, { capture: true });
+            const received = this.__clickReceived;
+            delete this.__clickReceived;
+            delete this.__ptrHandler;
+            delete this.__docHandler;
+            return received;
+          }`,
+          returnByValue: true
+        });
+      } catch (verifyError) {
+        // Context destroyed during verification means click likely triggered navigation
+        // Treat as successful click with navigation
+        if (isContextDestroyed(null, verifyError)) {
+          return { targetReceived: true, contextDestroyed: true };
+        }
+        throw verifyError;
+      }
 
-    const targetReceived = verifyResult.result.value === true;
-    const result = { targetReceived };
+      const targetReceived = verifyResult.result.value === true;
+      const result = { targetReceived };
 
-    // If click didn't reach target, get interceptor info
-    if (!targetReceived) {
-      const interceptor = await getInterceptorInfo(x, y, targetObjectId);
-      if (interceptor) {
-        result.interceptedBy = interceptor;
+      // If click didn't reach target, get interceptor info
+      if (!targetReceived) {
+        const interceptor = await getInterceptorInfo(x, y, targetObjectId);
+        if (interceptor) {
+          result.interceptedBy = interceptor;
+        }
       }
-    }
 
-    return result;
+      return result;
+    } finally {
+      // Always cleanup event listeners, even if click fails
+      try {
+        await session.send('Runtime.callFunctionOn', {
+          objectId: targetObjectId,
+          functionDeclaration: `function() {
+            this.removeEventListener('pointerdown', this.__ptrHandler);
+            document.removeEventListener('pointerdown', this.__docHandler, { capture: true });
+            delete this.__clickReceived;
+            delete this.__ptrHandler;
+            delete this.__docHandler;
+          }`,
+          returnByValue: true
+        });
+      } catch (cleanupError) {
+        // Ignore cleanup errors (element may be gone)
+      }
+    }
   }
 
   async function addNavigationAndDebugInfo(result, urlBeforeClick, debugData, opts) {
@@ -436,38 +498,66 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
   }
 
   async function clickWithVerificationByRef(ref, x, y) {
-    // Set up click listener on the ref element
-    await session.send('Runtime.evaluate', {
-      expression: `
+    // Use pointerdown for verification instead of click.
+    // React re-renders between mousedown and click, destroying the original DOM node.
+    // pointerdown fires synchronously before any re-render.
+    // Also uses document-level capture as fallback for descendant hits.
+    await session.send('Runtime.evaluate', frameEvalParams(`
         (function() {
-          const el = window.__ariaRefs && window.__ariaRefs.get(${JSON.stringify(ref)});
-          if (el) {
+          let el = window.__ariaRefs && window.__ariaRefs.get(${JSON.stringify(ref)});
+          if ((!el || !el.isConnected) && window.__ariaRefMeta) {
+            const meta = window.__ariaRefMeta.get(${JSON.stringify(ref)});
+            if (meta && meta.selector) {
+              try {
+                const candidate = document.querySelector(meta.selector);
+                if (candidate && candidate.isConnected) {
+                  el = candidate;
+                  if (window.__ariaRefs) window.__ariaRefs.set(${JSON.stringify(ref)}, el);
+                }
+              } catch (e) {}
+            }
+          }
+          if (el && el.isConnected) {
             el.__clickReceived = false;
-            el.__clickHandler = () => { el.__clickReceived = true; };
-            el.addEventListener('click', el.__clickHandler, { once: true });
+            el.__ptrHandler = () => { el.__clickReceived = true; };
+            el.addEventListener('pointerdown', el.__ptrHandler, { once: true });
+            el.__docHandler = (e) => {
+              if (el.contains(e.target) || e.target === el) {
+                el.__clickReceived = true;
+              }
+            };
+            document.addEventListener('pointerdown', el.__docHandler, { capture: true, once: true });
           }
         })()
-      `
-    });
+      `, false));
 
     await inputEmulator.click(x, y);
     await sleep(50);
 
-    // Check if click was received
-    const verifyResult = await session.send('Runtime.evaluate', {
-      expression: `
-        (function() {
-          const el = window.__ariaRefs && window.__ariaRefs.get(${JSON.stringify(ref)});
-          if (!el) return { targetReceived: false, reason: 'element not found' };
-          el.removeEventListener('click', el.__clickHandler);
-          const received = el.__clickReceived;
-          delete el.__clickReceived;
-          delete el.__clickHandler;
-          return { targetReceived: received };
-        })()
-      `,
-      returnByValue: true
-    });
+    // Check if pointerdown was received
+    let verifyResult;
+    try {
+      verifyResult = await session.send('Runtime.evaluate', frameEvalParams(`
+          (function() {
+            const el = window.__ariaRefs && window.__ariaRefs.get(${JSON.stringify(ref)});
+            if (!el) return { targetReceived: false, reason: 'element not found' };
+            if (el.__ptrHandler) el.removeEventListener('pointerdown', el.__ptrHandler);
+            if (el.__docHandler) document.removeEventListener('pointerdown', el.__docHandler, { capture: true });
+            const received = el.__clickReceived;
+            delete el.__clickReceived;
+            delete el.__ptrHandler;
+            delete el.__docHandler;
+            return { targetReceived: received };
+          })()
+        `, true));
+    } catch (verifyError) {
+      // Context destroyed during verification means click likely triggered navigation
+      // Treat as successful click with navigation
+      if (isContextDestroyed(null, verifyError)) {
+        return { targetReceived: true, contextDestroyed: true };
+      }
+      throw verifyError;
+    }
 
     return verifyResult.result.value || { targetReceived: false };
   }
@@ -493,12 +583,92 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
     }
 
     if (!force && refInfo.isVisible === false) {
+      // Special case: hidden radio/checkbox inputs — try to click associated label
+      const labelResult = await session.send('Runtime.evaluate', frameEvalParams(`
+        (function() {
+          const ref = ${JSON.stringify(ref)};
+          const el = window.__ariaRefs && window.__ariaRefs.get(ref);
+          if (!el) return { found: false };
+
+          const tag = el.tagName.toUpperCase();
+          const type = (el.type || '').toLowerCase();
+          if (tag === 'INPUT' && (type === 'radio' || type === 'checkbox')) {
+            // Look for associated label
+            let label = null;
+            if (el.id) {
+              label = document.querySelector('label[for="' + el.id + '"]');
+            }
+            if (!label) {
+              label = el.closest('label');
+            }
+
+            if (label) {
+              const rect = label.getBoundingClientRect();
+              const style = window.getComputedStyle(label);
+              const isVisible = style.display !== 'none' &&
+                                style.visibility !== 'hidden' &&
+                                style.opacity !== '0' &&
+                                rect.width > 0 && rect.height > 0;
+
+              if (isVisible) {
+                return {
+                  found: true,
+                  clickedLabel: true,
+                  box: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }
+                };
+              }
+            }
+          }
+          return { found: false };
+        })()
+      `, true));
+
+      const labelInfo = labelResult.result?.value || { found: false };
+      if (labelInfo.found && labelInfo.clickedLabel) {
+        // Click the label instead
+        const labelCenter = calculateVisibleCenter(labelInfo.box);
+        const urlBefore = await getCurrentUrl(session);
+        await inputEmulator.click(labelCenter.x, labelCenter.y);
+        const urlAfter = await getCurrentUrl(session);
+        const navigated = urlAfter !== urlBefore;
+
+        return {
+          clicked: true,
+          method: 'label-proxy',
+          ref,
+          warning: `Element ref:${ref} is a hidden radio/checkbox input. Clicked associated label instead.`,
+          navigated
+        };
+      }
+
+      // No label found or element isn't radio/checkbox — return original error
       return {
         clicked: false,
         warning: `Element ref:${ref} exists but is not visible. It may be hidden or have zero dimensions.`
       };
     }
 
+    // If element is outside viewport (e.g., inside an unscrolled container), scroll it into view first
+    const box = refInfo.box;
+    if (box && (box.x < 0 || box.y < 0 || box.x + box.width > 1920 || box.y + box.height > 1080)) {
+      try {
+        await session.send('Runtime.evaluate', frameEvalParams(`
+          (function() {
+            const el = window.__ariaRefs && window.__ariaRefs.get(${JSON.stringify(ref)});
+            if (el) el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
+          })()
+        `, true));
+        await sleep(100);
+        // Re-fetch element info after scroll
+        const updatedInfo = await ariaSnapshot.getElementByRef(ref);
+        if (updatedInfo && updatedInfo.box) {
+          refInfo.box = updatedInfo.box;
+        }
+      } catch {
+        // Scroll failed — proceed with original coordinates
+      }
+    }
+
     const urlBeforeClick = await getCurrentUrl(session);
 
     const point = calculateVisibleCenter(refInfo.box);
@@ -916,12 +1086,6 @@ export function createClickExecutor(session, elementLocator, inputEmulator, aria
       if (!scrollResult.found) {
         throw elementNotFoundError(selector, scrollOptions.timeout || 30000);
       }
-      // Release the objectId from scroll search since clickBySelector will find it again
-      if (scrollResult.objectId) {
-        try {
-          await releaseObject(session, scrollResult.objectId);
-        } catch { /* ignore cleanup errors */ }
-      }
       // Element found, now proceed with normal click
       // The scrollUntilVisible already scrolled it into view, so the actionability check should pass
     }

package/src/dom/element-locator.js

@@ -30,13 +30,27 @@ const MAX_TIMEOUT = TIMEOUTS.MAX;
  * @param {Object} session - CDP session
  * @param {Object} [options] - Configuration options
  * @param {number} [options.timeout=30000] - Default timeout in ms
+ * @param {Function} [options.getFrameContext] - Returns contextId when in a non-main frame
  * @returns {Object} Element locator interface
  */
 export function createElementLocator(session, options = {}) {
   if (!session) throw new Error('CDP session is required');
 
+  const getFrameContext = options.getFrameContext || null;
   let defaultTimeout = options.timeout || 30000;
 
+  /**
+   * Build Runtime.evaluate params, injecting contextId when in an iframe.
+   */
+  function evalParams(expression, returnByValue = false) {
+    const params = { expression, returnByValue };
+    if (getFrameContext) {
+      const contextId = getFrameContext();
+      if (contextId) params.contextId = contextId;
+    }
+    return params;
+  }
+
   function validateTimeout(timeout) {
     if (typeof timeout !== 'number' || !Number.isFinite(timeout)) return defaultTimeout;
     if (timeout < 0) return 0;
@@ -59,10 +73,9 @@ export function createElementLocator(session, options = {}) {
 
     let result;
     try {
-      result = await session.send('Runtime.evaluate', {
-        expression: `document.querySelector(${JSON.stringify(selector)})`,
-        returnByValue: false
-      });
+      result = await session.send('Runtime.evaluate',
+        evalParams(`document.querySelector(${JSON.stringify(selector)})`, false)
+      );
     } catch (error) {
       throw connectionError(error.message, 'Runtime.evaluate (querySelector)');
     }
@@ -89,10 +102,9 @@ export function createElementLocator(session, options = {}) {
 
     let result;
     try {
-      result = await session.send('Runtime.evaluate', {
-        expression: `Array.from(document.querySelectorAll(${JSON.stringify(selector)}))`,
-        returnByValue: false
-      });
+      result = await session.send('Runtime.evaluate',
+        evalParams(`Array.from(document.querySelectorAll(${JSON.stringify(selector)}))`, false)
+      );
     } catch (error) {
       throw connectionError(error.message, 'Runtime.evaluate (querySelectorAll)');
     }
@@ -175,10 +187,9 @@ export function createElementLocator(session, options = {}) {
     while (Date.now() - startTime < validatedTimeout) {
       let result;
       try {
-        result = await session.send('Runtime.evaluate', {
-          expression: checkExpr,
-          returnByValue: true
-        });
+        result = await session.send('Runtime.evaluate',
+          evalParams(checkExpr, true)
+        );
       } catch (error) {
         throw connectionError(error.message, 'Runtime.evaluate (waitForText)');
       }
@@ -278,10 +289,9 @@ export function createElementLocator(session, options = {}) {
 
     let result;
     try {
-      result = await session.send('Runtime.evaluate', {
-        expression,
-        returnByValue: false
-      });
+      result = await session.send('Runtime.evaluate',
+        evalParams(expression, false)
+      );
     } catch (error) {
       throw connectionError(error.message, 'Runtime.evaluate (queryByRole)');
     }
@@ -403,10 +413,9 @@ export function createElementLocator(session, options = {}) {
 
     let result;
     try {
-      result = await session.send('Runtime.evaluate', {
-        expression,
-        returnByValue: false
-      });
+      result = await session.send('Runtime.evaluate',
+        evalParams(expression, false)
+      );
     } catch (error) {
       throw connectionError(error.message, 'Runtime.evaluate (findElementByText)');
     }
@@ -514,10 +523,9 @@ export function createElementLocator(session, options = {}) {
 
     let result;
     try {
-      result = await session.send('Runtime.evaluate', {
-        expression,
-        returnByValue: false
-      });
+      result = await session.send('Runtime.evaluate',
+        evalParams(expression, false)
+      );
     } catch (error) {
       throw connectionError(error.message, 'Runtime.evaluate (findElementByTextWithinSelector)');
     }
@@ -582,6 +590,7 @@ export function createElementLocator(session, options = {}) {
     waitForElementByText,
     getBoundingBox,
     getDefaultTimeout: () => defaultTimeout,
-    setDefaultTimeout: (timeout) => { defaultTimeout = validateTimeout(timeout); }
+    setDefaultTimeout: (timeout) => { defaultTimeout = validateTimeout(timeout); },
+    get getFrameContext() { return getFrameContext; }
   };
 }