@apmantza/greedysearch-pi 1.4.0 → 1.4.2

package/extractors/consent.mjs

@@ -1,248 +1,255 @@
-// consent.mjs — auto-dismiss common cookie/consent banners and human-verification pages
-// Call dismissConsent(tab, cdpFn) after navigating to any page.
-
-const CONSENT_JS = `
-(function() {
-  // Google consent page (consent.google.com)
-  var g = document.querySelector('#L2AGLb, button[jsname="b3VHJd"], .tHlp8d');
-  if (g) { g.click(); return 'google'; }
-
-  // OneTrust (used by many sites including Stack Overflow)
-  var ot = document.querySelector('#onetrust-accept-btn-handler, .onetrust-accept-btn-handler');
-  if (ot) { ot.click(); return 'onetrust'; }
-
-  // Generic "accept all" / "agree" buttons
-  var btns = Array.from(document.querySelectorAll('button, a[role=button]'));
-  var accept = btns.find(b => /^(accept all|accept cookies|agree|i agree|got it|allow all|allow cookies)$/i.test(b.innerText?.trim()));
-  if (accept) { accept.click(); return 'generic:' + accept.innerText.trim(); }
-
-  return null;
-})()
-`;
-
-// Detect and auto-click human verification challenges (Google, Microsoft, Cloudflare)
-const VERIFY_DETECT_JS = `
-(function() {
-  var url = document.location.href;
-  
-  // --- Google "sorry" page (hard CAPTCHA, can't auto-solve) ---
-  if (url.includes('/sorry/') || url.includes('sorry.google')) return 'sorry-page';
-  
-  // --- Microsoft account verification page ---
-  if (url.includes('login.microsoftonline.com') || url.includes('login.live.com') || url.includes('account.microsoft.com')) {
-    // Look for "Verify" or "Continue" buttons on Microsoft auth pages
-    var msBtns = Array.from(document.querySelectorAll('button, input[type=submit], a'));
-    var msVerify = msBtns.find(b => /verify|continue|next/i.test(b.innerText?.trim() || b.value || ''));
-    if (msVerify) { msVerify.click(); return 'clicked-ms-verify:' + (msVerify.innerText?.trim() || msVerify.value); }
-  }
-  
-  // --- Bing Copilot / Microsoft "Verify you're human" interstitial ---
-  // Copilot sometimes shows a modal with "Continue" or "Verify" before allowing queries
-  if (url.includes('copilot.microsoft.com') || url.includes('bing.com/chat')) {
-    // Look for verification modal/dialog
-    var modal = document.querySelector('[role="dialog"], .b_modal, .bnp_hfly, [class*="verify"], [class*="challenge"]');
-    if (modal) {
-      // Find any actionable button in the modal
-      var modalBtns = Array.from(modal.querySelectorAll('button, a[role="button"], input[type="submit"]'));
-      var actionBtn = modalBtns.find(b => /^(continue|verify|submit|next|i agree|accept|got it)$/i.test(b.innerText?.trim() || b.value || ''));
-      if (actionBtn) { actionBtn.click(); return 'clicked-copilot-modal:' + actionBtn.innerText.trim(); }
-    }
-    
-    // Check for Turnstile iframe (Copilot uses Cloudflare Turnstile)
-    var turnstileIframe = document.querySelector('iframe[src*="challenges.cloudflare.com"], iframe[src*="turnstile"], iframe[title*="challenge"], iframe[title*="Widget"]');
-    if (turnstileIframe) {
-      // Try clicking the iframe container or nearby checkbox
-      var container = turnstileIframe.closest('[class*="turnstile"], [class*="challenge"], [id*="turnstile"]') || turnstileIframe.parentElement;
-      if (container) {
-        var checkbox = container.querySelector('input[type="checkbox"]');
-        if (checkbox && !checkbox.checked) {
-          checkbox.click();
-          return 'clicked-turnstile-in-iframe';
-        }
-        // Click the container itself (Turnstile often captures clicks on parent)
-        container.click();
-        return 'clicked-turnstile-container-near-iframe';
-      }
-    }
-  }
-  
-  // --- Cloudflare Turnstile (used by Copilot and many sites) ---
-  // Turnstile widget in iframe
-  var turnstileIframe = document.querySelector('iframe[src*="challenges.cloudflare.com"], iframe[src*="turnstile"]');
-  if (turnstileIframe) {
-    // Try to find and click the checkbox inside the iframe's container
-    var turnstileCheckbox = document.querySelector('#cf-turnstile-response, [data-turnstile-callback] input, .cf-turnstile input[type="checkbox"]');
-    if (turnstileCheckbox && !turnstileCheckbox.checked) {
-      turnstileCheckbox.click();
-      return 'clicked-turnstile-checkbox';
-    }
-    // Try clicking the turnstile container itself (some implementations)
-    var turnstileContainer = document.querySelector('.cf-turnstile, [data-sitekey]');
-    if (turnstileContainer) {
-      turnstileContainer.click();
-      return 'clicked-turnstile-container';
-    }
-  }
-  
-  // --- Cloudflare "Verify you are human" challenge page ---
-  if (url.includes('challenges.cloudflare.com') || document.querySelector('#challenge-running, #challenge-stage')) {
-    var cfCheckbox = document.querySelector('#cf-stage input[type="checkbox"], .ctp-checkbox-container input');
-    if (cfCheckbox) { cfCheckbox.click(); return 'clicked-cloudflare-checkbox'; }
-    var cfBtn = document.querySelector('#challenge-form button, .cf-challenge button');
-    if (cfBtn) { cfBtn.click(); return 'clicked-cloudflare-button'; }
-  }
-  
-  // --- Microsoft "I am human" / "Verify" challenge ---
-  // Microsoft uses various verification UIs
-  var msHumanBtn = document.querySelector('button[id*="i0"], button[id*="id__"]');
-  if (msHumanBtn && /verify|human|robot|continue/i.test(msHumanBtn.innerText?.trim())) {
-    msHumanBtn.click();
-    return 'clicked-ms-human:' + msHumanBtn.innerText.trim();
-  }
-  
-  // --- Generic verification buttons (catch-all) ---
-  var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
-  var verify = btns.find(b => /^(verify|verification|verify you are human|i am human|not a robot|continue|proceed)$/i.test(b.innerText?.trim() || b.value || ''));
-  if (verify && !document.querySelector('iframe[src*="recaptcha"]')) {
-    verify.click();
-    return 'clicked-verify:' + (verify.innerText?.trim() || verify.value);
-  }
-
-  // --- Google reCAPTCHA (no image challenge, just checkbox) ---
-  var recaptchaCheckbox = document.querySelector('.recaptcha-checkbox-unchecked, input[type=checkbox][id*="recaptcha"]');
-  if (recaptchaCheckbox) { recaptchaCheckbox.click(); return 'clicked-recaptcha'; }
-
-  return null;
-})()
-`;
-
-// Retry loop for verification — keeps checking and clicking until page changes or timeout
-const VERIFY_RETRY_JS = `
-(function() {
-  var url = document.location.href;
-  
-  // Check if we're still on a verification page
-  var isVerifyPage = url.includes('/sorry/') || 
-                     url.includes('challenges.cloudflare.com') ||
-                     url.includes('login.microsoftonline.com') ||
-                     document.querySelector('#challenge-running, #challenge-stage, .cf-turnstile, [role="dialog"]');
-  
-  if (!isVerifyPage) return 'cleared';
-  
-  // Try clicking any verify/continue button again
-  var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
-  var btn = btns.find(b => /^(verify|continue|next|i am human|not a robot|submit)$/i.test(b.innerText?.trim() || b.value || ''));
-  if (btn) { btn.click(); return 'clicked:' + (btn.innerText?.trim() || btn.value); }
-  
-  // Try Turnstile checkbox
-  var cf = document.querySelector('#cf-stage input[type="checkbox"], .cf-turnstile input');
-  if (cf && !cf.checked) { cf.click(); return 'clicked-turnstile'; }
-  
-  // Check for modal dialog with continue button (Copilot interstitial)
-  var modal = document.querySelector('[role="dialog"], .b_modal, [class*="verify"]');
-  if (modal) {
-    var modalBtn = modal.querySelector('button, a[role="button"]');
-    if (modalBtn) { modalBtn.click(); return 'clicked-modal-btn:' + modalBtn.innerText.trim(); }
-  }
-  
-  return 'still-verifying';
-})()
-`;
-
-export async function dismissConsent(tab, cdp) {
-  const result = await cdp(['eval', tab, CONSENT_JS]).catch(() => null);
-  if (result && result !== 'null') {
-    await new Promise(r => setTimeout(r, 1500));
-  }
-}
-
-// Get iframe bounding box for coordinate-based clicking (for cross-origin Turnstile)
-const GET_IFRAME_CENTER_JS = `
-(function() {
-  var iframe = document.querySelector('iframe[src*="challenges.cloudflare.com"], iframe[src*="turnstile"], iframe[title*="challenge"], iframe[title*="Widget"]');
-  if (!iframe) return null;
-  var rect = iframe.getBoundingClientRect();
-  // Click near the center-left where the checkbox usually is
-  return JSON.stringify({ x: rect.left + 30, y: rect.top + rect.height / 2 });
-})()
-`;
-
-// Returns 'clear' | 'clicked' | 'needs-human'
-export async function handleVerification(tab, cdp, waitMs = 60000) {
-  const result = await cdp(['eval', tab, VERIFY_DETECT_JS]).catch(() => null);
-
-  if (!result || result === 'null') return 'clear';
-
-  // Hard CAPTCHA page — wait for user to solve it manually
-  if (result === 'sorry-page') {
-    process.stderr.write(`[greedysearch] Google CAPTCHA detected — please solve it in the browser window (waiting up to ${Math.floor(waitMs / 1000)}s)...\n`);
-    const deadline = Date.now() + waitMs;
-    while (Date.now() < deadline) {
-      await new Promise(r => setTimeout(r, 2000));
-      const url = await cdp(['eval', tab, 'document.location.href']).catch(() => '');
-      if (!url.includes('/sorry/')) return 'cleared-by-user';
-    }
-    return 'needs-human';
-  }
-
-  // We clicked something — wait for page to update, then keep retrying
-  if (result.startsWith('clicked-')) {
-    process.stderr.write(`[greedysearch] Clicked verification: ${result}\n`);
-    await new Promise(r => setTimeout(r, 2000));
-    
-    // Keep checking if verification cleared, retry clicking for up to waitMs
-    const deadline = Date.now() + waitMs;
-    while (Date.now() < deadline) {
-      const retryResult = await cdp(['eval', tab, VERIFY_RETRY_JS]).catch(() => null);
-      
-      if (retryResult === 'cleared' || !retryResult || retryResult === 'null') {
-        process.stderr.write(`[greedysearch] Verification cleared.\n`);
-        await new Promise(r => setTimeout(r, 1000));
-        return 'clicked';
-      }
-      
-      if (retryResult.startsWith('clicked:')) {
-        process.stderr.write(`[greedysearch] Retrying verification click...\n`);
-        await new Promise(r => setTimeout(r, 2000));
-      }
-      
-      // If verification is stuck, try clicking the Turnstile iframe by coordinates
-      const iframeCenter = await cdp(['eval', tab, GET_IFRAME_CENTER_JS]).catch(() => null);
-      if (iframeCenter && iframeCenter !== 'null') {
-        try {
-          const { x, y } = JSON.parse(iframeCenter);
-          process.stderr.write(`[greedysearch] Trying coordinate click on Turnstile iframe at (${x}, ${y})...\n`);
-          await cdp(['clickxy', tab, String(x), String(y)]);
-          await new Promise(r => setTimeout(r, 3000));
-        } catch {}
-      }
-      
-      await new Promise(r => setTimeout(r, 1500));
-    }
-    
-    // Still stuck — might need user intervention
-    process.stderr.write(`[greedysearch] Verification may require manual intervention.\n`);
-    return 'needs-human';
-  }
-
-  // Detection didn't find anything initially, but check for Turnstile iframe with coordinates
-  if (result === 'null' || !result) {
-    const iframeCenter = await cdp(['eval', tab, GET_IFRAME_CENTER_JS]).catch(() => null);
-    if (iframeCenter && iframeCenter !== 'null') {
-      process.stderr.write(`[greedysearch] Found Turnstile iframe, attempting coordinate click...\n`);
-      try {
-        const { x, y } = JSON.parse(iframeCenter);
-        await cdp(['clickxy', tab, String(x), String(y)]);
-        await new Promise(r => setTimeout(r, 3000));
-        
-        // Check if it worked
-        const cleared = await cdp(['eval', tab, VERIFY_RETRY_JS]).catch(() => null);
-        if (cleared === 'cleared' || cleared === 'null') {
-          return 'clicked';
-        }
-      } catch {}
-    }
-  }
-
-  return 'clear';
-}
+// consent.mjs — auto-dismiss common cookie/consent banners and human-verification pages
+// Call dismissConsent(tab, cdpFn) after navigating to any page.
+
+const CONSENT_JS = `
+(function() {
+  // Google consent page (consent.google.com)
+  var g = document.querySelector('#L2AGLb, button[jsname="b3VHJd"], .tHlp8d');
+  if (g) { g.click(); return 'google'; }
+
+  // OneTrust (used by many sites including Stack Overflow)
+  var ot = document.querySelector('#onetrust-accept-btn-handler, .onetrust-accept-btn-handler');
+  if (ot) { ot.click(); return 'onetrust'; }
+
+  // Generic "accept all" / "agree" buttons
+  var btns = Array.from(document.querySelectorAll('button, a[role=button]'));
+  var accept = btns.find(b => /^(accept all|accept cookies|agree|i agree|got it|allow all|allow cookies)$/i.test(b.innerText?.trim()));
+  if (accept) { accept.click(); return 'generic:' + accept.innerText.trim(); }
+
+  return null;
+})()
+`;
+
+// Detect and auto-click human verification challenges (Google, Microsoft, Cloudflare)
+const VERIFY_DETECT_JS = `
+(function() {
+  var url = document.location.href;
+  
+  // --- Google "sorry" page (hard CAPTCHA, can't auto-solve) ---
+  if (url.includes('/sorry/') || url.includes('sorry.google')) return 'sorry-page';
+  
+  // --- Microsoft account verification page ---
+  if (url.includes('login.microsoftonline.com') || url.includes('login.live.com') || url.includes('account.microsoft.com')) {
+    // Look for "Verify" or "Continue" buttons on Microsoft auth pages
+    var msBtns = Array.from(document.querySelectorAll('button, input[type=submit], a'));
+    var msVerify = msBtns.find(b => /verify|continue|next/i.test(b.innerText?.trim() || b.value || ''));
+    if (msVerify) { msVerify.click(); return 'clicked-ms-verify:' + (msVerify.innerText?.trim() || msVerify.value); }
+  }
+  
+  // --- Bing Copilot / Microsoft "Verify you're human" interstitial ---
+  // Copilot sometimes shows a modal with "Continue" or "Verify" before allowing queries
+  if (url.includes('copilot.microsoft.com') || url.includes('bing.com/chat')) {
+    // Look for verification modal/dialog
+    var modal = document.querySelector('[role="dialog"], .b_modal, .bnp_hfly, [class*="verify"], [class*="challenge"]');
+    if (modal) {
+      // Find any actionable button in the modal
+      var modalBtns = Array.from(modal.querySelectorAll('button, a[role="button"], input[type="submit"]'));
+      var actionBtn = modalBtns.find(b => /^(continue|verify|submit|next|i agree|accept|got it)$/i.test(b.innerText?.trim() || b.value || ''));
+      if (actionBtn) { actionBtn.click(); return 'clicked-copilot-modal:' + actionBtn.innerText.trim(); }
+    }
+    
+    // Check for Turnstile iframe (Copilot uses Cloudflare Turnstile)
+    var turnstileIframe = document.querySelector('iframe[src*="challenges.cloudflare.com"], iframe[src*="turnstile"], iframe[title*="challenge"], iframe[title*="Widget"]');
+    if (turnstileIframe) {
+      // Try clicking the iframe container or nearby checkbox
+      var container = turnstileIframe.closest('[class*="turnstile"], [class*="challenge"], [id*="turnstile"]') || turnstileIframe.parentElement;
+      if (container) {
+        var checkbox = container.querySelector('input[type="checkbox"]');
+        if (checkbox && !checkbox.checked) {
+          checkbox.click();
+          return 'clicked-turnstile-in-iframe';
+        }
+        // Click the container itself (Turnstile often captures clicks on parent)
+        container.click();
+        return 'clicked-turnstile-container-near-iframe';
+      }
+    }
+  }
+  
+  // --- Cloudflare Turnstile (used by Copilot and many sites) ---
+  // Turnstile widget in iframe
+  var turnstileIframe = document.querySelector('iframe[src*="challenges.cloudflare.com"], iframe[src*="turnstile"]');
+  if (turnstileIframe) {
+    // Try to find and click the checkbox inside the iframe's container
+    var turnstileCheckbox = document.querySelector('#cf-turnstile-response, [data-turnstile-callback] input, .cf-turnstile input[type="checkbox"]');
+    if (turnstileCheckbox && !turnstileCheckbox.checked) {
+      turnstileCheckbox.click();
+      return 'clicked-turnstile-checkbox';
+    }
+    // Try clicking the turnstile container itself (some implementations)
+    var turnstileContainer = document.querySelector('.cf-turnstile, [data-sitekey]');
+    if (turnstileContainer) {
+      turnstileContainer.click();
+      return 'clicked-turnstile-container';
+    }
+  }
+  
+  // --- Cloudflare "Verify you are human" challenge page ---
+  if (url.includes('challenges.cloudflare.com') || document.querySelector('#challenge-running, #challenge-stage')) {
+    var cfCheckbox = document.querySelector('#cf-stage input[type="checkbox"], .ctp-checkbox-container input');
+    if (cfCheckbox) { cfCheckbox.click(); return 'clicked-cloudflare-checkbox'; }
+    var cfBtn = document.querySelector('#challenge-form button, .cf-challenge button');
+    if (cfBtn) { cfBtn.click(); return 'clicked-cloudflare-button'; }
+  }
+  
+  // --- Microsoft "I am human" / "Verify" challenge ---
+  // Microsoft uses various verification UIs
+  var msHumanBtn = document.querySelector('button[id*="i0"], button[id*="id__"]');
+  if (msHumanBtn && /verify|human|robot|continue/i.test(msHumanBtn.innerText?.trim())) {
+    msHumanBtn.click();
+    return 'clicked-ms-human:' + msHumanBtn.innerText.trim();
+  }
+  
+  // --- Generic verification buttons (catch-all) ---
+  var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
+  var verify = btns.find(b => {
+    var t = (b.innerText?.trim() || b.value || '').toLowerCase();
+    return (t.includes('verify') || t.includes('human') || t.includes('robot') || t.includes('continue') || t.includes('proceed')) &&
+           !t.includes('verified') && !document.querySelector('iframe[src*="recaptcha"]');
+  });
+  if (verify) {
+    verify.click();
+    return 'clicked-verify:' + (verify.innerText?.trim() || verify.value);
+  }
+
+  // --- Google reCAPTCHA (no image challenge, just checkbox) ---
+  var recaptchaCheckbox = document.querySelector('.recaptcha-checkbox-unchecked, input[type=checkbox][id*="recaptcha"]');
+  if (recaptchaCheckbox) { recaptchaCheckbox.click(); return 'clicked-recaptcha'; }
+
+  return null;
+})()
+`;
+
+// Retry loop for verification — keeps checking and clicking until page changes or timeout
+const VERIFY_RETRY_JS = `
+(function() {
+  var url = document.location.href;
+  
+  // Check if we're still on a verification page
+  var isVerifyPage = url.includes('/sorry/') || 
+                     url.includes('challenges.cloudflare.com') ||
+                     url.includes('login.microsoftonline.com') ||
+                     document.querySelector('#challenge-running, #challenge-stage, .cf-turnstile, [role="dialog"]');
+  
+  if (!isVerifyPage) return 'cleared';
+  
+  // Try clicking any verify/continue button again
+  var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
+  var btn = btns.find(b => {
+    var t = (b.innerText?.trim() || b.value || '').toLowerCase();
+    return t.includes('verify') || t.includes('human') || t.includes('robot') || t.includes('continue') || t.includes('next') || t.includes('submit');
+  });
+  if (btn) { btn.click(); return 'clicked:' + (btn.innerText?.trim() || btn.value); }
+  
+  // Try Turnstile checkbox
+  var cf = document.querySelector('#cf-stage input[type="checkbox"], .cf-turnstile input');
+  if (cf && !cf.checked) { cf.click(); return 'clicked-turnstile'; }
+  
+  // Check for modal dialog with continue button (Copilot interstitial)
+  var modal = document.querySelector('[role="dialog"], .b_modal, [class*="verify"]');
+  if (modal) {
+    var modalBtn = modal.querySelector('button, a[role="button"]');
+    if (modalBtn) { modalBtn.click(); return 'clicked-modal-btn:' + modalBtn.innerText.trim(); }
+  }
+  
+  return 'still-verifying';
+})()
+`;
+
+export async function dismissConsent(tab, cdp) {
+  const result = await cdp(['eval', tab, CONSENT_JS]).catch(() => null);
+  if (result && result !== 'null') {
+    await new Promise(r => setTimeout(r, 1500));
+  }
+}
+
+// Get iframe bounding box for coordinate-based clicking (for cross-origin Turnstile)
+const GET_IFRAME_CENTER_JS = `
+(function() {
+  var iframe = document.querySelector('iframe[src*="challenges.cloudflare.com"], iframe[src*="turnstile"], iframe[title*="challenge"], iframe[title*="Widget"]');
+  if (!iframe) return null;
+  var rect = iframe.getBoundingClientRect();
+  // Click near the center-left where the checkbox usually is
+  return JSON.stringify({ x: rect.left + 30, y: rect.top + rect.height / 2 });
+})()
+`;
+
+// Returns 'clear' | 'clicked' | 'needs-human'
+export async function handleVerification(tab, cdp, waitMs = 60000) {
+  const result = await cdp(['eval', tab, VERIFY_DETECT_JS]).catch(() => null);
+
+  if (!result || result === 'null') return 'clear';
+
+  // Hard CAPTCHA page — wait for user to solve it manually
+  if (result === 'sorry-page') {
+    process.stderr.write(`[greedysearch] Google CAPTCHA detected — please solve it in the browser window (waiting up to ${Math.floor(waitMs / 1000)}s)...\n`);
+    const deadline = Date.now() + waitMs;
+    while (Date.now() < deadline) {
+      await new Promise(r => setTimeout(r, 2000));
+      const url = await cdp(['eval', tab, 'document.location.href']).catch(() => '');
+      if (!url.includes('/sorry/')) return 'cleared-by-user';
+    }
+    return 'needs-human';
+  }
+
+  // We clicked something — wait for page to update, then keep retrying
+  if (result.startsWith('clicked-')) {
+    process.stderr.write(`[greedysearch] Clicked verification: ${result}\n`);
+    await new Promise(r => setTimeout(r, 2000));
+    
+    // Keep checking if verification cleared, retry clicking for up to waitMs
+    const deadline = Date.now() + waitMs;
+    while (Date.now() < deadline) {
+      const retryResult = await cdp(['eval', tab, VERIFY_RETRY_JS]).catch(() => null);
+      
+      if (retryResult === 'cleared' || !retryResult || retryResult === 'null') {
+        process.stderr.write(`[greedysearch] Verification cleared.\n`);
+        await new Promise(r => setTimeout(r, 1000));
+        return 'clicked';
+      }
+      
+      if (retryResult.startsWith('clicked:')) {
+        process.stderr.write(`[greedysearch] Retrying verification click...\n`);
+        await new Promise(r => setTimeout(r, 2000));
+      }
+      
+      // If verification is stuck, try clicking the Turnstile iframe by coordinates
+      const iframeCenter = await cdp(['eval', tab, GET_IFRAME_CENTER_JS]).catch(() => null);
+      if (iframeCenter && iframeCenter !== 'null') {
+        try {
+          const { x, y } = JSON.parse(iframeCenter);
+          process.stderr.write(`[greedysearch] Trying coordinate click on Turnstile iframe at (${x}, ${y})...\n`);
+          await cdp(['clickxy', tab, String(x), String(y)]);
+          await new Promise(r => setTimeout(r, 3000));
+        } catch {}
+      }
+      
+      await new Promise(r => setTimeout(r, 1500));
+    }
+    
+    // Still stuck — might need user intervention
+    process.stderr.write(`[greedysearch] Verification may require manual intervention.\n`);
+    return 'needs-human';
+  }
+
+  // Detection didn't find anything initially, but check for Turnstile iframe with coordinates
+  if (result === 'null' || !result) {
+    const iframeCenter = await cdp(['eval', tab, GET_IFRAME_CENTER_JS]).catch(() => null);
+    if (iframeCenter && iframeCenter !== 'null') {
+      process.stderr.write(`[greedysearch] Found Turnstile iframe, attempting coordinate click...\n`);
+      try {
+        const { x, y } = JSON.parse(iframeCenter);
+        await cdp(['clickxy', tab, String(x), String(y)]);
+        await new Promise(r => setTimeout(r, 3000));
+        
+        // Check if it worked
+        const cleared = await cdp(['eval', tab, VERIFY_RETRY_JS]).catch(() => null);
+        if (cleared === 'cleared' || cleared === 'null') {
+          return 'clicked';
+        }
+      } catch {}
+    }
+  }
+
+  return 'clear';
+}

package/extractors/gemini.mjs

@@ -44,13 +44,14 @@ function cdp(args, timeoutMs = 30000) {
 
 async function getOrOpenTab(tabPrefix) {
   if (tabPrefix) return tabPrefix;
-  if (existsSync(PAGES_CACHE)) {
-    const pages = JSON.parse(readFileSync(PAGES_CACHE, 'utf8'));
-    const existing = pages.find(p => p.url.includes('gemini.google.com'));
-    if (existing) return existing.targetId.slice(0, 8);
-  }
+  // Always open a fresh tab to avoid SPA navigation issues
   const list = await cdp(['list']);
-  return list.split('\n')[0].slice(0, 8);
+  const anchor = list.split('\n')[0]?.slice(0, 8);
+  if (!anchor) throw new Error('No Chrome tabs found. Is Chrome running with --remote-debugging-port=9222?');
+  const raw = await cdp(['evalraw', anchor, 'Target.createTarget', '{"url":"about:blank"}']);
+  const { targetId } = JSON.parse(raw);
+  await cdp(['list']); // refresh cache
+  return targetId.slice(0, 8);
 }
 
 async function typeIntoGemini(tab, text) {
@@ -111,53 +112,11 @@ async function extractAnswer(tab) {
   const answer = await cdp(['eval', tab, `window.__geminiClipboard || ''`]);
   if (!answer) throw new Error('Clipboard interceptor returned empty text');
 
-  // Click "Sources" button to open the sidebar with proper source cards
-  const sourceExcludeFilter = S.sourcesExclude.map(e => `!s.url.includes('${e}')`).join(' && ');
-  await cdp(['eval', tab, `
-    (function() {
-      var btn = document.querySelector('${S.sourcesSidebarButton}');
-      if (!btn) btn = Array.from(document.querySelectorAll('button')).find(b => b.innerText?.trim() === 'Sources');
-      if (btn) { btn.click(); return 'clicked'; }
-      return 'not-found';
-    })()
-  `]).catch(() => 'not-found');
-
-  // Wait for the sources sidebar to populate
-  await new Promise(r => setTimeout(r, 1500));
-
-  // Extract sources from the sidebar panel (has proper URLs + titles)
-  const raw = await cdp(['eval', tab, `
-    (function() {
-      // Find the Sources sidebar container by heading
-      var headings = Array.from(document.querySelectorAll('h1, h2, h3, [class*="header"]'));
-      var sourceHeading = headings.find(h => h.innerText?.trim() === 'Sources');
-      if (sourceHeading) {
-        var container = sourceHeading.closest('.container') || sourceHeading.parentElement;
-        var links = Array.from(container.querySelectorAll('a[href^="http"]'))
-          .map(a => ({ url: a.href.split('#')[0], title: a.innerText?.trim().split('\\n')[0] || '' }))
-          .filter(s => s.url && ${sourceExcludeFilter})
-          .filter((v, i, arr) => arr.findIndex(x => x.url === v.url) === i)
-          .slice(0, 8);
-        return JSON.stringify(links);
-      }
-      // Fallback: inline source cards with aria-labels
-      var cards = Array.from(document.querySelectorAll('${S.citationButtonPattern}'));
-      if (cards.length) {
-        return JSON.stringify(cards.map(b => {
-          var label = b.getAttribute('aria-label') || '';
-          var name = label.match(${S.citationNameRegex})?.[1] || label;
-          return { url: '', title: name };
-        }));
-      }
-      // Last resort: page-wide links (may include footer junk)
-      return JSON.stringify(Array.from(document.querySelectorAll('a[href^="http"]'))
-        .map(a => ({ url: a.href.split('#')[0], title: a.innerText?.trim().split('\\n')[0] || '' }))
-        .filter(s => s.url && !s.url.includes('gemini.google') && !s.url.includes('gstatic') && !s.url.includes('google.com/search'))
-        .filter((v, i, arr) => arr.findIndex(x => x.url === v.url) === i)
-        .slice(0, 8));
-    })()
-  `]).catch(() => '[]');
-  const sources = JSON.parse(raw);
+  // Regex parse Markdown links from clipboard — robust against DOM changes
+  const sources = Array.from(answer.matchAll(/\[([^\]]+)\]\((https?:\/\/[^\s\)]+)\)/g))
+    .map(m => ({ title: m[1], url: m[2] }))
+    .filter((v, i, arr) => arr.findIndex(x => x.url === v.url) === i)
+    .slice(0, 10);
 
   return { answer: answer.trim(), sources };
 }