@apmantza/greedysearch-pi 1.9.2 → 2.1.2

package/extractors/consent.mjs

@@ -57,12 +57,24 @@ const VERIFY_DETECT_JS = `
 
   // --- Cloudflare Turnstile widget inside closed shadow DOM (Copilot, etc.) ---
   // The iframe is not queryable from main document, but the host container
-  // (#cf-turnstile) and the hidden response input are.
-  var cfTurnstileHost = document.querySelector('#cf-turnstile, [id^="cf-chl-widget-"]');
+  // (#cf-turnstile) and the hidden response input are. When only the
+  // hidden response input matches (no #cf-turnstile host and no visible
+  // iframe), the actual challenge widget is rendered inside a closed
+  // shadow DOM and cannot be auto-clicked. Return a sentinel so callers
+  // know to surface this as needs-human verification instead of wasting
+  // time on a doomed waitForSelector.
+  var cfTurnstileHost = document.querySelector('#cf-turnstile');
   if (cfTurnstileHost) {
     var r2 = cfTurnstileHost.getBoundingClientRect();
     return JSON.stringify({t:'xy',x:r2.left+r2.width/2,y:r2.top+r2.height/2});
   }
+  // Hidden cf-chl-widget-*_response input present but no visible host:
+  // the widget is in closed shadow DOM. Signal this so handleVerification
+  // can return 'needs-human' rather than 'clear'.
+  var cfResponseInput = document.querySelector('input[name="cf-turnstile-response"], [id^="cf-chl-widget-"][id$="_response"]');
+  if (cfResponseInput && cfResponseInput.value === '') {
+    return 'cf-closed-shadow-dom';
+  }
 
   // --- Cloudflare challenge page ---
   var cfCheckbox = document.querySelector('#cf-stage input[type="checkbox"], .ctp-checkbox-container input');
@@ -77,15 +89,28 @@ const VERIFY_DETECT_JS = `
   }
 
   // --- Generic verify/continue/proceed buttons (catch-all) ---
-  // IMPORTANT: exclude sign-in / OAuth buttons (e.g. "Continue with Google")
+  // IMPORTANT: exclude sign-in / OAuth buttons (e.g. "Continue with Google",
+  // "Continue with email", "Login or sign up for free"). These appear on
+  // many sites (Perplexity, ChatGPT, etc.) when the user isn't logged in,
+  // and clicking them triggers a sign-in flow that takes us to a login
+  // wall — a much worse outcome than the original search failure we were
+  // trying to recover from. The exclusion list must cover both OAuth
+  // providers AND generic "sign in / log in / with email" patterns.
   var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
   var verify = btns.find(b => {
     var t = (b.innerText?.trim() || b.value || '').toLowerCase();
-    var isVerifyLike = (t.includes('verify') || t.includes('human') || t.includes('robot') || t.includes('continue') || t.includes('proceed')) &&
+    var isVerifyLike = (t === 'continue' || t === 'proceed' || t === 'next' ||
+                       t.startsWith('verify ') || t.startsWith('human ') || t === 'i am human' || t.includes('robot check')) &&
            !t.includes('verified') && !document.querySelector('iframe[src*="recaptcha"]');
     if (!isVerifyLike) return false;
     // Exclude OAuth / sign-in buttons to prevent accidental login flows
-    var isSignIn = /sign.in|log.in|google|microsoft|apple|facebook|github|auth/i.test(t);
+    // — covers "Continue with Google", "Continue with Apple", "Continue
+    // with email", "Login or sign up", "Log in", "Sign in", "Sign up",
+    // "Single sign-on", and the visible panel "Login or sign up for free"
+    // text. The previous list missed "email" and "sso" which let the
+    // auto-click land on the email/SSO sign-in buttons on Perplexity's
+    // anonymous-mode homepage, navigating us into a login flow.
+    var isSignIn = new RegExp("sign.?in|log.?in|sign.?up|with\\s+(google|apple|email|github|facebook|microsoft|sso)|sso|auth", "i").test(t);
     return !isSignIn;
   });
   if (verify) { verify.setAttribute('data-gs-verify','1'); return JSON.stringify({t:'sel',s:'[data-gs-verify="1"]',txt:verify.innerText?.trim()||verify.value}); }
@@ -327,16 +352,23 @@ export async function humanClickElement(tab, cdpFn, selector) {
 
 /**
  * Parse a detection result and perform a human click if it found something.
- * Returns true if a click was performed.
+ *
+ * Returns a tristate string:
+ *   - 'clicked'  — a click was successfully dispatched
+ *   - 'cant-click' — challenge was detected but we couldn't click it
+ *                    (zero-dimension element, OOPIF in closed shadow DOM, etc.)
+ *                    Caller should treat this as needs-human verification.
+ *   - 'no-challenge' — no challenge detected, nothing to click
  */
-async function tryHumanClick(tab, cdp, detectResult) {
+function tryHumanClick(tab, cdp, detectResult) {
 	if (
 		!detectResult ||
 		detectResult === "null" ||
 		detectResult === "cleared" ||
-		detectResult === "still-verifying"
+		detectResult === "still-verifying" ||
+		detectResult === "cf-closed-shadow-dom"
 	)
-		return false;
+		return Promise.resolve("no-challenge");
 
 	// JSON format: {t:"sel",s:"...",txt:"..."} or {t:"xy",x:...,y:...}
 	try {
@@ -345,26 +377,138 @@ async function tryHumanClick(tab, cdp, detectResult) {
 			process.stderr.write(
 				`[greedysearch] Human-clicking "${info.txt}" via CDP...\n`,
 			);
-			const r = await humanClickElement(tab, cdp, info.s);
-			return r !== null;
+			return humanClickElement(tab, cdp, info.s).then((r) =>
+				r !== null ? "clicked" : "cant-click",
+			);
 		}
 		if (info.t === "xy") {
 			// Skip zero/invalid coordinates — element is off-screen or not rendered
-			if (!info.x && !info.y) return false;
+			if (!info.x && !info.y) return Promise.resolve("cant-click");
 			process.stderr.write(
 				`[greedysearch] Human-clicking at (${info.x.toFixed(0)}, ${info.y.toFixed(0)})...\n`,
 			);
-			await humanClickXY(tab, cdp, info.x, info.y);
-			return true;
+			return humanClickXY(tab, cdp, info.x, info.y).then(() => "clicked");
 		}
 	} catch {}
 
-	return false;
+	return Promise.resolve("no-challenge");
 }
 
 export async function detectVerificationChallenge(tab, cdp) {
+	// Run the CDP-pierce probe FIRST so we get real click coordinates for
+	// Cloudflare iframes hidden inside closed shadow roots (chatgpt.com,
+	// perplexity.ai, etc.). The page-context probe falls back to a
+	// cf-closed-shadow-dom sentinel when the iframe is opaque to JS DOM
+	// queries, but that sentinel can't be auto-clicked.
+	const cfIframe = await findCloudflareIframeViaPierce(tab, cdp).catch(
+		() => null,
+	);
+	if (cfIframe) return cfIframe;
+
 	const result = await cdp(["eval", tab, VERIFY_DETECT_JS]).catch(() => null);
-	return result && result !== "null" ? result : null;
+	if (result && result !== "null") return result;
+
+	return null;
+}
+
+/**
+ * Walk the page DOM with pierce:true to locate a Cloudflare Turnstile
+ * iframe that's hidden inside a closed shadow root. Returns JSON of the
+ * shape `{t:'xy', x, y}` matching the main-document probe's convention,
+ * OR null if nothing was found.
+ *
+ * The returned coords target the **checkbox area** of the Turnstile widget
+ * (left ~25% of the 300x65 iframe, vertical center) rather than the
+ * iframe's geometric center, because the visible "Verify you are human"
+ * checkbox sits there in the standard widget layout.
+ */
+async function findCloudflareIframeViaPierce(tab, cdp) {
+	if (typeof cdp !== "function") return null;
+
+	// Step 1: enable DOM domain if needed (cheap idempotent call)
+	await cdp(["evalraw", tab, "DOM.enable", "{}"]).catch(() => {});
+
+	// Step 2: get the full DOM tree with pierce — walks closed shadow roots
+	const doc = await cdp(["evalraw", tab, "DOM.getDocument", JSON.stringify({ depth: -1, pierce: true })]).catch(
+		() => null,
+	);
+	if (!doc) return null;
+	let docParsed;
+	try {
+		docParsed = JSON.parse(doc);
+	} catch {
+		return null;
+	}
+	if (docParsed.error || !docParsed.root) return null;
+
+	// Step 3: recursive walk looking for an iframe whose src points at
+	// challenges.cloudflare.com / turnstile
+	const root = docParsed.root;
+	const found = await walkForCfIframe(root, tab, cdp);
+	return found;
+}
+
+async function walkForCfIframe(node, tab, cdp) {
+	if (!node) return null;
+	const children = [];
+	if (node.shadowRoots && node.shadowRoots.length > 0) {
+		for (const s of node.shadowRoots) {
+			children.push(s);
+		}
+	}
+	if (node.children) {
+		for (const c of node.children) children.push(c);
+	}
+	for (const child of children) {
+		if (child.nodeName === "IFRAME") {
+			const attrs = child.attributes || [];
+			const srcIdx = attrs.indexOf("src");
+			const src = srcIdx >= 0 ? attrs[srcIdx + 1] : "";
+			if (
+				src &&
+				/challenges\.cloudflare\.com|turnstile/i.test(src) &&
+				child.backendNodeId
+			) {
+				// Get bounding box via DOM.getBoxModel
+				const boxRes = await cdp([
+					"evalraw",
+					tab,
+					"DOM.getBoxModel",
+					JSON.stringify({ backendNodeId: child.backendNodeId }),
+				]).catch(() => null);
+				if (!boxRes) continue;
+				let boxParsed;
+				try {
+					boxParsed = JSON.parse(boxRes);
+				} catch {
+					continue;
+				}
+				const content =
+					boxParsed?.model?.content || boxParsed?.result?.model?.content;
+				if (!content || content.length < 8) continue;
+				// content = [x1, y1, x2, y2, x3, y3, x4, y4]
+				const x1 = content[0];
+				const y1 = content[1];
+				const x3 = content[4];
+				const y3 = content[5];
+				const width = x3 - x1;
+				const height = y3 - y1;
+				// Skip degenerate boxes (hidden iframes)
+				if (width < 50 || height < 20) continue;
+				// Click the checkbox: standard CF widget is 300x65 with the
+				// checkbox centered at ~25% width, 50% height.
+				const checkboxX = x1 + width * 0.25;
+				const checkboxY = y1 + height * 0.5;
+				process.stderr.write(
+					`[greedysearch] Found CF iframe via CDP pierce at (${x1.toFixed(0)}, ${y1.toFixed(0)}) ${width.toFixed(0)}x${height.toFixed(0)}, clicking checkbox at (${checkboxX.toFixed(0)}, ${checkboxY.toFixed(0)})\n`,
+				);
+				return JSON.stringify({ t: "xy", x: checkboxX, y: checkboxY });
+			}
+		}
+		const deeper = await walkForCfIframe(child, tab, cdp);
+		if (deeper) return deeper;
+	}
+	return null;
 }
 
 // Returns 'clear' | 'clicked' | 'needs-human'
@@ -389,9 +533,17 @@ export async function handleVerification(tab, cdp, waitMs = 30000) {
 		return "needs-human";
 	}
 
+	// Cloudflare Turnstile rendered inside a closed shadow root (e.g.
+	// chatgpt.com). detectVerificationChallenge now uses CDP-level
+	// DOM.getDocument({pierce:true}) to walk into the closed root and
+	// locate the iframe's screen-space bounding box. The result here is
+	// a normal {t:'xy',x,y} coordinate payload that flows through the
+	// regular click path. The historical "cf-closed-shadow-dom" sentinel
+	// is kept in VERIFY_DETECT_JS only as a safety net for unusual pages.
+
 	// Perform human click on detected element
-	const clicked = await tryHumanClick(tab, cdp, result);
-	if (clicked) {
+	const clickResult = await tryHumanClick(tab, cdp, result);
+	if (clickResult === "clicked") {
 		await new Promise((r) => setTimeout(r, 2000));
 
 		// Retry loop — keep checking until cleared or timeout
@@ -417,5 +569,17 @@ export async function handleVerification(tab, cdp, waitMs = 30000) {
 		return "needs-human";
 	}
 
+	// Challenge was detected but we couldn't auto-click it (zero-dimension
+	// element, OOPIF without coordinates, etc.). Surface this rather than
+	// silently returning 'clear' — the caller would otherwise proceed and
+	// fail downstream on a selector that won't appear until the challenge
+	// is solved.
+	if (clickResult === "cant-click") {
+		process.stderr.write(
+			"[greedysearch] Verification challenge detected but cannot be auto-clicked — please solve it manually in the visible browser window.\n",
+		);
+		return "needs-human";
+	}
+
 	return "clear";
 }