@duckmind/dm-darwin-x64 0.52.6 → 0.53.4

package/extensions/greedysearch-dm/extractors/consent.mjs

@@ -1,4 +1,4 @@
-import{randomInt as T}from"node:crypto";import{existsSync as Y,readFileSync as _}from"node:fs";import M from"node:http";var S=`
+import{randomInt as Y}from"node:crypto";import{existsSync as j,readFileSync as P}from"node:fs";import I from"node:http";var C=`
 (function() {
   // Google consent page (consent.google.com)
   var g = document.querySelector('#L2AGLb, button[jsname="b3VHJd"], .tHlp8d');
@@ -15,7 +15,7 @@ import{randomInt as T}from"node:crypto";import{existsSync as Y,readFileSync as _
 
   return null;
 })()
-`,E=`
+`,v=`
 (function() {
   var url = document.location.href;
 
@@ -46,12 +46,24 @@ import{randomInt as T}from"node:crypto";import{existsSync as Y,readFileSync as _
 
   // --- Cloudflare Turnstile widget inside closed shadow DOM (Copilot, etc.) ---
   // The iframe is not queryable from main document, but the host container
-  // (#cf-turnstile) and the hidden response input are.
-  var cfTurnstileHost = document.querySelector('#cf-turnstile, [id^="cf-chl-widget-"]');
+  // (#cf-turnstile) and the hidden response input are. When only the
+  // hidden response input matches (no #cf-turnstile host and no visible
+  // iframe), the actual challenge widget is rendered inside a closed
+  // shadow DOM and cannot be auto-clicked. Return a sentinel so callers
+  // know to surface this as needs-human verification instead of wasting
+  // time on a doomed waitForSelector.
+  var cfTurnstileHost = document.querySelector('#cf-turnstile');
   if (cfTurnstileHost) {
     var r2 = cfTurnstileHost.getBoundingClientRect();
     return JSON.stringify({t:'xy',x:r2.left+r2.width/2,y:r2.top+r2.height/2});
   }
+  // Hidden cf-chl-widget-*_response input present but no visible host:
+  // the widget is in closed shadow DOM. Signal this so handleVerification
+  // can return 'needs-human' rather than 'clear'.
+  var cfResponseInput = document.querySelector('input[name="cf-turnstile-response"], [id^="cf-chl-widget-"][id$="_response"]');
+  if (cfResponseInput && cfResponseInput.value === '') {
+    return 'cf-closed-shadow-dom';
+  }
 
   // --- Cloudflare challenge page ---
   var cfCheckbox = document.querySelector('#cf-stage input[type="checkbox"], .ctp-checkbox-container input');
@@ -66,15 +78,28 @@ import{randomInt as T}from"node:crypto";import{existsSync as Y,readFileSync as _
   }
 
   // --- Generic verify/continue/proceed buttons (catch-all) ---
-  // IMPORTANT: exclude sign-in / OAuth buttons (e.g. "Continue with Google")
+  // IMPORTANT: exclude sign-in / OAuth buttons (e.g. "Continue with Google",
+  // "Continue with email", "Login or sign up for free"). These appear on
+  // many sites (Perplexity, ChatGPT, etc.) when the user isn't logged in,
+  // and clicking them triggers a sign-in flow that takes us to a login
+  // wall — a much worse outcome than the original search failure we were
+  // trying to recover from. The exclusion list must cover both OAuth
+  // providers AND generic "sign in / log in / with email" patterns.
   var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
   var verify = btns.find(b => {
     var t = (b.innerText?.trim() || b.value || '').toLowerCase();
-    var isVerifyLike = (t.includes('verify') || t.includes('human') || t.includes('robot') || t.includes('continue') || t.includes('proceed')) &&
+    var isVerifyLike = (t === 'continue' || t === 'proceed' || t === 'next' ||
+                       t.startsWith('verify ') || t.startsWith('human ') || t === 'i am human' || t.includes('robot check')) &&
            !t.includes('verified') && !document.querySelector('iframe[src*="recaptcha"]');
     if (!isVerifyLike) return false;
     // Exclude OAuth / sign-in buttons to prevent accidental login flows
-    var isSignIn = /sign.in|log.in|google|microsoft|apple|facebook|github|auth/i.test(t);
+    // — covers "Continue with Google", "Continue with Apple", "Continue
+    // with email", "Login or sign up", "Log in", "Sign in", "Sign up",
+    // "Single sign-on", and the visible panel "Login or sign up for free"
+    // text. The previous list missed "email" and "sso" which let the
+    // auto-click land on the email/SSO sign-in buttons on Perplexity's
+    // anonymous-mode homepage, navigating us into a login flow.
+    var isSignIn = new RegExp("sign.?in|log.?in|sign.?up|with\\s+(google|apple|email|github|facebook|microsoft|sso)|sso|auth", "i").test(t);
     return !isSignIn;
   });
   if (verify) { verify.setAttribute('data-gs-verify','1'); return JSON.stringify({t:'sel',s:'[data-gs-verify="1"]',txt:verify.innerText?.trim()||verify.value}); }
@@ -85,7 +110,7 @@ import{randomInt as T}from"node:crypto";import{existsSync as Y,readFileSync as _
 
   return null;
 })()
-`,j=`
+`,k=`
 (function() {
   var url = document.location.href;
   var isVerifyPage = url.includes('/sorry/') ||
@@ -119,15 +144,17 @@ import{randomInt as T}from"node:crypto";import{existsSync as Y,readFileSync as _
 
   return 'still-verifying';
 })()
-`;async function F(B,G){let A=await G(["eval",B,S]).catch(()=>null);if(A&&A!=="null")await new Promise((z)=>setTimeout(z,1500))}function Q(B,G){return T(B*1000,G*1000)/1000}async function k(B,G){if(!globalThis.WebSocket)return;let A=process.env.CDP_PROFILE_DIR;if(!A)return;let z=`${A.replaceAll("\\","/")}/DevToolsActivePort`;if(!Y(z))return;let P=_(z,"utf8").trim().split(`
-`)[0],U=await new Promise((N,W)=>{let Z=M.get(`http://localhost:${P}/json/version`,(H)=>{let $="";H.on("data",(O)=>$+=O),H.on("end",()=>{try{N(JSON.parse($))}catch{W(Error("bad JSON"))}})});Z.on("error",W),Z.setTimeout(1000,()=>{Z.destroy(),W(Error("timeout"))})}),K=new globalThis.WebSocket(U.webSocketDebuggerUrl),L=0;await new Promise((N)=>{K.onopen=async()=>{let W=($,O)=>new Promise((V)=>{let X=++L,q=(I)=>{if(JSON.parse(I.data).id===X)K.removeEventListener("message",q),V()};K.addEventListener("message",q),K.send(JSON.stringify({id:X,method:$,params:O}))}),Z=B+Q(-2,2),H=G+Q(-2,2);await W("Input.dispatchMouseEvent",{type:"mouseMoved",x:Z,y:H,button:"none"}),await new Promise(($)=>setTimeout($,Q(80,160))),await W("Input.dispatchMouseEvent",{type:"mousePressed",x:Z,y:H,button:"left",clickCount:1}),await new Promise(($)=>setTimeout($,Q(30,80))),await W("Input.dispatchMouseEvent",{type:"mouseReleased",x:Z+Q(-1,1),y:H+Q(-1,1),button:"left",clickCount:1}),setTimeout(()=>{K.close(),N()},200)},K.onerror=()=>N(),setTimeout(N,3000)})}async function J(B,G,A,z){let P=Number.parseFloat(A),U=Number.parseFloat(z);if(Number.isNaN(P)||Number.isNaN(U))throw Error(`humanClickXY: invalid coordinates (${A}, ${z})`);let K={button:"left",clickCount:1,modifiers:0},L=P+Q(-3,3),N=U+Q(-3,3);await G(["evalraw",B,"Input.dispatchMouseEvent",JSON.stringify({...K,type:"mouseMoved",x:L,y:N})]),await new Promise((O)=>setTimeout(O,Q(80,180)));let W=P+Q(-2,2),Z=U+Q(-2,2);await G(["evalraw",B,"Input.dispatchMouseEvent",JSON.stringify({...K,type:"mousePressed",x:W,y:Z})]),await new Promise((O)=>setTimeout(O,Q(30,90)));let H=W+Q(-1,1),$=Z+Q(-1,1);return await G(["evalraw",B,"Input.dispatchMouseEvent",JSON.stringify({...K,type:"mouseReleased",x:H,y:$})]),await k(P,U).catch(()=>{}),await new Promise((O)=>setTimeout(O,Q(100,300))),`human-clicked at (${P.toFixed(0)}, ${U.toFixed(0)})`}async function C(B,G,A){let z=await G(["eval",B,`(function() {
-			var el = document.querySelector('${A.replace(/\\/g,"\\\\").replace(/'/g,"\\'")}');
+`;async function y(A,G){let B=await G(["eval",A,C]).catch(()=>null);if(B&&B!=="null")await new Promise((z)=>setTimeout(z,1500))}function Z(A,G){return Y(A*1000,G*1000)/1000}async function F(A,G){if(!globalThis.WebSocket)return;let B=process.env.CDP_PROFILE_DIR;if(!B)return;let z=`${B.replaceAll("\\","/")}/DevToolsActivePort`;if(!j(z))return;let K=P(z,"utf8").trim().split(`
+`)[0],U=await new Promise((q,O)=>{let $=I.get(`http://localhost:${K}/json/version`,(W)=>{let L="";W.on("data",(H)=>L+=H),W.on("end",()=>{try{q(JSON.parse(L))}catch{O(Error("bad JSON"))}})});$.on("error",O),$.setTimeout(1000,()=>{$.destroy(),O(Error("timeout"))})}),Q=new globalThis.WebSocket(U.webSocketDebuggerUrl),N=0;await new Promise((q)=>{Q.onopen=async()=>{let O=(L,H)=>new Promise((V)=>{let T=++N,D=(J)=>{if(JSON.parse(J.data).id===T)Q.removeEventListener("message",D),V()};Q.addEventListener("message",D),Q.send(JSON.stringify({id:T,method:L,params:H}))}),$=A+Z(-2,2),W=G+Z(-2,2);await O("Input.dispatchMouseEvent",{type:"mouseMoved",x:$,y:W,button:"none"}),await new Promise((L)=>setTimeout(L,Z(80,160))),await O("Input.dispatchMouseEvent",{type:"mousePressed",x:$,y:W,button:"left",clickCount:1}),await new Promise((L)=>setTimeout(L,Z(30,80))),await O("Input.dispatchMouseEvent",{type:"mouseReleased",x:$+Z(-1,1),y:W+Z(-1,1),button:"left",clickCount:1}),setTimeout(()=>{Q.close(),q()},200)},Q.onerror=()=>q(),setTimeout(q,3000)})}async function S(A,G,B,z){let K=Number.parseFloat(B),U=Number.parseFloat(z);if(Number.isNaN(K)||Number.isNaN(U))throw Error(`humanClickXY: invalid coordinates (${B}, ${z})`);let Q={button:"left",clickCount:1,modifiers:0},N=K+Z(-3,3),q=U+Z(-3,3);await G(["evalraw",A,"Input.dispatchMouseEvent",JSON.stringify({...Q,type:"mouseMoved",x:N,y:q})]),await new Promise((H)=>setTimeout(H,Z(80,180)));let O=K+Z(-2,2),$=U+Z(-2,2);await G(["evalraw",A,"Input.dispatchMouseEvent",JSON.stringify({...Q,type:"mousePressed",x:O,y:$})]),await new Promise((H)=>setTimeout(H,Z(30,90)));let W=O+Z(-1,1),L=$+Z(-1,1);return await G(["evalraw",A,"Input.dispatchMouseEvent",JSON.stringify({...Q,type:"mouseReleased",x:W,y:L})]),await F(K,U).catch(()=>{}),await new Promise((H)=>setTimeout(H,Z(100,300))),`human-clicked at (${K.toFixed(0)}, ${U.toFixed(0)})`}async function w(A,G,B){let z=await G(["eval",A,`(function() {
+			var el = document.querySelector('${B.replace(/\\/g,"\\\\").replace(/'/g,"\\'")}');
 			if (!el) return 'null';
 			var r = el.getBoundingClientRect();
 			return JSON.stringify({x: r.left + r.width / 2, y: r.top + r.height / 2, w: r.width, h: r.height});
-		})()`]).catch(()=>"null");if(!z||z==="null")return null;let P=JSON.parse(z);if(P.w===0||P.h===0||P.x===0&&P.y===0)return null;let{x:U,y:K}=P;return J(B,G,U,K)}async function D(B,G,A){if(!A||A==="null"||A==="cleared"||A==="still-verifying")return!1;try{let z=JSON.parse(A);if(z.t==="sel"&&z.s)return process.stderr.write(`[greedysearch] Human-clicking "${z.txt}" via CDP...
-`),await C(B,G,z.s)!==null;if(z.t==="xy"){if(!z.x&&!z.y)return!1;return process.stderr.write(`[greedysearch] Human-clicking at (${z.x.toFixed(0)}, ${z.y.toFixed(0)})...
-`),await J(B,G,z.x,z.y),!0}}catch{}return!1}async function h(B,G){let A=await G(["eval",B,E]).catch(()=>null);return A&&A!=="null"?A:null}async function m(B,G,A=30000){let z=await h(B,G);if(!z)return"clear";if(z==="sorry-page"){process.stderr.write(`[greedysearch] Google CAPTCHA detected — please solve it in the browser window (waiting up to ${Math.floor(A/1000)}s)...
-`);let U=Date.now()+A;while(Date.now()<U)if(await new Promise((L)=>setTimeout(L,2000)),!(await G(["eval",B,"document.location.href"]).catch(()=>"")).includes("/sorry/"))return"cleared-by-user";return"needs-human"}if(await D(B,G,z)){await new Promise((K)=>setTimeout(K,2000));let U=Date.now()+A;while(Date.now()<U){let K=await G(["eval",B,j]).catch(()=>null);if(K==="cleared"||!K||K==="null")return process.stderr.write(`[greedysearch] Verification cleared.
-`),"clicked";if(K!=="still-verifying")await D(B,G,K),await new Promise((L)=>setTimeout(L,2000));else await new Promise((L)=>setTimeout(L,1500))}return process.stderr.write(`[greedysearch] Verification may require manual intervention.
-`),"needs-human"}return"clear"}export{J as humanClickXY,C as humanClickElement,m as handleVerification,F as dismissConsent,h as detectVerificationChallenge};
+		})()`]).catch(()=>"null");if(!z||z==="null")return null;let K=JSON.parse(z);if(K.w===0||K.h===0||K.x===0&&K.y===0)return null;let{x:U,y:Q}=K;return S(A,G,U,Q)}function M(A,G,B){if(!B||B==="null"||B==="cleared"||B==="still-verifying"||B==="cf-closed-shadow-dom")return Promise.resolve("no-challenge");try{let z=JSON.parse(B);if(z.t==="sel"&&z.s)return process.stderr.write(`[greedysearch] Human-clicking "${z.txt}" via CDP...
+`),w(A,G,z.s).then((K)=>K!==null?"clicked":"cant-click");if(z.t==="xy"){if(!z.x&&!z.y)return Promise.resolve("cant-click");return process.stderr.write(`[greedysearch] Human-clicking at (${z.x.toFixed(0)}, ${z.y.toFixed(0)})...
+`),S(A,G,z.x,z.y).then(()=>"clicked")}}catch{}return Promise.resolve("no-challenge")}async function g(A,G){let B=await G(["eval",A,v]).catch(()=>null);if(B==="cf-closed-shadow-dom"){let z=await m(A,G).catch(()=>null);if(z)return z;return B}if(B&&B!=="null")return B;return null}async function m(A,G){if(typeof G!=="function")return null;await G(["evalraw",A,"DOM.enable","{}"]).catch(()=>{});let B=await G(["evalraw",A,"DOM.getDocument",JSON.stringify({depth:-1,pierce:!0})]).catch(()=>null);if(!B)return null;let z;try{z=JSON.parse(B)}catch{return null}if(z.error||!z.root)return null;let K=z.root;return await E(K,A,G)}async function E(A,G,B){if(!A)return null;let z=[];if(A.shadowRoots&&A.shadowRoots.length>0)for(let K of A.shadowRoots)z.push(K);if(A.children)for(let K of A.children)z.push(K);for(let K of z){if(K.nodeName==="IFRAME"){let Q=K.attributes||[],N=Q.indexOf("src"),q=N>=0?Q[N+1]:"";if(q&&/challenges\.cloudflare\.com|turnstile/i.test(q)&&K.backendNodeId){let O=await B(["evalraw",G,"DOM.getBoxModel",JSON.stringify({backendNodeId:K.backendNodeId})]).catch(()=>null);if(!O)continue;let $;try{$=JSON.parse(O)}catch{continue}let W=$?.model?.content||$?.result?.model?.content;if(!W||W.length<8)continue;let L=W[0],H=W[1],V=W[4],T=W[5],D=V-L,J=T-H;if(D<50||J<20)continue;let X=L+D*0.25,_=H+J*0.5;return process.stderr.write(`[greedysearch] Found CF iframe via CDP pierce at (${L.toFixed(0)}, ${H.toFixed(0)}) ${D.toFixed(0)}x${J.toFixed(0)}, clicking checkbox at (${X.toFixed(0)}, ${_.toFixed(0)})
+`),JSON.stringify({t:"xy",x:X,y:_})}}let U=await E(K,G,B);if(U)return U}return null}async function u(A,G,B=30000){let z=await g(A,G);if(!z)return"clear";if(z==="sorry-page"){process.stderr.write(`[greedysearch] Google CAPTCHA detected — please solve it in the browser window (waiting up to ${Math.floor(B/1000)}s)...
+`);let U=Date.now()+B;while(Date.now()<U)if(await new Promise((N)=>setTimeout(N,2000)),!(await G(["eval",A,"document.location.href"]).catch(()=>"")).includes("/sorry/"))return"cleared-by-user";return"needs-human"}let K=await M(A,G,z);if(K==="clicked"){await new Promise((Q)=>setTimeout(Q,2000));let U=Date.now()+B;while(Date.now()<U){let Q=await G(["eval",A,k]).catch(()=>null);if(Q==="cleared"||!Q||Q==="null")return process.stderr.write(`[greedysearch] Verification cleared.
+`),"clicked";if(Q!=="still-verifying")await M(A,G,Q),await new Promise((N)=>setTimeout(N,2000));else await new Promise((N)=>setTimeout(N,1500))}return process.stderr.write(`[greedysearch] Verification may require manual intervention.
+`),"needs-human"}if(K==="cant-click")return process.stderr.write(`[greedysearch] Verification challenge detected but cannot be auto-clicked — please solve it manually in the visible browser window.
+`),"needs-human";return"clear"}export{S as humanClickXY,w as humanClickElement,u as handleVerification,y as dismissConsent,g as detectVerificationChallenge};