@apmantza/greedysearch-pi 1.0.21 → 1.0.23

package/extractors/bing-copilot.mjs

@@ -136,21 +136,44 @@ async function main() {
 
     // Navigate to Copilot homepage and use the chat input
     await cdp(['nav', tab, 'https://copilot.microsoft.com/'], 35000);
-    await new Promise(r => setTimeout(r, 1500));
+    await new Promise(r => setTimeout(r, 2000));
     await dismissConsent(tab, cdp);
-    await handleVerification(tab, cdp, 60000);
+    
+    // Handle verification challenges (Cloudflare Turnstile, Microsoft auth, etc.)
+    const verifyResult = await handleVerification(tab, cdp, 90000);
+    if (verifyResult === 'needs-human') {
+      throw new Error('Copilot verification required — please solve it manually in the browser window');
+    }
+    
+    // After verification, page may have redirected or reloaded — wait for it to settle
+    if (verifyResult === 'clicked') {
+      await new Promise(r => setTimeout(r, 3000));
+      
+      // Re-navigate if we got redirected
+      const currentUrl = await cdp(['eval', tab, 'document.location.href']).catch(() => '');
+      if (!currentUrl.includes('copilot.microsoft.com')) {
+        await cdp(['nav', tab, 'https://copilot.microsoft.com/'], 35000);
+        await new Promise(r => setTimeout(r, 2000));
+        await dismissConsent(tab, cdp);
+      }
+    }
 
-    // Wait for React app to mount #userInput (up to 8s)
-    const deadline = Date.now() + 8000;
-    while (Date.now() < deadline) {
+    // Wait for React app to mount #userInput (up to 15s, longer after verification)
+    const inputDeadline = Date.now() + 15000;
+    while (Date.now() < inputDeadline) {
       const found = await cdp(['eval', tab, `!!document.querySelector('#userInput')`]).catch(() => 'false');
       if (found === 'true') break;
-      await new Promise(r => setTimeout(r, 400));
+      await new Promise(r => setTimeout(r, 500));
     }
     await new Promise(r => setTimeout(r, 300));
 
+    // Verify input is actually there before proceeding
+    const inputReady = await cdp(['eval', tab, `!!document.querySelector('#userInput')`]).catch(() => 'false');
+    if (inputReady !== 'true') {
+      throw new Error('Copilot input not found — verification may have failed or page is in unexpected state');
+    }
+
     await injectClipboardInterceptor(tab);
-    // Find input and type query
     await cdp(['click', tab, '#userInput']);
     await new Promise(r => setTimeout(r, 400));
     await cdp(['type', tab, query]);

package/extractors/consent.mjs

@@ -20,28 +20,98 @@ const CONSENT_JS = `
 })()
 `;
 
-// Detect Google's "verify you're human" / unusual traffic page
+// Detect and auto-click human verification challenges (Google, Microsoft, Cloudflare)
 const VERIFY_DETECT_JS = `
 (function() {
   var url = document.location.href;
+  
+  // --- Google "sorry" page (hard CAPTCHA, can't auto-solve) ---
   if (url.includes('/sorry/') || url.includes('sorry.google')) return 'sorry-page';
-
-  // Simple click-through verify button (not image CAPTCHA)
+  
+  // --- Microsoft account verification page ---
+  if (url.includes('login.microsoftonline.com') || url.includes('login.live.com') || url.includes('account.microsoft.com')) {
+    // Look for "Verify" or "Continue" buttons on Microsoft auth pages
+    var msBtns = Array.from(document.querySelectorAll('button, input[type=submit], a'));
+    var msVerify = msBtns.find(b => /verify|continue|next/i.test(b.innerText?.trim() || b.value || ''));
+    if (msVerify) { msVerify.click(); return 'clicked-ms-verify:' + (msVerify.innerText?.trim() || msVerify.value); }
+  }
+  
+  // --- Cloudflare Turnstile (used by Copilot and many sites) ---
+  // Turnstile widget in iframe
+  var turnstileIframe = document.querySelector('iframe[src*="challenges.cloudflare.com"], iframe[src*="turnstile"]');
+  if (turnstileIframe) {
+    // Try to find and click the checkbox inside the iframe's container
+    var turnstileCheckbox = document.querySelector('#cf-turnstile-response, [data-turnstile-callback] input, .cf-turnstile input[type=checkbox]');
+    if (turnstileCheckbox && !turnstileCheckbox.checked) {
+      turnstileCheckbox.click();
+      return 'clicked-turnstile-checkbox';
+    }
+    // Try clicking the turnstile container itself (some implementations)
+    var turnstileContainer = document.querySelector('.cf-turnstile, [data-sitekey]');
+    if (turnstileContainer) {
+      turnstileContainer.click();
+      return 'clicked-turnstile-container';
+    }
+  }
+  
+  // --- Cloudflare "Verify you are human" challenge page ---
+  if (url.includes('challenges.cloudflare.com') || document.querySelector('#challenge-running, #challenge-stage')) {
+    var cfCheckbox = document.querySelector('#cf-stage input[type="checkbox"], .ctp-checkbox-container input');
+    if (cfCheckbox) { cfCheckbox.click(); return 'clicked-cloudflare-checkbox'; }
+    var cfBtn = document.querySelector('#challenge-form button, .cf-challenge button');
+    if (cfBtn) { cfBtn.click(); return 'clicked-cloudflare-button'; }
+  }
+  
+  // --- Microsoft "I am human" / "Verify" challenge ---
+  // Microsoft uses various verification UIs
+  var msHumanBtn = document.querySelector('button[id*="i0"], button[id*="id__"]');
+  if (msHumanBtn && /verify|human|robot|continue/i.test(msHumanBtn.innerText?.trim())) {
+    msHumanBtn.click();
+    return 'clicked-ms-human:' + msHumanBtn.innerText.trim();
+  }
+  
+  // --- Generic verification buttons (catch-all) ---
   var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
-  var verify = btns.find(b => /verify|human|not a robot|continue/i.test(b.innerText?.trim() || b.value || ''));
+  var verify = btns.find(b => /^(verify|verification|verify you are human|i am human|not a robot|continue|proceed)$/i.test(b.innerText?.trim() || b.value || ''));
   if (verify && !document.querySelector('iframe[src*="recaptcha"]')) {
     verify.click();
     return 'clicked-verify:' + (verify.innerText?.trim() || verify.value);
   }
 
-  // Unchecked reCAPTCHA / Turnstile checkbox (no image challenge)
-  var checkbox = document.querySelector('.recaptcha-checkbox-unchecked, input[type=checkbox][id*="recaptcha"], #cf-stage input[type=checkbox]');
-  if (checkbox) { checkbox.click(); return 'clicked-checkbox'; }
+  // --- Google reCAPTCHA (no image challenge, just checkbox) ---
+  var recaptchaCheckbox = document.querySelector('.recaptcha-checkbox-unchecked, input[type=checkbox][id*="recaptcha"]');
+  if (recaptchaCheckbox) { recaptchaCheckbox.click(); return 'clicked-recaptcha'; }
 
   return null;
 })()
 `;
 
+// Retry loop for verification — keeps checking and clicking until page changes or timeout
+const VERIFY_RETRY_JS = `
+(function() {
+  var url = document.location.href;
+  
+  // Check if we're still on a verification page
+  var isVerifyPage = url.includes('/sorry/') || 
+                     url.includes('challenges.cloudflare.com') ||
+                     url.includes('login.microsoftonline.com') ||
+                     document.querySelector('#challenge-running, #challenge-stage, .cf-turnstile');
+  
+  if (!isVerifyPage) return 'cleared';
+  
+  // Try clicking any verify/continue button again
+  var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
+  var btn = btns.find(b => /^(verify|continue|next|i am human|not a robot)$/i.test(b.innerText?.trim() || b.value || ''));
+  if (btn) { btn.click(); return 'clicked:' + (btn.innerText?.trim() || btn.value); }
+  
+  // Try Turnstile checkbox
+  var cf = document.querySelector('#cf-stage input[type="checkbox"], .cf-turnstile input');
+  if (cf && !cf.checked) { cf.click(); return 'clicked-turnstile'; }
+  
+  return 'still-verifying';
+})()
+`;
+
 export async function dismissConsent(tab, cdp) {
   const result = await cdp(['eval', tab, CONSENT_JS]).catch(() => null);
   if (result && result !== 'null') {
@@ -55,9 +125,9 @@ export async function handleVerification(tab, cdp, waitMs = 60000) {
 
   if (!result || result === 'null') return 'clear';
 
+  // Hard CAPTCHA page — wait for user to solve it manually
   if (result === 'sorry-page') {
-    // Hard CAPTCHA page — wait for user to solve it manually
-    process.stderr.write(`[greedysearch] Google verification required — please solve it in the browser window (waiting up to ${waitMs / 1000}s)...\n`);
+    process.stderr.write(`[greedysearch] Google CAPTCHA detected — please solve it in the browser window (waiting up to ${Math.floor(waitMs / 1000)}s)...\n`);
     const deadline = Date.now() + waitMs;
     while (Date.now() < deadline) {
       await new Promise(r => setTimeout(r, 2000));
@@ -67,9 +137,33 @@ export async function handleVerification(tab, cdp, waitMs = 60000) {
     return 'needs-human';
   }
 
+  // We clicked something — wait for page to update, then keep retrying
   if (result.startsWith('clicked-')) {
+    process.stderr.write(`[greedysearch] Clicked verification: ${result}\n`);
     await new Promise(r => setTimeout(r, 2000));
-    return 'clicked';
+    
+    // Keep checking if verification cleared, retry clicking for up to waitMs
+    const deadline = Date.now() + waitMs;
+    while (Date.now() < deadline) {
+      const retryResult = await cdp(['eval', tab, VERIFY_RETRY_JS]).catch(() => null);
+      
+      if (retryResult === 'cleared' || !retryResult || retryResult === 'null') {
+        process.stderr.write(`[greedysearch] Verification cleared.\n`);
+        await new Promise(r => setTimeout(r, 1000));
+        return 'clicked';
+      }
+      
+      if (retryResult.startsWith('clicked:')) {
+        process.stderr.write(`[greedysearch] Retrying verification click...\n`);
+        await new Promise(r => setTimeout(r, 2000));
+      }
+      
+      await new Promise(r => setTimeout(r, 1500));
+    }
+    
+    // Still stuck — might need user intervention
+    process.stderr.write(`[greedysearch] Verification may require manual intervention.\n`);
+    return 'needs-human';
   }
 
   return 'clear';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apmantza/greedysearch-pi",
-  "version": "1.0.21",
+  "version": "1.0.23",
   "description": "Pi extension: search Perplexity, Bing Copilot, and Google AI in parallel with optional Gemini synthesis — grounded AI answers, not just links",
   "type": "module",
   "keywords": [

package/search.mjs

@@ -174,6 +174,80 @@ function pickTopSource(out) {
   return null;
 }
 
+function deduplicateSources(out) {
+  const seen = new Map(); // url -> { title, engines }
+  const engineOrder = ['perplexity', 'bing', 'google'];
+  
+  for (const engine of engineOrder) {
+    const r = out[engine];
+    if (!r?.sources) continue;
+    for (const s of r.sources) {
+      const url = s.url?.split('#')[0]?.replace(/\/$/, '');
+      if (!url || url.length < 10) continue;
+      if (!seen.has(url)) {
+        seen.set(url, { url: s.url, title: s.title || '', engines: [engine] });
+      } else {
+        const existing = seen.get(url);
+        if (!existing.engines.includes(engine)) {
+          existing.engines.push(engine);
+        }
+        if (!existing.title && s.title) existing.title = s.title;
+      }
+    }
+  }
+  
+  // Sort by consensus (most engines = highest confidence)
+  return Array.from(seen.values())
+    .sort((a, b) => b.engines.length - a.engines.length)
+    .slice(0, 10);
+}
+
+async function synthesizeWithGemini(query, results) {
+  // Build a prompt that includes all engine results
+  const sources = deduplicateSources(results);
+  
+  let prompt = `Based on the following search results from multiple AI engines, provide a single, synthesized answer to the user's question. Combine the information, resolve any conflicts, and present the most accurate and complete answer.\n\n`;
+  prompt += `User's question: "${query}"\n\n`;
+  
+  for (const engine of ['perplexity', 'bing', 'google']) {
+    const r = results[engine];
+    if (r?.error) {
+      prompt += `## ${engine} (failed)\nError: ${r.error}\n\n`;
+    } else if (r?.answer) {
+      prompt += `## ${engine}\n${r.answer.slice(0, 2000)}\n\n`;
+    }
+  }
+  
+  prompt += `Provide a synthesized answer that:\n`;
+  prompt += `1. Combines the best information from all sources\n`;
+  prompt += `2. Notes where sources agree or disagree\n`;
+  prompt += `3. Is clear and well-structured\n`;
+  prompt += `4. Includes key sources at the end\n`;
+  
+  // Run the query through Gemini extractor
+  return new Promise((resolve, reject) => {
+    const proc = spawn('node', [join(__dir, 'extractors', 'gemini.mjs'), prompt, '--short'], {
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let out = '';
+    let err = '';
+    proc.stdout.on('data', d => out += d);
+    proc.stderr.on('data', d => err += d);
+    const t = setTimeout(() => {
+      proc.kill();
+      reject(new Error('Gemini synthesis timed out after 120s'));
+    }, 120000);
+    proc.on('close', code => {
+      clearTimeout(t);
+      if (code !== 0) reject(new Error(err.trim() || 'gemini extractor failed'));
+      else {
+        try { resolve(JSON.parse(out.trim())); }
+        catch { reject(new Error(`bad JSON from gemini: ${out.slice(0, 100)}`)); }
+      }
+    });
+  });
+}
+
 function writeOutput(data, outFile) {
   const json = JSON.stringify(data, null, 2) + '\n';
   if (outFile) {
@@ -257,12 +331,14 @@ async function main() {
   const full        = args.includes('--full');
   const short       = !full;   // brief by default; --full opts into complete answers
   const fetchSource = args.includes('--fetch-top-source');
+  const synthesize  = args.includes('--synthesize');
   const outIdx      = args.indexOf('--out');
   const outFile     = outIdx !== -1 ? args[outIdx + 1] : null;
   const rest        = args.filter((a, i) =>
     a !== '--full' &&
     a !== '--short' &&  // keep accepting --short for back-compat
     a !== '--fetch-top-source' &&
+    a !== '--synthesize' &&
     a !== '--out' &&
     (outIdx === -1 || i !== outIdx + 1)
   );
@@ -310,6 +386,25 @@ async function main() {
       }
     }
 
+    // Deduplicate sources across all engines
+    out._sources = deduplicateSources(out);
+
+    // Synthesize with Gemini if requested
+    if (synthesize) {
+      process.stderr.write('[greedysearch] Synthesizing results with Gemini...\n');
+      try {
+        const synthesis = await synthesizeWithGemini(query, out);
+        out._synthesis = {
+          answer: synthesis.answer || '',
+          sources: synthesis.sources || [],
+          synthesized: true,
+        };
+      } catch (e) {
+        process.stderr.write(`[greedysearch] Synthesis failed: ${e.message}\n`);
+        out._synthesis = { error: e.message, synthesized: false };
+      }
+    }
+
     if (fetchSource) {
       const top = pickTopSource(out);
       if (top) out._topSource = await fetchTopSource(top.url);