@apmantza/greedysearch-pi 1.8.9 → 1.9.0

package/extractors/gemini.mjs

@@ -1,150 +1,217 @@
-#!/usr/bin/env node
-
-// extractors/gemini.mjs
-// Navigate gemini.google.com/app, submit query, wait for answer, return clean answer + sources.
-//
-// Usage:
-//   node extractors/gemini.mjs "<query>" [--tab <prefix>]
-//
-// Output (stdout): JSON { answer, sources, query, url }
-// Errors go to stderr only — stdout is always clean JSON for piping.
-
-import {
-	cdp,
-	formatAnswer,
-	getOrOpenTab,
-	handleError,
-	injectClipboardInterceptor,
-	jitter,
-	outputJson,
-	parseArgs,
-	parseSourcesFromMarkdown,
-	prepareArgs,
-	TIMING,
-	validateQuery,
-	waitForSelector,
-	waitForStreamComplete,
-} from "./common.mjs";
-import { dismissConsent, handleVerification } from "./consent.mjs";
-import { SELECTORS } from "./selectors.mjs";
-
-const S = SELECTORS.gemini;
-const GLOBAL_VAR = "__geminiClipboard";
-
-// ============================================================================
-// Gemini-specific helpers
-// ============================================================================
-
-async function typeIntoGemini(tab, text) {
-	await cdp([
-		"eval",
-		tab,
-		`
-    (function(t) {
-      var el = document.querySelector('${S.input}');
-      if (!el) return false;
-      el.focus();
-      document.execCommand('insertText', false, t);
-      return true;
-    })(${JSON.stringify(text)})
-  `,
-	]);
-}
-
-async function scrollToBottom(tab) {
-	await cdp([
-		"eval",
-		tab,
-		`(function() {
-			const chat = document.querySelector('chat-window, [role="main"], main') || document.body;
-			chat.scrollTo ? chat.scrollTo({ top: chat.scrollHeight, behavior: 'smooth' }) : window.scrollTo(0, document.body.scrollHeight);
-		})()`,
-	]);
-}
-
-async function extractAnswer(tab) {
-	// Click the LAST copy button (assistant's response at the bottom),
-	// not the first (which could be the user's echoed query).
-	await cdp([
-		"eval",
-		tab,
-		`(() => {
-			const buttons = document.querySelectorAll('${S.copyButton}');
-			buttons[buttons.length - 1]?.click();
-		})()`,
-	]);
-	await new Promise((r) => setTimeout(r, 400));
-
-	const answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
-	if (!answer) throw new Error("Clipboard interceptor returned empty text");
-
-	const sources = parseSourcesFromMarkdown(answer);
-	return { answer: answer.trim(), sources };
-}
-
-// ============================================================================
-// Main
-// ============================================================================
-
-const USAGE = 'Usage: node extractors/gemini.mjs "<query>" [--tab <prefix>]\n';
-
-async function main() {
-	const args = await prepareArgs(process.argv.slice(2));
-	validateQuery(args, USAGE);
-
-	const { query, tabPrefix, short } = parseArgs(args);
-
-	try {
-		await cdp(["list"]);
-		const tab = await getOrOpenTab(tabPrefix);
-
-		// Each search = fresh conversation
-		await cdp(["nav", tab, "https://gemini.google.com/app"], 20000);
-		await new Promise((r) => setTimeout(r, 600));
-		await dismissConsent(tab, cdp);
-		await handleVerification(tab, cdp, 10000);
-
-		// Wait for input to be ready
-		await waitForSelector(tab, S.input, 8000, TIMING.inputPoll);
-		await new Promise((r) => setTimeout(r, jitter(TIMING.postClick)));
-
-		await injectClipboardInterceptor(tab, GLOBAL_VAR);
-		await typeIntoGemini(tab, query);
-		await new Promise((r) => setTimeout(r, jitter(TIMING.postType)));
-
-		await cdp([
-			"eval",
-			tab,
-			`document.querySelector('${S.sendButton}')?.click()`,
-		]);
-
-		// Wait for Gemini's response to finish streaming before extracting.
-		// Periodic scrolling keeps lazy-loaded content triggered in the viewport.
-		let pollTick = 0;
-		const scrollInterval = setInterval(() => {
-			if (++pollTick % 10 === 0) scrollToBottom(tab).catch(() => null);
-		}, 6000);
-		try {
-			await waitForStreamComplete(tab, { timeout: 45000, minLength: 50 });
-		} finally {
-			clearInterval(scrollInterval);
-		}
-
-		const { answer, sources } = await extractAnswer(tab);
-		if (!answer) throw new Error("No answer captured from Gemini clipboard");
-
-		const finalUrl = await cdp(["eval", tab, "document.location.href"]).catch(
-			() => "https://gemini.google.com/app",
-		);
-		outputJson({
-			query,
-			url: finalUrl,
-			answer: formatAnswer(answer, short),
-			sources,
-		});
-	} catch (e) {
-		handleError(e);
-	}
-}
-
-main();
+#!/usr/bin/env node
+
+// extractors/gemini.mjs
+// Navigate gemini.google.com/app, submit query, wait for answer, return clean answer + sources.
+//
+// Usage:
+//   node extractors/gemini.mjs "<query>" [--tab <prefix>]
+//
+// Output (stdout): JSON { answer, sources, query, url }
+// Errors go to stderr only — stdout is always clean JSON for piping.
+
+import {
+	cdp,
+	formatAnswer,
+	getOrOpenTab,
+	handleError,
+	injectClipboardInterceptor,
+	jitter,
+	outputJson,
+	parseArgs,
+	parseSourcesFromMarkdown,
+	prepareArgs,
+	TIMING,
+	validateQuery,
+	waitForSelector,
+	waitForStreamComplete,
+} from "./common.mjs";
+import { dismissConsent, handleVerification } from "./consent.mjs";
+import { SELECTORS } from "./selectors.mjs";
+
+const S = SELECTORS.gemini;
+const GLOBAL_VAR = "__geminiClipboard";
+
+// ============================================================================
+// Gemini-specific helpers
+// ============================================================================
+
+async function typeIntoGemini(tab, text) {
+	// 1. Focus the input area via click (more reliable than eval focus for shadow-DOM editors)
+	await cdp(["click", tab, S.input]);
+	await new Promise((r) => setTimeout(r, jitter(200)));
+
+	// 2. Type using CDP Input.insertText (more reliable than document.execCommand)
+	await cdp(["type", tab, text]);
+	await new Promise((r) => setTimeout(r, jitter(300)));
+
+	// 3. Verify the text was actually inserted
+	const inserted = await cdp([
+		"eval",
+		tab,
+		`(function() {
+			var el = document.querySelector('${S.input}');
+			if (!el) return false;
+			var content = el.innerText || el.textContent || '';
+			return content.trim().length >= ${Math.floor(text.length * 0.8)};
+		})()`,
+	]);
+	if (inserted !== "true") {
+		throw new Error(
+			"Gemini input field did not accept text — input verification failed",
+		);
+	}
+}
+
+async function scrollToBottom(tab) {
+	await cdp([
+		"eval",
+		tab,
+		`(function() {
+			const chat = document.querySelector('chat-window, [role="main"], main') || document.body;
+			chat.scrollTo ? chat.scrollTo({ top: chat.scrollHeight, behavior: 'smooth' }) : window.scrollTo(0, document.body.scrollHeight);
+		})()`,
+	]);
+}
+
+async function extractAnswer(tab, query = "") {
+	const queryNorm = query.toLowerCase().trim();
+
+	// Wait for the assistant response copy button to appear.
+	// A fresh conversation has 1 copy button (user message); after the
+	// assistant responds there are 2+.  This prevents clicking the user's
+	// copy button before React hydrates the assistant's.
+	let copyReady = false;
+	const copyDeadline = Date.now() + 12000;
+	while (Date.now() < copyDeadline) {
+		const count = await cdp([
+			"eval",
+			tab,
+			`document.querySelectorAll('${S.copyButton}').length`,
+		]);
+		if (parseInt(count, 10) >= 2) {
+			copyReady = true;
+			break;
+		}
+		await new Promise((r) => setTimeout(r, 800));
+	}
+	if (!copyReady) {
+		console.error("[gemini] Warning: assistant copy button did not appear");
+	}
+
+	// Click the LAST copy button (assistant's response at the bottom)
+	await cdp([
+		"eval",
+		tab,
+		`(() => {
+			const buttons = document.querySelectorAll('${S.copyButton}');
+			buttons[buttons.length - 1]?.click();
+		})()`,
+	]);
+	await new Promise((r) => setTimeout(r, 600));
+
+	let answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
+
+	// Retry once if clipboard contains the user's query instead of the response.
+	// This can happen when the assistant response hasn't rendered its copy button yet.
+	if (
+		answer &&
+		queryNorm &&
+		(answer.toLowerCase().trim() === queryNorm ||
+			answer.trim().length < queryNorm.length)
+	) {
+		console.error("[gemini] Clipboard echoed query, retrying in 2s...");
+		await new Promise((r) => setTimeout(r, 2000));
+		await cdp([
+			"eval",
+			tab,
+			`(() => {
+				const buttons = document.querySelectorAll('${S.copyButton}');
+				buttons[buttons.length - 1]?.click();
+			})()`,
+		]);
+		await new Promise((r) => setTimeout(r, 600));
+		answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
+	}
+
+	if (!answer) throw new Error("Clipboard interceptor returned empty text");
+
+	const sources = parseSourcesFromMarkdown(answer);
+	return { answer: answer.trim(), sources };
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+const USAGE = 'Usage: node extractors/gemini.mjs "<query>" [--tab <prefix>]\n';
+
+async function main() {
+	const args = await prepareArgs(process.argv.slice(2));
+	validateQuery(args, USAGE);
+
+	const { query, tabPrefix, short } = parseArgs(args);
+
+	try {
+		await cdp(["list"]);
+		const tab = await getOrOpenTab(tabPrefix);
+
+		// Skip navigation if tab was pre-seeded to Gemini (e.g. by search.mjs
+		// opening the tab in parallel with source fetch to save ~4s nav time).
+		const currentUrl = await cdp(["eval", tab, "document.location.href"]).catch(() => "");
+		let onGemini = false;
+		try {
+			const host = new URL(currentUrl).hostname.toLowerCase();
+			onGemini = host === "gemini.google.com" || host.endsWith(".gemini.google.com");
+		} catch {}
+
+		if (!onGemini) {
+			await cdp(["nav", tab, "https://gemini.google.com/app"], 20000);
+			await new Promise((r) => setTimeout(r, 600));
+		}
+		await dismissConsent(tab, cdp);
+		await handleVerification(tab, cdp, 10000);
+
+		// Wait for input to be ready
+		await waitForSelector(tab, S.input, 8000, TIMING.inputPoll);
+		await new Promise((r) => setTimeout(r, jitter(TIMING.postClick)));
+
+		await injectClipboardInterceptor(tab, GLOBAL_VAR);
+		await typeIntoGemini(tab, query);
+		await new Promise((r) => setTimeout(r, jitter(TIMING.postType)));
+
+		await cdp([
+			"eval",
+			tab,
+			`document.querySelector('${S.sendButton}')?.click()`,
+		]);
+
+		// Wait for Gemini's response to finish streaming before extracting.
+		// Periodic scrolling keeps lazy-loaded content triggered in the viewport.
+		let pollTick = 0;
+		const scrollInterval = setInterval(() => {
+			if (++pollTick % 10 === 0) scrollToBottom(tab).catch(() => null);
+		}, 6000);
+		try {
+			await waitForStreamComplete(tab, { timeout: 45000, minLength: 50 });
+		} finally {
+			clearInterval(scrollInterval);
+		}
+
+		const { answer, sources } = await extractAnswer(tab, query);
+		if (!answer) throw new Error("No answer captured from Gemini clipboard");
+
+		const finalUrl = await cdp(["eval", tab, "document.location.href"]).catch(
+			() => "https://gemini.google.com/app",
+		);
+		outputJson({
+			query,
+			url: finalUrl,
+			answer: formatAnswer(answer, short),
+			sources,
+		});
+	} catch (e) {
+		handleError(e);
+	}
+}
+
+main();

package/extractors/perplexity.mjs

@@ -12,6 +12,7 @@
 // TODO: Refactor - this file has 42 lines duplicated with google-ai.mjs (line 28)
 
 import {
+	buildEnvelope,
 	cdp,
 	formatAnswer,
 	getOrOpenTab,
@@ -49,13 +50,14 @@ function findCopyButtonJsExpression() {
 // Extraction
 // ============================================================================
 
-async function extractAnswer(tab) {
+async function extractAnswer(tab, env) {
 	const copyBtnExpr = findCopyButtonJsExpression();
 
 	await cdp(["eval", tab, `${copyBtnExpr}?.click()`]);
 	await new Promise((r) => setTimeout(r, 400));
 
 	let answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
+	env.clipboardEmpty = !answer;
 
 	// Retry once if clipboard is empty (Perplexity might be slow to write)
 	if (!answer) {
@@ -63,6 +65,7 @@ async function extractAnswer(tab) {
 		await cdp(["eval", tab, `${copyBtnExpr}?.click()`]);
 		await new Promise((r) => setTimeout(r, 2000));
 		answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
+		env.clipboardEmpty = !answer;
 	}
 
 	if (!answer) throw new Error("Clipboard interceptor returned empty text");
@@ -83,6 +86,19 @@ async function main() {
 	validateQuery(args, USAGE);
 
 	const { query, tabPrefix, short } = parseArgs(args);
+	const startTime = Date.now();
+	const mode =
+		process.env.GREEDY_SEARCH_VISIBLE === "1" ? "visible" : "headless";
+
+	const env = {
+		engine: "perplexity",
+		mode,
+		clipboardEmpty: null,
+		fallbackUsed: null,
+		blockedBy: null,
+		verificationResult: null,
+		inputReady: null,
+	};
 
 	try {
 		// Only refresh page list when creating a fresh tab (no prefix provided)
@@ -107,6 +123,7 @@ async function main() {
 		}
 		// Handle verification challenges (Cloudflare Turnstile, etc.)
 		const verifyResult = await handleVerification(tab, cdp, 10000);
+		env.verificationResult = verifyResult;
 		if (verifyResult === "needs-human") {
 			throw new Error(
 				"Perplexity verification required — please solve it manually in the browser window",
@@ -114,14 +131,43 @@ async function main() {
 		}
 		await dismissConsent(tab, cdp);
 
-		// After verification, page may have redirected or reloaded — wait for it to settle
+		// After verification, page may have redirected — wait for it to settle
+		// then re-navigate to homepage if we ended up somewhere else.
 		if (verifyResult === "clicked") {
 			await new Promise((r) => setTimeout(r, TIMING.afterVerify));
-			await dismissConsent(tab, cdp);
+			const postVerifyUrl = await cdp(["eval", tab, "document.location.href"]).catch(() => "");
+			let onPerplexityAfter = false;
+			try {
+				const host = new URL(postVerifyUrl).hostname.toLowerCase();
+				onPerplexityAfter = host === "perplexity.ai" || host.endsWith(".perplexity.ai");
+			} catch {}
+			if (!onPerplexityAfter) {
+				await cdp(["nav", tab, "https://www.perplexity.ai/"], 20000);
+				await new Promise((r) => setTimeout(r, 800));
+				await dismissConsent(tab, cdp);
+			}
+		}
+
+		// In headless mode: snap the accessibility tree to detect Cloudflare
+		// before burning the selector wait. Perplexity is CF-protected in headless
+		// just like Bing — fast-fail triggers the visible retry.
+		if (process.env.GREEDY_SEARCH_HEADLESS === "1") {
+			const snap = await cdp(["snap", tab]).catch(() => "");
+			if (/cloudflare|challenge|security check/i.test(snap)) {
+				console.error("[perplexity] Cloudflare challenge in snap — fast-failing to visible retry");
+				env.blockedBy = "cloudflare";
+				throw new Error("Cloudflare challenge detected — headless blocked");
+			}
+		}
+
+		// Wait for React app to mount input (up to 15s — gives CF redirect + hydration time)
+		const inputReady = await waitForSelector(tab, S.input, 15000, 400);
+		env.inputReady = inputReady;
+
+		if (!inputReady) {
+			throw new Error("Perplexity input not found — page may not have loaded or is in unexpected state");
 		}
 
-		// Wait for React app to mount input (up to 5s)
-		await waitForSelector(tab, S.input, 5000, 400);
 		await new Promise((r) => setTimeout(r, jitter(300)));
 
 		await injectClipboardInterceptor(tab, GLOBAL_VAR);
@@ -144,7 +190,7 @@ async function main() {
 			selector: "document.body",
 		});
 
-		const { answer, sources } = await extractAnswer(tab);
+		const { answer, sources } = await extractAnswer(tab, env);
 
 		if (!answer)
 			throw new Error(
@@ -154,14 +200,17 @@ async function main() {
 		const finalUrl = await cdp(["eval", tab, "document.location.href"]).catch(
 			() => "",
 		);
+		env.durationMs = Date.now() - startTime;
 		outputJson({
 			query,
 			url: finalUrl,
 			answer: formatAnswer(answer, short),
 			sources,
+			_envelope: buildEnvelope(env),
 		});
 	} catch (e) {
-		handleError(e);
+		env.durationMs = Date.now() - startTime;
+		handleError(e, buildEnvelope(env));
 	}
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@apmantza/greedysearch-pi",
-	"version": "1.8.9",
+	"version": "1.9.0",
 	"description": "Headless multi-engine AI search (Perplexity, Bing Copilot, Google AI) via browser automation -- NO API KEYS needed. Extracts answers with sources, optional synthesis. Grounded AI answers from real browser interactions.",
 	"type": "module",
 	"keywords": [

package/src/search/chrome.mjs

@@ -113,6 +113,65 @@ function getPortPid() {
 	}
 }
 
+/**
+ * Send Browser.close via CDP WebSocket so Chrome flushes its cookie DB to disk
+ * before we force-kill it. Gives the process up to `graceMs` to exit on its own.
+ * Falls back to force-kill if Chrome is still running after the grace period.
+ * Returns true if the process is gone after the call.
+ */
+async function gracefulCloseChrome(graceMs = 1500) {
+	try {
+		const version = await new Promise((resolve, reject) => {
+			const req = http.get(
+				`http://localhost:${GREEDY_PORT}/json/version`,
+				(res) => {
+					let body = "";
+					res.on("data", (d) => (body += d));
+					res.on("end", () => {
+						try {
+							resolve(JSON.parse(body));
+						} catch {
+							reject(new Error("bad JSON"));
+						}
+					});
+				},
+			);
+			req.on("error", reject);
+			req.setTimeout(1000, () => {
+				req.destroy();
+				reject(new Error("timeout"));
+			});
+		});
+
+		const ws = new globalThis.WebSocket(version.webSocketDebuggerUrl);
+		await new Promise((resolve) => {
+			ws.onopen = () => {
+				ws.send(JSON.stringify({ id: 1, method: "Browser.close" }));
+				// Give Chrome a moment to receive the command before we close the socket
+				setTimeout(() => {
+					ws.close();
+					resolve();
+				}, 200);
+			};
+			ws.onerror = () => resolve();
+			setTimeout(resolve, 1000);
+		});
+	} catch {
+		// Chrome not reachable — skip to force-kill
+	}
+
+	// Wait for Chrome to exit gracefully (flushes SQLite cookie DB)
+	const deadline = Date.now() + graceMs;
+	while (Date.now() < deadline) {
+		const pid = getPortPid();
+		if (!pid) return true; // already gone
+		await new Promise((r) => setTimeout(r, 150));
+	}
+
+	// Still running — force-kill
+	return killProcessOnPort();
+}
+
 /**
  * Force-kill whatever process is listening on GREEDY_PORT.
  * Uses OS tools to find the PID (not the PID file — handles ghost processes).
@@ -159,7 +218,9 @@ export async function killChrome() {
 		return false;
 	}
 
-	const killed = killProcessOnPort();
+	// Graceful close: sends Browser.close so Chrome flushes its cookie DB,
+	// then force-kills if it doesn't exit within the grace period.
+	const killed = await gracefulCloseChrome(1500);
 
 	// Clean up tracking files regardless of kill success
 	try {

package/src/search/constants.mjs

@@ -21,23 +21,18 @@ export const ENGINE_DOMAINS = {
 
 export const ENGINES = {
 	perplexity: "perplexity.mjs",
-	pplx: "perplexity.mjs",
 	p: "perplexity.mjs",
 	bing: "bing-copilot.mjs",
-	bing2: "bing-aria.mjs",
-	copilot: "bing-copilot.mjs",
 	b: "bing-copilot.mjs",
 	google: "google-ai.mjs",
 	g: "google-ai.mjs",
 	gemini: "gemini.mjs",
 	gem: "gemini.mjs",
-	googlesearch: "google-search.mjs",
-	gs: "google-search.mjs",
 };
 
 export const SOURCE_FETCH_CONCURRENCY = Math.max(
 	1,
-	Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY || "4", 10) || 4,
+	Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY || "5", 10) || 5,
 );
 
 // Tell cdp.mjs to prefer the GreedySearch Chrome profile's DevToolsActivePort