@apmantza/greedysearch-pi 1.9.2 → 2.0.0

package/bin/launch.mjs

@@ -76,6 +76,15 @@ const BASE_CHROME_FLAGS = [
 	"--window-size=1920,1080",
 	"--lang=en-US",
 	"--force-color-profile=srgb",
+	// Background-tab throttling kills parallel extractions: Chrome clamps
+	// setTimeout to ~1Hz in unfocused tabs, so a streaming response that
+	// finishes in 5s solo takes 60s+ when 4 engines share one Chrome.
+	// The trio below restores full-speed JS in every tab. Safe for our
+	// anti-bot stealth — Cloudflare detects CDP/webdriver artifacts, not
+	// timer-throttling behavior. Same flags Playwright/Puppeteer add.
+	"--disable-background-timer-throttling",
+	"--disable-renderer-backgrounding",
+	"--disable-backgrounding-occluded-windows",
 ];
 
 function getChromeVersion(chromePath) {

package/bin/search.mjs

@@ -20,7 +20,7 @@
 //   node search.mjs gem "latest React features"
 //   node search.mjs all "how does TCP congestion control work"
 
-import { existsSync, readFileSync } from "node:fs";
+import { appendFileSync, existsSync, readFileSync } from "node:fs";
 // Config file for user defaults
 import { homedir } from "node:os";
 import { join } from "node:path";
@@ -33,7 +33,12 @@ import {
 	openNewTab,
 	touchActivity,
 } from "../src/search/chrome.mjs";
-import { ALL_ENGINES, ENGINES } from "../src/search/constants.mjs";
+import {
+	ALL_ENGINES,
+	ENGINES,
+	SYNTHESIZER,
+	VISIBLE_RECOVERY_LOG,
+} from "../src/search/constants.mjs";
 import { runExtractor } from "../src/search/engines.mjs";
 import {
 	fetchMultipleSources,
@@ -51,7 +56,11 @@ import {
 	mergeFetchDataIntoSources,
 } from "../src/search/sources.mjs";
 import { buildConfidence } from "../src/search/synthesis.mjs";
-import { synthesizeWithGemini } from "../src/search/synthesis-runner.mjs";
+import {
+	getSynthesisStartUrl,
+	normalizeSynthesizer,
+	synthesizeResults,
+} from "../src/search/synthesis-runner.mjs";
 import { normalizeQuery } from "../src/search/query.mjs";
 import { runResearchMode } from "../src/search/research.mjs";
 
@@ -69,6 +78,18 @@ function loadUserConfig() {
 	return {};
 }
 
+function logVisibleRecovery(event) {
+	try {
+		appendFileSync(
+			VISIBLE_RECOVERY_LOG,
+			`${JSON.stringify({ at: new Date().toISOString(), ...event })}\n`,
+			"utf8",
+		);
+	} catch {
+		// Best-effort diagnostics only. Never fail a search because logging failed.
+	}
+}
+
 /** Read query/prompt from stdin (used with --stdin to avoid command-line leakage) */
 async function readStdin() {
 	return new Promise((resolve) => {
@@ -89,16 +110,20 @@ async function main() {
 			`${[
 				'Usage: node search.mjs <engine> "<query>"',
 				"",
-				"Engines: perplexity (p), bing (b), google (g), gemini (gem), all",
+				"Engines: all, perplexity (p), google (g), chatgpt (gpt), gemini (gem), semantic-scholar (s2), logically (log), bing (b)",
 				"",
 				"Flags:",
-				"  --fast              Quick mode: no source fetching or synthesis",
-				"  --synthesize        Deprecated: synthesis is now default for multi-engine",
-				"  --deep-research     Deprecated: source fetching is now default",
+				"  --synthesize        For engine=all: synthesize fetched sources",
+				"  --synthesizer <engine>  Synthesis engine (default from ~/.pi/greedyconfig)",
+				"  --fast              Legacy quick mode: no source fetching or synthesis",
+				"  --depth <mode>      Legacy: fast|standard|deep aliases, or research",
+				"  --deep-research     Deprecated alias for --research",
 				"  --research          Iterative query/learnings loop (alias: --depth research)",
 				"  --breadth <n>       Research mode query breadth, 1-5 (default: 3)",
 				"  --iterations <n>    Research mode rounds, 1-3 (default: 2)",
 				"  --max-sources <n>   Research mode fetched source cap, 3-12",
+				"  --research-out-dir <dir>  Write research bundle to a specific directory",
+				"  --no-research-bundle     Disable the default .pi/greedysearch-research bundle",
 				"  --fetch-top-source  Fetch content from top source",
 				"  --inline            Output JSON to stdout (for piping)",
 				"  --locale <lang>     Force results language (en, de, fr, etc.)",
@@ -112,10 +137,11 @@ async function main() {
 				"  GREEDY_SEARCH_LOCALE          Default locale (default: en)",
 				"",
 				"Examples:",
-				'  node search.mjs all "Node.js streams"           # Default: sources + synthesis',
-				'  node search.mjs all "quick check" --fast        # Fast: no sources/synthesis',
+				'  node search.mjs all "Node.js streams"              # Grounded: engines + fetched sources',
+				'  node search.mjs all "Node.js streams" --synthesize # Add Gemini synthesis',
+				'  node search.mjs all "quick check" --fast           # Legacy fast: no sources/synthesis',
 				'  node search.mjs all "browser automation" --research --breadth 3 --iterations 2',
-				'  node search.mjs p "what is memoization"         # Single engine: fast mode',
+				'  node search.mjs p "what is memoization"            # Single engine search',
 			].join("\n")}\n`,
 		);
 		process.exit(1);
@@ -129,6 +155,11 @@ async function main() {
 		process.env.GREEDY_SEARCH_VISIBLE = "1";
 		process.env.GREEDY_SEARCH_ALWAYS_VISIBLE = "1";
 		delete process.env.GREEDY_SEARCH_HEADLESS;
+	} else if (process.env.GREEDY_SEARCH_VISIBLE !== "1") {
+		// Establish the desired mode BEFORE ensureChrome() so a stale visible
+		// recovery browser is switched back to headless before research planning
+		// and Gemini synthesis tabs are opened.
+		process.env.GREEDY_SEARCH_HEADLESS = "1";
 	}
 
 	await ensureChrome();
@@ -136,41 +167,44 @@ async function main() {
 	// Track activity for headless idle timeout
 	touchActivity();
 
-	// Depth modes: fast (no synthesis/fetch), standard (synthesis+fetch 5 sources)
 	const depthIdx = args.indexOf("--depth");
-	let depth = "standard"; // DEFAULT: synthesis + source fetch
-
-	if (depthIdx !== -1 && args[depthIdx + 1]) {
-		depth = args[depthIdx + 1];
-	} else if (args.includes("--fast")) {
-		depth = "fast"; // Explicit fast mode requested
-	}
-
-	// For single engine (not "all"), default to fast unless explicit
+	const legacyDepth =
+		depthIdx !== -1 && args[depthIdx + 1]
+			? args[depthIdx + 1].toLowerCase()
+			: null;
 	const engineArg = args.find((a) => !a.startsWith("--"))?.toLowerCase();
-	if (engineArg !== "all" && depthIdx === -1 && !args.includes("--fast")) {
-		depth = "fast";
-	}
+	const researchMode =
+		args.includes("--research") ||
+		args.includes("--deep-research") ||
+		legacyDepth === "research";
+	const legacyFast = args.includes("--fast") || legacyDepth === "fast";
+	const legacySynthesisDepth =
+		legacyDepth === "standard" ||
+		legacyDepth === "deep" ||
+		args.includes("--deep");
+	const shouldFetchSources = engineArg === "all" && !legacyFast;
+	const shouldSynthesize =
+		engineArg === "all" &&
+		!legacyFast &&
+		(args.includes("--synthesize") || legacySynthesisDepth);
+	const groundedSynthesis = legacyDepth === "deep" || args.includes("--deep");
 
-	// --deep-research / --deep flags map to deep mode (backward compat)
 	if (args.includes("--deep-research")) {
-		depth = "standard";
 		process.stderr.write(
-			"[greedysearch] --deep-research is deprecated; use --depth standard (now default)\n",
+			"[greedysearch] --deep-research is deprecated; use --research or --depth research\n",
 		);
 	}
-	if (args.includes("--deep")) {
-		depth = "deep";
-	}
-	if (args.includes("--research")) {
-		depth = "research";
-	}
-	if (args.includes("--synthesize")) {
+	if (legacySynthesisDepth) {
 		process.stderr.write(
-			"[greedysearch] --synthesize is deprecated; synthesis is now default for multi-engine\n",
+			"[greedysearch] depth fast|standard|deep is deprecated; use default grounded search plus --synthesize when needed\n",
 		);
 	}
 
+	const synthesizerIdx = args.indexOf("--synthesizer");
+	const synthesizer = normalizeSynthesizer(
+		synthesizerIdx === -1 ? SYNTHESIZER : args[synthesizerIdx + 1],
+	);
+
 	const full = args.includes("--full");
 	const short = !full;
 	const fetchSource = args.includes("--fetch-top-source");
@@ -183,9 +217,10 @@ async function main() {
 		iterationsIdx === -1 ? undefined : args[iterationsIdx + 1];
 	const researchMaxSources =
 		maxSourcesIdx === -1 ? undefined : args[maxSourcesIdx + 1];
-	// Headless is the default — only disable if GREEDY_SEARCH_VISIBLE=1
-	if (process.env.GREEDY_SEARCH_VISIBLE !== "1")
-		process.env.GREEDY_SEARCH_HEADLESS = "1";
+	const researchOutDirIdx = args.indexOf("--research-out-dir");
+	const researchOutDir =
+		researchOutDirIdx === -1 ? undefined : args[researchOutDirIdx + 1];
+	const writeResearchBundle = !args.includes("--no-research-bundle");
 	const outIdx = args.indexOf("--out");
 	const outFile = outIdx === -1 ? null : args[outIdx + 1];
 
@@ -218,18 +253,23 @@ async function main() {
 			a !== "--visible" &&
 			a !== "--always-visible" &&
 			a !== "--depth" &&
+			a !== "--synthesizer" &&
 			a !== "--out" &&
 			a !== "--locale" &&
 			a !== "--breadth" &&
 			a !== "--iterations" &&
 			a !== "--max-sources" &&
+			a !== "--research-out-dir" &&
+			a !== "--no-research-bundle" &&
 			a !== "--help" &&
 			(depthIdx === -1 || i !== depthIdx + 1) &&
+			(synthesizerIdx === -1 || i !== synthesizerIdx + 1) &&
 			(outIdx === -1 || i !== outIdx + 1) &&
 			(localeIdx === -1 || i !== localeIdx + 1) &&
 			(breadthIdx === -1 || i !== breadthIdx + 1) &&
 			(iterationsIdx === -1 || i !== iterationsIdx + 1) &&
-			(maxSourcesIdx === -1 || i !== maxSourcesIdx + 1),
+			(maxSourcesIdx === -1 || i !== maxSourcesIdx + 1) &&
+			(researchOutDirIdx === -1 || i !== researchOutDirIdx + 1),
 	);
 	const engine = rest[0]?.toLowerCase();
 	// Read query from stdin when --stdin flag is set (avoids leaking query in process table)
@@ -241,7 +281,7 @@ async function main() {
 		query = rest.slice(1).join(" ");
 	}
 
-	if (depth === "research") {
+	if (researchMode) {
 		if (engine !== "all") {
 			process.stderr.write(
 				`[greedysearch] Research mode uses all engines; ignoring engine "${engine}".\n`,
@@ -254,6 +294,8 @@ async function main() {
 			maxSources: researchMaxSources,
 			locale,
 			short,
+			writeBundle: writeResearchBundle,
+			researchOutDir,
 		});
 		writeOutput(out, outFile, {
 			inline,
@@ -270,8 +312,11 @@ async function main() {
 		// engine homepage so extractors can skip the initial navigation.
 		const ENGINE_START_URLS = {
 			perplexity: "https://www.perplexity.ai/",
-			bing: "https://copilot.microsoft.com/",
 			google: "https://www.google.com/",
+			"semantic-scholar": "https://www.semanticscholar.org/",
+			semanticscholar: "https://www.semanticscholar.org/",
+			s2: "https://www.semanticscholar.org/",
+			logically: "https://logically.app/research-assistant/",
 		};
 		const engineTabs = await Promise.all(
 			ALL_ENGINES.map((e) => openNewTab(ENGINE_START_URLS[e])),
@@ -280,11 +325,10 @@ async function main() {
 		await cdp(["list"]);
 
 		// Time-bounded per-engine extraction so slow engines don't stall the batch.
-		// Bing can take a little longer than Google/Perplexity under CDP contention;
-		// keep fast mode bounded while avoiding most false recovery trips.
 		const engineTimeoutFor = (engineName) => {
-			if (depth !== "fast") return 55000;
-			return engineName === "bing" ? 40000 : 30000;
+			if (!legacyFast) return 70000;
+			// ChatGPT needs ~25-30s solo; under CDP contention needs more headroom
+			return engineName === "chatgpt" ? 60000 : 35000;
 		};
 
 		try {
@@ -316,7 +360,19 @@ async function main() {
 				if (r.status === "fulfilled") {
 					out[r.value.engine] = r.value;
 				} else {
-					out[ALL_ENGINES[i]] = { error: r.reason?.message || "unknown error" };
+					const err = r.reason;
+					const msg = err?.message || "unknown error";
+					out[ALL_ENGINES[i]] = { error: msg };
+					if (err?.lastStage) {
+						process.stderr.write(
+							`[greedysearch] ${ALL_ENGINES[i]} failed at stage '${err.lastStage}': ${msg}\n`,
+						);
+					}
+					if (err?.partialErr) {
+						process.stderr.write(
+							`[greedysearch] ${ALL_ENGINES[i]} tail stderr:\n${err.partialErr}\n`,
+						);
+					}
 				}
 			}
 
@@ -331,6 +387,20 @@ async function main() {
 				recoveryCandidates.length > 0 &&
 				process.env.GREEDY_SEARCH_VISIBLE !== "1"
 			) {
+				logVisibleRecovery({
+					scope: "all",
+					phase: "start",
+					engines: recoveryCandidates,
+					reasons: Object.fromEntries(
+						recoveryCandidates.map((engineName) => [
+							engineName,
+							{
+								error: out[engineName]?.error || null,
+								envelope: out[engineName]?._envelope || null,
+							},
+						]),
+					),
+				});
 				process.stderr.write(
 					`[greedysearch] 🔓 Headless ${recoveryCandidates.join(", ")} search hit timeout/verification/antibot signals — retrying visible to establish cookies...\n`,
 				);
@@ -431,6 +501,23 @@ async function main() {
 						stillBlocked.push(...secondStillBlocked);
 					}
 
+					logVisibleRecovery({
+						scope: "all",
+						phase: stillBlocked.length > 0 ? "needs-human" : "success",
+						engines: recoveryCandidates,
+						results: Object.fromEntries(
+							recoveryCandidates.map((engineName) => [
+								engineName,
+								{
+									mode: out[engineName]?._envelope?.mode || null,
+									durationMs: out[engineName]?._envelope?.durationMs || null,
+									lastStage: out[engineName]?._envelope?.lastStage || null,
+									error: out[engineName]?.error || null,
+								},
+							]),
+						),
+					});
+
 					if (stillBlocked.length > 0) {
 						for (const blockedEngine of stillBlocked) {
 							process.stderr.write(`PROGRESS:${blockedEngine}:needs-human\n`);
@@ -488,18 +575,9 @@ async function main() {
 			// Build a canonical source registry across all engines
 			out._sources = buildSourceRegistry(out, query);
 
-			// Pre-navigate Gemini tab in parallel with source fetch so the page
-			// is already loaded when synthesis starts — saves ~4s of nav time.
-			let geminiTabPromise = null;
-			if (depth !== "fast") {
-				geminiTabPromise = openNewTab("https://gemini.google.com/app").catch(
-					() => null,
-				);
-			}
-
 			// Source fetching: default for all "all" searches
 			// Fetch all sources in a single batch (concurrency = source count).
-			if (depth !== "fast" && out._sources.length > 0) {
+			if (shouldFetchSources && out._sources.length > 0) {
 				process.stderr.write("PROGRESS:source-fetch:start\n");
 				const fetchedSources = await fetchMultipleSources(
 					out._sources,
@@ -512,29 +590,41 @@ async function main() {
 				process.stderr.write("PROGRESS:source-fetch:done\n");
 			}
 
-			// Synthesize with Gemini for all non-fast modes
-			if (depth !== "fast") {
+			// Optional engine-agnostic synthesis for multi-engine searches.
+			// Open the synthesizer tab HERE (after source fetch) instead of
+			// pre-opening before source fetch. Pre-opening was fragile: Chrome could
+			// be killed during visible recovery or idle-timeout between source fetch
+			// and synthesis, leaving a stale tab ID that causes "No target matching prefix".
+			if (shouldSynthesize) {
 				process.stderr.write("PROGRESS:synthesis:start\n");
 				process.stderr.write(
-					"[greedysearch] Synthesizing results with Gemini...\n",
+					`[greedysearch] Synthesizing results with ${synthesizer}...\n`,
 				);
+				let synthesisTab = null;
 				try {
-					const geminiTab = (await geminiTabPromise) ?? (await openNewTab());
-					const synthesis = await synthesizeWithGemini(query, out, {
-						grounded: depth === "deep",
-						tabPrefix: geminiTab,
+					synthesisTab = await openNewTab(getSynthesisStartUrl(synthesizer));
+					const synthesis = await synthesizeResults(query, out, {
+						grounded: groundedSynthesis,
+						tabPrefix: synthesisTab,
+						visible: process.env.GREEDY_SEARCH_VISIBLE === "1",
+						synthesizer,
 					});
 					out._synthesis = {
 						...synthesis,
 						synthesized: true,
 					};
-					await closeTab(geminiTab);
 					process.stderr.write("PROGRESS:synthesis:done\n");
 				} catch (e) {
 					process.stderr.write(
 						`[greedysearch] Synthesis failed: ${e.message}\n`,
 					);
-					out._synthesis = { error: e.message, synthesized: false };
+					out._synthesis = {
+						error: e.message,
+						synthesized: false,
+						synthesizedBy: synthesizer,
+					};
+				} finally {
+					if (synthesisTab) await closeTab(synthesisTab);
 				}
 			}
 
@@ -544,12 +634,12 @@ async function main() {
 					out._topSource = await fetchTopSource(top.canonicalUrl || top.url);
 			}
 
-			// Always include confidence metrics for non-fast searches
-			if (depth !== "fast") out._confidence = buildConfidence(out);
+			// Include confidence metrics for grounded multi-engine searches.
+			if (!legacyFast) out._confidence = buildConfidence(out);
 
 			writeOutput(out, outFile, {
 				inline,
-				synthesize: depth !== "fast",
+				synthesize: shouldSynthesize,
 				query,
 			});
 			return;
@@ -585,13 +675,31 @@ async function main() {
 			? "bing"
 			: script.includes("perplexity")
 				? "perplexity"
-				: null;
+				: script.includes("chatgpt")
+					? "chatgpt"
+					: script.includes("semantic-scholar")
+						? "semantic-scholar"
+						: script.includes("logically")
+							? "logically"
+							: null;
 		const canRetryVisible =
 			recoveryEngine &&
 			process.env.GREEDY_SEARCH_VISIBLE !== "1" &&
 			isHeadlessBlockedResult(e);
 
 		if (canRetryVisible) {
+			logVisibleRecovery({
+				scope: "single",
+				phase: "start",
+				engines: [recoveryEngine],
+				reasons: {
+					[recoveryEngine]: {
+						error: e.message || null,
+						envelope: e.envelope || null,
+						lastStage: e.lastStage || null,
+					},
+				},
+			});
 			process.stderr.write(
 				`[greedysearch] 🔓 ${recoveryEngine} blocked in headless — retrying visible to establish cookies...\n`,
 			);
@@ -612,12 +720,33 @@ async function main() {
 					null,
 					locale,
 				);
+				logVisibleRecovery({
+					scope: "single",
+					phase: "success",
+					engines: [recoveryEngine],
+					result: {
+						engine: recoveryEngine,
+						mode: result._envelope?.mode || null,
+						durationMs: result._envelope?.durationMs || null,
+						lastStage: result._envelope?.lastStage || null,
+					},
+				});
 				if (fetchSource && result.sources?.length > 0) {
 					result.topSource = await fetchTopSource(result.sources[0].url);
 				}
 				writeOutput(result, outFile, { inline, synthesize: false, query });
 				return;
 			} catch (retryErr) {
+				logVisibleRecovery({
+					scope: "single",
+					phase: "needs-human",
+					engines: [recoveryEngine],
+					result: {
+						engine: recoveryEngine,
+						error: retryErr.message || String(retryErr),
+						envelope: retryErr.envelope || null,
+					},
+				});
 				// Any visible retry failure: keep Chrome open so user can solve Turnstile.
 				// Once solved, cookies are stored in the shared profile for future headless runs.
 				keepVisibleForHuman = true;

package/extractors/bing-copilot.mjs

@@ -41,6 +41,24 @@ const GLOBAL_VAR = "__bingClipboard";
 // Bing Copilot-specific helpers
 // ============================================================================
 
+async function detectSignInWall(tab) {
+	// Language-agnostic: if the chat input is absent but the page hosts
+	// known OAuth provider endpoints, we're on the Copilot login wall.
+	const code = `(() => {
+		if (document.querySelector('#userInput')) return false;
+		const links = Array.from(document.querySelectorAll('a[href], button'));
+		const hasOAuth = links.some(el => {
+			const h = (el.href || el.getAttribute('formaction') || '').toLowerCase();
+			return h.includes('login.microsoftonline.com')
+				|| h.includes('appleid.apple.com')
+				|| h.includes('accounts.google.com');
+		});
+		return hasOAuth;
+	})()`;
+	const result = await cdp(["eval", tab, code]).catch(() => "false");
+	return result === "true";
+}
+
 async function extractAnswer(tab, env, query = "") {
 	// In headless mode: snap the accessibility tree before spending ~18s on
 	// clipboard polls. Copilot loads its input fine in headless but renders
@@ -181,10 +199,15 @@ async function extractFromAccessibilityTree(tab, query = "") {
 		const snap = await cdp(["snap", tab]).catch(() => "");
 		if (!snap || (await detectVerificationChallenge(tab, cdp))) return "";
 
-		const articleLines = snap
-			.split("\n")
-			.map((line) => line.match(/^\s*\[article\]\s+(.+)$/i)?.[1])
-			.filter(Boolean);
+		// Linear article extraction — no regex. Avoids the ReDoS-prone
+		// /^\s*\[article\]\s+(.+)$/i pattern (SonarCloud hotspot js:S5852).
+		const articleLines = [];
+		for (const line of snap.split("\n")) {
+			const trimmed = line.trimStart();
+			if (!trimmed.toLowerCase().startsWith("[article]")) continue;
+			const after = trimmed.slice("[article]".length).trimStart();
+			if (after) articleLines.push(after);
+		}
 		if (articleLines.length === 0) return "";
 
 		const answer = pickAnswerArticle(articleLines, query);
@@ -419,12 +442,27 @@ async function main() {
 			}
 		}
 
+		// Detect sign-in wall before burning time waiting for an input that
+		// will never appear. Copilot now gates the chat behind Microsoft/Apple/Google
+		// login on fresh sessions.
+		if (await detectSignInWall(tab)) {
+			throw new Error(
+				"Copilot requires sign-in — please sign in with Microsoft, Apple, or Google in the visible browser window. Once signed in, cookies persist for future runs.",
+			);
+		}
+
 		// Wait for React app to mount input (up to 15s, longer after verification)
 		const inputReady = await waitForSelector(tab, S.input, 15000, 500);
 		env.inputReady = inputReady;
 		await new Promise((r) => setTimeout(r, jitter(300)));
 
 		if (!inputReady) {
+			// If input still missing, double-check we didn't land on the login wall
+			if (await detectSignInWall(tab)) {
+				throw new Error(
+					"Copilot requires sign-in — please sign in with Microsoft, Apple, or Google in the visible browser window. Once signed in, cookies persist for future runs.",
+				);
+			}
 			throw new Error(
 				"Copilot input not found — verification may have failed or page is in unexpected state",
 			);