@apmantza/greedysearch-pi 1.9.2 → 2.1.2

package/src/search/constants.mjs

@@ -1,39 +1,155 @@
-// src/search/constants.mjs — Shared constants for GreedySearch search pipeline
-
-import { tmpdir } from "node:os";
-
-export const GREEDY_PORT = 9222;
-export const GREEDY_PROFILE_DIR = `${tmpdir().replaceAll("\\", "/")}/greedysearch-chrome-profile`;
-export const ACTIVE_PORT_FILE = `${GREEDY_PROFILE_DIR}/DevToolsActivePort`;
-export const PAGES_CACHE = `${tmpdir().replaceAll("\\", "/")}/cdp-pages.json`;
-export const CHROME_MODE_FILE = `${tmpdir().replaceAll("\\", "/")}/greedysearch-chrome-mode`;
-
-// ALL_ENGINES drives the "all" fan-out. Add engines here to include them in multi-engine searches.
-// Engines in ENGINES but not in ALL_ENGINES are available for explicit use only.
-export const ALL_ENGINES = ["perplexity", "bing", "google"];
-
-export const ENGINE_DOMAINS = {
-	perplexity: "perplexity.ai",
-	bing: "copilot.microsoft.com",
-	google: "google.com",
-	gemini: "gemini.google.com",
-};
-
-export const ENGINES = {
-	perplexity: "perplexity.mjs",
-	p: "perplexity.mjs",
-	bing: "bing-copilot.mjs",
-	b: "bing-copilot.mjs",
-	google: "google-ai.mjs",
-	g: "google-ai.mjs",
-	gemini: "gemini.mjs",
-	gem: "gemini.mjs",
-};
-
-export const SOURCE_FETCH_CONCURRENCY = Math.max(
-	1,
-	Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY || "5", 10) || 5,
-);
-
-// Tell cdp.mjs to prefer the GreedySearch Chrome profile's DevToolsActivePort
-process.env.CDP_PROFILE_DIR = GREEDY_PROFILE_DIR;
+// src/search/constants.mjs — Shared constants for GreedySearch search pipeline
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+export const GREEDY_PORT = 9222;
+export const GREEDY_PROFILE_DIR = `${tmpdir().replaceAll("\\", "/")}/greedysearch-chrome-profile`;
+export const ACTIVE_PORT_FILE = `${GREEDY_PROFILE_DIR}/DevToolsActivePort`;
+export const PAGES_CACHE = `${tmpdir().replaceAll("\\", "/")}/cdp-pages.json`;
+export const CHROME_MODE_FILE = `${tmpdir().replaceAll("\\", "/")}/greedysearch-chrome-mode`;
+export const VISIBLE_RECOVERY_LOG = `${tmpdir().replaceAll("\\", "/")}/greedysearch-visible-recovery.jsonl`;
+
+// ── User config: ~/.pi/greedyconfig ────────────────────────────────────────
+// Users can override which engines participate in the "all" fan-out and which
+// engine performs optional synthesis.
+// Default engines: perplexity, google, chatgpt; synthesizer: gemini
+
+const CONFIG_DIR = join(homedir(), ".pi");
+const CONFIG_FILE = join(CONFIG_DIR, "greedyconfig");
+
+// Default engines that participate in the "all" fan-out for normal
+// (non-research) searches. Opt-in research/academic engines like
+// `semantic-scholar` are deliberately excluded — they belong in research
+// mode, not casual web search. Users who want them in normal `engine:all`
+// runs can add them via ~/.pi/greedyconfig (see ensureDefaultConfig()).
+export const DEFAULT_ENGINES = ["perplexity", "google", "chatgpt"];
+export const DEFAULT_SYNTHESIZER = "gemini";
+
+function loadUserEngines() {
+	try {
+		if (existsSync(CONFIG_FILE)) {
+			const raw = readFileSync(CONFIG_FILE, "utf8");
+			const config = JSON.parse(raw);
+			if (
+				Array.isArray(config.engines) &&
+				config.engines.length > 0 &&
+				config.engines.every((e) => typeof e === "string")
+			) {
+				// Validate each engine exists in ENGINES. Unknown names are
+				// silently dropped — but at least once we tell the user about
+				// it so a typo in ~/.pi/greedyconfig doesn't quietly shrink
+				// the all-search fan-out.
+				const valid = config.engines.filter((e) => ENGINES[e]);
+				const invalid = config.engines.filter((e) => !ENGINES[e]);
+				if (invalid.length > 0) {
+					process.stderr.write(
+						`[greedysearch] Warning: ignoring unknown engine(s) in ${CONFIG_FILE}: ${invalid.join(", ")}\n` +
+							`[greedysearch] Available engines: ${Object.keys(ENGINES).join(", ")}\n`,
+					);
+				}
+				if (valid.length > 0) return valid;
+				process.stderr.write(
+					`[greedysearch] Warning: no valid engines in ${CONFIG_FILE}, falling back to defaults: ${DEFAULT_ENGINES.join(", ")}\n`,
+				);
+			}
+		}
+	} catch {
+		// Ignore parse/read errors — fall through to default
+	}
+	return DEFAULT_ENGINES;
+}
+
+function ensureDefaultConfig() {
+	try {
+		if (!existsSync(CONFIG_DIR)) mkdirSync(CONFIG_DIR, { recursive: true });
+		if (!existsSync(CONFIG_FILE)) {
+			writeFileSync(
+				CONFIG_FILE,
+				JSON.stringify(
+					{ engines: DEFAULT_ENGINES, synthesizer: DEFAULT_SYNTHESIZER },
+					null,
+					2,
+				) + "\n",
+				"utf8",
+			);
+		}
+	} catch {
+		// Best-effort — don't crash if we can't write the config file
+	}
+}
+
+ensureDefaultConfig();
+
+export const SUPPORTED_SYNTHESIZERS = ["gemini", "chatgpt"];
+
+function loadUserSynthesizer() {
+	try {
+		if (existsSync(CONFIG_FILE)) {
+			const raw = readFileSync(CONFIG_FILE, "utf8");
+			const config = JSON.parse(raw);
+			if (typeof config.synthesizer === "string") {
+				const normalized = config.synthesizer.toLowerCase();
+				if (SUPPORTED_SYNTHESIZERS.includes(normalized)) return normalized;
+				process.stderr.write(
+					`[greedysearch] Warning: unknown synthesizer "${config.synthesizer}" in ${CONFIG_FILE}\n` +
+						`[greedysearch] Available synthesizers: ${SUPPORTED_SYNTHESIZERS.join(", ")}\n` +
+						`[greedysearch] Falling back to default: ${DEFAULT_SYNTHESIZER}\n`,
+				);
+			}
+		}
+	} catch {
+		// Ignore parse/read errors — fall through to default
+	}
+	return DEFAULT_SYNTHESIZER;
+}
+
+export const ENGINE_DOMAINS = {
+	perplexity: "perplexity.ai",
+	bing: "copilot.microsoft.com",
+	google: "google.com",
+	gemini: "gemini.google.com",
+	chatgpt: "chatgpt.com",
+	"semantic-scholar": "semanticscholar.org",
+	semanticscholar: "semanticscholar.org",
+	s2: "semanticscholar.org",
+	logically: "logically.app",
+};
+
+export const ENGINES = {
+	perplexity: "perplexity.mjs",
+	p: "perplexity.mjs",
+	bing: "bing-copilot.mjs",
+	b: "bing-copilot.mjs",
+	google: "google-ai.mjs",
+	g: "google-ai.mjs",
+	gemini: "gemini.mjs",
+	gem: "gemini.mjs",
+	chatgpt: "chatgpt.mjs",
+	gpt: "chatgpt.mjs",
+	"semantic-scholar": "semantic-scholar.mjs",
+	semanticscholar: "semantic-scholar.mjs",
+	s2: "semantic-scholar.mjs",
+	logically: "logically.mjs",
+	log: "logically.mjs",
+};
+
+// ALL_ENGINES drives the "all" fan-out. Edit ~/.pi/greedyconfig to customize.
+export const ALL_ENGINES = loadUserEngines();
+
+// Research child searches intentionally reuse the normal configured fan-out.
+// Gemini remains the research planner/final-report synthesizer.
+export const RESEARCH_ENGINES = ALL_ENGINES;
+
+// SYNTHESIZER drives optional all-search synthesis. Edit ~/.pi/greedyconfig to customize.
+export const SYNTHESIZER = loadUserSynthesizer();
+
+export const SOURCE_FETCH_CONCURRENCY = Math.max(
+	1,
+	Number.parseInt(process.env.GREEDY_FETCH_CONCURRENCY || "5", 10) || 5,
+);
+
+// Tell cdp.mjs to prefer the GreedySearch Chrome profile's DevToolsActivePort
+process.env.CDP_PROFILE_DIR = GREEDY_PROFILE_DIR;

package/src/search/engines.mjs

@@ -1,76 +1,114 @@
-// src/search/engines.mjs — Extractor runner
-//
-// Engine map lives in constants.mjs; this module re-exports it for
-// backward compatibility and provides the runExtractor() function.
-
-import { spawn } from "node:child_process";
-import { join } from "node:path";
-import { ENGINES, GREEDY_PROFILE_DIR } from "./constants.mjs";
-
-export { ENGINES };
-
-const __dir =
-	import.meta.dirname ||
-	new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
-
-export function runExtractor(
-	script,
-	query,
-	tabPrefix = null,
-	short = false,
-	timeoutMs = null,
-	locale = null,
-) {
-	// Gemini synthesis: 70s budget (45s stream + ~25s nav/settle overhead)
-	// Other engines: 60s budget
-	if (timeoutMs === null) {
-		timeoutMs = script.includes("gemini") ? 70000 : 60000;
-	}
-	const extraArgs = [
-		...(tabPrefix ? ["--tab", tabPrefix] : []),
-		...(short ? ["--short"] : []),
-		...(locale ? ["--locale", locale] : []),
-	];
-	return new Promise((resolve, reject) => {
-		const proc = spawn(
-			process.execPath,
-			[join(__dir, "..", "..", "extractors", script), "--stdin", ...extraArgs],
-			{
-				stdio: ["pipe", "pipe", "pipe"],
-				env: { ...process.env, CDP_PROFILE_DIR: GREEDY_PROFILE_DIR },
-			},
-		);
-		// Pipe query via stdin to avoid leaking it in process table command-line
-		proc.stdin.write(query);
-		proc.stdin.end();
-		let out = "";
-		let err = "";
-		proc.stdout.on("data", (d) => (out += d));
-		proc.stderr.on("data", (d) => (err += d));
-		const t = setTimeout(() => {
-			proc.kill();
-			reject(new Error(`${script} timed out after ${timeoutMs / 1000}s`));
-		}, timeoutMs);
-		proc.on("close", (code) => {
-			clearTimeout(t);
-			if (code === 0) {
-				try {
-					resolve(JSON.parse(out.trim()));
-				} catch {
-					reject(new Error(`bad JSON from ${script}: ${out.slice(0, 100)}`));
-				}
-			} else {
-				// Try to parse structured error envelope from stdout before falling back
-				let envelope = null;
-				try {
-					const parsed = JSON.parse(out.trim());
-					if (parsed._envelope) envelope = parsed._envelope;
-				} catch {}
-				const msg = err.trim() || `extractor exit ${code}`;
-				const errObj = new Error(msg);
-				if (envelope) errObj.envelope = envelope;
-				reject(errObj);
-			}
-		});
-	});
-}
+// src/search/engines.mjs — Extractor runner
+//
+// Engine map lives in constants.mjs; this module re-exports it for
+// backward compatibility and provides the runExtractor() function.
+
+import { spawn } from "node:child_process";
+import { join } from "node:path";
+import { ENGINES, GREEDY_PROFILE_DIR } from "./constants.mjs";
+
+export { ENGINES };
+
+const __dir =
+	import.meta.dirname ||
+	new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
+
+export function runExtractor(
+	script,
+	query,
+	tabPrefix = null,
+	short = false,
+	timeoutMs = null,
+	locale = null,
+) {
+	// Gemini synthesis: 70s budget (45s stream + ~25s nav/settle overhead)
+	// ChatGPT can use a 30s in-page stream wait plus a 35s node-side fallback.
+	// Logically research answers can run academic + web searches before streaming.
+	// Other engines: 60s budget
+	if (timeoutMs === null) {
+		timeoutMs = script.includes("logically")
+			? 120000
+			: script.includes("chatgpt")
+				? 80000
+				: script.includes("gemini")
+					? 70000
+					: 60000;
+	}
+	const extraArgs = [
+		...(tabPrefix ? ["--tab", tabPrefix] : []),
+		...(short ? ["--short"] : []),
+		...(locale ? ["--locale", locale] : []),
+	];
+	return new Promise((resolve, reject) => {
+		const proc = spawn(
+			process.execPath,
+			[join(__dir, "..", "..", "extractors", script), "--stdin", ...extraArgs],
+			{
+				stdio: ["pipe", "pipe", "pipe"],
+				env: { ...process.env, CDP_PROFILE_DIR: GREEDY_PROFILE_DIR },
+			},
+		);
+		// Pipe query via stdin to avoid leaking it in process table command-line
+		proc.stdin.write(query);
+		proc.stdin.end();
+		let out = "";
+		let err = "";
+		proc.stdout.on("data", (d) => (out += d));
+		// Forward child stderr to parent so [engine] stage: lines are visible
+		// in real time. Also retain the buffer for the timeout diagnostic path.
+		proc.stderr.on("data", (d) => {
+			err += d;
+			if (process.env.GREEDY_SEARCH_CHILD_STDERR !== "0") {
+				process.stderr.write(d);
+			}
+		});
+		const t = setTimeout(() => {
+			proc.kill();
+			// Surface as much diagnostic info as the killed child produced so the
+			// caller can see *which stage* the extractor was in. handleError()
+			// emits `{ _envelope, error }` JSON to stdout on graceful failure,
+			// but a hard kill discards whatever was buffered.
+			const tailLines = (s, n = 20) =>
+				String(s ?? "")
+					.split(/\r?\n/)
+					.filter(Boolean)
+					.slice(-n)
+					.join("\n");
+			let envelope = null;
+			try {
+				const parsed = JSON.parse(out.trim());
+				if (parsed._envelope) envelope = parsed._envelope;
+			} catch {}
+			const errObj = new Error(
+				`${script} timed out after ${timeoutMs / 1000}s` +
+					(envelope?.lastStage ? ` (last stage: ${envelope.lastStage})` : ""),
+			);
+			errObj.engineScript = script;
+			errObj.lastStage = envelope?.lastStage || null;
+			errObj.partialErr = tailLines(err);
+			errObj.partialOut = tailLines(out);
+			reject(errObj);
+		}, timeoutMs);
+		proc.on("close", (code) => {
+			clearTimeout(t);
+			if (code === 0) {
+				try {
+					resolve(JSON.parse(out.trim()));
+				} catch {
+					reject(new Error(`bad JSON from ${script}: ${out.slice(0, 100)}`));
+				}
+			} else {
+				// Try to parse structured error envelope from stdout before falling back
+				let envelope = null;
+				try {
+					const parsed = JSON.parse(out.trim());
+					if (parsed._envelope) envelope = parsed._envelope;
+				} catch {}
+				const msg = err.trim() || `extractor exit ${code}`;
+				const errObj = new Error(msg);
+				if (envelope) errObj.envelope = envelope;
+				reject(errObj);
+			}
+		});
+	});
+}