@apmantza/greedysearch-pi 1.9.2 → 2.1.2

package/src/search/sources.mjs

@@ -132,6 +132,16 @@ export function classifySourceType(domain, title = "", rawUrl = "") {
 	const lowerUrl = rawUrl.toLowerCase();
 
 	if (domain === "github.com" || domain === "gitlab.com") return "repo";
+	if (
+		domain === "arxiv.org" ||
+		domain === "doi.org" ||
+		domain === "semanticscholar.org" ||
+		domain.endsWith(".semanticscholar.org") ||
+		lowerUrl.includes("/paper/") ||
+		lowerUrl.includes("/pdf/")
+	) {
+		return "academic";
+	}
 	if (matchesDomain(domain, SOCIAL_HOSTS)) return "social";
 	if (matchesDomain(domain, COMMUNITY_HOSTS)) return "community";
 	if (matchesDomain(domain, NEWS_HOSTS)) return "news";
@@ -160,6 +170,8 @@ export function sourceTypePriority(sourceType) {
 			return 5;
 		case "repo":
 			return 4;
+		case "academic":
+			return 4;
 		case "maintainer-blog":
 			return 3;
 		case "website":
@@ -335,7 +347,9 @@ export function domainMatches(hostname, candidate) {
 
 export function buildSourceRegistry(out, query = "") {
 	const seen = new Map();
-	const engineOrder = ["perplexity", "bing", "google"];
+	const engineOrder = Object.keys(out || {}).filter(
+		(key) => !key.startsWith("_"),
+	);
 
 	// Get preferred domains for this query
 	const preferredDomains = inferPreferredDomains(query);
@@ -379,11 +393,16 @@ export function buildSourceRegistry(out, query = "") {
 			// Penalize discussion/social sites for technical queries — high noise,
 			// hard to fetch cleanly, and rarely canonical. Q&A sites (StackOverflow,
 			// StackExchange) are excluded from the community penalty.
+			//
+			// Social penalty is now −20 (was −12). The original −12 wasn't enough
+			// to overcome the +10 preferred-domain boost + clean rank, so a single
+			// social citation could land as S1. The post-sort demotion below
+			// is the hard guardrail on top.
 			const queryTargetsSocialHost = preferredDomains.some((pd) =>
 				domainMatches(domain, pd),
 			);
 			if (sourceType === "social" && !queryTargetsSocialHost) {
-				smartScore -= 12;
+				smartScore -= 20;
 			}
 			if (preferredDomains.length > 0) {
 				if (matchesDomain(domain, DISCUSSION_HOSTS)) {
@@ -432,27 +451,38 @@ export function buildSourceRegistry(out, query = "") {
 		}
 	}
 
-	const sources = Array.from(seen.values())
-		.map((source) => ({
-			...source,
-			engineCount: source.engines.length,
-		}))
-		.sort((a, b) => {
-			// Single composite score so all signals contribute simultaneously.
-			// Avoids rank being ignored when engineCount differs, and smartScore
-			// dominating even when rank/type signal would break the tie better.
-			const diff = computeCompositeScore(b) - computeCompositeScore(a);
-			if (diff !== 0) return diff;
-			return a.domain.localeCompare(b.domain);
-		})
-		.slice(0, 12)
-		.map((source, index) => ({
-			...source,
-			id: `S${index + 1}`,
-			title: source.title || source.domain || source.canonicalUrl,
-		}));
+	const sources = Array.from(seen.values()).map((source) => ({
+		...source,
+		engineCount: source.engines.length,
+	}));
+
+	// Social hard guardrail: when the query doesn't explicitly target a
+	// social host (rare — only happens for queries like "latest twitter
+	// announcement"), keep social sources OUT of the composite sort and
+	// pin them to the end of the registry. The smartScore −20 penalty
+	// above handles the "bare social gets a +10 boost" case, but a
+	// clean multi-engine social citation can still occasionally outscore
+	// a noisy single-engine academic source. This sort is the final say
+	// on what becomes S1, S2, etc.
+	const nonSocial = sources.filter((s) => s.sourceType !== "social");
+	const socialSources = sources.filter((s) => s.sourceType === "social");
+	nonSocial.sort((a, b) => {
+		const diff = computeCompositeScore(b) - computeCompositeScore(a);
+		if (diff !== 0) return diff;
+		return a.domain.localeCompare(b.domain);
+	});
+	socialSources.sort((a, b) => {
+		const diff = computeCompositeScore(b) - computeCompositeScore(a);
+		if (diff !== 0) return diff;
+		return a.domain.localeCompare(b.domain);
+	});
+	const ordered = [...nonSocial, ...socialSources];
 
-	return sources;
+	return ordered.slice(0, 12).map((source, index) => ({
+		...source,
+		id: `S${index + 1}`,
+		title: source.title || source.domain || source.canonicalUrl,
+	}));
 }
 
 export function mergeFetchDataIntoSources(sources, fetchedSources) {

package/src/search/synthesis-runner.mjs

@@ -1,36 +1,81 @@
-// src/search/synthesis-runner.mjs — Run Gemini synthesis via CDP
+// src/search/synthesis-runner.mjs — Engine-agnostic synthesis via CDP extractors
 //
-// Extracted from search.mjs.
+// The all-search synthesis layer builds a neutral prompt and can route it to a
+// configured browser engine. Gemini remains the default synthesizer; ChatGPT is
+// supported for users who opt in via ~/.pi/greedyconfig or --synthesizer.
 
 import { spawn } from "node:child_process";
 import { join } from "node:path";
-import { GREEDY_PROFILE_DIR } from "./constants.mjs";
+import { GREEDY_PROFILE_DIR, SUPPORTED_SYNTHESIZERS } from "./constants.mjs";
 import {
 	buildSynthesisPrompt,
 	normalizeSynthesisPayload,
 	parseStructuredJson,
 } from "./synthesis.mjs";
+import { buildSourceRegistry } from "./sources.mjs";
 
 const __dir =
 	import.meta.dirname ||
 	new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
 
-export async function runGeminiPrompt(
+const SYNTHESIS_EXTRACTORS = {
+	gemini: "gemini.mjs",
+	chatgpt: "chatgpt.mjs",
+};
+
+const SYNTHESIS_START_URLS = {
+	gemini: "https://gemini.google.com/app",
+	chatgpt: "https://chatgpt.com/",
+};
+
+export function normalizeSynthesizer(synthesizer = "gemini") {
+	const normalized = String(synthesizer || "gemini").toLowerCase();
+	if (normalized === "gem") return "gemini";
+	if (normalized === "gpt") return "chatgpt";
+	return normalized;
+}
+
+export function getSynthesisStartUrl(synthesizer = "gemini") {
+	return (
+		SYNTHESIS_START_URLS[normalizeSynthesizer(synthesizer)] || "about:blank"
+	);
+}
+
+export async function runSynthesisPrompt(
+	synthesizer,
 	prompt,
-	{ tabPrefix = null, timeoutMs = 180000 } = {},
+	{ tabPrefix = null, timeoutMs = 180000, visible = null } = {},
 ) {
+	const normalizedSynthesizer = normalizeSynthesizer(synthesizer);
+	const script = SYNTHESIS_EXTRACTORS[normalizedSynthesizer];
+	if (!script || !SUPPORTED_SYNTHESIZERS.includes(normalizedSynthesizer)) {
+		throw new Error(
+			`Unsupported synthesizer "${synthesizer}". Supported: ${SUPPORTED_SYNTHESIZERS.join(", ")}`,
+		);
+	}
+
 	return new Promise((resolve, reject) => {
 		const extraArgs = tabPrefix ? ["--tab", String(tabPrefix)] : [];
+		// Strip inherited visible-mode flags so a stale GREEDY_SEARCH_VISIBLE=1
+		// in the parent process doesn't force visible Chrome. Callers that
+		// genuinely want visible synthesis should pass visible: true explicitly.
+		const childEnv = {
+			...process.env,
+			CDP_PROFILE_DIR: GREEDY_PROFILE_DIR,
+		};
+		if (visible !== true) {
+			delete childEnv.GREEDY_SEARCH_VISIBLE;
+			delete childEnv.GREEDY_SEARCH_ALWAYS_VISIBLE;
+		} else {
+			childEnv.GREEDY_SEARCH_VISIBLE = "1";
+			childEnv.GREEDY_SEARCH_ALWAYS_VISIBLE = "1";
+		}
 		const proc = spawn(
 			process.execPath,
-			[
-				join(__dir, "..", "..", "extractors", "gemini.mjs"),
-				"--stdin",
-				...extraArgs,
-			],
+			[join(__dir, "..", "..", "extractors", script), "--stdin", ...extraArgs],
 			{
 				stdio: ["pipe", "pipe", "pipe"],
-				env: { ...process.env, CDP_PROFILE_DIR: GREEDY_PROFILE_DIR },
+				env: childEnv,
 			},
 		);
 		// Pipe prompts via stdin to avoid leaking them in process tables.
@@ -42,40 +87,63 @@ export async function runGeminiPrompt(
 		proc.stderr.on("data", (d) => (err += d));
 		const t = setTimeout(() => {
 			proc.kill();
-			reject(new Error(`Gemini prompt timed out after ${timeoutMs / 1000}s`));
+			reject(
+				new Error(
+					`${normalizedSynthesizer} prompt timed out after ${timeoutMs / 1000}s`,
+				),
+			);
 		}, timeoutMs);
 		proc.on("close", (code) => {
 			clearTimeout(t);
 			if (code !== 0) {
-				reject(new Error(err.trim() || "gemini extractor failed"));
+				reject(
+					new Error(err.trim() || `${normalizedSynthesizer} extractor failed`),
+				);
 				return;
 			}
 			try {
 				resolve(JSON.parse(out.trim()));
 			} catch {
-				reject(new Error(`bad JSON from gemini: ${out.slice(0, 100)}`));
+				reject(
+					new Error(
+						`bad JSON from ${normalizedSynthesizer}: ${out.slice(0, 100)}`,
+					),
+				);
 			}
 		});
 	});
 }
 
-export async function synthesizeWithGemini(
+// Backward-compatible Gemini helper used by research mode internals.
+export async function runGeminiPrompt(prompt, options = {}) {
+	return runSynthesisPrompt("gemini", prompt, options);
+}
+
+export async function synthesizeResults(
 	query,
 	results,
-	{ grounded = false, tabPrefix = null } = {},
+	{
+		grounded = false,
+		tabPrefix = null,
+		visible = null,
+		synthesizer = "gemini",
+	} = {},
 ) {
+	const normalizedSynthesizer = normalizeSynthesizer(synthesizer);
 	const sources = Array.isArray(results._sources)
 		? results._sources
 		: buildSourceRegistry(results);
 	const prompt = buildSynthesisPrompt(query, results, sources, { grounded });
 
-	const raw = await runGeminiPrompt(prompt, { tabPrefix, timeoutMs: 180000 });
+	const raw = await runSynthesisPrompt(normalizedSynthesizer, prompt, {
+		tabPrefix,
+		timeoutMs: 180000,
+		visible,
+	});
 	let structured = parseStructuredJson(raw.answer || "");
 
-	// Detect if Gemini echoed back the engine summaries instead of a synthesis.
-	// Happens when Gemini can't synthesize (e.g. only 1 engine responded) and
-	// echoes the prompt JSON. The engine summary JSON has per-engine keys
-	// (perplexity/bing/google) but no synthesis fields (answer/agreement).
+	// Detect if the synthesizer echoed back the engine summaries instead of a
+	// synthesis. This can happen when it can't synthesize and mirrors prompt JSON.
 	const SYNTHESIS_FIELDS = [
 		"answer",
 		"agreement",
@@ -86,17 +154,28 @@ export async function synthesizeWithGemini(
 	const hasSynthesisFields =
 		structured && SYNTHESIS_FIELDS.some((f) => f in structured);
 	const hasEngineKeys =
-		structured && ["perplexity", "bing", "google"].some((e) => e in structured);
+		structured &&
+		["perplexity", "bing", "google", "chatgpt", "gemini"].some(
+			(e) => e in structured,
+		);
 	if (hasEngineKeys && !hasSynthesisFields) {
-		structured = null; // Treat as parse failure — Gemini echoed input
+		structured = null; // Treat as parse failure — synthesizer echoed input
 	}
 
 	return {
 		...normalizeSynthesisPayload(structured, sources, raw.answer || ""),
 		rawAnswer: raw.answer || "",
-		geminiSources: raw.sources || [],
+		synthesizedBy: normalizedSynthesizer,
+		synthesizerSources: raw.sources || [],
+		// Backward-compatible field for existing consumers.
+		geminiSources: normalizedSynthesizer === "gemini" ? raw.sources || [] : [],
 	};
 }
 
-// Need to import buildSourceRegistry for fallback
-import { buildSourceRegistry } from "./sources.mjs";
+// Backward-compatible all-search synthesis helper.
+export async function synthesizeWithGemini(query, results, options = {}) {
+	return synthesizeResults(query, results, {
+		...options,
+		synthesizer: "gemini",
+	});
+}