@apmantza/greedysearch-pi 1.9.1 → 2.0.0

package/src/search/sources.mjs

@@ -37,6 +37,16 @@ export const NEWS_HOSTS = [
 	"zdnet.com",
 ];
 
+export const SOCIAL_HOSTS = [
+	"facebook.com",
+	"instagram.com",
+	"linkedin.com",
+	"pinterest.com",
+	"tiktok.com",
+	"twitter.com",
+	"x.com",
+];
+
 export function trimText(text = "", maxChars = 240) {
 	const clean = String(text).replaceAll(/\s+/g, " ").trim();
 	if (clean.length <= maxChars) return clean;
@@ -122,6 +132,17 @@ export function classifySourceType(domain, title = "", rawUrl = "") {
 	const lowerUrl = rawUrl.toLowerCase();
 
 	if (domain === "github.com" || domain === "gitlab.com") return "repo";
+	if (
+		domain === "arxiv.org" ||
+		domain === "doi.org" ||
+		domain === "semanticscholar.org" ||
+		domain.endsWith(".semanticscholar.org") ||
+		lowerUrl.includes("/paper/") ||
+		lowerUrl.includes("/pdf/")
+	) {
+		return "academic";
+	}
+	if (matchesDomain(domain, SOCIAL_HOSTS)) return "social";
 	if (matchesDomain(domain, COMMUNITY_HOSTS)) return "community";
 	if (matchesDomain(domain, NEWS_HOSTS)) return "news";
 	if (
@@ -149,6 +170,8 @@ export function sourceTypePriority(sourceType) {
 			return 5;
 		case "repo":
 			return 4;
+		case "academic":
+			return 4;
 		case "maintainer-blog":
 			return 3;
 		case "website":
@@ -157,6 +180,8 @@ export function sourceTypePriority(sourceType) {
 			return 1;
 		case "news":
 			return 0;
+		case "social":
+			return -6;
 		default:
 			return 0;
 	}
@@ -308,6 +333,10 @@ export function inferPreferredDomains(query) {
 	if (normalized.includes("gemini") || normalized.includes("google ai")) {
 		matches.push("ai.google.dev", "developers.google.com");
 	}
+	for (const socialHost of SOCIAL_HOSTS) {
+		const bareName = socialHost.replace(/\.com$/, "");
+		if (normalized.includes(bareName)) matches.push(socialHost);
+	}
 
 	return [...new Set(matches)];
 }
@@ -318,7 +347,9 @@ export function domainMatches(hostname, candidate) {
 
 export function buildSourceRegistry(out, query = "") {
 	const seen = new Map();
-	const engineOrder = ["perplexity", "bing", "google"];
+	const engineOrder = Object.keys(out || {}).filter(
+		(key) => !key.startsWith("_"),
+	);
 
 	// Get preferred domains for this query
 	const preferredDomains = inferPreferredDomains(query);
@@ -359,10 +390,20 @@ export function buildSourceRegistry(out, query = "") {
 				smartScore += 2;
 			}
 
-			// Penalize discussion forums for technical queries — high noise, rarely canonical.
-			// Q&A sites (stackoverflow, stackexchange) are excluded: they often have the
-			// best practical answer and shouldn't be penalised just because an official
-			// domain also exists.
+			// Penalize discussion/social sites for technical queries — high noise,
+			// hard to fetch cleanly, and rarely canonical. Q&A sites (StackOverflow,
+			// StackExchange) are excluded from the community penalty.
+			//
+			// Social penalty is now −20 (was −12). The original −12 wasn't enough
+			// to overcome the +10 preferred-domain boost + clean rank, so a single
+			// social citation could land as S1. The post-sort demotion below
+			// is the hard guardrail on top.
+			const queryTargetsSocialHost = preferredDomains.some((pd) =>
+				domainMatches(domain, pd),
+			);
+			if (sourceType === "social" && !queryTargetsSocialHost) {
+				smartScore -= 20;
+			}
 			if (preferredDomains.length > 0) {
 				if (matchesDomain(domain, DISCUSSION_HOSTS)) {
 					smartScore -= 3;
@@ -410,27 +451,38 @@ export function buildSourceRegistry(out, query = "") {
 		}
 	}
 
-	const sources = Array.from(seen.values())
-		.map((source) => ({
-			...source,
-			engineCount: source.engines.length,
-		}))
-		.sort((a, b) => {
-			// Single composite score so all signals contribute simultaneously.
-			// Avoids rank being ignored when engineCount differs, and smartScore
-			// dominating even when rank/type signal would break the tie better.
-			const diff = computeCompositeScore(b) - computeCompositeScore(a);
-			if (diff !== 0) return diff;
-			return a.domain.localeCompare(b.domain);
-		})
-		.slice(0, 12)
-		.map((source, index) => ({
-			...source,
-			id: `S${index + 1}`,
-			title: source.title || source.domain || source.canonicalUrl,
-		}));
+	const sources = Array.from(seen.values()).map((source) => ({
+		...source,
+		engineCount: source.engines.length,
+	}));
+
+	// Social hard guardrail: when the query doesn't explicitly target a
+	// social host (rare — only happens for queries like "latest twitter
+	// announcement"), keep social sources OUT of the composite sort and
+	// pin them to the end of the registry. The smartScore −20 penalty
+	// above handles the "bare social gets a +10 boost" case, but a
+	// clean multi-engine social citation can still occasionally outscore
+	// a noisy single-engine academic source. This sort is the final say
+	// on what becomes S1, S2, etc.
+	const nonSocial = sources.filter((s) => s.sourceType !== "social");
+	const socialSources = sources.filter((s) => s.sourceType === "social");
+	nonSocial.sort((a, b) => {
+		const diff = computeCompositeScore(b) - computeCompositeScore(a);
+		if (diff !== 0) return diff;
+		return a.domain.localeCompare(b.domain);
+	});
+	socialSources.sort((a, b) => {
+		const diff = computeCompositeScore(b) - computeCompositeScore(a);
+		if (diff !== 0) return diff;
+		return a.domain.localeCompare(b.domain);
+	});
+	const ordered = [...nonSocial, ...socialSources];
 
-	return sources;
+	return ordered.slice(0, 12).map((source, index) => ({
+		...source,
+		id: `S${index + 1}`,
+		title: source.title || source.domain || source.canonicalUrl,
+	}));
 }
 
 export function mergeFetchDataIntoSources(sources, fetchedSources) {

package/src/search/synthesis-runner.mjs

@@ -1,45 +1,84 @@
-// src/search/synthesis-runner.mjs — Run Gemini synthesis via CDP
+// src/search/synthesis-runner.mjs — Engine-agnostic synthesis via CDP extractors
 //
-// Extracted from search.mjs.
+// The all-search synthesis layer builds a neutral prompt and can route it to a
+// configured browser engine. Gemini remains the default synthesizer; ChatGPT is
+// supported for users who opt in via ~/.pi/greedyconfig or --synthesizer.
 
 import { spawn } from "node:child_process";
 import { join } from "node:path";
-import { GREEDY_PROFILE_DIR } from "./constants.mjs";
+import { GREEDY_PROFILE_DIR, SUPPORTED_SYNTHESIZERS } from "./constants.mjs";
 import {
 	buildSynthesisPrompt,
 	normalizeSynthesisPayload,
 	parseStructuredJson,
 } from "./synthesis.mjs";
+import { buildSourceRegistry } from "./sources.mjs";
 
 const __dir =
 	import.meta.dirname ||
 	new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
 
-export async function synthesizeWithGemini(
-	query,
-	results,
-	{ grounded = false, tabPrefix = null } = {},
+const SYNTHESIS_EXTRACTORS = {
+	gemini: "gemini.mjs",
+	chatgpt: "chatgpt.mjs",
+};
+
+const SYNTHESIS_START_URLS = {
+	gemini: "https://gemini.google.com/app",
+	chatgpt: "https://chatgpt.com/",
+};
+
+export function normalizeSynthesizer(synthesizer = "gemini") {
+	const normalized = String(synthesizer || "gemini").toLowerCase();
+	if (normalized === "gem") return "gemini";
+	if (normalized === "gpt") return "chatgpt";
+	return normalized;
+}
+
+export function getSynthesisStartUrl(synthesizer = "gemini") {
+	return (
+		SYNTHESIS_START_URLS[normalizeSynthesizer(synthesizer)] || "about:blank"
+	);
+}
+
+export async function runSynthesisPrompt(
+	synthesizer,
+	prompt,
+	{ tabPrefix = null, timeoutMs = 180000, visible = null } = {},
 ) {
-	const sources = Array.isArray(results._sources)
-		? results._sources
-		: buildSourceRegistry(results);
-	const prompt = buildSynthesisPrompt(query, results, sources, { grounded });
+	const normalizedSynthesizer = normalizeSynthesizer(synthesizer);
+	const script = SYNTHESIS_EXTRACTORS[normalizedSynthesizer];
+	if (!script || !SUPPORTED_SYNTHESIZERS.includes(normalizedSynthesizer)) {
+		throw new Error(
+			`Unsupported synthesizer "${synthesizer}". Supported: ${SUPPORTED_SYNTHESIZERS.join(", ")}`,
+		);
+	}
 
 	return new Promise((resolve, reject) => {
 		const extraArgs = tabPrefix ? ["--tab", String(tabPrefix)] : [];
+		// Strip inherited visible-mode flags so a stale GREEDY_SEARCH_VISIBLE=1
+		// in the parent process doesn't force visible Chrome. Callers that
+		// genuinely want visible synthesis should pass visible: true explicitly.
+		const childEnv = {
+			...process.env,
+			CDP_PROFILE_DIR: GREEDY_PROFILE_DIR,
+		};
+		if (visible !== true) {
+			delete childEnv.GREEDY_SEARCH_VISIBLE;
+			delete childEnv.GREEDY_SEARCH_ALWAYS_VISIBLE;
+		} else {
+			childEnv.GREEDY_SEARCH_VISIBLE = "1";
+			childEnv.GREEDY_SEARCH_ALWAYS_VISIBLE = "1";
+		}
 		const proc = spawn(
 			process.execPath,
-			[
-				join(__dir, "..", "..", "extractors", "gemini.mjs"),
-				"--stdin",
-				...extraArgs,
-			],
+			[join(__dir, "..", "..", "extractors", script), "--stdin", ...extraArgs],
 			{
 				stdio: ["pipe", "pipe", "pipe"],
-				env: { ...process.env, CDP_PROFILE_DIR: GREEDY_PROFILE_DIR },
+				env: childEnv,
 			},
 		);
-		// Pipe synthesis prompt via stdin to avoid leaking the full prompt in process table
+		// Pipe prompts via stdin to avoid leaking them in process tables.
 		proc.stdin.write(prompt);
 		proc.stdin.end();
 		let out = "";
@@ -48,49 +87,95 @@ export async function synthesizeWithGemini(
 		proc.stderr.on("data", (d) => (err += d));
 		const t = setTimeout(() => {
 			proc.kill();
-			reject(new Error("Gemini synthesis timed out after 180s"));
-		}, 180000);
+			reject(
+				new Error(
+					`${normalizedSynthesizer} prompt timed out after ${timeoutMs / 1000}s`,
+				),
+			);
+		}, timeoutMs);
 		proc.on("close", (code) => {
 			clearTimeout(t);
-			if (code !== 0)
-				reject(new Error(err.trim() || "gemini extractor failed"));
-			else {
-				try {
-					const raw = JSON.parse(out.trim());
-					let structured = parseStructuredJson(raw.answer || "");
-
-					// Detect if Gemini echoed back the engine summaries instead of a synthesis.
-					// Happens when Gemini can't synthesize (e.g. only 1 engine responded) and
-					// echoes the prompt JSON. The engine summary JSON has per-engine keys
-					// (perplexity/bing/google) but no synthesis fields (answer/agreement).
-					const SYNTHESIS_FIELDS = [
-						"answer",
-						"agreement",
-						"claims",
-						"differences",
-						"caveats",
-					];
-					const hasSynthesisFields =
-						structured && SYNTHESIS_FIELDS.some((f) => f in structured);
-					const hasEngineKeys =
-						structured &&
-						["perplexity", "bing", "google"].some((e) => e in structured);
-					if (hasEngineKeys && !hasSynthesisFields) {
-						structured = null; // Treat as parse failure — Gemini echoed input
-					}
-
-					resolve({
-						...normalizeSynthesisPayload(structured, sources, raw.answer || ""),
-						rawAnswer: raw.answer || "",
-						geminiSources: raw.sources || [],
-					});
-				} catch {
-					reject(new Error(`bad JSON from gemini: ${out.slice(0, 100)}`));
-				}
+			if (code !== 0) {
+				reject(
+					new Error(err.trim() || `${normalizedSynthesizer} extractor failed`),
+				);
+				return;
+			}
+			try {
+				resolve(JSON.parse(out.trim()));
+			} catch {
+				reject(
+					new Error(
+						`bad JSON from ${normalizedSynthesizer}: ${out.slice(0, 100)}`,
+					),
+				);
 			}
 		});
 	});
 }
 
-// Need to import buildSourceRegistry for fallback
-import { buildSourceRegistry } from "./sources.mjs";
+// Backward-compatible Gemini helper used by research mode internals.
+export async function runGeminiPrompt(prompt, options = {}) {
+	return runSynthesisPrompt("gemini", prompt, options);
+}
+
+export async function synthesizeResults(
+	query,
+	results,
+	{
+		grounded = false,
+		tabPrefix = null,
+		visible = null,
+		synthesizer = "gemini",
+	} = {},
+) {
+	const normalizedSynthesizer = normalizeSynthesizer(synthesizer);
+	const sources = Array.isArray(results._sources)
+		? results._sources
+		: buildSourceRegistry(results);
+	const prompt = buildSynthesisPrompt(query, results, sources, { grounded });
+
+	const raw = await runSynthesisPrompt(normalizedSynthesizer, prompt, {
+		tabPrefix,
+		timeoutMs: 180000,
+		visible,
+	});
+	let structured = parseStructuredJson(raw.answer || "");
+
+	// Detect if the synthesizer echoed back the engine summaries instead of a
+	// synthesis. This can happen when it can't synthesize and mirrors prompt JSON.
+	const SYNTHESIS_FIELDS = [
+		"answer",
+		"agreement",
+		"claims",
+		"differences",
+		"caveats",
+	];
+	const hasSynthesisFields =
+		structured && SYNTHESIS_FIELDS.some((f) => f in structured);
+	const hasEngineKeys =
+		structured &&
+		["perplexity", "bing", "google", "chatgpt", "gemini"].some(
+			(e) => e in structured,
+		);
+	if (hasEngineKeys && !hasSynthesisFields) {
+		structured = null; // Treat as parse failure — synthesizer echoed input
+	}
+
+	return {
+		...normalizeSynthesisPayload(structured, sources, raw.answer || ""),
+		rawAnswer: raw.answer || "",
+		synthesizedBy: normalizedSynthesizer,
+		synthesizerSources: raw.sources || [],
+		// Backward-compatible field for existing consumers.
+		geminiSources: normalizedSynthesizer === "gemini" ? raw.sources || [] : [],
+	};
+}
+
+// Backward-compatible all-search synthesis helper.
+export async function synthesizeWithGemini(query, results, options = {}) {
+	return synthesizeResults(query, results, {
+		...options,
+		synthesizer: "gemini",
+	});
+}