@apmantza/greedysearch-pi 1.9.2 → 2.0.0

package/extractors/semantic-scholar.mjs

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+
+// extractors/semantic-scholar.mjs
+// Search Semantic Scholar without API keys and return paper/PDF sources for
+// GreedySearch's source fetcher and research synthesizer.
+
+import {
+	buildEnvelope,
+	cdp,
+	formatAnswer,
+	getOrOpenTab,
+	handleError,
+	logStage,
+	outputJson,
+	parseArgs,
+	prepareArgs,
+	validateQuery,
+	waitForSelector,
+} from "./common.mjs";
+
+const USAGE =
+	'Usage: node extractors/semantic-scholar.mjs "<query>" [--tab <prefix>]\n';
+const RESULT_SELECTOR = ".cl-paper-row[data-paper-id]";
+
+function semanticScholarSearchUrl(query) {
+	// Semantic Scholar docs note hyphenated terms can reduce matches; use spaces.
+	const normalized = String(query || "").replaceAll("-", " ");
+	return `https://www.semanticscholar.org/search?q=${encodeURIComponent(normalized)}&sort=relevance`;
+}
+
+async function dismissCookieBanner(tab) {
+	await cdp([
+		"eval",
+		tab,
+		String.raw`
+		(() => {
+			const selectors = [
+				'.osano-cm-dialog__close',
+				'.osano-cm-denyAll',
+				'.osano-cm-accept-all',
+				'button[aria-label*="Close" i]',
+			];
+			for (const selector of selectors) {
+				const btn = document.querySelector(selector);
+				if (btn) { btn.click(); return selector; }
+			}
+			return null;
+		})()
+	`,
+	]).catch(() => null);
+}
+
+async function extractPapers(tab, { limit = 8 } = {}) {
+	const raw = await cdp([
+		"eval",
+		tab,
+		String.raw`
+		((limit) => {
+			function clean(value) {
+				return String(value || '').replace(/\s+/g, ' ').trim();
+			}
+			function absolutize(href) {
+				try { return new URL(href, location.href).href; } catch { return ''; }
+			}
+			function isDirectPdf(url) {
+				return /\.pdf(?:[?#]|$)/i.test(url) || /\/pdf\//i.test(url);
+			}
+			const rows = Array.from(document.querySelectorAll('.cl-paper-row[data-paper-id]')).slice(0, limit);
+			return JSON.stringify(rows.map((row, index) => {
+				const titleLink = row.querySelector('a[data-test-id="title-link"][href], a[href*="/paper/"][href]');
+				const paperUrl = absolutize(titleLink?.getAttribute('href') || '');
+				const title = clean(titleLink?.innerText || row.querySelector('.cl-paper-title')?.innerText || '');
+				const authors = Array.from(row.querySelectorAll('[data-test-id="author-list"] a, .cl-paper-authors a'))
+					.map((a) => clean(a.innerText))
+					.filter(Boolean)
+					.slice(0, 8);
+				const field = clean(row.querySelector('.cl-paper-fos')?.innerText || '');
+				const venue = clean(row.querySelector('[data-test-id="normalized-venue-link"], .cl-paper-venue')?.innerText || '');
+				const date = clean(row.querySelector('.cl-paper-pubdates')?.innerText || '');
+				const tldrNode = row.querySelector('.tldr-abstract-replacement');
+				let tldr = clean(tldrNode?.innerText || '');
+				tldr = tldr.replace(/^TLDR\s*/i, '').replace(/\s*Expand$/i, '').trim();
+				const citationNode = row.querySelector('[data-test-id="total-citations-stat"]');
+				const citationLabel = citationNode?.getAttribute('aria-label') || citationNode?.innerText || '';
+				const citationMatch = clean(citationLabel).match(/[\d,]+/);
+				const citationCount = citationMatch ? Number.parseInt(citationMatch[0].replace(/,/g, ''), 10) : null;
+				const externalLinks = Array.from(row.querySelectorAll('a[data-test-id="paper-link"][href], a.cl-paper-view-paper[href]'))
+					.map((a) => ({
+						url: absolutize(a.getAttribute('href')),
+						label: clean(a.innerText),
+					}))
+					.filter((link) => link.url);
+				const directPdf = externalLinks.find((link) => isDirectPdf(link.url));
+				const primaryExternal = directPdf || externalLinks[0] || null;
+				const sourceUrl = primaryExternal?.url || paperUrl;
+				return {
+					rank: index + 1,
+					paperId: row.getAttribute('data-paper-id') || '',
+					title,
+					url: sourceUrl,
+					semanticScholarUrl: paperUrl,
+					pdfUrl: directPdf?.url || '',
+					externalUrl: primaryExternal?.url || '',
+					externalLabel: primaryExternal?.label || '',
+					authors,
+					field,
+					venue,
+					date,
+					tldr,
+					citationCount,
+				};
+			}));
+		})(${limit})
+	`,
+	]);
+	try {
+		return JSON.parse(raw);
+	} catch {
+		return [];
+	}
+}
+
+function formatPaperSummary(papers) {
+	if (!papers.length) return "Semantic Scholar returned no paper results.";
+	return papers
+		.map((paper) => {
+			const parts = [];
+			if (paper.authors?.length) parts.push(paper.authors.join(", "));
+			if (paper.venue) parts.push(paper.venue);
+			if (paper.date) parts.push(paper.date);
+			if (Number.isFinite(paper.citationCount)) {
+				parts.push(`${paper.citationCount.toLocaleString()} citations`);
+			}
+			const meta = parts.length ? ` — ${parts.join(" · ")}` : "";
+			const tldr = paper.tldr ? `\n   TLDR: ${paper.tldr}` : "";
+			return `${paper.rank}. ${paper.title}${meta}${tldr}`;
+		})
+		.join("\n\n");
+}
+
+async function main() {
+	const args = await prepareArgs(process.argv.slice(2));
+	validateQuery(args, USAGE);
+	const { query, tabPrefix, short } = parseArgs(args);
+	const startTime = Date.now();
+	const mode =
+		process.env.GREEDY_SEARCH_VISIBLE === "1" ? "visible" : "headless";
+	const env = {
+		engine: "semantic-scholar",
+		mode,
+		blockedBy: null,
+		verificationResult: null,
+		inputReady: null,
+	};
+
+	try {
+		if (!tabPrefix) await cdp(["list"]);
+		const tab = await getOrOpenTab(tabPrefix);
+		logStage(env, "nav", startTime);
+		await cdp(["nav", tab, semanticScholarSearchUrl(query)], 25000);
+		await new Promise((r) => setTimeout(r, 800));
+
+		logStage(env, "consent", startTime);
+		await dismissCookieBanner(tab);
+
+		logStage(env, "results-wait", startTime);
+		const inputReady = await waitForSelector(tab, RESULT_SELECTOR, 15000, 500);
+		env.inputReady = inputReady;
+		if (!inputReady) {
+			const body = await cdp([
+				"eval",
+				tab,
+				"document.body?.innerText || ''",
+			]).catch(() => "");
+			if (/captcha|cloudflare|verify|robot|blocked/i.test(body)) {
+				env.blockedBy = "verification";
+				env.verificationResult = "needs-human";
+				throw new Error(
+					"Semantic Scholar verification required — please solve it in the visible browser window",
+				);
+			}
+			throw new Error("Semantic Scholar results not found");
+		}
+
+		logStage(env, "extract", startTime);
+		const papers = await extractPapers(tab, { limit: short ? 5 : 8 });
+		const sources = papers
+			.filter((paper) => paper.title && paper.url)
+			.map((paper) => ({
+				title: paper.pdfUrl ? `${paper.title} (PDF)` : paper.title,
+				url: paper.url,
+				semanticScholarUrl: paper.semanticScholarUrl,
+				paperId: paper.paperId,
+				citationCount: paper.citationCount,
+				venue: paper.venue,
+				year: paper.date,
+			}));
+		const answer = formatPaperSummary(papers);
+		const durationMs = Date.now() - startTime;
+		outputJson({
+			answer: formatAnswer(answer, short),
+			sources,
+			query,
+			url: semanticScholarSearchUrl(query),
+			papers,
+			_envelope: buildEnvelope({ ...env, durationMs }),
+		});
+	} catch (error) {
+		handleError(
+			error,
+			buildEnvelope({
+				...env,
+				durationMs: Date.now() - startTime,
+			}),
+		);
+	}
+}
+
+main();

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@apmantza/greedysearch-pi",
-	"version": "1.9.2",
-	"description": "Headless multi-engine AI search (Perplexity, Bing Copilot, Google AI) via browser automation -- NO API KEYS needed. Extracts answers with sources, optional synthesis. Grounded AI answers from real browser interactions.",
+	"version": "2.0.0",
+	"description": "Headless multi-engine AI search (Perplexity, Google AI, ChatGPT, Gemini) via browser automation. NO API KEYS needed. Grounded all-engine search fetches sources by default, with optional configurable synthesis and deep research.",
 	"type": "module",
 	"keywords": [
 		"pi-package"
@@ -15,11 +15,14 @@
 	"license": "MIT",
 	"scripts": {
 		"test": "node test.mjs",
+		"test:unit": "node test.mjs unit",
 		"test:quick": "node test.mjs quick",
 		"test:smoke": "node test.mjs smoke",
 		"test:bash": "./test.sh",
 		"test:bash:quick": "./test.sh quick",
-		"test:bash:smoke": "./test.sh smoke"
+		"test:bash:smoke": "./test.sh smoke",
+		"lint": "node scripts/lint.mjs",
+		"check:lockfile": "node scripts/check-lockfile.mjs"
 	},
 	"engines": {
 		"node": ">=20.11.0"
@@ -47,6 +50,7 @@
 		"@mozilla/readability": "^0.6.0",
 		"@sinclair/typebox": "^0.34.48",
 		"jsdom": "^24.0.0",
+		"pdf-parse": "^2.4.5",
 		"turndown": "^7.1.2"
 	},
 	"peerDependencies": {

package/skills/greedy-search/skill.md

@@ -1,11 +1,17 @@
 ---
 name: greedy-search
-description: Web search via Perplexity, Bing, Google AI & Gemini. Current docs, recent changes, dependency choices. NOT codebase search.
+description: Web/search plus opt-in research via Perplexity, Google AI, ChatGPT, Gemini, Semantic Scholar, and Logically. Grounded all-engine search fetches sources by default; optional configurable synthesis; deep research as separate workflow. Configurable via ~/.pi/greedyconfig. Bing Copilot available for signed-in users. Current docs, recent changes, dependency choices. NOT codebase search.
 ---
 
-`greedy_search({ query, engine: "all"|"perplexity"|"bing"|"google"|"gemini", depth: "fast"|"standard"|"deep"|"research", breadth: 1-5, iterations: 1-3, maxSources: 3-12, visible: bool })`
+`greedy_search({ query, engine: "all"|"perplexity"|"google"|"chatgpt"|"gemini"|"semantic-scholar"|"logically"|"bing", synthesize?: bool, synthesizer?: "gemini"|"chatgpt", depth?: "research", breadth: 1-5, iterations: 1-3, maxSources: 3-12, researchOutDir?: string, writeResearchBundle?: bool, visible: bool })`
 
-**Depth:** `fast`(15-30s, no synthesis) · `standard`(30-90s, all+synthesis+sources) · `deep`(60-180s, stronger grounding) · `research`(slowest, iterative planning+follow-ups+learning extraction; uses breadth/iterations/maxSources)
+**Modes:** individual engine search · grounded `engine:"all"` search with fetched sources · optional `synthesize:true` using the configured synthesizer over all-engine results · `depth:"research"` for the iterative deep-research workflow.
+
+**Config:** `~/.pi/greedyconfig` supports `{ "engines": ["perplexity", "google", "chatgpt", "gemini", "semantic-scholar", "logically"], "synthesizer": "gemini" }`. Gemini is a normal search engine; Semantic Scholar and Logically are opt-in research engines. Any configured engine can participate in `engine:"all"`; deep research child searches reuse the same configured `engines` list and stdin-safe query passing. Normal all-search synthesis remains controlled separately by `synthesizer`; research planning/final synthesis uses Gemini.
+
+**Compatibility:** legacy `depth:"fast"|"standard"|"deep"` is still accepted. `fast` skips source fetching; `standard`/`deep` alias `synthesize:true`. Prefer `synthesize:true`, optional `synthesizer`, and `depth:"research"` going forward.
+
+**Research output:** `depth:"research"` writes a dataroom-style bundle by default under `.pi/greedysearch-research/<timestamp>_<query>/` with `STATUS.md`, `OUTLINE.md`, `reports/SUMMARY.md`, `reports/CLAIMS.md`, `reports/GAPS.md`, `sources/`, and `data/manifest.json`. Pass `researchOutDir` to choose the directory or `writeResearchBundle:false` to disable disk output.
 
 **Auto-recovery:** Headless default. Bing/Perplexity auto-retry visible on CF block. Manual CAPTCHA → visible stays open; solve then rerun.
 

package/src/fetcher.mjs

@@ -63,6 +63,10 @@ const PRIVATE_URL_PATTERNS = [
  * @param {string} url - URL to check
  * @returns {{blocked: boolean, reason?: string}}
  */
+export function defaultFetchHeaders(overrides = {}) {
+	return { ...DEFAULT_HEADERS, ...overrides };
+}
+
 export function isPrivateUrl(url) {
 	try {
 		const parsed = new URL(url);
@@ -594,7 +598,10 @@ export function checkContentQuality(extracted) {
 			desc: "access denied in content",
 		},
 		{
-			check: () => /^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(markdown),
+			check: () =>
+				/^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(
+					markdown,
+				),
 			desc: "login form only",
 		},
 	];

package/src/formatters/results.ts

@@ -1,128 +1,163 @@
-/**
- * Search results formatters
- * Extracted from index.ts
- */
-
-import { formatEngineName } from "../utils/helpers.js";
-import { renderSynthesis } from "./synthesis.js";
-
-/**
- * Format search results based on engine type
- */
-export function formatResults(
-	engine: string,
-	data: Record<string, unknown>,
-): string {
-	const lines: string[] = [];
-
-	if (engine === "all") {
-		return formatAllEnginesResult(data, lines);
-	}
-
-	return formatSingleEngineResult(data, lines);
-}
-
-/**
- * Format multi-engine results with synthesis
- */
-function formatAllEnginesResult(
-	data: Record<string, unknown>,
-	lines: string[],
-): string {
-	const synthesis = data._synthesis as Record<string, unknown> | undefined;
-	const dedupedSources = data._sources as
-		| Array<Record<string, unknown>>
-		| undefined;
-	const needsHuman = data._needsHumanVerification as
-		| Record<string, unknown>
-		| undefined;
-
-	if (needsHuman) {
-		const engines = Array.isArray(needsHuman.engines)
-			? needsHuman.engines.join(", ")
-			: "one or more engines";
-		lines.push("## Manual verification required");
-		lines.push(
-			String(
-				needsHuman.message ||
-					"Visible Chrome is open. Solve the verification challenge, then rerun the same search.",
-			),
-		);
-		lines.push(`Engines: ${engines}`);
-		lines.push("");
-	}
-
-	// If we have a synthesis answer, render it
-	if (synthesis?.answer) {
-		renderSynthesis(lines, synthesis, dedupedSources || [], 6);
-		lines.push("*Synthesized from Perplexity, Bing Copilot, and Google AI*\n");
-		return lines.join("\n").trim();
-	}
-
-	// Fallback: render individual engine results
-	for (const [eng, result] of Object.entries(data)) {
-		if (eng.startsWith("_")) continue;
-		lines.push(`\n## ${formatEngineName(eng)}`);
-		formatEngineResult(result as Record<string, unknown>, lines, 3);
-	}
-
-	return lines.join("\n").trim();
-}
-
-/**
- * Format single engine result
- */
-function formatSingleEngineResult(
-	data: Record<string, unknown>,
-	lines: string[],
-): string {
-	const needsHuman = data._needsHumanVerification as
-		| Record<string, unknown>
-		| undefined;
-	if (needsHuman) {
-		const engines = Array.isArray(needsHuman.engines)
-			? needsHuman.engines.join(", ")
-			: "this engine";
-		lines.push("## Manual verification required");
-		lines.push(
-			String(
-				needsHuman.message ||
-					"Visible Chrome is open. Solve the verification challenge, then rerun the same search.",
-			),
-		);
-		lines.push(`Engines: ${engines}`);
-		lines.push("");
-	}
-	formatEngineResult(data, lines, 5);
-	return lines.join("\n").trim();
-}
-
-/**
- * Format a single engine's result (answer + sources)
- */
-function formatEngineResult(
-	data: Record<string, unknown>,
-	lines: string[],
-	maxSources: number,
-): void {
-	if (data.error) {
-		lines.push(`Error: ${data.error}`);
-		return;
-	}
-
-	if (data.answer) {
-		lines.push(String(data.answer));
-	}
-
-	const sources = data.sources as Array<Record<string, string>> | undefined;
-	if (Array.isArray(sources) && sources.length > 0) {
-		lines.push("\nSources:");
-		for (const s of sources.slice(0, maxSources)) {
-			lines.push(`- [${s.title || s.url}](${s.url})`);
-		}
-	}
-}
-
-/**
- * Format deep research results with confidence metrics
- */
+/**
+ * Search results formatters
+ * Extracted from index.ts
+ */
+
+import { formatEngineName } from "../utils/helpers.js";
+import { renderSynthesis } from "./synthesis.js";
+
+/**
+ * Format search results based on engine type
+ */
+export function formatResults(
+	engine: string,
+	data: Record<string, unknown>,
+): string {
+	const lines: string[] = [];
+
+	if (engine === "all") {
+		return formatAllEnginesResult(data, lines);
+	}
+
+	return formatSingleEngineResult(data, lines);
+}
+
+/**
+ * Format multi-engine results with synthesis
+ */
+function formatAllEnginesResult(
+	data: Record<string, unknown>,
+	lines: string[],
+): string {
+	const synthesis = data._synthesis as Record<string, unknown> | undefined;
+	const dedupedSources = data._sources as
+		| Array<Record<string, unknown>>
+		| undefined;
+	const needsHuman = data._needsHumanVerification as
+		| Record<string, unknown>
+		| undefined;
+	const research = data._research as Record<string, unknown> | undefined;
+
+	if (needsHuman) {
+		const engines = Array.isArray(needsHuman.engines)
+			? needsHuman.engines.join(", ")
+			: "one or more engines";
+		lines.push("## Manual verification required");
+		lines.push(
+			String(
+				needsHuman.message ||
+					"Visible Chrome is open. Solve the verification challenge, then rerun the same search.",
+			),
+		);
+		lines.push(`Engines: ${engines}`);
+		lines.push("");
+	}
+
+	// If we have a synthesis answer, render it
+	if (synthesis?.answer) {
+		if (research?.mode === "iterative") renderResearchHeader(lines, research);
+		renderSynthesis(lines, synthesis, dedupedSources || [], 6);
+		const synthesizedBy = String(
+			synthesis.synthesizedBy || "configured synthesizer",
+		);
+		lines.push(
+			research?.mode === "iterative"
+				? "*Research mode: iterative planning, source fetching, citation audit, and bundle output*\n"
+				: `*Synthesized by ${synthesizedBy} from multi-engine results and fetched sources*\n`,
+		);
+		return lines.join("\n").trim();
+	}
+
+	// Fallback: render individual engine results
+	for (const [eng, result] of Object.entries(data)) {
+		if (eng.startsWith("_")) continue;
+		lines.push(`\n## ${formatEngineName(eng)}`);
+		formatEngineResult(result as Record<string, unknown>, lines, 3);
+	}
+
+	return lines.join("\n").trim();
+}
+
+function renderResearchHeader(
+	lines: string[],
+	research: Record<string, unknown>,
+): void {
+	const floor = research.floor as Record<string, unknown> | undefined;
+	const metrics = floor?.metrics as Record<string, unknown> | undefined;
+	const bundle = research.bundle as Record<string, unknown> | undefined;
+	const manifest = research.manifest as Record<string, unknown> | undefined;
+	lines.push("## Research Run");
+	lines.push(
+		`- Status: ${floor?.floorMet ? "floor met" : "partial / floor unmet"}`,
+	);
+	if (manifest?.terminationReason)
+		lines.push(`- Stop reason: ${String(manifest.terminationReason)}`);
+	if (metrics) {
+		lines.push(
+			`- Evidence: ${metrics.fetchedOk || 0} fetched sources, ${metrics.primarySources || 0} primary/official, ${metrics.claims || 0} claims, ${metrics.cited || 0} citations`,
+		);
+		lines.push(
+			`- Questions: ${metrics.closedQuestions || 0}/${metrics.totalQuestions || 0} closed${metrics.openQuestions ? `, ${metrics.openQuestions} open` : ""}`,
+		);
+	}
+	if (bundle?.dir) lines.push(`- Bundle: ${String(bundle.dir)}`);
+	lines.push("");
+}
+
+/**
+ * Format single engine result
+ */
+function formatSingleEngineResult(
+	data: Record<string, unknown>,
+	lines: string[],
+): string {
+	const needsHuman = data._needsHumanVerification as
+		| Record<string, unknown>
+		| undefined;
+	if (needsHuman) {
+		const engines = Array.isArray(needsHuman.engines)
+			? needsHuman.engines.join(", ")
+			: "this engine";
+		lines.push("## Manual verification required");
+		lines.push(
+			String(
+				needsHuman.message ||
+					"Visible Chrome is open. Solve the verification challenge, then rerun the same search.",
+			),
+		);
+		lines.push(`Engines: ${engines}`);
+		lines.push("");
+	}
+	formatEngineResult(data, lines, 5);
+	return lines.join("\n").trim();
+}
+
+/**
+ * Format a single engine's result (answer + sources)
+ */
+function formatEngineResult(
+	data: Record<string, unknown>,
+	lines: string[],
+	maxSources: number,
+): void {
+	if (data.error) {
+		lines.push(`Error: ${data.error}`);
+		return;
+	}
+
+	if (data.answer) {
+		lines.push(String(data.answer));
+	}
+
+	const sources = data.sources as Array<Record<string, string>> | undefined;
+	if (Array.isArray(sources) && sources.length > 0) {
+		lines.push("\nSources:");
+		for (const s of sources.slice(0, maxSources)) {
+			lines.push(`- [${s.title || s.url}](${s.url})`);
+		}
+	}
+}
+
+/**
+ * Format deep research results with confidence metrics
+ */

package/src/search/browser-lifecycle.mjs

@@ -116,13 +116,35 @@ function getProcessCommandLine(pid) {
  * @param {number} debugPort - expected debug port
  * @returns {boolean}
  */
-export function verifyBrowserProcess(pid, tempDir, debugPort = GREEDY_PORT) {
-	const cmdLine = getProcessCommandLine(pid);
+export function commandLineMatchesGreedyChrome(
+	cmdLine,
+	tempDir,
+	debugPort = GREEDY_PORT,
+) {
 	if (!cmdLine) return false;
+	// Windows may report Chrome command lines with backslashes while the shared
+	// GREEDY_PROFILE_DIR constant is normalized to forward slashes. Compare a
+	// normalized form so child processes do not misclassify a live GreedySearch
+	// Chrome as a ghost and kill it during cleanupStaleSessions().
+	const normalize = (value) =>
+		String(value || "")
+			.replaceAll("\\", "/")
+			.toLowerCase();
+	const normalizedCmdLine = normalize(cmdLine);
+	const normalizedTempDir = normalize(tempDir);
+
 	return (
-		cmdLine.includes(tempDir) &&
-		cmdLine.includes(`--remote-debugging-port=${debugPort}`) &&
-		!cmdLine.includes("--type=")
+		normalizedCmdLine.includes(normalizedTempDir) &&
+		normalizedCmdLine.includes(`--remote-debugging-port=${debugPort}`) &&
+		!normalizedCmdLine.includes("--type=")
+	);
+}
+
+export function verifyBrowserProcess(pid, tempDir, debugPort = GREEDY_PORT) {
+	return commandLineMatchesGreedyChrome(
+		getProcessCommandLine(pid),
+		tempDir,
+		debugPort,
 	);
 }