@apmantza/greedysearch-pi 1.7.7 → 1.8.0

package/src/search/chrome.mjs

@@ -0,0 +1,223 @@
+// src/search/chrome.mjs — Chrome launch, probe, port file management, and CDP wrapper
+//
+// Extracted from search.mjs to reduce file complexity.
+// Also used by coding-task.mjs (via import).
+//
+// cdp() is re-exported from extractors/common.mjs to avoid duplication.
+
+import { spawn } from "node:child_process";
+import {
+	existsSync,
+	readFileSync,
+	renameSync,
+	unlinkSync,
+	writeFileSync,
+} from "node:fs";
+import http from "node:http";
+import { join } from "node:path";
+
+import { GREEDY_PORT, ACTIVE_PORT_FILE, PAGES_CACHE } from "./constants.mjs";
+import { cdp as _cdp } from "../../extractors/common.mjs";
+
+const __dir = import.meta.dirname || new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
+
+/** Re-export cdp() from the canonical location in extractors/common.mjs */
+export const cdp = _cdp;
+
+export async function getAnyTab() {
+	const list = await cdp(["list"]);
+	const first = list.split("\n")[0];
+	if (!first) throw new Error("No Chrome tabs found");
+	return first.slice(0, 8);
+}
+
+export async function openNewTab() {
+	const anchor = await getAnyTab();
+	const raw = await cdp([
+		"evalraw",
+		anchor,
+		"Target.createTarget",
+		'{"url":"about:blank"}',
+	]);
+	const { targetId } = JSON.parse(raw);
+	return targetId;
+}
+
+export async function activateTab(targetId) {
+	try {
+		const anchor = await getAnyTab();
+		await cdp([
+			"evalraw",
+			anchor,
+			"Target.activateTarget",
+			JSON.stringify({ targetId }),
+		]);
+	} catch {
+		// best-effort
+	}
+}
+
+export async function closeTab(targetId) {
+	try {
+		const anchor = await getAnyTab();
+		await cdp([
+			"evalraw",
+			anchor,
+			"Target.closeTarget",
+			JSON.stringify({ targetId }),
+		]);
+	} catch {
+		/* best-effort */
+	}
+}
+
+export async function closeTabs(targetIds = []) {
+	for (const tid of targetIds) {
+		if (!tid) continue;
+		await closeTab(tid);
+	}
+	if (targetIds.length > 0) {
+		await new Promise((r) => setTimeout(r, 300));
+		await cdp(["list"]).catch(() => null);
+	}
+}
+
+export function getFullTabFromCache(engine, engineDomains) {
+	try {
+		if (!existsSync(PAGES_CACHE)) return null;
+		const pages = JSON.parse(readFileSync(PAGES_CACHE, "utf8"));
+		const found = pages.find((p) => p.url.includes(engineDomains[engine]));
+		return found ? found.targetId : null;
+	} catch {
+		return null;
+	}
+}
+
+export function probeGreedyChrome(timeoutMs = 3000) {
+	return new Promise((resolve) => {
+		const req = http.get(
+			`http://localhost:${GREEDY_PORT}/json/version`,
+			(res) => {
+				res.resume();
+				resolve(res.statusCode === 200);
+			},
+		);
+		req.on("error", () => resolve(false));
+		req.setTimeout(timeoutMs, () => {
+			req.destroy();
+			resolve(false);
+		});
+	});
+}
+
+export async function refreshPortFile() {
+	const LOCK_FILE = `${ACTIVE_PORT_FILE}.lock`;
+	const TEMP_FILE = `${ACTIVE_PORT_FILE}.tmp`;
+	const LOCK_STALE_MS = 5000;
+	const LOCK_WAIT_MS = 1000;
+
+	// File-based lock with exclusive create + stale lock recovery
+	const lockAcquired = await new Promise((resolve) => {
+		const start = Date.now();
+		const tryLock = () => {
+			try {
+				const payload = JSON.stringify({ pid: process.pid, ts: Date.now() });
+				writeFileSync(LOCK_FILE, payload, { encoding: "utf8", flag: "wx" });
+				resolve(true);
+			} catch (e) {
+				if (e?.code !== "EEXIST") {
+					if (Date.now() - start < LOCK_WAIT_MS) {
+						setTimeout(tryLock, 50);
+					} else {
+						resolve(false);
+					}
+					return;
+				}
+
+				try {
+					const lockRaw = readFileSync(LOCK_FILE, "utf8").trim();
+					const parsed = lockRaw.startsWith("{")
+						? JSON.parse(lockRaw)
+						: { ts: Number(lockRaw) };
+					const lockTime = Number(parsed?.ts) || 0;
+
+					if (lockTime > 0 && Date.now() - lockTime > LOCK_STALE_MS) {
+						try {
+							unlinkSync(LOCK_FILE);
+						} catch {}
+					}
+
+					if (Date.now() - start < LOCK_WAIT_MS) {
+						setTimeout(tryLock, 50);
+					} else {
+						resolve(false);
+					}
+				} catch {
+					if (Date.now() - start < LOCK_WAIT_MS) {
+						setTimeout(tryLock, 50);
+					} else {
+						resolve(false);
+					}
+				}
+			}
+		};
+		tryLock();
+	});
+
+	try {
+		const body = await new Promise((res, rej) => {
+			const req = http.get(
+				`http://localhost:${GREEDY_PORT}/json/version`,
+				(r) => {
+					let b = "";
+					r.on("data", (d) => (b += d));
+					r.on("end", () => res(b));
+				},
+			);
+			req.on("error", rej);
+			req.setTimeout(3000, () => {
+				req.destroy();
+				rej(new Error("timeout"));
+			});
+		});
+		const { webSocketDebuggerUrl } = JSON.parse(body);
+		const wsPath = new URL(webSocketDebuggerUrl).pathname;
+
+		// Atomic write: write to temp file, then rename
+		if (lockAcquired) {
+			writeFileSync(TEMP_FILE, `${GREEDY_PORT}\n${wsPath}`, "utf8");
+			try {
+				unlinkSync(ACTIVE_PORT_FILE);
+			} catch {}
+			renameSync(TEMP_FILE, ACTIVE_PORT_FILE);
+		}
+	} catch {
+		/* best-effort — launch.mjs already wrote the file on first start */
+	} finally {
+		if (lockAcquired) {
+			try {
+				unlinkSync(LOCK_FILE);
+			} catch {}
+		}
+	}
+}
+
+export async function ensureChrome() {
+	const ready = await probeGreedyChrome();
+	if (!ready) {
+		process.stderr.write(
+			`GreedySearch Chrome not running on port ${GREEDY_PORT} — auto-launching...\n`,
+		);
+		await new Promise((resolve, reject) => {
+			const proc = spawn("node", [join(__dir, "..", "..", "bin", "launch.mjs")], {
+				stdio: ["ignore", process.stderr, process.stderr],
+			});
+			proc.on("close", (code) =>
+				code === 0 ? resolve() : reject(new Error("launch.mjs failed")),
+			);
+		});
+	} else {
+		// Chrome already running — refresh the port file
+		await refreshPortFile();
+	}
+}

package/src/search/constants.mjs

@@ -0,0 +1,38 @@
+// src/search/constants.mjs — Shared constants for GreedySearch search pipeline
+
+import { tmpdir } from "node:os";
+
+export const GREEDY_PORT = 9222;
+export const GREEDY_PROFILE_DIR = `${tmpdir().replace(/\\/g, "/")}/greedysearch-chrome-profile`;
+export const ACTIVE_PORT_FILE = `${GREEDY_PROFILE_DIR}/DevToolsActivePort`;
+export const PAGES_CACHE = `${tmpdir().replace(/\\/g, "/")}/cdp-pages.json`;
+
+export const ALL_ENGINES = ["perplexity", "bing", "google"];
+
+export const ENGINE_DOMAINS = {
+	perplexity: "perplexity.ai",
+	bing: "copilot.microsoft.com",
+	google: "google.com",
+	gemini: "gemini.google.com",
+};
+
+export const ENGINES = {
+	perplexity: "perplexity.mjs",
+	pplx: "perplexity.mjs",
+	p: "perplexity.mjs",
+	bing: "bing-copilot.mjs",
+	copilot: "bing-copilot.mjs",
+	b: "bing-copilot.mjs",
+	google: "google-ai.mjs",
+	g: "google-ai.mjs",
+	gemini: "gemini.mjs",
+	gem: "gemini.mjs",
+};
+
+export const SOURCE_FETCH_CONCURRENCY = Math.max(
+	1,
+	parseInt(process.env.GREEDY_FETCH_CONCURRENCY || "2", 10) || 2,
+);
+
+// Tell cdp.mjs to prefer the GreedySearch Chrome profile's DevToolsActivePort
+process.env.CDP_PROFILE_DIR = GREEDY_PROFILE_DIR;

package/src/search/defaults.mjs

@@ -0,0 +1,15 @@
+// src/search/defaults.mjs — Shared default values and timeouts
+//
+// Centralizes magic numbers used across the codebase.
+// Import from here instead of hardcoding values.
+
+export const DEFAULTS = {
+	CDP_TIMEOUT: 30000,             // Default CDP command timeout (ms)
+	CDP_TIMEOUT_SHORT: 15000,       // Short CDP timeout for search operations (ms)
+	NAV_TIMEOUT: 35000,             // Navigation timeout (ms)
+	STREAM_TIMEOUT: 30000,          // Stream completion timeout (ms)
+	COPY_TIMEOUT: 60000,            // Copy button appearance timeout (ms)
+	CODING_TASK_TIMEOUT: 180000,    // Coding task max duration (ms)
+	MAX_SOURCE_FETCH: 10,           // Max concurrent source fetches
+	DESCRIPTION_MAX_LENGTH: 300,    // Max answer length in truncated mode
+};

package/src/search/engines.mjs

@@ -0,0 +1,58 @@
+// src/search/engines.mjs — Extractor runner
+//
+// Engine map lives in constants.mjs; this module re-exports it for
+// backward compatibility and provides the runExtractor() function.
+
+import { spawn } from "node:child_process";
+import { join } from "node:path";
+import { ENGINES, GREEDY_PROFILE_DIR } from "./constants.mjs";
+
+export { ENGINES };
+
+const __dir = import.meta.dirname || new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
+
+export function runExtractor(
+	script,
+	query,
+	tabPrefix = null,
+	short = false,
+	timeoutMs = null,
+) {
+	// Gemini is slower - use longer timeout
+	if (timeoutMs === null) {
+		timeoutMs = script.includes("gemini") ? 180000 : 90000;
+	}
+	const extraArgs = [
+		...(tabPrefix ? ["--tab", tabPrefix] : []),
+		...(short ? ["--short"] : []),
+	];
+	return new Promise((resolve, reject) => {
+		const proc = spawn(
+			"node",
+			[join(__dir, "..", "..", "extractors", script), query, ...extraArgs],
+			{
+				stdio: ["ignore", "pipe", "pipe"],
+				env: { ...process.env, CDP_PROFILE_DIR: GREEDY_PROFILE_DIR },
+			},
+		);
+		let out = "";
+		let err = "";
+		proc.stdout.on("data", (d) => (out += d));
+		proc.stderr.on("data", (d) => (err += d));
+		const t = setTimeout(() => {
+			proc.kill();
+			reject(new Error(`${script} timed out after ${timeoutMs / 1000}s`));
+		}, timeoutMs);
+		proc.on("close", (code) => {
+			clearTimeout(t);
+			if (code !== 0) reject(new Error(err.trim() || `extractor exit ${code}`));
+			else {
+				try {
+					resolve(JSON.parse(out.trim()));
+				} catch {
+					reject(new Error(`bad JSON from ${script}: ${out.slice(0, 100)}`));
+				}
+			}
+		});
+	});
+}

package/src/search/fetch-source.mjs

@@ -0,0 +1,230 @@
+// src/search/fetch-source.mjs — HTTP and browser-based source content fetching
+//
+// Extracted from search.mjs. Uses fetchSourceHttp from src/fetcher.mjs
+// with browser fallback via CDP, plus GitHub content fetching.
+
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fetchSourceHttp, shouldUseBrowser } from "../fetcher.mjs";
+import { fetchGitHubContent, parseGitHubUrl } from "../github.mjs";
+import { trimContentHeadTail } from "../utils/content.mjs";
+import { cdp } from "./chrome.mjs";
+import { openNewTab, closeTab, closeTabs } from "./chrome.mjs";
+import { SOURCE_FETCH_CONCURRENCY } from "./constants.mjs";
+import { trimText } from "./sources.mjs";
+
+export async function fetchSourceContent(url, maxChars = 8000) {
+	const start = Date.now();
+
+	// Check if it's a GitHub URL
+	if (parseGitHubUrl(url)) {
+		const parsed = parseGitHubUrl(url);
+		if (
+			parsed &&
+			(parsed.type === "root" ||
+				parsed.type === "tree" ||
+				(parsed.type === "blob" && !parsed.path?.includes(".")))
+		) {
+			const ghResult = await fetchGitHubContent(url);
+			if (ghResult.ok) {
+				const content = trimContentHeadTail(ghResult.content, maxChars);
+				return {
+					url,
+					finalUrl: url,
+					status: 200,
+					contentType: "text/markdown",
+					lastModified: "",
+					title: ghResult.title,
+					snippet: content.slice(0, 320),
+					content,
+					contentChars: content.length,
+					source: "github-api",
+					...(ghResult.tree && { tree: ghResult.tree }),
+					duration: Date.now() - start,
+				};
+			}
+			process.stderr.write(
+				`[greedysearch] GitHub API fetch failed, trying HTTP: ${ghResult.error}\n`,
+			);
+		}
+	}
+
+	// Try HTTP first
+	const httpResult = await fetchSourceHttp(url, { timeoutMs: 15000 });
+
+	if (httpResult.ok) {
+		const content = trimContentHeadTail(httpResult.markdown, maxChars);
+		return {
+			url,
+			finalUrl: httpResult.finalUrl,
+			status: httpResult.status,
+			contentType: "text/markdown",
+			lastModified: httpResult.lastModified || "",
+			publishedTime: httpResult.publishedTime || "",
+			byline: httpResult.byline || "",
+			siteName: httpResult.siteName || "",
+			lang: httpResult.lang || "",
+			title: httpResult.title,
+			snippet: httpResult.excerpt,
+			content,
+			contentChars: content.length,
+			source: "http",
+			duration: Date.now() - start,
+		};
+	}
+
+	// HTTP failed — fall back to browser
+	process.stderr.write(
+		`[greedysearch] HTTP failed for ${url.slice(0, 60)}, trying browser...\n`,
+	);
+	return await fetchSourceContentBrowser(url, maxChars);
+}
+
+async function fetchSourceContentBrowser(url, maxChars = 8000) {
+	const start = Date.now();
+	const tab = await openNewTab();
+
+	try {
+		await cdp(["nav", tab, url], 30000);
+		await new Promise((r) => setTimeout(r, 1500));
+
+		const content = await cdp([
+			"eval",
+			tab,
+			`
+			(function(){
+				var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
+				var text = (el || document.body).innerText;
+				return JSON.stringify({
+					title: document.title,
+					content: text.replace(/\\s+/g, ' ').trim(),
+					url: location.href
+				});
+			})()
+		`,
+		]);
+
+		const parsed = JSON.parse(content);
+		const finalContent = trimContentHeadTail(parsed.content, maxChars);
+
+		return {
+			url,
+			finalUrl: parsed.url || url,
+			status: 200,
+			contentType: "text/plain",
+			lastModified: "",
+			title: parsed.title,
+			snippet: trimText(finalContent, 320),
+			content: finalContent,
+			contentChars: finalContent.length,
+			source: "browser",
+			duration: Date.now() - start,
+		};
+	} catch (error) {
+		return {
+			url,
+			title: "",
+			content: null,
+			snippet: "",
+			contentChars: 0,
+			error: error.message,
+			source: "browser",
+			duration: Date.now() - start,
+		};
+	} finally {
+		await closeTab(tab);
+	}
+}
+
+export async function fetchMultipleSources(
+	sources,
+	maxSources = 5,
+	maxChars = 8000,
+	concurrency = SOURCE_FETCH_CONCURRENCY,
+) {
+	const toFetch = sources.slice(0, maxSources);
+	if (toFetch.length === 0) return [];
+
+	const workerCount = Math.min(
+		toFetch.length,
+		Math.max(1, parseInt(String(concurrency), 10) || SOURCE_FETCH_CONCURRENCY),
+	);
+
+	process.stderr.write(
+		`[greedysearch] Fetching content from ${toFetch.length} sources via HTTP (concurrency ${workerCount})...\n`,
+	);
+
+	const fetched = new Array(toFetch.length);
+	let nextIndex = 0;
+	let completed = 0;
+
+	async function worker() {
+		while (true) {
+			const index = nextIndex++;
+			if (index >= toFetch.length) return;
+
+			const s = toFetch[index];
+			const url = s.canonicalUrl || s.url;
+			process.stderr.write(
+				`[greedysearch] [${index + 1}/${toFetch.length}] Fetching: ${url.slice(0, 60)}...\n`,
+			);
+
+			const result = await fetchSourceContent(url, maxChars);
+			fetched[index] = {
+				id: s.id,
+				...result,
+			};
+
+			if (result.content && result.content.length > 100) {
+				process.stderr.write(
+					`[greedysearch] ✓ ${result.source}: ${result.content.length} chars\n`,
+				);
+			} else if (result.error) {
+				process.stderr.write(`[greedysearch] ✗ ${result.error.slice(0, 80)}\n`);
+			}
+
+			completed += 1;
+			process.stderr.write(`PROGRESS:fetch:${completed}/${toFetch.length}\n`);
+		}
+	}
+
+	await Promise.all(Array.from({ length: workerCount }, () => worker()));
+
+	// Log summary
+	const successful = fetched.filter((f) => f.content && f.content.length > 100);
+	const httpCount = fetched.filter((f) => f.source === "http").length;
+	const browserCount = fetched.filter((f) => f.source === "browser").length;
+
+	process.stderr.write(
+		`[greedysearch] Fetched ${successful.length}/${fetched.length} sources ` +
+			`(HTTP: ${httpCount}, Browser: ${browserCount})\n`,
+	);
+
+	return fetched;
+}
+
+export async function fetchTopSource(url) {
+	const tab = await openNewTab();
+	await cdp(["list"]); // refresh cache
+	try {
+		await cdp(["nav", tab, url], 30000);
+		await new Promise((r) => setTimeout(r, 1500));
+		const content = await cdp([
+			"eval",
+			tab,
+			`
+      (function(){
+        var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
+        var text = (el || document.body).innerText;
+        return text.replace(/\\s+/g, ' ').trim();
+      })()
+    `,
+		]);
+		return { url, content };
+	} catch (e) {
+		return { url, content: null, error: e.message };
+	} finally {
+		await closeTab(tab);
+	}
+}

package/src/search/output.mjs

@@ -0,0 +1,59 @@
+// src/search/output.mjs — Output serialization for search results
+//
+// Extracted from search.mjs.
+
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+const __dir = import.meta.dirname || new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
+
+export function slugify(query) {
+	return query
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, "-")
+		.replace(/^-|-$/g, "")
+		.slice(0, 60);
+}
+
+export function resultsDir() {
+	const dir = join(__dir, "..", "..", "results");
+	mkdirSync(dir, { recursive: true });
+	return dir;
+}
+
+export function writeOutput(
+	data,
+	outFile,
+	{ inline = false, synthesize = false, query = "" } = {},
+) {
+	const json = `${JSON.stringify(data, null, 2)}\n`;
+
+	if (outFile) {
+		writeFileSync(outFile, json, "utf8");
+		process.stderr.write(`Results written to ${outFile}\n`);
+		return;
+	}
+
+	if (inline) {
+		process.stdout.write(json);
+		return;
+	}
+
+	const ts = new Date()
+		.toISOString()
+		.replace("T", "_")
+		.replace(/[:.]/g, "-")
+		.slice(0, 19);
+	const slug = slugify(query);
+	const base = join(resultsDir(), `${ts}_${slug}`);
+
+	writeFileSync(`${base}.json`, json, "utf8");
+
+	if (synthesize && data._synthesis?.answer) {
+		writeFileSync(`${base}-synthesis.md`, data._synthesis.answer, "utf8");
+		process.stdout.write(`${base}-synthesis.md\n`);
+	} else {
+		process.stdout.write(`${base}.json\n`);
+	}
+}