@apmantza/greedysearch-pi 1.9.2 → 2.1.2

package/src/search/pdf.mjs

@@ -0,0 +1,68 @@
+// src/search/pdf.mjs — PDF text extraction helpers
+//
+// Adapted from pi-webaio's PDF pipeline. `pdf-parse` is loaded lazily so the
+// package remains importable even when optional native canvas bindings are not
+// available. PDF extraction is only attempted for actual PDF source fetches.
+
+function ensurePdfDomPolyfills() {
+	if (typeof globalThis.DOMMatrix === "undefined") {
+		globalThis.DOMMatrix = class DOMMatrix {
+			constructor(_init = undefined) {}
+			multiplySelf() {
+				return this;
+			}
+			preMultiplySelf() {
+				return this;
+			}
+			translateSelf() {
+				return this;
+			}
+			scaleSelf() {
+				return this;
+			}
+			rotateSelf() {
+				return this;
+			}
+		};
+	}
+	if (typeof globalThis.ImageData === "undefined") {
+		globalThis.ImageData = class ImageData {
+			constructor(data = undefined, width = 0, height = 0) {
+				this.data = data;
+				this.width = width;
+				this.height = height;
+			}
+		};
+	}
+	if (typeof globalThis.Path2D === "undefined") {
+		globalThis.Path2D = class Path2D {
+			constructor(_path = undefined) {}
+		};
+	}
+}
+
+async function loadPdfParseCtor() {
+	ensurePdfDomPolyfills();
+	const mod = await import("pdf-parse");
+	const ctor = mod.PDFParse ?? mod.default;
+	if (!ctor) throw new Error("pdf-parse did not export PDFParse");
+	return ctor;
+}
+
+export async function extractPdfMarkdown(buffer, url) {
+	try {
+		const PDFParseCtor = await loadPdfParseCtor();
+		const parser = new PDFParseCtor({ data: new Uint8Array(buffer) });
+		await parser.load();
+		const data = await parser.getText();
+		const text = data.text?.trim();
+		if (!text) return null;
+		return {
+			title: new URL(url).pathname.split("/").pop() || "Document.pdf",
+			content: `## PDF Content (${data.total} pages)\n\n${text}`,
+			pages: data.total,
+		};
+	} catch (error) {
+		return { error: error.message || String(error) };
+	}
+}

package/src/search/progress.mjs

@@ -0,0 +1,145 @@
+// src/search/progress.mjs — Progress bar with ETA for long-running research
+//
+// Tracks per-action and per-round timing, prints a progress bar to stderr
+// after each step with an ETA based on rolling average. Inspired by pi-webaio's
+// streaming progress output.
+//
+// Usage:
+//   const tracker = createProgressTracker({ totalActions: 6, totalRounds: 2 });
+//   tracker.startRound(1);
+//   tracker.startAction('search', 'what is X');
+//   ... do work ...
+//   tracker.endAction();
+//   tracker.startAction('fetch', 'https://...');
+//   ... do work ...
+//   tracker.endAction();
+//   tracker.endRound();
+//   tracker.print(); // prints bar to stderr
+
+const BAR_WIDTH = 20;
+
+/**
+ * Format seconds as a human-readable duration (e.g. "1m 23s", "45s", "0s")
+ */
+function formatDuration(ms) {
+	if (ms < 1000) return "0s";
+	const totalSeconds = Math.round(ms / 1000);
+	if (totalSeconds < 60) return `${totalSeconds}s`;
+	const minutes = Math.floor(totalSeconds / 60);
+	const seconds = totalSeconds % 60;
+	return `${minutes}m ${seconds}s`;
+}
+
+/**
+ * Render a progress bar string.
+ * Example: [████████████░░░░░░░░] 12/20 (ETA 1m 30s)
+ */
+function renderBar(progress, width = BAR_WIDTH) {
+	const filled = Math.round(progress * width);
+	const empty = width - filled;
+	return "[" + "█".repeat(filled) + "░".repeat(empty) + "]";
+}
+
+/**
+ * Create a progress tracker.
+ * @param {object} opts
+ * @param {number} opts.totalActions - Total expected actions across all rounds
+ * @param {number} opts.totalRounds - Total expected rounds
+ * @param {number} opts.totalFetches - Total expected source fetches
+ * @param {boolean} [opts.silent] - Suppress stderr output (for tests)
+ */
+export function createProgressTracker({
+	totalActions = 0,
+	totalRounds = 0,
+	totalFetches = 0,
+	silent = false,
+} = {}) {
+	const startedAt = Date.now();
+	let completedActions = 0;
+	let completedRounds = 0;
+	let completedFetches = 0;
+	const actionTimings = []; // rolling window of recent action durations
+	let currentActionStart = null;
+	let currentActionLabel = null;
+	let lastPrintAt = 0;
+	const MIN_PRINT_INTERVAL_MS = 500; // throttle to avoid spam
+
+	function recordAction(durationMs) {
+		actionTimings.push(durationMs);
+		// keep only last 5 for rolling average
+		if (actionTimings.length > 5) actionTimings.shift();
+	}
+
+	function avgActionMs() {
+		if (actionTimings.length === 0) return null;
+		return actionTimings.reduce((a, b) => a + b, 0) / actionTimings.length;
+	}
+
+	function buildStatus(phase) {
+		const elapsed = Date.now() - startedAt;
+		const total = totalActions + totalFetches + totalRounds;
+		const done = completedActions + completedFetches + completedRounds;
+		const progress = total > 0 ? Math.min(1, done / total) : 0;
+		const bar = renderBar(progress);
+		const avg = avgActionMs();
+		const remaining = Math.max(0, total - done);
+		const etaMs = avg ? avg * remaining : null;
+		const eta = etaMs ? formatDuration(etaMs) : "—";
+		const label = currentActionLabel ? ` ${currentActionLabel}` : "";
+		return `${bar} ${done}/${total} (${phase}${label}, ETA ${eta})`;
+	}
+
+	function print(phase) {
+		if (silent) return;
+		const now = Date.now();
+		// throttle to avoid spamming
+		if (now - lastPrintAt < MIN_PRINT_INTERVAL_MS && phase !== "done") return;
+		lastPrintAt = now;
+		process.stderr.write(`[greedysearch] ${buildStatus(phase)}\n`);
+	}
+
+	return {
+		startRound(n) {
+			completedRounds = n - 1; // will be incremented when endRound fires
+		},
+		endRound() {
+			completedRounds++;
+			print("round");
+		},
+		startAction(type, label) {
+			currentActionStart = Date.now();
+			currentActionLabel = `${type}:${(label || "").slice(0, 40)}`;
+			print(type);
+		},
+		endAction() {
+			if (currentActionStart) {
+				recordAction(Date.now() - currentActionStart);
+				currentActionStart = null;
+			}
+			completedActions++;
+			print("action");
+		},
+		startFetch(label) {
+			currentActionStart = Date.now();
+			currentActionLabel = `fetch:${(label || "").slice(0, 40)}`;
+			print("fetch");
+		},
+		endFetch(ok = true) {
+			if (currentActionStart) {
+				recordAction(Date.now() - currentActionStart);
+				currentActionStart = null;
+			}
+			completedFetches++;
+			print(ok ? "fetch" : "fetch-failed");
+		},
+		print() {
+			print("progress");
+		},
+		finish() {
+			print("done");
+		},
+		getElapsedMs() {
+			return Date.now() - startedAt;
+		},
+	};
+}

package/src/search/recovery.mjs

@@ -1,45 +1,73 @@
-// src/search/recovery.mjs — Headless-block detection and visible recovery policy
-
-// Only these engines use automatic headless → visible recovery. Google is
-// intentionally excluded for now; see issue #9 discussion / maintainer choice.
-export const HEADLESS_RECOVERY_ENGINES = ["perplexity", "bing"];
-
-const HEADLESS_BLOCKED_PATTERN =
-	/timed out|timeout|verification|captcha|cloudflare|turnstile|input not found|ask-input|clipboard|copy button hidden/i;
-
-const MANUAL_VERIFICATION_PATTERN =
-	/needs-human|verification required|please solve|captcha|cloudflare|turnstile|could not be completed automatically|manual intervention/i;
-
-export function isHeadlessBlockedError(error) {
-	return HEADLESS_BLOCKED_PATTERN.test(String(error || ""));
-}
-
-export function isManualVerificationError(error) {
-	return MANUAL_VERIFICATION_PATTERN.test(String(error || ""));
-}
-
-export function findHeadlessBlockedEngines(resultsByEngine) {
-	return HEADLESS_RECOVERY_ENGINES.filter((engine) => {
-		const result = resultsByEngine?.[engine];
-		if (!result) return false;
-		// Data-driven: check envelope first (zero regex cost)
-		if (result._envelope?.blockedBy) return true;
-		if (result._envelope?.verificationResult === "needs-human") return true;
-		// Fallback: legacy string matching for errors passed as plain strings
-		const error = result.error;
-		return error && isHeadlessBlockedError(error);
-	});
-}
-
-/**
- * Check if an extractor Error carries a structured envelope indicating
- * headless blocking. Used in single-engine recovery paths where the Error
- * object is caught directly rather than parsed from a result record.
- */
-export function isHeadlessBlockedResult(error) {
-	if (!error) return false;
-	const env = error.envelope;
-	if (env?.blockedBy) return true;
-	if (env?.verificationResult === "needs-human") return true;
-	return isHeadlessBlockedError(error.message);
-}
+// src/search/recovery.mjs — Headless-block detection and visible recovery policy
+
+// Only these engines use automatic headless → visible recovery. Google is
+// intentionally excluded for now; see issue #9 discussion / maintainer choice.
+export const HEADLESS_RECOVERY_ENGINES = [
+	"perplexity",
+	"bing",
+	"chatgpt",
+	"semantic-scholar",
+	"logically",
+];
+
+// blockedBy values that indicate visible-mode cookies CANNOT bypass the block.
+// These still match the "headless blocked" shape but should NOT trigger
+// visible recovery — the block is account-level (rate limit, ban) or
+// structural (page redesign), not session-level.
+const NON_RECOVERABLE_BLOCKED_BY = new Set(["rate-limit"]);
+
+const HEADLESS_BLOCKED_PATTERN =
+	/timed out|timeout|verification|captcha|cloudflare|turnstile|input not found|ask-input|copy button hidden|sign.in|login required/i;
+
+const MANUAL_VERIFICATION_PATTERN =
+	/needs-human|verification required|please solve|captcha|cloudflare|turnstile|could not be completed automatically|manual intervention|sign.in|login required/i;
+
+export function isHeadlessBlockedError(error) {
+	return HEADLESS_BLOCKED_PATTERN.test(String(error || ""));
+}
+
+export function isManualVerificationError(error) {
+	return MANUAL_VERIFICATION_PATTERN.test(String(error || ""));
+}
+
+/**
+ * Check if a blockedBy value is non-recoverable (visible retry won't help).
+ */
+export function isNonRecoverableBlockedBy(blockedBy) {
+	return NON_RECOVERABLE_BLOCKED_BY.has(blockedBy);
+}
+
+export function findHeadlessBlockedEngines(resultsByEngine) {
+	return HEADLESS_RECOVERY_ENGINES.filter((engine) => {
+		const result = resultsByEngine?.[engine];
+		if (!result) return false;
+		// Data-driven: check envelope first (zero regex cost)
+		const blockedBy = result._envelope?.blockedBy;
+		if (blockedBy) {
+			// Skip non-recoverable blocks (rate-limit, ban, etc.)
+			if (NON_RECOVERABLE_BLOCKED_BY.has(blockedBy)) return false;
+			return true;
+		}
+		if (result._envelope?.verificationResult === "needs-human") return true;
+		// Fallback: legacy string matching for errors passed as plain strings
+		const error = result.error;
+		return error && isHeadlessBlockedError(error);
+	});
+}
+
+/**
+ * Check if an extractor Error carries a structured envelope indicating
+ * headless blocking. Used in single-engine recovery paths where the Error
+ * object is caught directly rather than parsed from a result record.
+ */
+export function isHeadlessBlockedResult(error) {
+	if (!error) return false;
+	const env = error.envelope;
+	if (env?.blockedBy) {
+		// Skip non-recoverable blocks (rate-limit, ban, etc.)
+		if (NON_RECOVERABLE_BLOCKED_BY.has(env.blockedBy)) return false;
+		return true;
+	}
+	if (env?.verificationResult === "needs-human") return true;
+	return isHeadlessBlockedError(error.message);
+}