@apmantza/greedysearch-pi 2.0.0 → 2.1.2

package/src/tools/shared.ts

@@ -52,6 +52,19 @@ export function errorResult(prefix: string, e: unknown): ToolResult {
 	};
 }
 
+/** Progress update for a single engine finishing/failing */
+type EngineProgress = {
+	type: "engine";
+	engine: string;
+	status: "done" | "error" | "needs-human";
+};
+
+/** Free-form progress text (e.g. research bar + ETA) */
+type TextProgress = {
+	type: "text";
+	text: string;
+};
+
 /**
  * Spawn search.mjs and collect JSON results, with progress streaming via stderr.
  * Shared by GreedySearch tool handlers.
@@ -62,10 +75,7 @@ export function runSearch(
 	flags: string[],
 	searchBin: string,
 	signal?: AbortSignal,
-	onProgress?: (
-		engine: string,
-		status: "done" | "error" | "needs-human",
-	) => void,
+	onProgress?: (update: EngineProgress | TextProgress) => void,
 	options: { headless?: boolean } = {},
 ): Promise<Record<string, unknown>> {
 	return new Promise((resolve, reject) => {
@@ -108,20 +118,48 @@ export function runSearch(
 				// Engine progress: any known engine
 				const engineMatch = line.match(ENGINE_PROGRESS_RE);
 				if (engineMatch && onProgress) {
-					onProgress(
-						engineMatch[1],
-						engineMatch[2] as "done" | "error" | "needs-human",
-					);
+					onProgress({
+						type: "engine",
+						engine: engineMatch[1],
+						status: engineMatch[2] as "done" | "error" | "needs-human",
+					});
 				}
 				// Synthesis progress: skipped (manual verification) or done/error
 				const synthMatch = line.match(
 					/^PROGRESS:synthesis:(done|error|skipped)$/,
 				);
 				if (synthMatch && onProgress) {
-					onProgress(
-						"synthesis",
-						synthMatch[1] as "done" | "error" | "needs-human",
-					);
+					onProgress({
+						type: "engine",
+						engine: "synthesis",
+						status: synthMatch[1] as "done" | "error" | "needs-human",
+					});
+				}
+				// Research progress markers (planning/fetching/synthesizing)
+				const researchMatch = line.match(/^PROGRESS:research:(.+)$/);
+				if (researchMatch && onProgress) {
+					onProgress({
+						type: "text",
+						text: researchMatch[1],
+					});
+				}
+				// Progress bar + ETA lines from createProgressTracker
+				const barMatch = line.match(/^\[greedysearch\] (\[.+?\] .+)$/);
+				if (barMatch && onProgress) {
+					onProgress({
+						type: "text",
+						text: barMatch[1],
+					});
+				}
+				// Single-engine stage lines: "[perplexity] stage: nav (+563ms)"
+				const stageMatch = line.match(
+					/^\[(perplexity|google|chatgpt|bing|gemini|semantic-scholar|logically)\] stage: (.+) \(\+\d+ms\)$/,
+				);
+				if (stageMatch && onProgress) {
+					onProgress({
+						type: "text",
+						text: `${stageMatch[1]}: ${stageMatch[2]}`,
+					});
 				}
 			}
 		});
@@ -132,6 +170,15 @@ export function runSearch(
 			if (code !== 0) {
 				reject(new Error(err.trim() || `search.mjs exited with code ${code}`));
 			} else {
+				// For single-engine calls, signal completion so the progress
+				// tracker can mark the engine as done.
+				if (onProgress && engine !== "all") {
+					onProgress({
+						type: "engine",
+						engine,
+						status: "done" as const,
+					});
+				}
 				try {
 					resolve(JSON.parse(out.trim()));
 				} catch {
@@ -144,20 +191,72 @@ export function runSearch(
 	});
 }
 
+/**
+ * Render a Unicode progress bar.
+ * Example: [████████████░░░░] for 75%
+ */
+function renderBar(done: number, total: number): string {
+	if (total <= 0) return "";
+	const width = 16;
+	const filled = Math.round((done / total) * width);
+	return (
+		"[" +
+		"█".repeat(Math.min(filled, width)) +
+		"░".repeat(Math.max(0, width - filled)) +
+		"]"
+	);
+}
+
+/**
+ * Format milliseconds as a short human duration.
+ * e.g. "—" / "45s" / "1m 30s"
+ */
+function fmtDuration(ms: number): string {
+	if (ms < 1000) return "—";
+	const s = Math.round(ms / 1000);
+	if (s < 60) return `${s}s`;
+	return `${Math.floor(s / 60)}m ${s % 60}s`;
+}
+
 /**
  * Build a progress callback that tracks completed engines.
  * Returns an onProgress function suitable for runSearch.
+ *
+ * For multi-engine calls (research or not) this shows a bar + ETA
+ * line that tracks fraction of engines completed.  The bar always
+ * appears; the research-path bar from `createProgressTracker` takes
+ * priority when present.
  */
 export function makeProgressTracker(
 	engines: readonly string[],
 	onUpdate: ((update: ProgressUpdate) => void) | undefined,
 	suffix: "Searching" | "Researching",
 	showSynthesis: boolean,
+	query?: string,
 ) {
+	const startedAt = Date.now();
 	const completed = new Map<string, "done" | "error" | "needs-human">();
+	let latestBarText = "";
+
+	function render() {
+		const lines: string[] = [];
+		lines.push(`**${suffix}...** ${query || ""}`.trim());
+
+		const done = completed.size;
+
+		// Multi-engine bar + ETA (unless research supplies its own bar)
+		if (engines.length > 1 && done > 0 && !latestBarText) {
+			const elapsed = Date.now() - startedAt;
+			const frac = Math.min(1, done / engines.length);
+			const etaMs = frac > 0.01 ? Math.round(elapsed / frac - elapsed) : null;
+			const bar = renderBar(done, engines.length);
+			const eta = etaMs != null && etaMs > 0 ? fmtDuration(etaMs) : "—";
+			lines.push(`${bar} ${done}/${engines.length} engines (ETA ${eta})`);
+		}
+
+		// Research mode bar from createProgressTracker has priority
+		if (latestBarText) lines.push(latestBarText);
 
-	return (eng: string, status: "done" | "error" | "needs-human") => {
-		completed.set(eng, status);
 		const parts: string[] = [];
 		for (const e of engines) {
 			const s = completed.get(e);
@@ -167,21 +266,47 @@ export function makeProgressTracker(
 				parts.push(`🔓 ${e} needs manual verification`);
 			else parts.push(`⏳ ${e}`);
 		}
-		// Synthesis status is shown only when the caller explicitly requested
-		// Gemini synthesis for a multi-engine search.
-		if (showSynthesis && completed.size >= engines.length) {
+		if (showSynthesis && done >= engines.length) {
 			const synStatus = completed.get("synthesis");
 			if (synStatus === "done") parts.push("✅ synthesized");
 			else if (synStatus === "error") parts.push("❌ synthesis failed");
 			else if (synStatus === "needs-human") parts.push("⏭️ synthesis skipped");
 			else parts.push("🔄 synthesizing");
 		}
+		if (parts.length > 0) {
+			// Engine status line: 5 engines with emoji+separator runs ~110
+			// chars (visible width 116+ because emoji take 2 cols each),
+			// which is over the 112-char terminal width. The TUI's
+			// Text.render can't wrap a single line and crashes with
+			//   "Rendered line N exceeds terminal width (W > W-4)"
+			// if a single rendered line is wider than the terminal.
+			// Truncate at 90 chars (well under 100 visible-width to leave
+			// padding-room for variable emoji widths).
+			const statusLine = parts.join(" · ");
+			lines.push(
+				statusLine.length > 90
+					? statusLine.slice(0, 88) + "…"
+					: statusLine,
+			);
+		}
 
 		onUpdate?.({
-			content: [
-				{ type: "text", text: `**${suffix}...** ${parts.join(" · ")}` },
-			],
+			content: [{ type: "text", text: lines.join("\n") }],
 			details: { _progress: true },
 		} satisfies ProgressUpdate);
+	}
+
+	return (update: EngineProgress | TextProgress) => {
+		if (update.type === "text") {
+			if (update.text.startsWith("[")) {
+				latestBarText = update.text;
+			}
+			render();
+			return;
+		}
+
+		const { engine, status } = update;
+		completed.set(engine, status);
+		render();
 	};
 }

package/test.mjs

@@ -98,7 +98,12 @@ if (["", "all", "unit", "quick", "smoke", "synth"].includes(mode)) {
 	section("🧪 Unit Tests");
 
 	subsection("stripQuotes — param double-escaping workaround (issue #2)");
-	const { stripQuotes } = await import("./src/tools/shared.ts");
+	// Inlined from src/tools/shared.ts — importing the .ts file from
+	// test.mjs works at the project root (Node strips types) but fails
+	// when test.mjs runs from the installed tarball in node_modules
+	// (ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING). Keep in sync with
+	// src/tools/shared.ts.
+	const stripQuotes = (val) => String(val ?? "").replace(/^"|"$/g, "");
 
 	const stripCases = [
 		// [input, expected, label]
@@ -167,24 +172,24 @@ if (["", "all", "unit", "quick", "smoke", "synth"].includes(mode)) {
 		["VERIFICATION REQUIRED", true, 'legacy pattern: "VERIFICATION REQUIRED"'],
 		["verification failed", true, 'extended: "verification" in sentence'],
 		[
-			"Clipboard interceptor returned empty text",
+			"Cloudflare Turnstile challenge detected in closed shadow DOM",
 			true,
-			"new: clipboard error (headless Cloudflare block)",
+			"new: CF closed-shadow-dom block triggers visible retry",
 		],
 		[
-			"[bing] Clipboard empty, retrying in 2s...",
+			"Copilot verification required — please solve it manually in the browser window",
 			true,
-			"new: clipboard empty retry message",
+			"new: per-engine 'verification required' triggers visible retry",
 		],
 		[
-			"Cloudflare challenge detected — content blocked in headless",
+			"Network timeout after 30000ms",
 			true,
-			"new: Cloudflare detection triggers visible retry",
+			"new: timeout triggers visible retry",
 		],
 		[
-			"Network timeout after 30000ms",
+			"Perplexity input not found — page may be blocked or in unexpected state",
 			true,
-			"new: timeout triggers visible retry",
+			"new: 'input not found' triggers visible retry",
 		],
 		["", false, "empty string"],
 	];
@@ -218,7 +223,7 @@ if (["", "all", "unit", "quick", "smoke", "synth"].includes(mode)) {
 	}
 
 	const retryEngines = findHeadlessBlockedEngines({
-		perplexity: { error: "Clipboard interceptor returned empty text" },
+		perplexity: { error: "Perplexity input not found — page may be blocked or in unexpected state" },
 		bing: { error: "Copilot verification required" },
 		google: { error: "Google verification required" },
 	});
@@ -233,11 +238,16 @@ if (["", "all", "unit", "quick", "smoke", "synth"].includes(mode)) {
 	const pplxTestCases = [
 		["ask-input selector not found", true, 'legacy: "ask-input"'],
 		[
-			"Clipboard interceptor returned empty text",
+			"Perplexity input not found — page may be blocked or in unexpected state",
 			true,
-			"new: clipboard also triggers for perplexity",
+			"new: 'input not found' triggers for perplexity",
 		],
 		["Perplexity timeout", true, "timeout triggers visible retry"],
+		[
+			"Clipboard interceptor returned empty text",
+			false,
+			"new: 'clipboard' substring no longer triggers (was too broad — fired on routine DOM-fallback failures)",
+		],
 	];
 	for (const [error, expected, label] of pplxTestCases) {
 		const matched = isHeadlessBlockedError(error);
@@ -976,6 +986,115 @@ END_JSON`,
 		failMsg("citation audit: S2 should be flagged as unfetched");
 	}
 
+	subsection("Citation URL Reachability — checkCitationUrls");
+	const { checkCitationUrls, runCitationUrlCheck } = await import(
+		"./src/search/research.mjs"
+	);
+
+	// Empty sources → ok
+	const emptyResult = await checkCitationUrls([]);
+	if (emptyResult.ok && emptyResult.reachable.length === 0) {
+		passMsg("checkCitationUrls: empty sources returns ok");
+	} else {
+		failMsg(
+			`checkCitationUrls: empty sources unexpected: ${JSON.stringify(emptyResult)}`,
+		);
+	}
+
+	// Non-HTTP URLs are skipped
+	const nonHttpResult = await checkCitationUrls([
+		{ id: "S1", url: "ftp://example.com/file" },
+		{ id: "S2", url: "not-a-url" },
+	]);
+	if (
+		nonHttpResult.ok &&
+		nonHttpResult.skipped.length === 2 &&
+		nonHttpResult.reachable.length === 0
+	) {
+		passMsg("checkCitationUrls: non-HTTP URLs are skipped");
+	} else {
+		failMsg(
+			`checkCitationUrls: non-HTTP unexpected: ${JSON.stringify(nonHttpResult)}`,
+		);
+	}
+
+	// Concurrency guard: concurrency=0 should not infinite loop
+	// Skip in CI — makes a real HEAD request to example.com which may be
+	// blocked in sandboxed CI environments
+	if (!process.env.CI) {
+		const concurrencyResult = await checkCitationUrls(
+			[{ id: "S1", url: "https://example.com" }],
+			{ concurrency: 0, timeoutMs: 2000 },
+		);
+		if (concurrencyResult.ok || concurrencyResult.dead.length > 0) {
+			passMsg("checkCitationUrls: concurrency=0 does not infinite loop");
+		} else {
+			failMsg(
+				`checkCitationUrls: concurrency=0 unexpected: ${JSON.stringify(concurrencyResult)}`,
+			);
+		}
+	}
+
+	// runCitationUrlCheck returns null on error (non-throwing)
+	const runResult = await runCitationUrlCheck([]);
+	if (runResult && runResult.ok) {
+		passMsg("runCitationUrlCheck: empty sources returns ok");
+	} else {
+		failMsg(
+			`runCitationUrlCheck: empty sources unexpected: ${JSON.stringify(runResult)}`,
+		);
+	}
+
+	subsection("Provenance Sidecar — writeProvenanceSidecar");
+	const { writeProvenanceSidecar } = await import("./src/search/research.mjs");
+	const { existsSync, rmSync } = await import("node:fs");
+	const { tmpdir } = await import("node:os");
+
+	const testProvenanceDir = join(
+		tmpdir(),
+		`greedysearch-test-provenance-${Date.now()}`,
+	);
+	mkdirSync(testProvenanceDir, { recursive: true });
+
+	try {
+		writeProvenanceSidecar(testProvenanceDir, {
+			query: "test query",
+			rounds: [{ round: 1, actions: [], learnings: [], gaps: [] }],
+			sources: [{ id: "S1", title: "Test Source" }],
+			fetchedSources: [{ id: "S1", contentChars: 500 }],
+			citationAudit: { ok: true, cited: ["S1"], missing: [], unfetched: [] },
+			citationUrls: { reachable: [], dead: [], skipped: [], ok: true },
+			floor: { floorMet: true, checks: { citationsPresent: true } },
+			manifest: {
+				startedAt: "2026-01-01",
+				finishedAt: "2026-01-01",
+				durationMs: 1000,
+			},
+		});
+
+		const provenancePath = join(testProvenanceDir, "provenance.md");
+		if (existsSync(provenancePath)) {
+			const content = readFileSync(provenancePath, "utf8");
+			if (content.includes("test query") && content.includes("S1")) {
+				passMsg(
+					"writeProvenanceSidecar: writes provenance.md with query and sources",
+				);
+			} else {
+				failMsg(
+					"writeProvenanceSidecar: provenance.md missing expected content",
+				);
+			}
+		} else {
+			failMsg("writeProvenanceSidecar: provenance.md not created");
+		}
+	} catch (e) {
+		failMsg(`writeProvenanceSidecar: threw error: ${e.message}`);
+	} finally {
+		try {
+			rmSync(testProvenanceDir, { recursive: true, force: true });
+		} catch {}
+	}
+
 	subsection("Research Floor and Question Ledger");
 	const { computeResearchFloor, createQuestionLedger, updateQuestionLedger } =
 		await import("./src/search/research.mjs");
@@ -1060,6 +1179,35 @@ trailing note`);
 			`structured JSON: failed to repair ${JSON.stringify(parsedLooseJson)}`,
 		);
 	}
+
+	subsection("Progress tracker — bar rendering and ETA");
+	const { createProgressTracker } = await import("./src/search/progress.mjs");
+	const silentTracker = createProgressTracker({
+		totalActions: 4,
+		silent: true,
+	});
+	silentTracker.startAction("search", "test");
+	silentTracker.endAction();
+	silentTracker.startAction("fetch", "https://example.com");
+	silentTracker.endAction();
+	if (silentTracker.getElapsedMs() >= 0) {
+		passMsg("progress: tracker records action timing");
+	} else {
+		failMsg("progress: tracker elapsed time invalid");
+	}
+	// Test bar formatting indirectly via duration
+	const tracker2 = createProgressTracker({
+		totalActions: 2,
+		totalRounds: 1,
+		silent: true,
+	});
+	tracker2.startAction("search", "q1");
+	tracker2.endAction();
+	if (tracker2.getElapsedMs() >= 0) {
+		passMsg("progress: round tracking works");
+	} else {
+		failMsg("progress: round tracking broken");
+	}
 }
 
 // ─────────────────────────────────────────────────────────────────────────────