@apmantza/greedysearch-pi 2.0.0 → 2.1.3

package/src/search/research.mjs

@@ -20,6 +20,9 @@ import {
 import { parseStructuredJson } from "./synthesis.mjs";
 import { RESEARCH_ENGINES } from "./constants.mjs";
 import { runGeminiPrompt } from "./synthesis-runner.mjs";
+import { classifyResearchComplexity } from "./scale-aware.mjs";
+import { runSimpleResearchMode } from "./simple-research.mjs";
+import { createProgressTracker } from "./progress.mjs";
 
 const __dir = fileURLToPath(new URL(".", import.meta.url)).replace(
 	/^\/([A-Z]:)/,
@@ -984,7 +987,7 @@ function normalizeEvidenceExtractions(payload, fetchedSources) {
 		);
 }
 
-async function extractEvidenceFromSources({
+export async function extractEvidenceFromSources({
 	query,
 	questions,
 	fetchedSources,
@@ -1071,7 +1074,7 @@ function buildLearningPrompt(
 	].join("\n");
 }
 
-function buildFinalReportPrompt(
+export function buildFinalReportPrompt(
 	originalQuery,
 	rounds,
 	sources,
@@ -1151,7 +1154,7 @@ function buildFinalReportPrompt(
  * structured learnings (for example when Gemini's input field rejected the
  * per-round learning prompt but the goal-based extraction step succeeded).
  */
-function buildSynthesisFromEvidencePrompt(
+export function buildSynthesisFromEvidencePrompt(
 	originalQuery,
 	sources = [],
 	questions = [],
@@ -1401,6 +1404,142 @@ export function auditCitations(answer, sources) {
 	};
 }
 
+/**
+ * Check reachability of cited source URLs via HEAD requests.
+ * Returns { reachable, dead, skipped } with per-URL status.
+ */
+export async function checkCitationUrls(
+	sources,
+	{ timeoutMs = 6000, concurrency = 4 } = {},
+) {
+	const safeConcurrency = Math.max(1, Math.floor(concurrency || 1));
+	const citedSources = (sources || []).filter(
+		(s) => s?.id && (s?.canonicalUrl || s?.finalUrl || s?.url),
+	);
+	if (citedSources.length === 0) {
+		return { reachable: [], dead: [], skipped: [], ok: true };
+	}
+
+	const reachable = [];
+	const dead = [];
+	const skipped = [];
+
+	// Process in batches to avoid overwhelming
+	for (let i = 0; i < citedSources.length; i += safeConcurrency) {
+		const batch = citedSources.slice(i, i + safeConcurrency);
+		const results = await Promise.allSettled(
+			batch.map(async (source) => {
+				const url =
+					source.fetch?.finalUrl ||
+					source.canonicalUrl ||
+					source.finalUrl ||
+					source.url;
+				if (!url) return { id: source.id, url: "", status: "skipped" };
+
+				// Skip non-HTTP URLs and known-unreachable patterns
+				try {
+					const parsed = new URL(url);
+					if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+						return { id: source.id, url, status: "skipped" };
+					}
+				} catch {
+					return { id: source.id, url, status: "skipped" };
+				}
+
+				try {
+					const controller = new AbortController();
+					const timer = setTimeout(() => controller.abort(), timeoutMs);
+					try {
+						const response = await fetch(url, {
+							method: "HEAD",
+							redirect: "follow",
+							signal: controller.signal,
+							headers: {
+								"User-Agent":
+									"Mozilla/5.0 (compatible; GreedySearch/2.0; +https://github.com/apmantza/greedysearch-pi)",
+							},
+						});
+						clearTimeout(timer);
+						const ok = response.status >= 200 && response.status < 400;
+						return {
+							id: source.id,
+							url,
+							status: ok ? "reachable" : "dead",
+							httpStatus: response.status,
+						};
+					} catch (fetchError) {
+						clearTimeout(timer);
+						return {
+							id: source.id,
+							url,
+							status: "dead",
+							error:
+								fetchError.name === "AbortError"
+									? "timeout"
+									: fetchError.message,
+						};
+					}
+				} catch (error) {
+					return {
+						id: source.id,
+						url,
+						status: "dead",
+						error: error.message,
+					};
+				}
+			}),
+		);
+
+		for (const result of results) {
+			const value =
+				result.status === "fulfilled"
+					? result.value
+					: {
+							id: "?",
+							url: "",
+							status: "dead",
+							error: result.reason?.message || "unknown",
+						};
+			if (value.status === "reachable") reachable.push(value);
+			else if (value.status === "dead") dead.push(value);
+			else skipped.push(value);
+		}
+	}
+
+	return {
+		reachable,
+		dead,
+		skipped,
+		ok: dead.length === 0,
+	};
+}
+
+/**
+ * Shared orchestration: run citation URL check with logging.
+ * Used by both runResearchMode() and runSimpleResearchMode() to avoid
+ * duplicating the try/catch/logging block.
+ */
+export async function runCitationUrlCheck(combinedSources) {
+	process.stderr.write("PROGRESS:research:check-urls\n");
+	try {
+		const citationUrls = await checkCitationUrls(combinedSources, {
+			timeoutMs: 6000,
+			concurrency: 4,
+		});
+		if (!citationUrls.ok) {
+			process.stderr.write(
+				`[greedysearch] ${citationUrls.dead.length} dead citation URL(s) detected\n`,
+			);
+		}
+		return citationUrls;
+	} catch (error) {
+		process.stderr.write(
+			`[greedysearch] URL reachability check failed: ${error.message}\n`,
+		);
+		return null;
+	}
+}
+
 export function computeResearchFloor({
 	sources = [],
 	fetchedSources = [],
@@ -1437,7 +1576,13 @@ export function computeResearchFloor({
 		(q) => !q.createdRound || q.reason === "Original research question",
 	);
 	const requiredQuestionStats = questionProgress(requiredQuestions);
-	const minFetched = Math.min(4, Math.max(2, Number(maxSources) || 8));
+	// Scale the minimum fetched sources by the number of rounds. The
+	// simple research path runs 1 round with fewer sources, so requiring
+	// 2-4 sources would be too strict. Iterative research (2+ rounds)
+	// gets the full minFetched requirement.
+	const roundCount = (rounds || []).length;
+	const baseMin = Math.min(4, Math.max(2, Number(maxSources) || 8));
+	const minFetched = roundCount <= 1 ? Math.min(2, baseMin) : baseMin;
 	const checks = {
 		roundsRun: rounds.length >= 1,
 		fetchedSources: fetchedOk.length >= minFetched,
@@ -1689,7 +1834,11 @@ function pickAcademicFetchTargets(combinedSources, usedUrls) {
 	return targets.slice(0, 2);
 }
 
-function reconcileQuestionsFromSynthesis(questions, synthesis, citationAudit) {
+export function reconcileQuestionsFromSynthesis(
+	questions,
+	synthesis,
+	citationAudit,
+) {
 	if (!synthesis?.answer || citationAudit?.ok !== true) return questions;
 	const claims = Array.isArray(synthesis.claims) ? synthesis.claims : [];
 	const citedIds = Array.isArray(citationAudit.cited)
@@ -1740,7 +1889,120 @@ function markdownList(items, fallback = "None recorded.") {
 		: fallback;
 }
 
-async function writeResearchBundle({
+/**
+ * Write a human-readable provenance sidecar next to the research bundle.
+ * Records date, rounds, sources, verification status, and floor results.
+ */
+export function writeProvenanceSidecar(
+	dir,
+	{
+		query,
+		rounds,
+		sources,
+		fetchedSources,
+		citationAudit,
+		citationUrls,
+		floor,
+		manifest,
+	},
+) {
+	const fetchedOk = (fetchedSources || []).filter(
+		(s) => s?.contentChars > 100 || s?.fetch?.ok,
+	);
+	const primarySources = (sources || []).filter((s) =>
+		["official-docs", "repo", "maintainer-blog", "academic"].includes(
+			String(s?.sourceType || ""),
+		),
+	);
+	const citedIds = new Set(citationAudit?.cited || []);
+	const citedSources = (sources || []).filter((s) => citedIds.has(s?.id));
+
+	const lines = [
+		`# Provenance: ${query}`,
+		"",
+		`- **Date:** ${manifest?.startedAt || new Date().toISOString()}`,
+		`- **Duration:** ${manifest?.durationMs ? `${(manifest.durationMs / 1000).toFixed(1)}s` : "unknown"}`,
+		`- **Mode:** ${manifest?.terminationReason === "simple_single_pass" ? "simple (single-pass)" : "iterative"}`,
+		`- **Rounds:** ${manifest?.rounds || rounds?.length || 1}`,
+		"",
+		"## Sources",
+		"",
+		`- **Consulted:** ${sources?.length || 0}`,
+		`- **Fetched successfully:** ${fetchedOk.length}`,
+		`- **Primary sources:** ${primarySources.length}`,
+		`- **Cited in report:** ${citedSources.length}`,
+		"",
+	];
+
+	// Cited source details
+	if (citedSources.length > 0) {
+		lines.push("### Cited sources", "");
+		for (const source of citedSources) {
+			const url = source.canonicalUrl || source.finalUrl || source.url || "";
+			const fetched = source.fetch?.ok ? "✓" : "✗";
+			lines.push(
+				`- **${source.id}:** [${source.title || url}](${url}) (${source.sourceType || "unknown"}, fetched: ${fetched})`,
+			);
+		}
+		lines.push("");
+	}
+
+	// URL reachability
+	if (
+		citationUrls &&
+		(citationUrls.reachable.length > 0 || citationUrls.dead.length > 0)
+	) {
+		lines.push("## URL reachability", "");
+		if (citationUrls.dead.length > 0) {
+			lines.push("");
+			lines.push("**Dead links:**");
+			for (const d of citationUrls.dead) {
+				lines.push(
+					`- ${d.id}: ${d.url} (${d.httpStatus || d.error || "unknown"})`,
+				);
+			}
+		}
+		if (citationUrls.reachable.length > 0) {
+			lines.push("");
+			lines.push(
+				`**Reachable:** ${citationUrls.reachable.length}/${citationUrls.reachable.length + citationUrls.dead.length}`,
+			);
+		}
+		lines.push("");
+	}
+
+	// Verification status
+	const verificationStatus = !citationAudit
+		? "NOT CHECKED"
+		: citationAudit.ok && (citationUrls?.ok ?? true)
+			? "PASS"
+			: citationAudit.ok === false
+				? "FAIL (missing citations)"
+				: "FAIL (dead links)";
+
+	lines.push(
+		"## Verification",
+		"",
+		`- **Citations:** ${citationAudit?.ok ? "PASS" : `FAIL — missing: ${(citationAudit?.missing || []).join(", ")}`}`,
+		`- **URL reachability:** ${citationUrls ? (citationUrls.ok ? "PASS" : `FAIL — ${citationUrls.dead.length} dead`) : "SKIPPED"}`,
+		`- **Floor:** ${floor?.floorMet ? "PASS" : "PARTIAL"}`,
+		`- **Overall:** ${verificationStatus}`,
+		"",
+	);
+
+	// Floor checks
+	if (floor?.checks) {
+		lines.push("## Floor checks", "");
+		for (const [name, ok] of Object.entries(floor.checks)) {
+			lines.push(`- [${ok ? "x" : " "}] ${name}`);
+		}
+		lines.push("");
+	}
+
+	writeFileSync(join(dir, "provenance.md"), lines.join("\n"), "utf8");
+}
+
+export async function writeResearchBundle({
 	query,
 	rounds,
 	sources,
@@ -1752,6 +2014,7 @@ async function writeResearchBundle({
 	manifest,
 	allGaps = [],
 	questions = [],
+	citationUrls = null,
 	outDir = null,
 }) {
 	const stamp = new Date().toISOString().replaceAll(/[:.]/g, "-").slice(0, 19);
@@ -1807,6 +2070,7 @@ async function writeResearchBundle({
 			"- `reports/CLAIMS.md` — extracted claims with support/source IDs",
 			"- `reports/EVIDENCE.md` — goal-based source evidence",
 			"- `reports/GAPS.md` — remaining caveats and uncertainties",
+			"- `provenance.md` — human-readable run metadata and verification",
 			"- `sources/` — fetched source markdown files",
 			"- `data/manifest.json` — machine-readable run metadata",
 			"- `data/rounds.json` — per-round actions/learnings/gaps",
@@ -1915,11 +2179,31 @@ async function writeResearchBundle({
 		].join("\n"),
 		"utf8",
 	);
+
+	// Provenance sidecar — human-readable run metadata (non-critical)
+	try {
+		writeProvenanceSidecar(dir, {
+			query,
+			rounds,
+			sources,
+			fetchedSources,
+			citationAudit,
+			citationUrls,
+			floor,
+			manifest,
+		});
+	} catch (sidecarError) {
+		process.stderr.write(
+			`[greedysearch] Provenance sidecar write failed (non-critical): ${sidecarError.message}\n`,
+		);
+	}
+
 	return {
 		dir,
 		statusPath: join(dir, "STATUS.md"),
 		summaryPath: join(reportsDir, "SUMMARY.md"),
 		manifestPath: join(dataDir, "manifest.json"),
+		provenancePath: join(dir, "provenance.md"),
 		sourceCount: sourceFiles.length,
 		sourceFiles,
 	};
@@ -1937,6 +2221,50 @@ export async function runResearchMode({
 	researchOutDir = null,
 } = {}) {
 	const options = clampResearchOptions({ breadth, iterations, maxSources });
+
+	// ── Scale-aware fast path ────────────────────────────────────────────────
+	// When breadth and iterations are at defaults (not user-specified), classify
+	// the query complexity. Simple queries bypass the iterative loop entirely
+	// for ~70% faster results and lower API cost.
+	const userSpecifiedBreadth = typeof breadth === "number";
+	const userSpecifiedIterations = typeof iterations === "number";
+	const atDefaults = !userSpecifiedBreadth && !userSpecifiedIterations;
+
+	if (atDefaults) {
+		try {
+			const classification = await classifyResearchComplexity(query);
+			process.stderr.write(
+				`[greedysearch] Complexity: ${classification.complexity} (${classification.reasoning})\n`,
+			);
+			if (classification.complexity === "simple") {
+				process.stderr.write(
+					`[greedysearch] Simple query detected — using fast single-pass path\n`,
+				);
+				return runSimpleResearchMode({
+					query,
+					locale,
+					maxSources: Math.min(maxSources ?? 5, 5),
+					qualityThreshold,
+					writeBundle,
+					researchOutDir,
+				});
+			}
+			// For moderate/complex: use classifier suggestions as hints if user
+			// didn't specify values. This tightens the loop for moderate queries
+			// without changing the user-explicit path.
+			if (!userSpecifiedBreadth) {
+				options.breadth = classification.suggestedBreadth;
+			}
+			if (!userSpecifiedIterations) {
+				options.iterations = classification.suggestedIterations;
+			}
+		} catch (error) {
+			process.stderr.write(
+				`[greedysearch] Scale classification failed, using defaults: ${error.message}\n`,
+			);
+		}
+	}
+
 	const rounds = [];
 	let allLearnings = [];
 	let allGaps = [];
@@ -1959,6 +2287,18 @@ export async function runResearchMode({
 	let totalFetches = 0;
 	const engineFailures = [];
 
+	// Progress bar with ETA — pre-compute totals from plan so the bar
+	// reflects the full run, not just the current round. The actual
+	// actions per round come from Gemini's plan; we estimate 1 fetch
+	// per academic source found.
+	const progressTracker = createProgressTracker({
+		totalActions: options.iterations * options.breadth,
+		totalRounds: options.iterations,
+		totalFetches: options.iterations, // estimate: ~1 fetch per round
+		silent: process.env.GREEDY_RESEARCH_QUIET === "1",
+	});
+	progressTracker.startRound(1);
+
 	process.stderr.write(
 		`[greedysearch] Research mode: breadth ${options.breadth}, iterations ${options.iterations}, qualityThreshold ${qualityThreshold}, engines ${RESEARCH_ENGINES.join(",")}, synthesizer gemini\n`,
 	);
@@ -2072,6 +2412,10 @@ export async function runResearchMode({
 			process.stderr.write(
 				`[greedysearch] Action ${i + 1}/${roundActions.length} [${action.type}]: ${(action.query || action.url).slice(0, 80)}\n`,
 			);
+			progressTracker.startAction(
+				action.type,
+				(action.query || action.url || "").slice(0, 60),
+			);
 			const run = await executeResearchAction(action, {
 				locale,
 				short,
@@ -2079,10 +2423,14 @@ export async function runResearchMode({
 				usedUrls,
 				maxChars: 8000,
 			});
+			progressTracker.endAction();
 			actionRuns.push(run);
 			totalActionsRun++;
 			if (action.type === "search") totalSearches++;
-			if (action.type === "fetchUrl") totalFetches++;
+			if (action.type === "fetchUrl") {
+				totalFetches++;
+				progressTracker.endFetch(run.ok);
+			}
 			if (!run.ok) {
 				engineFailures.push({
 					round: roundNumber,
@@ -2243,6 +2591,10 @@ export async function runResearchMode({
 
 		// Quality evaluation
 		process.stderr.write(`PROGRESS:research:round-${roundNumber}:evaluating\n`);
+		progressTracker.endRound();
+		if (roundNumber < options.iterations) {
+			progressTracker.startRound(roundNumber + 1);
+		}
 		const evaluation = await evaluateResearchQuality(
 			query,
 			rounds,
@@ -2436,6 +2788,10 @@ export async function runResearchMode({
 	// Citation audit + final question reconciliation + deterministic completion floor
 	process.stderr.write("PROGRESS:research:audit-citations\n");
 	const citationAudit = auditCitations(synthesis.answer || "", combinedSources);
+
+	// Citation URL reachability check
+	const citationUrls = await runCitationUrlCheck(combinedSources);
+
 	reconcileQuestionsFromSynthesis(questions, synthesis, citationAudit);
 	const floor = computeResearchFloor({
 		sources: combinedSources,
@@ -2483,6 +2839,7 @@ export async function runResearchMode({
 				evidenceItems,
 				synthesis,
 				citationAudit,
+				citationUrls,
 				floor,
 				manifest,
 				allGaps,
@@ -2500,6 +2857,7 @@ export async function runResearchMode({
 	}
 
 	process.stderr.write("PROGRESS:research:done\n");
+	progressTracker.finish();
 
 	return {
 		query,
@@ -2522,6 +2880,7 @@ export async function runResearchMode({
 			manifest,
 		},
 		_citationAudit: citationAudit,
+		_citationUrls: citationUrls,
 		_sources: combinedSources,
 		_fetchedSources: fetchedFiles,
 		_synthesis: synthesis,

package/src/search/scale-aware.mjs

@@ -0,0 +1,93 @@
+// src/search/scale-aware.mjs — Complexity classification and fast-path research
+//
+// Before entering the full iterative loop, classify the query complexity to
+// decide whether the expensive multi-round path is warranted. Simple "what is
+// X" queries get a fast single-pass path; complex/multi-faceted queries get
+// the full iterative treatment (possibly with adjusted breadth/iterations).
+
+import { trimText } from "./sources.mjs";
+import { runGeminiPrompt } from "./synthesis-runner.mjs";
+import { parseStructuredJson } from "./synthesis.mjs";
+
+const COMPLEXITY_PROMPT_TIMEOUT_MS = 30_000;
+
+function clampInt(value, min, max, fallback) {
+	const n = Number.parseInt(String(value ?? ""), 10);
+	if (!Number.isFinite(n)) return fallback;
+	return Math.min(max, Math.max(min, n));
+}
+
+/**
+ * Classify a research query as simple, moderate, or complex.
+ * Returns { complexity, reasoning, suggestedBreadth, suggestedIterations, needsAcademicSources }.
+ */
+export async function classifyResearchComplexity(query) {
+	const prompt = [
+		"You are a research complexity classifier.",
+		"Classify the following query by research complexity.",
+		"",
+		"- simple: A narrow factual question (what is X, define X, how does X work).",
+		"  Answerable with 1-3 search queries and a short synthesis. No sub-questions.",
+		"- moderate: A focused comparison, recent change, or best-practice lookup.",
+		"  Needs 2-4 angles but stays within one domain.",
+		"- complex: Multi-faceted survey, landscape analysis, or cross-domain investigation.",
+		"  Benefits from parallel research directions and iterative deepening.",
+		"",
+		"Respond ONLY with JSON wrapped in BEGIN_JSON / END_JSON markers:",
+		"BEGIN_JSON",
+		JSON.stringify(
+			{
+				complexity: "simple",
+				reasoning: "narrow factual question",
+				suggestedBreadth: 1,
+				suggestedIterations: 1,
+				needsAcademicSources: false,
+			},
+			null,
+			2,
+		),
+		"END_JSON",
+		"",
+		"Query: " + query,
+	].join("\n");
+
+	try {
+		const raw = await runGeminiPrompt(prompt, {
+			timeoutMs: COMPLEXITY_PROMPT_TIMEOUT_MS,
+		});
+		const parsed = parseStructuredJson(raw?.answer || "") || {};
+		const complexity = ["simple", "moderate", "complex"].includes(
+			parsed.complexity,
+		)
+			? parsed.complexity
+			: "moderate";
+		return {
+			complexity,
+			reasoning: trimText(parsed.reasoning || "", 200),
+			suggestedBreadth: clampInt(
+				parsed.suggestedBreadth,
+				1,
+				5,
+				complexity === "simple" ? 1 : 3,
+			),
+			suggestedIterations: clampInt(
+				parsed.suggestedIterations,
+				1,
+				3,
+				complexity === "simple" ? 1 : 2,
+			),
+			needsAcademicSources: parsed.needsAcademicSources === true,
+		};
+	} catch (error) {
+		process.stderr.write(
+			`[greedysearch] Complexity classification failed, defaulting to moderate: ${error.message}\n`,
+		);
+		return {
+			complexity: "moderate",
+			reasoning: "classification failed",
+			suggestedBreadth: 3,
+			suggestedIterations: 2,
+			needsAcademicSources: false,
+		};
+	}
+}