@apmantza/greedysearch-pi 2.1.3 → 2.1.5

package/src/search/research.mjs

@@ -1461,11 +1461,20 @@ export async function checkCitationUrls(
 						});
 						clearTimeout(timer);
 						const ok = response.status >= 200 && response.status < 400;
+						const botProtectedOrHeadlessHost = [401, 403, 405, 429].includes(
+							response.status,
+						);
+						let status = "dead";
+						if (ok) status = "reachable";
+						else if (botProtectedOrHeadlessHost) status = "skipped";
 						return {
 							id: source.id,
 							url,
-							status: ok ? "reachable" : "dead",
+							status,
 							httpStatus: response.status,
+							reason: botProtectedOrHeadlessHost
+								? "bot-protected-or-head-disallowed"
+								: undefined,
 						};
 					} catch (fetchError) {
 						clearTimeout(timer);
@@ -1519,10 +1528,17 @@ export async function checkCitationUrls(
  * Used by both runResearchMode() and runSimpleResearchMode() to avoid
  * duplicating the try/catch/logging block.
  */
-export async function runCitationUrlCheck(combinedSources) {
+export async function runCitationUrlCheck(
+	combinedSources,
+	citationAudit = null,
+) {
 	process.stderr.write("PROGRESS:research:check-urls\n");
 	try {
-		const citationUrls = await checkCitationUrls(combinedSources, {
+		const citedIds = new Set(citationAudit?.cited || []);
+		const sourcesToCheck = citedIds.size
+			? (combinedSources || []).filter((source) => citedIds.has(source?.id))
+			: combinedSources;
+		const citationUrls = await checkCitationUrls(sourcesToCheck, {
 			timeoutMs: 6000,
 			concurrency: 4,
 		});
@@ -1587,7 +1603,9 @@ export function computeResearchFloor({
 		roundsRun: rounds.length >= 1,
 		fetchedSources: fetchedOk.length >= minFetched,
 		primarySources: primarySources.length >= 1,
-		qualityScore: qualityScore >= Math.min(qualityThreshold, 8),
+		qualityScore:
+			qualityScore >= Math.min(qualityThreshold, 8) ||
+			(requireCitations && claims.length > 0 && citedCount > 0),
 		claimsExtracted: !requireCitations || claims.length > 0,
 		citationsPresent: !requireCitations || citedCount > 0,
 		citationsValid: !requireCitations || citationAudit?.ok === true,
@@ -2211,8 +2229,8 @@ export async function writeResearchBundle({
 
 export async function runResearchMode({
 	query,
-	breadth = 3,
-	iterations = 2,
+	breadth,
+	iterations,
 	maxSources,
 	locale = null,
 	short = false,
@@ -2226,8 +2244,9 @@ export async function runResearchMode({
 	// When breadth and iterations are at defaults (not user-specified), classify
 	// the query complexity. Simple queries bypass the iterative loop entirely
 	// for ~70% faster results and lower API cost.
-	const userSpecifiedBreadth = typeof breadth === "number";
-	const userSpecifiedIterations = typeof iterations === "number";
+	const userSpecifiedBreadth = breadth !== undefined && breadth !== null;
+	const userSpecifiedIterations =
+		iterations !== undefined && iterations !== null;
 	const atDefaults = !userSpecifiedBreadth && !userSpecifiedIterations;
 
 	if (atDefaults) {
@@ -2475,9 +2494,33 @@ export async function runResearchMode({
 		);
 		if (remainingFetchBudget > 0 && combinedSources.length > 0) {
 			process.stderr.write(`PROGRESS:research:round-${roundNumber}:fetching\n`);
+			const fetchedUrlKeys = new Set();
+			const safeNormalizeFetchUrl = (value) => {
+				try {
+					return normalizeUrl(value || "");
+				} catch {
+					return "";
+				}
+			};
+			for (const source of fetchedSources) {
+				for (const value of [
+					source?.url,
+					source?.finalUrl,
+					source?.canonicalUrl,
+				]) {
+					const key = safeNormalizeFetchUrl(value);
+					if (key) fetchedUrlKeys.add(key);
+				}
+			}
+			const fetchCandidates = combinedSources.filter((source) => {
+				const keys = [source?.canonicalUrl, source?.finalUrl, source?.url]
+					.map((value) => safeNormalizeFetchUrl(value))
+					.filter(Boolean);
+				return keys.length > 0 && keys.every((key) => !fetchedUrlKeys.has(key));
+			});
 			const fetched = await fetchMultipleResearchSources(
-				combinedSources,
-				Math.min(remainingFetchBudget, combinedSources.length),
+				fetchCandidates,
+				Math.min(remainingFetchBudget, fetchCandidates.length),
 				8000,
 				Math.min(3, remainingFetchBudget || 1),
 			);
@@ -2790,7 +2833,10 @@ export async function runResearchMode({
 	const citationAudit = auditCitations(synthesis.answer || "", combinedSources);
 
 	// Citation URL reachability check
-	const citationUrls = await runCitationUrlCheck(combinedSources);
+	const citationUrls = await runCitationUrlCheck(
+		combinedSources,
+		citationAudit,
+	);
 
 	reconcileQuestionsFromSynthesis(questions, synthesis, citationAudit);
 	const floor = computeResearchFloor({

package/src/search/simple-research.mjs

@@ -387,7 +387,10 @@ export async function runSimpleResearchMode({
 	const citationAudit = auditCitations(synthesis.answer || "", combinedSources);
 
 	// Citation URL reachability check
-	const citationUrls = await runCitationUrlCheck(combinedSources);
+	const citationUrls = await runCitationUrlCheck(
+		combinedSources,
+		citationAudit,
+	);
 
 	reconcileQuestionsFromSynthesis(questions, synthesis, citationAudit);
 	const allGaps = uniqueStrings(synthesis.caveats || []);

package/test.mjs

@@ -223,12 +223,15 @@ if (["", "all", "unit", "quick", "smoke", "synth"].includes(mode)) {
 	}
 
 	const retryEngines = findHeadlessBlockedEngines({
-		perplexity: { error: "Perplexity input not found — page may be blocked or in unexpected state" },
+		perplexity: {
+			error:
+				"Perplexity input not found — page may be blocked or in unexpected state",
+		},
 		bing: { error: "Copilot verification required" },
 		google: { error: "Google verification required" },
 	});
 	if (retryEngines.join(",") === "perplexity,bing") {
-		passMsg("visible retry engines: perplexity and bing only");
+		passMsg("visible retry engines: google excluded from recovery");
 	} else {
 		failMsg(
 			`visible retry engines: expected perplexity,bing, got ${retryEngines.join(",")}`,
@@ -715,7 +718,17 @@ proc.on('close', code => {
 			const tmp = join(resultsDir, `_synth_${synthesizer}.mjs`);
 			writeFileSync(tmp, script, "utf8");
 			await runNode([tmp], 240);
-			const data = JSON.parse(readFileSync(outFile, "utf8"));
+			let data;
+			try {
+				data = JSON.parse(readFileSync(outFile, "utf8"));
+			} catch (e) {
+				return {
+					synthesized: false,
+					synthesizedBy: null,
+					parseError: e.message,
+					rawOut: "",
+				};
+			}
 			let parsed = null;
 			try {
 				parsed = JSON.parse(data.out);

package/skills/greedy-search/skill.md

@@ -1,20 +0,0 @@
----
-name: greedy-search
-description: Web/search plus opt-in research via Perplexity, Google AI, ChatGPT, Gemini, Semantic Scholar, and Logically. Grounded all-engine search fetches sources by default; optional configurable synthesis; deep research as separate workflow. Configurable via ~/.pi/greedyconfig. Bing Copilot available for signed-in users. Current docs, recent changes, dependency choices. NOT codebase search.
----
-
-`greedy_search({ query, engine: "all"|"perplexity"|"google"|"chatgpt"|"gemini"|"semantic-scholar"|"logically"|"bing", synthesize?: bool, synthesizer?: "gemini"|"chatgpt", depth?: "research", breadth: 1-5, iterations: 1-3, maxSources: 3-12, researchOutDir?: string, writeResearchBundle?: bool, visible: bool })`
-
-**Modes:** individual engine search · grounded `engine:"all"` search with fetched sources · optional `synthesize:true` using the configured synthesizer over all-engine results · `depth:"research"` for the iterative deep-research workflow.
-
-**Config:** `~/.pi/greedyconfig` supports `{ "engines": ["perplexity", "google", "chatgpt", "gemini"], "synthesizer": "gemini" }` by default. `semantic-scholar` and `logically` are opt-in academic/research engines — add them to `engines` only when you want academic paper discovery or research-assistant workflows in the normal all-search fan-out. Without explicit opt-in, `engine:"all"` excludes them because their results are noisy for casual web search; they shine in `depth:"research"` mode. Any configured engine can participate in `engine:"all"`; deep research child searches reuse the same configured `engines` list and stdin-safe query passing. Normal all-search synthesis remains controlled separately by `synthesizer`; research planning/final synthesis uses Gemini.
-
-**Compatibility:** legacy `depth:"fast"|"standard"|"deep"` is still accepted. `fast` skips source fetching; `standard`/`deep` alias `synthesize:true`. Prefer `synthesize:true`, optional `synthesizer`, and `depth:"research"` going forward.
-
-**Research output:** `depth:"research"` writes a dataroom-style bundle by default under `.pi/greedysearch-research/<timestamp>_<query>/` with `STATUS.md`, `OUTLINE.md`, `reports/SUMMARY.md`, `reports/CLAIMS.md`, `reports/GAPS.md`, `sources/`, and `data/manifest.json`. Pass `researchOutDir` to choose the directory or `writeResearchBundle:false` to disable disk output.
-
-**Scale-aware research:** When `breadth` and `iterations` are not explicitly set, the classifier auto-detects query complexity. Simple queries ("what is X") use a fast single-pass path (~70% faster). Moderate queries get tighter breadth/iterations. Complex queries use the full loop. Explicit `breadth`/`iterations` always override the classifier.
-
-**Auto-recovery:** Headless default. Bing/Perplexity auto-retry visible on CF block. Manual CAPTCHA → visible stays open; solve then rerun.
-
-**CDP safety:** Use `bin/cdp-greedy.mjs` only. Never raw `bin/cdp.mjs`.