@companion-ai/feynman 0.5.43 → 0.5.45

package/RELEASES.md

@@ -6,6 +6,17 @@ GitHub release notes are generated from the matching `## vX.Y.Z` section in this
 
 ## Unreleased
 
+## v0.5.45 - 2026-10-13
+
+- **Paper lookups recover instead of failing.**
+  - A topic sent to the arXiv source, which only looks papers up by ID, now gets OpenAlex semantic search results with a note saying so, instead of an error the model had to recover from.
+  - When arXiv times out or is overloaded, arXiv ID lookups come from Semantic Scholar, with a note saying so.
+
+## v0.5.44 - 2026-10-13
+
+- **`feynman search set` works with every search provider pi-web-access supports.** It accepted only `auto`, `exa`, `perplexity` and `gemini`; it now takes all 39, including Brave, Tavily, Kagi, Serper, You.com, SearXNG and DuckDuckGo, and saves each key where pi-web-access reads it. `feynman search status` and `feynman doctor` name the provider you chose and list every saved key, and no longer describe an outdated automatic search order.
+- **pi-web-access 0.40.0.** Asking about a YouTube or local video works again when a thumbnail comes back with the transcript, excerpts no longer cut an emoji in half, and a new opt-in setting (`fetchContent.redirects: "same-origin"`) refuses redirects to another site.
+
 ## v0.5.43 - 2026-10-13
 
 - **Feynman reads and writes Pi's configuration only through Pi.** Settings now go through Pi's settings manager, which locks the file and changes only the keys Feynman sets, so a change Pi or another Feynman window makes at the same moment is no longer overwritten. Model checks, local-model detection and `feynman doctor` read your models through Pi, so comments in `models.json`, a `baseUrl` set on one model, and a key saved with `feynman model login` count the same way they do when Feynman runs.

package/dist/cli.js

@@ -24,7 +24,7 @@ import { BUNDLED_PI_PACKAGES, buildPiEnv, findWindowsBash, WINDOWS_BASH_MISSING_
 import { getConfiguredServiceTier, normalizeServiceTier, setConfiguredServiceTier } from "./model/service-tier.js";
 import { authenticateModelProvider, getCurrentModelSpec, isLocalModelProvider, loginModelProvider, logoutModelProvider, printModelList, selectDefaultModel, setDefaultModelSpec, } from "./model/commands.js";
 import { buildModelStatusSnapshotFromRecords, getAuthenticatedModelRecords, isProClassModelSpec, getSupportedModelRecords, } from "./model/catalog.js";
-import { clearSearchConfig, printSearchStatus, setSearchProvider } from "./search/commands.js";
+import { clearSearchConfig, printSearchStatus, searchSetUsage, setSearchProvider } from "./search/commands.js";
 import { fetchLatestFeynmanVersion, getFeynmanUpgradeLines, isNewerVersion } from "./system/self-update.js";
 import { runDoctor, runStatus } from "./setup/doctor.js";
 import { setupPreviewDependencies } from "./setup/preview.js";
@@ -307,12 +307,9 @@ function handleSearchCommand(subcommand, args) {
         return;
     }
     if (subcommand === "set") {
-        const provider = args[0];
-        const validProviders = ["auto", "perplexity", "exa", "gemini"];
-        if (!provider || !validProviders.includes(provider)) {
-            throw new Error("Usage: feynman search set <auto|perplexity|exa|gemini> [api-key]");
-        }
-        setSearchProvider(provider, args[1]);
+        if (!args[0])
+            throw new Error(searchSetUsage());
+        setSearchProvider(args[0], args[1]);
         return;
     }
     if (subcommand === "clear") {

package/dist/pi/web-access.js

@@ -2,13 +2,33 @@ import { chmodSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { readJsonFile } from "../config/json-file.js";
 import { getFeynmanAgentDir, getFeynmanHome } from "../config/paths.js";
+// pi-web-access's search providers (SEARCH_PROVIDERS in its gemini-search.ts;
+// a test keeps this list in step with the installed package).
+export const PI_WEB_SEARCH_PROVIDERS = [
+    "auto", "all", "openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit",
+    "tavily", "you", "firecrawl", "jina", "searxng", "degoog", "duckduckgo", "perplexity", "gemini", "kimi", "exa",
+    "serpdive", "kagi", "ollama", "anysearch", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply",
+    "valyu", "bocha", "xcrawl", "baizhi", "zai", "keenable", "ceramic",
+];
+// The web-search.json field holding a provider's key (searxng takes its base URL);
+// undefined for providers with nothing to set.
+export function searchProviderKeyField(provider) {
+    if (provider === "auto" || provider === "all" || provider === "duckduckgo" || provider === "kimi")
+        return undefined;
+    if (provider === "parallel-mcp")
+        return "parallelApiKey";
+    if (provider === "searxng")
+        return "searxngBaseUrl";
+    return `${provider}ApiKey`;
+}
+const ROUTE_LABELS = { auto: "Auto", perplexity: "Perplexity", exa: "Exa", gemini: "Gemini" };
 // pi-web-access reads web-search.json from PI_CODING_AGENT_DIR.
 export function getPiWebSearchConfigPath(home) {
     const feynmanHome = home ? resolve(home, ".feynman") : getFeynmanHome();
     return resolve(getFeynmanAgentDir(feynmanHome), "web-search.json");
 }
 function normalizeProvider(value) {
-    return value === "auto" || value === "perplexity" || value === "exa" || value === "gemini" ? value : undefined;
+    return PI_WEB_SEARCH_PROVIDERS.includes(value) ? value : undefined;
 }
 function normalizeWorkflow(value) {
     return value === "none" || value === "summary-review" ? value : undefined;
@@ -48,29 +68,12 @@ export function savePiWebAccessConfig(updates, configPath = getPiWebSearchConfig
         // ignore permission errors (best-effort)
     }
 }
-function formatRouteLabel(provider) {
-    switch (provider) {
-        case "perplexity":
-            return "Perplexity";
-        case "exa":
-            return "Exa";
-        case "gemini":
-            return "Gemini";
-        default:
-            return "Auto";
-    }
-}
 function formatRouteNote(provider) {
-    switch (provider) {
-        case "perplexity":
-            return "Pi web-access will use Perplexity for search.";
-        case "exa":
-            return "Pi web-access will use Exa for search.";
-        case "gemini":
-            return "Pi web-access will use Gemini API. Browser-cookie fallback is opt-in.";
-        default:
-            return "Pi web-access will try Exa, then Perplexity, then Gemini API. Browser-cookie fallback is opt-in.";
-    }
+    if (provider === "auto")
+        return "Pi web-access picks from the search providers you have keys or sign-ins for. Browser-cookie fallback is opt-in.";
+    if (provider === "all")
+        return "Pi web-access searches every eligible provider at once.";
+    return `Pi web-access will use ${ROUTE_LABELS[provider] ?? provider} for search.`;
 }
 export function getPiWebAccessStatus(config = loadPiWebAccessConfig(), configPath = getPiWebSearchConfigPath()) {
     const searchProvider = normalizeProvider(config.searchProvider) ?? normalizeProvider(config.route) ?? normalizeProvider(config.provider) ?? "auto";
@@ -82,6 +85,10 @@ export function getPiWebAccessStatus(config = loadPiWebAccessConfig(), configPat
     const chromeProfile = normalizeNonEmptyString(config.browserCookies?.profile);
     const geminiBrowserEnabled = config.allowBrowserCookies === true;
     const effectiveProvider = searchProvider;
+    const configuredKeys = PI_WEB_SEARCH_PROVIDERS.filter((provider) => {
+        const field = searchProviderKeyField(provider);
+        return field !== undefined && field.endsWith("ApiKey") && Boolean(normalizeNonEmptyString(config[field]));
+    }).filter((provider) => provider !== "parallel-mcp");
     return {
         configPath,
         configExists: existsSync(configPath),
@@ -93,7 +100,8 @@ export function getPiWebAccessStatus(config = loadPiWebAccessConfig(), configPat
         geminiApiConfigured,
         chromeProfile,
         geminiBrowserEnabled,
-        routeLabel: formatRouteLabel(effectiveProvider),
+        configuredKeys,
+        routeLabel: ROUTE_LABELS[effectiveProvider] ?? effectiveProvider,
         note: formatRouteNote(effectiveProvider),
     };
 }
@@ -104,18 +112,16 @@ export function formatPiWebAccessDoctorLines(status = getPiWebAccessStatus()) {
         `  search route: ${status.routeLabel}`,
         `  request route: ${status.requestProvider}`,
         `  search workflow: ${status.workflow}`,
-        `  perplexity api: ${status.perplexityConfigured ? "configured" : "not configured"}`,
-        `  exa api: ${status.exaConfigured ? "configured" : "not configured"}`,
-        `  gemini api: ${status.geminiApiConfigured ? "configured" : "not configured"}`,
+        `  saved api keys: ${status.configuredKeys.join(", ") || "none"}`,
         `  gemini browser fallback: ${status.geminiBrowserEnabled ? "enabled" : "disabled"}`,
         `  config path: ${status.configPath}${configPathSuffix}`,
         `  note: ${status.note}`,
     ];
     if (status.geminiBrowserEnabled && status.chromeProfile) {
-        lines.splice(8, 0, `  gemini browser profile: ${status.chromeProfile}`);
+        lines.splice(6, 0, `  gemini browser profile: ${status.chromeProfile}`);
     }
     if (!status.configExists) {
-        lines.push("  hint: run `feynman search set <auto|perplexity|exa|gemini> [api-key]` to configure web search");
+        lines.push("  hint: run `feynman search set <provider> [api-key]` to configure web search (`feynman search set` lists providers)");
     }
     return lines;
 }

package/dist/search/commands.js

@@ -1,20 +1,15 @@
-import { getPiWebAccessStatus, savePiWebAccessConfig, } from "../pi/web-access.js";
+import { getPiWebAccessStatus, PI_WEB_SEARCH_PROVIDERS, savePiWebAccessConfig, searchProviderKeyField, } from "../pi/web-access.js";
 import { printInfo } from "../ui/terminal.js";
-const SEARCH_PROVIDERS = ["auto", "perplexity", "exa", "gemini"];
-const PROVIDER_API_KEY_FIELDS = {
-    perplexity: "perplexityApiKey",
-    exa: "exaApiKey",
-    gemini: "geminiApiKey",
-};
+export function searchSetUsage() {
+    return `Usage: feynman search set <provider> [api-key]\nProviders: ${PI_WEB_SEARCH_PROVIDERS.join(", ")}`;
+}
 export function printSearchStatus(status = getPiWebAccessStatus()) {
     const configPathSuffix = status.configExists ? "" : " (not created yet)";
     printInfo("Managed by: pi-web-access");
     printInfo(`Search route: ${status.routeLabel}`);
     printInfo(`Request route: ${status.requestProvider}`);
     printInfo(`Search workflow: ${status.workflow}`);
-    printInfo(`Perplexity API configured: ${status.perplexityConfigured ? "yes" : "no"}`);
-    printInfo(`Exa API configured: ${status.exaConfigured ? "yes" : "no"}`);
-    printInfo(`Gemini API configured: ${status.geminiApiConfigured ? "yes" : "no"}`);
+    printInfo(`Saved API keys: ${status.configuredKeys.join(", ") || "none"}`);
     printInfo(`Gemini browser fallback: ${status.geminiBrowserEnabled ? "enabled" : "disabled"}`);
     if (status.geminiBrowserEnabled && status.chromeProfile) {
         printInfo(`Gemini browser profile: ${status.chromeProfile}`);
@@ -23,17 +18,16 @@ export function printSearchStatus(status = getPiWebAccessStatus()) {
     if (!status.configExists) {
         printInfo("Not configured yet. Run one of:");
         printInfo("  feynman search set auto");
-        printInfo("  feynman search set perplexity <api-key>");
-        printInfo("  feynman search set exa <api-key>");
-        printInfo("  feynman search set gemini <api-key>");
+        printInfo("  feynman search set <provider> <api-key>   (for example exa, brave, tavily, perplexity)");
     }
 }
 export function setSearchProvider(provider, apiKey) {
-    if (!SEARCH_PROVIDERS.includes(provider)) {
-        throw new Error(`Usage: feynman search set <${SEARCH_PROVIDERS.join("|")}> [api-key]`);
+    if (!PI_WEB_SEARCH_PROVIDERS.includes(provider)) {
+        throw new Error(searchSetUsage());
     }
-    if (apiKey !== undefined && provider === "auto") {
-        throw new Error("The auto provider does not use an API key. Usage: feynman search set auto");
+    const keyField = searchProviderKeyField(provider);
+    if (apiKey !== undefined && !keyField) {
+        throw new Error(`${provider} takes no API key. Usage: feynman search set ${provider}`);
     }
     const updates = {
         provider,
@@ -42,9 +36,8 @@ export function setSearchProvider(provider, apiKey) {
         allowBrowserCookies: false,
         route: undefined,
     };
-    const apiKeyField = PROVIDER_API_KEY_FIELDS[provider];
-    if (apiKeyField && apiKey !== undefined) {
-        updates[apiKeyField] = apiKey;
+    if (keyField && apiKey !== undefined) {
+        updates[keyField] = apiKey;
     }
     savePiWebAccessConfig(updates);
     const status = getPiWebAccessStatus();

package/extensions/research-tools/science-database-arxiv.ts

@@ -35,6 +35,20 @@ function cleanQuery(query: string): string {
 const arxivPacer = createRequestPacer();
 export const ARXIV_MIN_GAP_MS = 3000;
 
+// The query held no arXiv ID; the caller searches another index for `topic`.
+export class ArxivTopicQueryError extends Error {
+	constructor(message: string, readonly topic: string) {
+		super(message);
+	}
+}
+
+// arXiv timed out or answered 429/5xx; the caller can look `ids` up elsewhere.
+export class ArxivUnavailableError extends Error {
+	constructor(message: string, readonly ids: string[]) {
+		super(message);
+	}
+}
+
 async function fetchText(url: URL, accept: string): Promise<string> {
 	for (let attempt = 0; ; attempt += 1) {
 		const result = await arxivPacer(ARXIV_MIN_GAP_MS, async () => {
@@ -157,13 +171,25 @@ async function getArxivPapers(query: string): Promise<Record<string, unknown>> {
 		else notFound.push(raw);
 	}
 	if (!ids.length) {
-		throw new Error(`No arXiv IDs found in "${query}". The arxiv source only looks up papers by ID; search topics with source semanticscholar or openalex.`);
+		throw new ArxivTopicQueryError(
+			`No arXiv IDs found in "${query}". The arxiv source only looks up papers by ID; search topics with source semanticscholar or openalex.`,
+			parsed.text || stripGetPapersCommand(query),
+		);
 	}
 	const rows: Record<string, unknown>[] = [];
 	const duplicates: Array<Record<string, unknown>> = [];
 	const url = new URL(ARXIV_QUERY_URL);
 	url.search = new URLSearchParams({ id_list: ids.join(","), max_results: String(ids.length) }).toString();
-	const xml = await fetchText(url, "application/atom+xml, application/xml;q=0.9, text/xml;q=0.8");
+	let xml: string;
+	try {
+		xml = await fetchText(url, "application/atom+xml, application/xml;q=0.9, text/xml;q=0.8");
+	} catch (error) {
+		const reason = error instanceof Error && error.name === "AbortError" ? `no answer within ${REQUEST_TIMEOUT_MS / 1000} s` : error instanceof Error ? error.message : String(error);
+		if (error instanceof Error && (error.name === "AbortError" || /request failed: (?:429|5\d\d)\b/.test(error.message))) {
+			throw new ArxivUnavailableError(reason, ids);
+		}
+		throw error;
+	}
 	const feed = recordValue(xmlParser.parse(xml).feed);
 	const byId = new Map<string, Record<string, unknown>>();
 	for (const row of arxivFeedEntries(feed).map(parseArxivEntry).filter((item): item is Record<string, unknown> => Boolean(item))) {

package/extensions/research-tools/science-databases.ts

@@ -1,6 +1,6 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { searchArxiv } from "./science-database-arxiv.js";
+import { ArxivTopicQueryError, ArxivUnavailableError, searchArxiv } from "./science-database-arxiv.js";
 import { searchOpenAlex } from "./science-database-openalex.js";
 import { searchPreprints } from "./science-database-preprints.js";
 import { searchPubMed } from "./science-database-pubmed.js";
@@ -322,6 +322,29 @@ function semanticScholarSearchUrl(query: string, limit: number, sort: string, ye
 
 // Bulk search sorted by citation count surfaces seminal papers that relevance
 // ranking misses; it returns up to 1,000 rows, so only the top `limit` are kept.
+function semanticScholarRecord(item: unknown): Record<string, unknown>[] {
+	const record = recordValue(item);
+	const paperId = stringValue(record.paperId);
+	if (!paperId) return [];
+	const externalIds = recordValue(record.externalIds);
+	const doi = stringValue(externalIds.DOI);
+	return [{
+		paperId,
+		title: stringValue(record.title),
+		year: numberValue(record.year),
+		authors: arrayValue(record.authors).map((author) => stringValue(recordValue(author).name)).filter(Boolean).slice(0, 8),
+		venue: stringValue(record.venue),
+		citationCount: numberValue(record.citationCount),
+		doi,
+		arxivId: stringValue(externalIds.ArXiv),
+		pmid: stringValue(externalIds.PubMed),
+		openAccessPdf: stringValue(recordValue(record.openAccessPdf).url),
+		abstract: truncateText(stringValue(record.abstract), SEMANTIC_SCHOLAR_ABSTRACT_CHARS),
+		url: `https://www.semanticscholar.org/paper/${paperId}`,
+		...(doiUrl(doi) ? { doiUrl: doiUrl(doi) } : {}),
+	}];
+}
+
 async function searchSemanticScholar(params: ScienceDatabaseSearchParams): Promise<Record<string, unknown>> {
 	const { query, year } = parseSemanticScholarQuery(cleanQuery(params.query));
 	const limit = safeLimit(params.limit);
@@ -340,28 +363,7 @@ async function searchSemanticScholar(params: ScienceDatabaseSearchParams): Promi
 		note = `Relevance search was rate-limited (HTTP 429) on Semantic Scholar's shared anonymous pool, so these are citation-sorted bulk results. A free SEMANTIC_SCHOLAR_API_KEY avoids this: ${SEMANTIC_SCHOLAR_KEY_URL}`;
 		payload = recordValue(await fetchSemanticScholar(endpoints[1]!));
 	}
-	const results = arrayValue(payload.data).slice(0, limit).flatMap((item) => {
-		const record = recordValue(item);
-		const paperId = stringValue(record.paperId);
-		if (!paperId) return [];
-		const externalIds = recordValue(record.externalIds);
-		const doi = stringValue(externalIds.DOI);
-		return [{
-			paperId,
-			title: stringValue(record.title),
-			year: numberValue(record.year),
-			authors: arrayValue(record.authors).map((author) => stringValue(recordValue(author).name)).filter(Boolean).slice(0, 8),
-			venue: stringValue(record.venue),
-			citationCount: numberValue(record.citationCount),
-			doi,
-			arxivId: stringValue(externalIds.ArXiv),
-			pmid: stringValue(externalIds.PubMed),
-			openAccessPdf: stringValue(recordValue(record.openAccessPdf).url),
-			abstract: truncateText(stringValue(record.abstract), SEMANTIC_SCHOLAR_ABSTRACT_CHARS),
-			url: `https://www.semanticscholar.org/paper/${paperId}`,
-			...(doiUrl(doi) ? { doiUrl: doiUrl(doi) } : {}),
-		}];
-	});
+	const results = arrayValue(payload.data).slice(0, limit).flatMap(semanticScholarRecord);
 	return {
 		schema: "feynman.scienceDatabaseSearch.v1",
 		source: "semanticscholar",
@@ -418,8 +420,49 @@ async function withIndexFallback(
 	}
 }
 
+// arXiv IDs looked up on Semantic Scholar, one request per ID (at most 10).
+async function semanticScholarByArxivIds(ids: string[]): Promise<Record<string, unknown>> {
+	const endpoints: string[] = [];
+	const results: Record<string, unknown>[] = [];
+	const notFound: string[] = [];
+	for (const id of ids.slice(0, 10)) {
+		const bare = id.replace(/v\d+$/, "");
+		const url = new URL(`${SEMANTIC_SCHOLAR_BASE}/paper/ARXIV:${bare}`);
+		url.searchParams.set("fields", SEMANTIC_SCHOLAR_FIELDS);
+		endpoints.push(url.toString());
+		try {
+			results.push(...semanticScholarRecord(await fetchSemanticScholar(url)));
+		} catch (error) {
+			if (!(error instanceof ScienceDatabaseRequestError) || error.status !== 404) throw error;
+			notFound.push(id);
+		}
+	}
+	return { schema: "feynman.scienceDatabaseSearch.v1", source: "semanticscholar", query: ids.join(","), not_found: notFound, results, provenance: { endpoints } };
+}
+
+async function searchArxivWithFallback(params: ScienceDatabaseSearchParams): Promise<Record<string, unknown>> {
+	try {
+		return await searchArxiv(params);
+	} catch (error) {
+		if (error instanceof ArxivTopicQueryError) {
+			const result = await searchOpenAlex({ limit: params.limit, query: `semantic: ${error.topic}`, source: "openalex" });
+			return { fallbackNote: "The arxiv source only looks up papers by ID, so this topic was searched with OpenAlex semantic search instead.", ...result };
+		}
+		if (error instanceof ArxivUnavailableError) {
+			let result: Record<string, unknown>;
+			try {
+				result = await semanticScholarByArxivIds(error.ids);
+			} catch {
+				throw error;
+			}
+			return { fallbackNote: `arXiv was unavailable (${error.message}); these records are from Semantic Scholar.`, ...result };
+		}
+		throw error;
+	}
+}
+
 async function scienceDatabaseSearch(params: ScienceDatabaseSearchParams): Promise<Record<string, unknown>> {
-	if (params.source === "arxiv") return searchArxiv(params);
+	if (params.source === "arxiv") return searchArxivWithFallback(params);
 	if (params.source === "biorxiv") return searchPreprints(params, "biorxiv");
 	if (params.source === "medrxiv") return searchPreprints(params, "medrxiv");
 	if (params.source === "pubmed") return searchPubMed(params);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@companion-ai/feynman",
-  "version": "0.5.43",
+  "version": "0.5.45",
   "description": "The open source AI research agent.",
   "license": "MIT",
   "type": "module",
@@ -82,11 +82,11 @@
     "@earendil-works/pi-codemode": "1.2.0",
     "@earendil-works/pi-mcp": "1.2.0",
     "@earendil-works/pi-telemetry": "1.2.0",
-    "fast-xml-parser": "5.11.1",
+    "fast-xml-parser": "5.11.2",
     "pi-btw": "0.8.0",
     "pi-docparser": "4.0.0",
     "pi-subagents": "0.78.0",
-    "pi-web-access": "0.39.0",
+    "pi-web-access": "0.40.0",
     "posthog-node": "^5.53.0"
   },
   "peerDependencies": {