@companion-ai/feynman 0.5.44 → 0.5.46

package/RELEASES.md

@@ -6,6 +6,17 @@ GitHub release notes are generated from the matching `## vX.Y.Z` section in this
 
 ## Unreleased
 
+## v0.5.46 - 2026-10-13
+
+- **Recent arXiv papers resolve by DOI.** OpenAlex adds new arXiv preprints days late, so looking up an arXiv DOI (`10.48550/arXiv.<id>`) it doesn't have yet failed. It is now looked up on arXiv itself, with a note.
+- When a topic sent to the arXiv source is written in arXiv's own query syntax (`search_query=all:...`, `submittedDate:[...]`), only the search words are passed on to OpenAlex.
+
+## v0.5.45 - 2026-10-13
+
+- **Paper lookups recover instead of failing.**
+  - A topic sent to the arXiv source, which only looks papers up by ID, now gets OpenAlex semantic search results with a note saying so, instead of an error the model had to recover from.
+  - When arXiv times out or is overloaded, arXiv ID lookups come from Semantic Scholar, with a note saying so.
+
 ## v0.5.44 - 2026-10-13
 
 - **`feynman search set` works with every search provider pi-web-access supports.** It accepted only `auto`, `exa`, `perplexity` and `gemini`; it now takes all 39, including Brave, Tavily, Kagi, Serper, You.com, SearXNG and DuckDuckGo, and saves each key where pi-web-access reads it. `feynman search status` and `feynman doctor` name the provider you chose and list every saved key, and no longer describe an outdated automatic search order.

package/extensions/research-tools/science-database-arxiv.ts

@@ -35,6 +35,20 @@ function cleanQuery(query: string): string {
 const arxivPacer = createRequestPacer();
 export const ARXIV_MIN_GAP_MS = 3000;
 
+// The query held no arXiv ID; the caller searches another index for `topic`.
+export class ArxivTopicQueryError extends Error {
+	constructor(message: string, readonly topic: string) {
+		super(message);
+	}
+}
+
+// arXiv timed out or answered 429/5xx; the caller can look `ids` up elsewhere.
+export class ArxivUnavailableError extends Error {
+	constructor(message: string, readonly ids: string[]) {
+		super(message);
+	}
+}
+
 async function fetchText(url: URL, accept: string): Promise<string> {
 	for (let attempt = 0; ; attempt += 1) {
 		const result = await arxivPacer(ARXIV_MIN_GAP_MS, async () => {
@@ -145,6 +159,19 @@ function arxivFeedEntries(feed: Record<string, unknown>): Record<string, unknown
 	return entries;
 }
 
+// Plain search words from a query written in arXiv's API syntax
+// (search_query=all:..., AND submittedDate:[...], &max_results=...).
+export function arxivTopicText(query: string): string {
+	return query
+		.replace(/&?\b(?:max_results|start|sortBy|sortOrder)=[^&\s]*/gi, " ")
+		.replace(/\b(?:AND|OR|ANDNOT)\s+submittedDate:\[[^\]]*\]/gi, " ")
+		.replace(/\bsearch_query[=_:]+/gi, " ")
+		.replace(/\b(?:all|ti|abs|au|cat|co|jr):/gi, " ")
+		.replace(/[&"()]/g, " ")
+		.replace(/\s+/g, " ")
+		.trim();
+}
+
 async function getArxivPapers(query: string): Promise<Record<string, unknown>> {
 	const parsed = parseToolKeyValueQuery(stripGetPapersCommand(query));
 	const requested = (parsed.flags.ids ?? parsed.text).split(/[,\s]+/).map((id) => id.trim()).filter(Boolean).slice(0, MAX_ARXIV_EXACT_RESULTS);
@@ -157,13 +184,25 @@ async function getArxivPapers(query: string): Promise<Record<string, unknown>> {
 		else notFound.push(raw);
 	}
 	if (!ids.length) {
-		throw new Error(`No arXiv IDs found in "${query}". The arxiv source only looks up papers by ID; search topics with source semanticscholar or openalex.`);
+		throw new ArxivTopicQueryError(
+			`No arXiv IDs found in "${query}". The arxiv source only looks up papers by ID; search topics with source semanticscholar or openalex.`,
+			arxivTopicText(parsed.text || stripGetPapersCommand(query)),
+		);
 	}
 	const rows: Record<string, unknown>[] = [];
 	const duplicates: Array<Record<string, unknown>> = [];
 	const url = new URL(ARXIV_QUERY_URL);
 	url.search = new URLSearchParams({ id_list: ids.join(","), max_results: String(ids.length) }).toString();
-	const xml = await fetchText(url, "application/atom+xml, application/xml;q=0.9, text/xml;q=0.8");
+	let xml: string;
+	try {
+		xml = await fetchText(url, "application/atom+xml, application/xml;q=0.9, text/xml;q=0.8");
+	} catch (error) {
+		const reason = error instanceof Error && error.name === "AbortError" ? `no answer within ${REQUEST_TIMEOUT_MS / 1000} s` : error instanceof Error ? error.message : String(error);
+		if (error instanceof Error && (error.name === "AbortError" || /request failed: (?:429|5\d\d)\b/.test(error.message))) {
+			throw new ArxivUnavailableError(reason, ids);
+		}
+		throw error;
+	}
 	const feed = recordValue(xmlParser.parse(xml).feed);
 	const byId = new Map<string, Record<string, unknown>>();
 	for (const row of arxivFeedEntries(feed).map(parseArxivEntry).filter((item): item is Record<string, unknown> => Boolean(item))) {

package/extensions/research-tools/science-databases.ts

@@ -1,6 +1,6 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { searchArxiv } from "./science-database-arxiv.js";
+import { ArxivTopicQueryError, ArxivUnavailableError, searchArxiv } from "./science-database-arxiv.js";
 import { searchOpenAlex } from "./science-database-openalex.js";
 import { searchPreprints } from "./science-database-preprints.js";
 import { searchPubMed } from "./science-database-pubmed.js";
@@ -322,6 +322,29 @@ function semanticScholarSearchUrl(query: string, limit: number, sort: string, ye
 
 // Bulk search sorted by citation count surfaces seminal papers that relevance
 // ranking misses; it returns up to 1,000 rows, so only the top `limit` are kept.
+function semanticScholarRecord(item: unknown): Record<string, unknown>[] {
+	const record = recordValue(item);
+	const paperId = stringValue(record.paperId);
+	if (!paperId) return [];
+	const externalIds = recordValue(record.externalIds);
+	const doi = stringValue(externalIds.DOI);
+	return [{
+		paperId,
+		title: stringValue(record.title),
+		year: numberValue(record.year),
+		authors: arrayValue(record.authors).map((author) => stringValue(recordValue(author).name)).filter(Boolean).slice(0, 8),
+		venue: stringValue(record.venue),
+		citationCount: numberValue(record.citationCount),
+		doi,
+		arxivId: stringValue(externalIds.ArXiv),
+		pmid: stringValue(externalIds.PubMed),
+		openAccessPdf: stringValue(recordValue(record.openAccessPdf).url),
+		abstract: truncateText(stringValue(record.abstract), SEMANTIC_SCHOLAR_ABSTRACT_CHARS),
+		url: `https://www.semanticscholar.org/paper/${paperId}`,
+		...(doiUrl(doi) ? { doiUrl: doiUrl(doi) } : {}),
+	}];
+}
+
 async function searchSemanticScholar(params: ScienceDatabaseSearchParams): Promise<Record<string, unknown>> {
 	const { query, year } = parseSemanticScholarQuery(cleanQuery(params.query));
 	const limit = safeLimit(params.limit);
@@ -340,28 +363,7 @@ async function searchSemanticScholar(params: ScienceDatabaseSearchParams): Promi
 		note = `Relevance search was rate-limited (HTTP 429) on Semantic Scholar's shared anonymous pool, so these are citation-sorted bulk results. A free SEMANTIC_SCHOLAR_API_KEY avoids this: ${SEMANTIC_SCHOLAR_KEY_URL}`;
 		payload = recordValue(await fetchSemanticScholar(endpoints[1]!));
 	}
-	const results = arrayValue(payload.data).slice(0, limit).flatMap((item) => {
-		const record = recordValue(item);
-		const paperId = stringValue(record.paperId);
-		if (!paperId) return [];
-		const externalIds = recordValue(record.externalIds);
-		const doi = stringValue(externalIds.DOI);
-		return [{
-			paperId,
-			title: stringValue(record.title),
-			year: numberValue(record.year),
-			authors: arrayValue(record.authors).map((author) => stringValue(recordValue(author).name)).filter(Boolean).slice(0, 8),
-			venue: stringValue(record.venue),
-			citationCount: numberValue(record.citationCount),
-			doi,
-			arxivId: stringValue(externalIds.ArXiv),
-			pmid: stringValue(externalIds.PubMed),
-			openAccessPdf: stringValue(recordValue(record.openAccessPdf).url),
-			abstract: truncateText(stringValue(record.abstract), SEMANTIC_SCHOLAR_ABSTRACT_CHARS),
-			url: `https://www.semanticscholar.org/paper/${paperId}`,
-			...(doiUrl(doi) ? { doiUrl: doiUrl(doi) } : {}),
-		}];
-	});
+	const results = arrayValue(payload.data).slice(0, limit).flatMap(semanticScholarRecord);
 	return {
 		schema: "feynman.scienceDatabaseSearch.v1",
 		source: "semanticscholar",
@@ -418,8 +420,65 @@ async function withIndexFallback(
 	}
 }
 
+// arXiv IDs looked up on Semantic Scholar, one request per ID (at most 10).
+async function semanticScholarByArxivIds(ids: string[]): Promise<Record<string, unknown>> {
+	const endpoints: string[] = [];
+	const results: Record<string, unknown>[] = [];
+	const notFound: string[] = [];
+	for (const id of ids.slice(0, 10)) {
+		const bare = id.replace(/v\d+$/, "");
+		const url = new URL(`${SEMANTIC_SCHOLAR_BASE}/paper/ARXIV:${bare}`);
+		url.searchParams.set("fields", SEMANTIC_SCHOLAR_FIELDS);
+		endpoints.push(url.toString());
+		try {
+			results.push(...semanticScholarRecord(await fetchSemanticScholar(url)));
+		} catch (error) {
+			if (!(error instanceof ScienceDatabaseRequestError) || error.status !== 404) throw error;
+			notFound.push(id);
+		}
+	}
+	return { schema: "feynman.scienceDatabaseSearch.v1", source: "semanticscholar", query: ids.join(","), not_found: notFound, results, provenance: { endpoints } };
+}
+
+async function searchArxivWithFallback(params: ScienceDatabaseSearchParams): Promise<Record<string, unknown>> {
+	try {
+		return await searchArxiv(params);
+	} catch (error) {
+		if (error instanceof ArxivTopicQueryError) {
+			const result = await searchOpenAlex({ limit: params.limit, query: `semantic: ${error.topic}`, source: "openalex" });
+			return { fallbackNote: "The arxiv source only looks up papers by ID, so this topic was searched with OpenAlex semantic search instead.", ...result };
+		}
+		if (error instanceof ArxivUnavailableError) {
+			let result: Record<string, unknown>;
+			try {
+				result = await semanticScholarByArxivIds(error.ids);
+			} catch {
+				throw error;
+			}
+			return { fallbackNote: `arXiv was unavailable (${error.message}); these records are from Semantic Scholar.`, ...result };
+		}
+		throw error;
+	}
+}
+
+// OpenAlex indexes new arXiv preprints days late; arXiv itself answers for its
+// own DOIs (10.48550/arXiv.<id>).
+async function withArxivDoiFallback(
+	params: ScienceDatabaseSearchParams,
+	run: () => Promise<Record<string, unknown>>,
+): Promise<Record<string, unknown>> {
+	try {
+		return await run();
+	} catch (error) {
+		const arxivId = error instanceof Error ? /^OpenAlex has no work for doi:10\.48550\/arxiv\.(.+)$/i.exec(error.message)?.[1] : undefined;
+		if (!arxivId) throw error;
+		const result = await searchArxivWithFallback({ ...params, source: "arxiv", query: arxivId });
+		return { fallbackNote: `OpenAlex has no record of this arXiv DOI yet, so it was looked up on arXiv (${arxivId}).`, ...result };
+	}
+}
+
 async function scienceDatabaseSearch(params: ScienceDatabaseSearchParams): Promise<Record<string, unknown>> {
-	if (params.source === "arxiv") return searchArxiv(params);
+	if (params.source === "arxiv") return searchArxivWithFallback(params);
 	if (params.source === "biorxiv") return searchPreprints(params, "biorxiv");
 	if (params.source === "medrxiv") return searchPreprints(params, "medrxiv");
 	if (params.source === "pubmed") return searchPubMed(params);
@@ -430,11 +489,11 @@ async function scienceDatabaseSearch(params: ScienceDatabaseSearchParams): Promi
 		return withIndexFallback(semanticScholar, openAlex, (reason) => `Semantic Scholar was unavailable (${reason}); these results are from OpenAlex.`);
 	}
 	if (params.source === "openalex") {
-		return withIndexFallback(
+		return withArxivDoiFallback(params, () => withIndexFallback(
 			openAlex,
 			OPENALEX_LOOKUP_QUERY.test(params.query.trim()) ? undefined : semanticScholar,
 			(reason) => `OpenAlex was unavailable (${reason}); these results are from Semantic Scholar.`,
-		);
+		));
 	}
 	return searchEuropePmc(params);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@companion-ai/feynman",
-  "version": "0.5.44",
+  "version": "0.5.46",
   "description": "The open source AI research agent.",
   "license": "MIT",
   "type": "module",
@@ -82,7 +82,7 @@
     "@earendil-works/pi-codemode": "1.2.0",
     "@earendil-works/pi-mcp": "1.2.0",
     "@earendil-works/pi-telemetry": "1.2.0",
-    "fast-xml-parser": "5.11.1",
+    "fast-xml-parser": "5.11.2",
     "pi-btw": "0.8.0",
     "pi-docparser": "4.0.0",
     "pi-subagents": "0.78.0",