one-code-extension 0.4.1 → 0.5.0

package/extensions/web/index.ts

@@ -4,6 +4,11 @@
  * `web_search` is registered here with Claude Code's schema (`query`,
  * `allowed_domains`, `blocked_domains`) and routes by what the session can do:
  *
+ *   0. Anthropic's server-side `web_search` with dynamic filtering, in our own
+ *      nested call, on a first-party Anthropic API-key session with a Claude
+ *      4.6+ model available (lib/anthropic-server-tools.ts). pi-web-search's
+ *      Anthropic call uses the older `web_search_20250305`, so it is the next
+ *      step down, not this one.
  *   1. Provider-native search through the community `pi-web-search` package
  *      when the current model's provider has one (Anthropic, OpenAI/Codex,
  *      Gemini, xAI) — no third-party key, and as close to Claude Code's
@@ -23,18 +28,24 @@
  * WebFetch lives in `extensions/web-fetch` (our own); this file only owns search.
  */
 
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { defineTool, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { homedir } from "node:os";
 import { Type } from "typebox";
 import webSearchPackage from "pi-web-search/src/index.ts";
-import { getProviderKind } from "pi-web-search/src/api.ts";
-import { webSearch as nativeWebSearch } from "pi-web-search/src/web_search.ts";
+import { callApiStream, getConfig, getProviderKind } from "pi-web-search/src/api.ts";
 import { getWebSearchModel } from "pi-web-search/src/utils.ts";
+import { tryNativeWeb } from "../lib/anthropic-server-call.ts";
+import { CUT_OFF_NOTE, nativeSearchBody, searchOutcome, searchSources, sourceLine, type ThinkingFields } from "../lib/anthropic-server-tools.ts";
 import { readJsonFile } from "../lib/atomic-write.ts";
-import { DEFER_CHANNEL } from "../lib/deferred.ts";
+import { recordUsage } from "../lib/usage-bus.ts";
+import { DEFER_CHANNEL, WITHHOLD_CHANNEL } from "../lib/deferred.ts";
 import { oneCodeSettingsPath } from "../lib/one-code-settings.ts";
 import { persistIfLarge, sessionResultsDir } from "../lib/persisted-output.ts";
 import { ccToolRenderers } from "../lib/tui-render.ts";
+import { registerFormTool } from "../lib/tool-variants.ts";
+import { webSearchDescription } from "./description.ts";
+import { formatNativeSearchResult } from "./native-results.ts";
+import { runNativeSearch } from "./native-call.ts";
 import {
 	DEFAULT_MAX_RESULTS,
 	formatSearchResults,
@@ -68,16 +79,15 @@ export default function webExtension(pi: ExtensionAPI) {
 
 	// Once per session: the user is told the first time a search goes keyless.
 	let keylessNoticeShown = false;
+	// Per-session: the thinking-off fields a native-call model turned out to need.
+	const learnedNativeThinking = new Map<string, ThinkingFields>();
 
-	pi.registerTool({
+	// Short or long text by the model's tier (description.ts, lib/tool-variants.ts).
+	const tool = defineTool({
 		name: "web_search",
 		label: "Web Search",
 		...ccToolRenderers<{ query?: string }>("Web Search", { title: (args) => args?.query }),
-		description:
-			"Search the web. Returns result blocks with titles and URLs.\n\n" +
-			"- `allowed_domains` / `blocked_domains` filter results (enforced on Brave/Tavily/Exa; with provider-native search they become `site:` operators in the query, best-effort).\n" +
-			'- After answering from results, end with a "Sources:" list of the URLs you used as markdown links.\n' +
-			"- Uses the model provider's own search when it has one; otherwise a configured search API (Brave, Tavily) or, with no key, a free rate-limited endpoint — the result names which.",
+		description: webSearchDescription("short"),
 		parameters: Type.Object({
 			query: Type.String({ minLength: 2, description: "The search query to use" }),
 			allowed_domains: Type.Optional(Type.Array(Type.String(), { description: "Only include search results from these domains" })),
@@ -86,37 +96,90 @@ export default function webExtension(pi: ExtensionAPI) {
 		async execute(toolCallId, params, signal, onUpdate, ctx) {
 			const filters = { allowed: params.allowed_domains, blocked: params.blocked_domains };
 
-			// 1. Provider-native search (pi-web-search) when the current model — or a
-			// web-search.json model — supports it. Its result text is passed through;
-			// it reports failures as "Failed: …" text with details.error and no
-			// isError, which a weak model can read as a successful search with odd
-			// content, so isError is stamped here.
+			// 0. Anthropic's server-side search with dynamic filtering, on a
+			// first-party Anthropic API-key session. The domain filters are enforced
+			// by the search itself; it takes one list or the other, so a call with
+			// both goes down the paths below. Anything short of an answer from real
+			// results falls through, with a note saying why.
+			let nativeNote: string | undefined;
+			const bothFilters = Boolean(params.allowed_domains?.length && params.blocked_domains?.length);
+			if (!bothFilters) {
+				const native = await tryNativeWeb(ctx, {
+					body: (model) =>
+						nativeSearchBody({ model, query: params.query, allowedDomains: params.allowed_domains, blockedDomains: params.blocked_domains }),
+					outcome: searchOutcome,
+					onUsage: (usage) => recordUsage(pi, "web-search", usage),
+					learned: learnedNativeThinking,
+					onStart: () =>
+						onUpdate?.({
+							content: [{ type: "text", text: `Searching with Anthropic's web search for "${params.query}"...` }],
+							details: { query: params.query } as SearchDetails,
+						}),
+					signal,
+				});
+				if (native.kind === "cancelled") {
+					return {
+						content: [{ type: "text", text: `Search for "${params.query}" was cancelled.` }],
+						details: { query: params.query } as SearchDetails,
+						isError: true,
+					};
+				}
+				if (native.kind === "answered") {
+					const sources = searchSources(native.result);
+					const filtered = Boolean(params.allowed_domains?.length || params.blocked_domains?.length);
+					const text = [
+						native.text,
+						...(native.cutOff ? ["", CUT_OFF_NOTE] : []),
+						"",
+						"Sources:",
+						...sources.map(sourceLine),
+						"",
+						`(Searched with Anthropic's server-side web search with dynamic filtering, on ${native.spec}${filtered ? "; the domain filters were enforced by the search" : ""}.)`,
+					].join("\n");
+					return {
+						content: [{ type: "text", text: persistIfLarge(text, { dir: sessionResultsDir(ctx), id: toolCallId }) }],
+						details: { query: params.query, native: true, backend: `anthropic web_search via ${native.spec}`, resultCount: sources.length } as SearchDetails,
+					};
+				}
+				if (native.kind === "fell-back") {
+					nativeNote = `(Anthropic's server-side web search did not answer (${native.reason}); the results below come from the next search route.)`;
+				}
+			}
+			const withNativeNote = (text: string) => nativeNote ? `${text}\n\n${nativeNote}` : text;
+
+			// 1. Reuse pi-web-search's transport, not its tool wrapper: that wrapper
+			// truncates before we can persist the full answer and its sources.
 			const nativeModel = await getWebSearchModel(ctx);
 			if (nativeModel) {
-				const result = await nativeWebSearch(
-					toolCallId,
-					{ query: withSiteOperators(params.query, filters) },
-					signal ?? new AbortController().signal,
-					onUpdate,
-					ctx,
-				);
-				// pi-web-search exposes no domain parameters and returns a synthesized
-				// answer (nothing to post-filter), so the filters ride as `site:`
-				// operators only — say so, rather than let the model assume enforcement.
-				const hasFilters = Boolean(params.allowed_domains?.length || params.blocked_domains?.length);
-				const content =
-					hasFilters && !result.details?.error
-						? [
-								...result.content,
-								{ type: "text" as const, text: "(Domain filters were applied as `site:` operators in the query — best-effort with provider-native search; verify the sources' hosts.)" },
-							]
-						: result.content;
-				return {
-					...result,
-					content,
-					details: { ...result.details, query: params.query, native: true, backend: `${nativeModel.provider}/${nativeModel.id}` } as SearchDetails,
-					isError: Boolean(result.isError) || Boolean(result.details?.error),
-				};
+				const backend = `${nativeModel.provider}/${nativeModel.id}`;
+				try {
+					const config = getConfig(nativeModel);
+					const response = await runNativeSearch((nativeSignal) => callApiStream(ctx, nativeModel, {
+						contents: [{ role: "user", parts: [{ text: withSiteOperators(params.query, filters) }] }],
+						...(config.kind === "google" ? { tools: [{ [config.searchTool!]: {} }] } : {}),
+					}, (update) => { if (!nativeSignal.aborted) onUpdate?.(update); }, nativeSignal), signal);
+					const result = formatNativeSearchResult(response);
+					// This route returns a synthesized answer, not rows we can filter.
+					const hasFilters = Boolean(params.allowed_domains?.length || params.blocked_domains?.length);
+					const text = withNativeNote([
+						result.text,
+						`(Searched with ${backend}.)`,
+						...(hasFilters ? ["(Domain filters were applied as `site:` operators in the query — best-effort with provider-native search; verify the sources' hosts.)"] : []),
+					].join("\n\n"));
+					return {
+						content: [{ type: "text", text: persistIfLarge(text, { dir: sessionResultsDir(ctx), id: toolCallId }) }],
+						details: { ...result.details, query: params.query, native: true, backend } as SearchDetails,
+					};
+				} catch (error) {
+					if (signal?.aborted) {
+						return {
+							content: [{ type: "text", text: `Search for "${params.query}" was cancelled.` }],
+							details: { query: params.query } as SearchDetails,
+							isError: true,
+						};
+					}
+					nativeNote = withNativeNote(`(Provider-native search on ${backend} failed (${error instanceof Error ? error.message : String(error)}); the next search route was tried.)`);
+				}
 			}
 
 			// 2. Third-party chain.
@@ -133,7 +196,7 @@ export default function webExtension(pi: ExtensionAPI) {
 				}
 				// Bounded by result count, but a provider's error body (quoted in
 				// "Fell back after") or long snippets can still be large: persist, never slice.
-				const text = persistIfLarge(formatSearchResults(params.query, outcome), { dir: sessionResultsDir(ctx), id: toolCallId });
+				const text = persistIfLarge(withNativeNote(formatSearchResults(params.query, outcome)), { dir: sessionResultsDir(ctx), id: toolCallId });
 				return {
 					content: [{ type: "text", text }],
 					details: {
@@ -147,7 +210,7 @@ export default function webExtension(pi: ExtensionAPI) {
 			} catch (error) {
 				const message = (error as Error).message;
 				const text = persistIfLarge(
-					`web_search failed on ${ctx.model?.provider}/${ctx.model?.id} (no native web search on this provider).\n${message}`,
+					withNativeNote(`web_search failed on ${ctx.model?.provider}/${ctx.model?.id}.\n${message}`),
 					{ dir: sessionResultsDir(ctx), id: toolCallId },
 				);
 				return {
@@ -158,6 +221,7 @@ export default function webExtension(pi: ExtensionAPI) {
 			}
 		},
 	});
+	registerFormTool(pi, tool, webSearchDescription);
 
 	// url_context reports failures ("Failed: …" text, details.error set) without
 	// isError. Stamp it here rather than patching the vendor package.
@@ -179,20 +243,21 @@ export default function webExtension(pi: ExtensionAPI) {
 	// still advertised it in the names-only reminder (and as a defer_loading
 	// definition) on every provider, so a non-Gemini model would load it and burn
 	// two rounds discovering it cannot run (TOOL-FIDELITY-REVIEW-2026-09-07 M5).
-	// Mirror pi-web-search's gate: register it once the first Gemini model is
-	// seen. The one-shot flag keeps later session_tree/model_select events from
-	// re-emitting (each emit past session_start takes the DEFER handler's
-	// late-arrival path — deactivate + reschedule the listing rewrite — which is a
-	// no-op once url_context is already registered).
+	// Mirror that gate in the deferred registry too: withdraw on leaving Gemini
+	// and re-defer on returning. Emit only when availability changes, since a
+	// repeated defer would deactivate a tool already loaded through tool_search.
+	// The registry owns announcements and keeps the cached listing frozen.
 	let urlContextDeferred = false;
 	const deferUrlContextIfGemini = (model: { provider?: string; api?: string } | undefined) => {
-		// ctx.model is undefined until a model resolves (session_start can fire
-		// first), and getProviderKind dereferences model.provider without a guard —
-		// so an undefined model must short-circuit here, as pi-web-search's own
-		// `!!model && getProviderKind(...)` does.
-		if (urlContextDeferred || !model || getProviderKind(model) !== "google") return;
-		urlContextDeferred = true;
-		pi.events.emit(DEFER_CHANNEL, { name: "url_context", keywords: ["url", "analyze page", "gemini"] });
+		// session_start can run before a model resolves; getProviderKind requires one.
+		const available = !!model && getProviderKind(model) === "google";
+		if (available === urlContextDeferred) return;
+		urlContextDeferred = available;
+		if (available) {
+			pi.events.emit(DEFER_CHANNEL, { name: "url_context", keywords: ["url", "analyze page", "gemini"] });
+		} else {
+			pi.events.emit(WITHHOLD_CHANNEL, { name: "url_context" });
+		}
 	};
 	pi.on("session_start", (_event, ctx) => deferUrlContextIfGemini(ctx.model));
 	pi.on("session_tree", (_event, ctx) => deferUrlContextIfGemini(ctx.model));

package/extensions/web/native-call.ts

@@ -0,0 +1,15 @@
+import { runWithDeadline } from "../lib/operation-deadline.ts";
+
+/** Bound the provider-native transport, including auth, streamed bodies and source resolution. */
+export const NATIVE_SEARCH_TIMEOUT_MS = 120_000;
+
+export async function runNativeSearch<T>(
+	call: (signal: AbortSignal) => Promise<T>,
+	signal: AbortSignal | undefined,
+): Promise<T> {
+	return runWithDeadline(call, {
+		signal,
+		timeoutMs: NATIVE_SEARCH_TIMEOUT_MS,
+		message: "Provider-native search did not answer within 120 seconds.",
+	});
+}

package/extensions/web/native-results.ts

@@ -0,0 +1,41 @@
+/** Provider-native search formatting without pi-web-search's lossy truncateHead wrapper. */
+import { applyCitations, type StreamResult } from "pi-web-search/src/api.ts";
+
+export function formatNativeSearchResult(result: StreamResult) {
+	if (result.nativeSearchUsed === false) throw new Error("The provider returned a response without performing a web search.");
+	// The vendor substitutes this placeholder when the stream contains no answer text.
+	if (!result.text.trim() || result.text.trim() === "No answer available.") throw new Error("The provider returned an empty web-search answer.");
+	const cited = applyCitations(result.text, result.groundingMetadata);
+	const sources = result.sources?.length ? result.sources : cited.sources;
+	const failedSearch = result.searchResults?.find((item) => item.type === "web_search_tool_result_error");
+	if (failedSearch) throw new Error(`The provider's search returned ${failedSearch.status ?? "an unspecified tool error"}.`);
+	if (!sources.length && !result.searchResults?.some((item) => item.url)) throw new Error("The provider returned no search results.");
+	const seen = new Set<string>();
+	const extraResults = (result.searchResults ?? []).filter((item) => {
+		if (!item.url || sources.some((source) => source.url === item.url)) return false;
+		const key = `${item.title ?? ""}\t${item.url}`;
+		if (seen.has(key)) return false;
+		seen.add(key);
+		return true;
+	});
+	const parts = [cited.text];
+	if (sources.length) parts.push(`## Sources\n${sources.map((source, i) => `${i + 1}. [${source.title}](${source.url})`).join("\n")}`);
+	if (extraResults.length) {
+		parts.push(`## Additional Search Results\n${extraResults.map((item, i) => `${i + 1}. ${item.title || item.url} - ${item.url}`).join("\n")}`);
+	}
+	return {
+		text: parts.join("\n\n"),
+		details: {
+			sources,
+			providerKind: result.providerKind,
+			nativeSearchUsed: result.nativeSearchUsed,
+			nativeSearchEvents: result.nativeSearchEvents,
+			nativeSearchCalls: result.nativeSearchCalls,
+			searchQueries: result.searchQueries ?? result.groundingMetadata?.webSearchQueries,
+			searchResults: result.searchResults,
+			citations: result.citations,
+			grounded: sources.length > 0 || (result.searchResults?.length ?? 0) > 0,
+			resultCount: result.searchResults?.length || sources.length,
+		},
+	};
+}

package/extensions/web-fetch/description.ts

@@ -0,0 +1,42 @@
+/**
+ * web_fetch's model-facing description (pure).
+ *
+ * The short form, on frontier and workhorse models, is One Code's own: it
+ * describes what this fetch does that Claude Code's does not (an optional
+ * `prompt`, windowed pages, Anthropic's server-side fetch). The long form, on
+ * cheap and tiny models (lib/tool-variants.ts), is the text Claude Code writes
+ * for its weakest models, with One Code's tool names, without the claude.ai
+ * artifact line, with `prompt` optional and with the paging note in place of
+ * "Results may be summarized".
+ */
+
+import type { DescriptionForm } from "../lib/tool-variants.ts";
+
+export const WEB_FETCH_SHORT_DESCRIPTION =
+	"Fetch a URL and return its readable content as markdown. Navigation and boilerplate are stripped. Pass `prompt` to have a small fast model answer it from the full page instead of returning the page itself — prefer that for long pages (on an Anthropic API-key session, Anthropic's server-side fetch answers it). Without `prompt`, long pages are windowed; pass `offset` to continue reading. Responses are cached for 15 minutes. Cross-host redirects are reported instead of followed; call again with the new URL to follow one.";
+
+export const WEB_FETCH_LONG_DESCRIPTION = `IMPORTANT: web_fetch WILL FAIL for authenticated or private URLs. Before using this tool, check if the URL points to an authenticated service (e.g. Google Docs, Confluence, Jira, GitHub). If so, look for a specialized MCP tool that provides authenticated access.
+
+- Fetches content from a specified URL and processes it using an AI model
+- Takes a URL and an optional prompt as input
+- Fetches the URL content, converts HTML to markdown
+- Processes the content with the prompt using a small, fast model
+- Returns the model's response about the content
+- Use this tool when you need to retrieve and analyze web content
+
+Usage notes:
+  - IMPORTANT: If an MCP-provided web fetch tool is available, prefer using that tool instead of this one, as it may have fewer restrictions.
+  - The URL must be a fully-formed valid URL
+  - HTTP URLs will be automatically upgraded to HTTPS
+  - localhost and other hostnames without a dot are not supported; for a local server, use curl via bash
+  - The prompt should describe what information you want to extract from the page
+  - This tool is read-only and does not modify any files
+  - Without a prompt, a long page is returned in windows; pass \`offset\` to continue reading
+  - Includes a self-cleaning cache (entries expire after 15 minutes) for faster responses when repeatedly accessing the same URL
+  - When a URL redirects to a different host, the tool will inform you and provide the redirect URL in a special format. You should then make a new web_fetch request with the redirect URL to fetch the content.
+  - For GitHub URLs, prefer using the gh CLI via bash instead (e.g., gh pr view, gh issue view, gh api).
+`;
+
+export function webFetchDescription(form: DescriptionForm): string {
+	return form === "long" ? WEB_FETCH_LONG_DESCRIPTION : WEB_FETCH_SHORT_DESCRIPTION;
+}

package/extensions/web-fetch/extract.ts

@@ -47,19 +47,25 @@ export function normalizeUrl(input: string): NormalizedUrl {
 		throw new Error(`Not a valid URL: ${input}`);
 	}
 
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+		throw new Error(`Unsupported URL scheme "${parsed.protocol}" — only http and https are fetched.`);
+	}
+	if (parsed.username || parsed.password) throw new Error("URLs with embedded credentials are not supported.");
+	if (!parsed.hostname.replace(/\.+$/, "").includes(".")) {
+		throw new Error("web_fetch cannot fetch localhost or other hostnames without a dot. To reach a local server, use bash with curl instead.");
+	}
 	if (parsed.protocol === "http:") {
 		parsed.protocol = "https:";
 		return { url: parsed.toString(), note: "Upgraded http to https." };
 	}
-	if (parsed.protocol !== "https:") {
-		throw new Error(`Unsupported URL scheme "${parsed.protocol}" — only http and https are fetched.`);
-	}
 	return { url: parsed.toString() };
 }
 
 export function isSameHost(a: string, b: string): boolean {
 	try {
-		return new URL(a).host === new URL(b).host;
+		const target = new URL(a);
+		const source = new URL(b);
+		return target.host === source.host && target.protocol === source.protocol && !target.username && !target.password;
 	} catch {
 		return false;
 	}
@@ -88,11 +94,22 @@ export interface ExtractResult {
  * first (it strips navigation and boilerplate); if it finds no article — common
  * for API references and landing pages — the whole body is converted instead.
  */
-export async function htmlToMarkdown(html: string, _url: string): Promise<ExtractResult> {
+export async function htmlToMarkdown(html: string, url: string): Promise<ExtractResult> {
 	const { Readability, parseHTML, Turndown } = await loadLibs();
 	const { document } = parseHTML(html);
 	const turndown = createTurndown(Turndown);
+	// linkedom does not resolve relative attributes against the fetched URL.
+	let base = url;
+	try { base = new URL(document.querySelector("base[href]")?.getAttribute("href") ?? url, url).href; } catch { /* use the fetched URL */ }
+	for (const [selector, attribute] of [["a[href]", "href"], ["img[src]", "src"]]) {
+		for (const element of document.querySelectorAll(selector)) {
+			try { element.setAttribute(attribute, new URL(element.getAttribute(attribute)!, base).href); } catch { /* keep malformed links as written */ }
+		}
+	}
 
+	// Readability mutates the document, and linkedom can create an empty body
+	// for a fragment. Keep the original content for the whole-page fallback.
+	const body = document.querySelector("body")?.innerHTML ?? document.toString();
 	// linkedom's Document is structurally compatible with what Readability needs
 	// but is not the DOM lib's Document type, hence the cast.
 	let article: { title?: string | null; content?: string | null } | null = null;
@@ -110,7 +127,6 @@ export async function htmlToMarkdown(html: string, _url: string): Promise<Extrac
 		};
 	}
 
-	const body = document.body?.innerHTML ?? html;
 	return {
 		title: document.title || undefined,
 		markdown: turndown.turndown(body).trim(),
@@ -137,3 +153,34 @@ export function paginate(text: string, offset: number, maxChars: number): Page {
 		totalChars: text.length,
 	};
 }
+
+/** A redirect to another host: reported, not followed (Claude Code's rule). Not a failure. */
+export class CrossHostRedirect extends Error {
+	constructor(
+		readonly redirectUrl: string,
+		readonly status: number,
+	) {
+		super(`Redirects to a different host: ${redirectUrl}`);
+	}
+}
+
+/**
+ * Claude Code's WebFetch redirect result, with our tool name, and the prompt
+ * line only when the call had a prompt (ours is optional). Before it, the
+ * redirect read as a fetch failure ("Verify the URL and host, retry, or use
+ * web_search instead"), advice that does not apply.
+ */
+export function redirectMessage(originalUrl: string, redirect: CrossHostRedirect, prompt: string | undefined): string {
+	const statusText = redirect.status === 301 ? "Moved Permanently" : redirect.status === 308 ? "Permanent Redirect" : redirect.status === 307 ? "Temporary Redirect" : "Found";
+	return [
+		"REDIRECT DETECTED: The URL redirects to a different host.",
+		"",
+		`Original URL: ${originalUrl}`,
+		`Redirect URL: ${redirect.redirectUrl}`,
+		`Status: ${redirect.status} ${statusText}`,
+		"",
+		"To complete your request, I need to fetch content from the redirected URL. Please use web_fetch again with these parameters:",
+		`- url: "${redirect.redirectUrl}"`,
+		...(prompt ? [`- prompt: "${prompt}"`] : []),
+	].join("\n");
+}