@shanepadgett/tau-agent 0.33.1 → 0.34.0

package/extensions/tau/index.ts

@@ -64,6 +64,52 @@ async function init(ctx: ExtensionCommandContext, scope: "global" | "project"):
 	ctx.ui.notify(`tau init wrote ${path}`, "info");
 }
 
+function pushExtensionFindings(
+	findings: Finding[],
+	level: Finding["level"],
+	path: string,
+	extensions: unknown,
+	specs: Awaited<ReturnType<typeof discoverTauSettingsSpecs>>,
+): void {
+	if (extensions !== undefined && (!extensions || typeof extensions !== "object" || Array.isArray(extensions))) {
+		findings.push({ level, message: `extensions must be an object: ${path}`, startup: true });
+		return;
+	}
+	const extensionRecord = (extensions ?? {}) as Record<string, unknown>;
+	for (const spec of specs) {
+		const section = extensionRecord[spec.key];
+		if (section === undefined) continue;
+		if (!Value.Check(spec.schema, section)) {
+			findings.push({ level, message: `invalid extensions.${spec.key}: ${path}`, startup: true });
+		}
+	}
+}
+
+async function inspectPath(
+	findings: Finding[],
+	item: { level: Finding["level"]; path: string; trusted: boolean },
+	specs: Awaited<ReturnType<typeof discoverTauSettingsSpecs>>,
+): Promise<void> {
+	const status = await readJsonStatus(item.path);
+	if (!status.exists) return;
+	if (!item.trusted) {
+		findings.push({
+			level: item.level,
+			message: `project Tau settings ignored because project is not trusted: ${item.path}. Run /trust, then /reload.`,
+			startup: true,
+		});
+		return;
+	}
+	if (!status.ok) {
+		findings.push({ level: item.level, message: `malformed JSON: ${item.path}: ${status.error}`, startup: true });
+		return;
+	}
+	if (typeof status.value.$schema !== "string") {
+		findings.push({ level: item.level, message: `missing $schema: ${item.path}`, startup: false });
+	}
+	pushExtensionFindings(findings, item.level, item.path, status.value.extensions, specs);
+}
+
 async function inspect(ctx: Pick<ExtensionContext, "cwd" | "isProjectTrusted">): Promise<Finding[]> {
 	const findings: Finding[] = [];
 	const specs = await discoverTauSettingsSpecs(ctx.cwd);
@@ -71,44 +117,7 @@ async function inspect(ctx: Pick<ExtensionContext, "cwd" | "isProjectTrusted">):
 		{ level: "global" as const, path: globalTauSettingsPath(), trusted: true },
 		{ level: "project" as const, path: await projectTauSettingsPath(ctx.cwd), trusted: ctx.isProjectTrusted() },
 	];
-
-	for (const item of paths) {
-		const status = await readJsonStatus(item.path);
-		if (!status.exists) continue;
-		if (!item.trusted) {
-			findings.push({
-				level: item.level,
-				message: `project Tau settings ignored because project is not trusted: ${item.path}. Run /trust, then /reload.`,
-				startup: true,
-			});
-			continue;
-		}
-		if (!status.ok) {
-			findings.push({ level: item.level, message: `malformed JSON: ${item.path}: ${status.error}`, startup: true });
-			continue;
-		}
-		if (typeof status.value.$schema !== "string") {
-			findings.push({ level: item.level, message: `missing $schema: ${item.path}`, startup: false });
-		}
-		const extensions = status.value.extensions;
-		if (extensions !== undefined && (!extensions || typeof extensions !== "object" || Array.isArray(extensions))) {
-			findings.push({ level: item.level, message: `extensions must be an object: ${item.path}`, startup: true });
-			continue;
-		}
-		const extensionRecord = (extensions ?? {}) as Record<string, unknown>;
-		for (const spec of specs) {
-			const section = extensionRecord[spec.key];
-			if (section === undefined) continue;
-			if (!Value.Check(spec.schema, section)) {
-				findings.push({
-					level: item.level,
-					message: `invalid extensions.${spec.key}: ${item.path}`,
-					startup: true,
-				});
-			}
-		}
-	}
-
+	for (const item of paths) await inspectPath(findings, item, specs);
 	return findings;
 }
 

package/extensions/tau-help/help.md

@@ -6,6 +6,10 @@ Tau is a coding agent built to be a reliable partner during software work. It ca
 
 Gives the agent macOS window discovery, screenshots, and app activation tools for visual validation. Requires macOS 14 and Screen & System Audio Recording permission.
 
+## aside
+
+Adds `/aside <question>` for a one-off question to the current model without putting the question or answer in the conversation. Choose the current conversation branch or no context. A thinking widget clears when the answer opens. Run `/aside` to reopen the latest answer and `/aside clear` to cancel or clear it.
+
 ## attention
 
 Shows attention state when Tau needs the user to look at the chat, finishes compacting a session, or summarizes an abandoned branch.
@@ -32,15 +36,15 @@ Adds `/commit` for semantic commit grouping, review, and committing selected rep
 
 ## context
 
-Adds `/context` to set branch-local reusable repository work scopes from `.pi/contexts`, and `/context-sync` or `/context-sync <nudge>` for human-driven catalog sync. Active entries produce one ephemeral per-call projection instead of transcript messages. Entry `read` paths supply exact contents, `outline` paths use Explore, and `references` stay unloaded. Clear all selections and confirm to remove active context. Escape cancels a running manual sync. When `sync.automation` is on, coding agent can also run `context-sync` after meaningful uncommitted work. Sync catalogs durable code and long-lived documentation; recurring scratch, planning, interview, and rough-idea paths belong in `validation.ignoreGlobs`. `sync.enabled` is master switch for command, automation, and validation auto-run. Context validation is off by default; when on (and sync enabled), Tau auto-runs context-sync on failure. Folder names are tabs, TOML files are concepts, and TOML sections are selectable entries.
+Adds `/context` to inject reusable repository work scopes from `.pi/contexts`, and `/context-sync` or `/context-sync <nudge>` for human-driven catalog sync. Selecting entries injects them once into the conversation: `read` paths as complete files, `show` targets as current declaration slices, `outline` paths as Explore structures, and one hidden note listing `references` plus instructions to treat the injected material as current. Run `/context` again to inject more. Manual sync replaces the editor with a status panel; Escape or Ctrl+C cancels. When `sync.automation` is on, coding agent can also run `context-sync` after meaningful uncommitted work. Sync catalogs durable code and long-lived documentation; recurring scratch, planning, interview, and rough-idea paths belong in `validation.ignoreGlobs`. `sync.enabled` is master switch for command, automation, and validation auto-run. Context validation is off by default; when on (and sync enabled), Tau auto-runs context-sync on failure. Domain folders are `NN_slug` tabs (ordered by the two-digit prefix; UI shows the slug), TOML files are concepts, and TOML sections are selectable entries.
 
-## working-memory
+## checkpoint
 
-Gives agent `working_memory` for selective hard checkpoints. Every checkpoint retains at least one useful user message or visible assistant text. Requested source files return as structural outlines, deferred files remain cheap conditional reminders, and a continuation note carries conclusions extracted from exploration. Tool history and full file reads leave future model input without changing saved session. Advisory reminders begin at 40k active-context tokens. Run `/prune` to request reassessment manually.
+Adds the `checkpoint` tool for retaining exact user and assistant messages, current file reads and outlines, durable work state, an agent-written resume directive, and deferred file paths as a rolling continuation context. Checkpoint rows are hidden by default. Set `extensions.checkpoint.showToolRows` to `true` before `/reload` to watch checkpoint and newly injected-file rows while working; existing injected-file rows keep their saved display state. The agent receives hidden checkpoint nudges at 50% and 75% of `extensions.checkpoint.checkpointTokenLimit` (150,000 by default); non-checkpoint tools are blocked at the limit until checkpoint succeeds.
 
 ## effort
 
-Adds `/effort [low|medium|high]` to select effort and a provider from current logins. Tau selects provider’s best available model for tier, then tries its configured model fallback. `Ctrl+Shift+E` cycles tiers on current provider. Footer derives effort from current provider, model, and thinking level, and hides it when no configured tier matches.
+Adds `/effort [quick|standard|deep]` to select effort and a provider from current logins. Tau selects provider’s best available model for tier, then tries its configured model fallback. `Ctrl+Shift+E` cycles tiers on current provider. Footer derives effort from current provider, model, and thinking level, and hides it when no configured tier matches.
 
 ## explore
 
@@ -74,6 +78,10 @@ Replaces separate edit/write operations with one multi-file `patch` tool. It can
 
 Adds `/qna` for when the agent has asked you several questions in chat and you want a friendly UI for answering them on your own terms. It is only active when you manually run the command.
 
+## ready
+
+Adds `/ready` to scan agent-readiness rails (cold start, toolchain, verify, lint/entropy, policy, standards, context, and related signals). Choose Markdown or HTML; Tau writes a timestamped report under `.pi/tau/ready/` and notifies with the path. Scan-only in v1 — no model judgment and no scores.
+
 ## review
 
 Adds `/review` for explicit isolated review of current Git changes. Choose `simplify`, `architecture`, or `correctness`, or run a mode directly. Results stay outside agent context until you send them from result view, and can be exported under `.pi/tau/reviews/`. `/review show` reopens latest result on current session branch.
@@ -92,7 +100,7 @@ Supplies the agent with the current local date and an initial root directory sna
 
 ## script-runner
 
-Gives the agent a first-class `script_runner` tool to execute Python 3 and TypeScript instead of bash. On failure it returns a `scriptId`; the agent retries with targeted `{oldText,newText}` edits against the script it already wrote rather than resending the whole script. Languages are detected from the environment (Python 3 via `python3`; TypeScript via Node `--experimental-strip-types`, Node 22.6+). The tool registers only available languages and is hidden from the prompt if neither is present.
+Gives the agent a first-class `script_runner` tool to execute Python 3, Node.js, and Deno scripts instead of bash. On failure it returns a `scriptId`; the agent retries with targeted `{oldText,newText}` edits against the script it already wrote rather than resending the whole script. Runtimes are detected from the environment (Python 3 via `python3`; `node` is the local Node.js runtime with `--experimental-strip-types`, Node 22.6+ — full Node APIs, TypeScript with erasable syntax or plain JavaScript; `deno` via `deno run -A` — full permissions, native TypeScript/JavaScript, Deno APIs). The tool registers only available runtimes and is hidden from the prompt if none are present.
 
 ## silent-command-runner
 
@@ -120,7 +128,7 @@ Adds `/tau`, `/tau init [--global|--project]`, and `/tau doctor` for Tau setup a
 
 ## tool-loader
 
-Progressively exposes specialist tools through `load_tools`. Tau normally loads the fixed `web`, `image`, and `appshot` groups itself when needed; supported providers can preserve more prompt-cache reuse.
+Progressively exposes registered specialist tool groups through `load_tools`. Tau registers `web`, `image`, and `appshot`; project or global package extensions can add groups with `registerDeferredToolGroup()` from `@shanepadgett/tau-agent`. Supported providers can preserve more prompt-cache reuse.
 
 ## web
 

package/extensions/tool-loader/README.md

@@ -1,13 +1,15 @@
 # Tool Loader
 
-Tau progressively exposes specialist tools. Most coding turns do not need web, image, or macOS application schemas, so Pi can load those tools later without discarding supported provider cache prefixes.
+Tau progressively exposes registered specialist tool groups. Most coding turns do not need web, image, macOS application, or application-specific schemas, so Pi can load those tools later without discarding supported provider cache prefixes.
 
-The agent normally calls `load_tools` itself. Users can also ask Tau to load one of these groups:
+The agent normally calls `load_tools` itself. Tau's built-in groups are:
 
 - `web` for public web and implementation research
 - `image` for raster image generation and editing
 - `appshot` for macOS window discovery, capture, and activation
 
+Project and global package extensions can add groups with `registerDeferredToolGroup()` from `@shanepadgett/tau-agent`. The group description is included in the loader catalog so the agent can select it when a task needs that capability.
+
 Supported models optimize prompt caching when a group loads. Other models keep the same functional behavior.
 
 After changing this extension during development, run `/reload` before testing.

package/extensions/tool-loader/index.ts

@@ -1,21 +1,12 @@
-import { StringEnum } from "@earendil-works/pi-ai";
 import { defineTool, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { type Static, Type } from "typebox";
-
-const CAPABILITIES = ["web", "image", "appshot"] as const;
-type Capability = (typeof CAPABILITIES)[number];
-
-const CAPABILITY_TOOLS: Record<Capability, readonly string[]> = {
-	web: ["webfetch", "websearch", "codesearch"],
-	image: ["image_gen"],
-	appshot: ["list_windows", "screenshot_window", "activate_app"],
-};
-const SPECIALIST_TOOLS = CAPABILITIES.flatMap((capability) => CAPABILITY_TOOLS[capability]);
+import { registeredDeferredToolGroups, type DeferredToolGroupInfo } from "../../src/tool-loading/index.ts";
 
 const loadToolsSchema = Type.Object(
 	{
-		capability: StringEnum(CAPABILITIES, {
-			description: "Specialist group to load: web, image, or appshot",
+		capability: Type.String({
+			minLength: 1,
+			description: "Registered specialist group ID, such as web, image, appshot, or a package-provided group",
 		}),
 	},
 	{ additionalProperties: false },
@@ -25,50 +16,85 @@ type LoadToolsParams = Static<typeof loadToolsSchema>;
 
 interface LoadToolsDetails {
 	version: 1;
-	capability: Capability;
+	capability: string;
 	requestedToolNames: string[];
 	addedToolNames: string[];
 }
 
 export default function toolLoaderExtension(pi: ExtensionAPI): void {
 	let managed = false;
-	let allowedSpecialistNames = new Set<string>();
+	let allowedToolNames = new Map<string, ReadonlySet<string>>();
+	let managedToolNames = new Set<string>();
+
+	pi.registerTool(createLoadToolsTool(pi, []));
+
+	pi.on("session_start", (_event, ctx) => {
+		const groups = registeredDeferredToolGroups(pi);
+		pi.registerTool(createLoadToolsTool(pi, groups));
+
+		const initial = pi.getActiveTools();
+		const initialSet = new Set(initial);
+		allowedToolNames = new Map(
+			groups.map((group) => [group.id, new Set(group.toolNames.filter((name) => initialSet.has(name)))]),
+		);
+		managedToolNames = new Set(groups.flatMap((group) => group.toolNames));
+		managed = initialSet.has("load_tools") && groups.length > 0;
+		if (managed) restoreActiveTools(pi, initial, loadedCapabilities(ctx.sessionManager.getBranch()), groups);
+	});
 
-	pi.registerTool(
-		defineTool<typeof loadToolsSchema, LoadToolsDetails>({
+	pi.on("session_tree", (_event, ctx) => {
+		if (managed) {
+			restoreActiveTools(
+				pi,
+				pi.getActiveTools(),
+				loadedCapabilities(ctx.sessionManager.getBranch()),
+				registeredDeferredToolGroups(pi),
+			);
+		}
+	});
+
+	function createLoadToolsTool(pi: ExtensionAPI, groups: readonly DeferredToolGroupInfo[]) {
+		return defineTool<typeof loadToolsSchema, LoadToolsDetails>({
 			name: "load_tools",
 			label: "Load Tools",
-			description:
-				"Load one Tau specialist tool group for the current session. Groups: web for public web and implementation research; image for raster generation and editing; appshot for macOS window discovery, capture, and activation.",
-			promptSnippet: "Load a specialist Tau tool group for web research, image generation, or macOS app inspection",
+			description: `Load one registered Tau specialist tool group for the current session.${formatGroupCatalog(groups)}`,
+			promptSnippet: "Load a registered specialist tool group when the current tools cannot perform the task",
 			promptGuidelines: [
-				"Use load_tools before attempting a specialist capability whose tools are not currently available.",
+				"Use load_tools before attempting a registered specialist capability whose tools are not currently available.",
 			],
 			parameters: loadToolsSchema,
 			async execute(_toolCallId, params: LoadToolsParams) {
+				const group = registeredDeferredToolGroups(pi).find((candidate) => candidate.id === params.capability);
+				if (group === undefined) {
+					throw new Error(
+						`Unknown specialist tool group: ${params.capability}.${formatGroupCatalog(registeredDeferredToolGroups(pi))}`,
+					);
+				}
+
 				const before = pi.getActiveTools();
-				const requested = [...CAPABILITY_TOOLS[params.capability]];
+				const requested = [...group.toolNames];
 				const registered = new Set(pi.getAllTools().map((tool) => tool.name));
-				const loadable = requested.filter((name) => registered.has(name) && allowedSpecialistNames.has(name));
+				const allowed = allowedToolNames.get(group.id) ?? new Set<string>();
+				const loadable = requested.filter((name) => registered.has(name) && allowed.has(name));
 				if (loadable.length === 0) {
 					throw new Error(`No ${params.capability} tools are available in this session's tool configuration.`);
 				}
+
 				const beforeSet = new Set(before);
-				const next = [...before, ...loadable.filter((name) => !beforeSet.has(name))];
-				pi.setActiveTools(next);
+				pi.setActiveTools([...before, ...loadable.filter((name) => !beforeSet.has(name))]);
 				const after = pi.getActiveTools();
-				const addedToolNames = after.filter((name) => !beforeSet.has(name));
+				const addedToolNames = requested.filter((name) => !beforeSet.has(name) && after.includes(name));
 				const available = requested.filter((name) => after.includes(name));
 				const unavailable = requested.filter((name) => !after.includes(name));
-				const label = `${params.capability[0]?.toUpperCase()}${params.capability.slice(1)}`;
 				const text =
 					addedToolNames.length > 0
 						? `Loaded ${params.capability} tools: ${addedToolNames.join(", ")}.`
-						: `${label} tools are already loaded: ${available.join(", ")}.`;
+						: `${params.capability} tools are already loaded: ${available.join(", ")}.`;
+
 				return {
 					content: [
 						{
-							type: "text",
+							type: "text" as const,
 							text: unavailable.length ? `${text} Unavailable: ${unavailable.join(", ")}.` : text,
 						},
 					],
@@ -80,33 +106,33 @@ export default function toolLoaderExtension(pi: ExtensionAPI): void {
 					},
 				};
 			},
-		}),
-	);
-
-	pi.on("session_start", (_event, ctx) => {
-		const initial = pi.getActiveTools();
-		const initialSet = new Set(initial);
-		allowedSpecialistNames = new Set(SPECIALIST_TOOLS.filter((name) => initialSet.has(name)));
-		managed = initialSet.has("load_tools") && SPECIALIST_TOOLS.every((name) => initialSet.has(name));
-		if (managed) restoreActiveTools(pi, initial, loadedCapabilities(ctx.sessionManager.getBranch()));
-	});
+		});
+	}
 
-	pi.on("session_tree", (_event, ctx) => {
-		if (managed) restoreActiveTools(pi, pi.getActiveTools(), loadedCapabilities(ctx.sessionManager.getBranch()));
-	});
+	function restoreActiveTools(
+		pi: ExtensionAPI,
+		current: readonly string[],
+		loaded: ReadonlySet<string>,
+		groups: readonly DeferredToolGroupInfo[],
+	): void {
+		const next = current.filter((name) => !managedToolNames.has(name));
+		for (const group of groups) {
+			if (!loaded.has(group.id)) continue;
+			const allowed = allowedToolNames.get(group.id) ?? new Set<string>();
+			next.push(...group.toolNames.filter((name) => allowed.has(name)));
+		}
+		pi.setActiveTools([...new Set(next)]);
+	}
 }
 
-function restoreActiveTools(pi: ExtensionAPI, current: readonly string[], loaded: ReadonlySet<Capability>): void {
-	const specialist = new Set(SPECIALIST_TOOLS);
-	const next = current.filter((name) => !specialist.has(name));
-	for (const capability of CAPABILITIES) {
-		if (loaded.has(capability)) next.push(...CAPABILITY_TOOLS[capability]);
-	}
-	pi.setActiveTools([...new Set(next)]);
+function formatGroupCatalog(groups: readonly DeferredToolGroupInfo[]): string {
+	if (groups.length === 0) return " No specialist groups are registered.";
+	const catalog = groups.map((group) => `${group.id}: ${group.description}`).join("; ");
+	return ` Registered groups: ${catalog}.`;
 }
 
-function loadedCapabilities(entries: readonly unknown[]): Set<Capability> {
-	const loaded = new Set<Capability>();
+function loadedCapabilities(entries: readonly unknown[]): Set<string> {
+	const loaded = new Set<string>();
 	for (const value of entries) {
 		if (!value || typeof value !== "object") continue;
 		const entry = value as Record<string, unknown>;
@@ -125,7 +151,6 @@ function isLoadToolsDetails(value: unknown): value is LoadToolsDetails {
 	return (
 		details.version === 1 &&
 		typeof details.capability === "string" &&
-		CAPABILITIES.includes(details.capability as Capability) &&
 		Array.isArray(details.requestedToolNames) &&
 		details.requestedToolNames.every((name) => typeof name === "string") &&
 		Array.isArray(details.addedToolNames) &&

package/extensions/web/index.ts

@@ -1,13 +1,16 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { createToolRowStateStore } from "../../shared/tool-row-state.js";
+import { registerDeferredToolGroup } from "../../src/tool-loading/index.ts";
 import { createCodeSearchTool } from "./codesearch.ts";
 import { createWebFetchTool } from "./webfetch.ts";
 import { createWebSearchTool } from "./websearch.ts";
 
 export default function webExtension(pi: ExtensionAPI): void {
 	const rowState = createToolRowStateStore(pi, "web.tool-row-state");
-	pi.registerTool(createWebFetchTool(rowState));
-	pi.registerTool(createWebSearchTool(rowState));
-	pi.registerTool(createCodeSearchTool(rowState));
+	registerDeferredToolGroup(pi, {
+		id: "web",
+		description: "Public web and implementation research",
+		tools: [createWebFetchTool(rowState), createWebSearchTool(rowState), createCodeSearchTool(rowState)],
+	});
 	pi.on("session_start", () => rowState.clear());
 }

package/extensions/web/webfetch.ts

@@ -27,14 +27,22 @@ const webFetchParams = Type.Object(
 );
 
 type WebFetchParams = Static<typeof webFetchParams>;
+type WebFetchFormat = "markdown" | "text" | "html";
 interface WebFetchDetails {
 	url: string;
-	format: "markdown" | "text" | "html";
+	format: WebFetchFormat;
 	mime: string;
 	bytes: number;
 	truncation?: TruncationResult;
 }
 
+type WebFetchContent = { type: "text"; text: string } | { type: "image"; data: string; mimeType: string };
+
+interface WebFetchResult {
+	content: WebFetchContent[];
+	details: WebFetchDetails;
+}
+
 async function readResponseBody(response: FetchResponse): Promise<Uint8Array> {
 	if (!response.body) return new Uint8Array();
 	const reader = response.body.getReader();
@@ -61,6 +69,110 @@ function renderCallSummary(args: WebFetchParams): string {
 	return truncateCallSummary((args.url ?? "").trim());
 }
 
+function parseFetchUrl(raw: string): URL {
+	let url: URL;
+	try {
+		url = new URL(raw);
+	} catch {
+		throw new Error(`Invalid URL: ${raw}`);
+	}
+	if (url.protocol !== "http:" && url.protocol !== "https:") {
+		throw new Error("URL must use http:// or https://");
+	}
+	return url;
+}
+
+function acceptHeaderForFormat(format: WebFetchFormat): string {
+	if (format === "markdown") return "text/markdown;q=1.0, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1";
+	if (format === "text") return "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1";
+	return "text/html;q=1.0, application/xhtml+xml;q=0.9, */*;q=0.1";
+}
+
+async function fetchWithChallengeRetry(
+	url: string,
+	headers: Record<string, string>,
+	signal: AbortSignal,
+): Promise<FetchResponse> {
+	const first = (await fetch(url, { method: "GET", headers, signal })) as FetchResponse;
+	if (first.status !== 403 || first.headers.get("cf-mitigated")?.toLowerCase() !== "challenge") return first;
+	await first.body?.cancel().catch(() => undefined);
+	return (await fetch(url, {
+		method: "GET",
+		headers: { ...headers, "User-Agent": "pi" },
+		signal,
+	})) as FetchResponse;
+}
+
+async function assertDeclaredBodySizeOk(response: FetchResponse): Promise<void> {
+	const declaredLength = response.headers.get("content-length");
+	if (declaredLength === null) return;
+	const bytes = Number.parseInt(declaredLength, 10);
+	if (!Number.isFinite(bytes) || bytes <= MAX_RESPONSE_BYTES) return;
+	await response.body?.cancel().catch(() => undefined);
+	throw new Error("Response too large (limit is 5MB)");
+}
+
+function formatFetchedText(raw: string, format: WebFetchFormat, mime: string): string {
+	if (format === "html") return raw;
+	const isHtml = mime === "text/html" || mime === "application/xhtml+xml";
+	if (!isHtml) return raw;
+	return format === "text" ? htmlToText(raw) : htmlToMarkdown(raw);
+}
+
+function buildWebFetchResult(url: string, format: WebFetchFormat, mime: string, body: Uint8Array): WebFetchResult {
+	const details = { url, format, mime, bytes: body.byteLength } satisfies WebFetchDetails;
+	if (mime.startsWith("image/") && mime !== "image/svg+xml") {
+		return {
+			content: [
+				{ type: "text", text: `Fetched image from ${url} (${mime})` },
+				{ type: "image", data: Buffer.from(body).toString("base64"), mimeType: mime },
+			],
+			details,
+		};
+	}
+	const truncated = truncateToolOutput(formatFetchedText(new TextDecoder().decode(body), format, mime));
+	return {
+		content: [{ type: "text", text: truncated.text }],
+		details: {
+			...details,
+			...(truncated.truncation ? { truncation: truncated.truncation } : {}),
+		},
+	};
+}
+
+async function executeWebFetch(
+	params: WebFetchParams,
+	signal: AbortSignal | undefined,
+	onUpdate: ((update: { content: WebFetchContent[]; details: undefined }) => void | Promise<void>) | undefined,
+): Promise<WebFetchResult> {
+	const url = parseFetchUrl(params.url);
+	const format = params.format ?? "markdown";
+	const timeout = normalizeTimeout(params.timeout, 30);
+	await onUpdate?.({ content: [{ type: "text", text: "Fetching page..." }], details: undefined });
+	const timeoutSignal = AbortSignal.timeout(timeout * 1000);
+	const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+
+	try {
+		const headers = {
+			"User-Agent":
+				"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
+			Accept: acceptHeaderForFormat(format),
+			"Accept-Language": "en-US,en;q=0.9",
+		};
+		const response = await fetchWithChallengeRetry(url.toString(), headers, requestSignal);
+		if (!response.ok) throw new Error(`Request failed with status ${response.status}`);
+		await assertDeclaredBodySizeOk(response);
+		const body = await readResponseBody(response);
+		const mime = (response.headers.get("content-type")?.split(";", 1)[0] ?? "").trim().toLowerCase();
+		return buildWebFetchResult(url.toString(), format, mime, body);
+	} catch (error) {
+		if (timeoutSignal.aborted && signal?.aborted !== true) {
+			throw new Error(`Web fetch timed out after ${timeout}s`);
+		}
+		throw error;
+	}
+}
+
 export function createWebFetchTool(rowState: ToolRowStateStore) {
 	return defineTool<typeof webFetchParams, WebFetchDetails | undefined>({
 		name: "webfetch",
@@ -69,91 +181,7 @@ export function createWebFetchTool(rowState: ToolRowStateStore) {
 			"Fetch a known HTTP(S) URL as Markdown, text, or HTML. Use webfetch when you already have a URL; use websearch for broad discovery and codesearch for implementation-oriented lookups. Use a separate research workflow when several searches, fetches, and synthesis are needed. Supports inline images, limits response bodies to 5 MB, and truncates text to 2,000 lines or 50 KB.",
 		parameters: webFetchParams,
 		async execute(_toolCallId, params, signal, onUpdate) {
-			let url: URL;
-			try {
-				url = new URL(params.url);
-			} catch {
-				throw new Error(`Invalid URL: ${params.url}`);
-			}
-			if (url.protocol !== "http:" && url.protocol !== "https:") {
-				throw new Error("URL must use http:// or https://");
-			}
-
-			const format = params.format ?? "markdown";
-			const timeout = normalizeTimeout(params.timeout, 30);
-			await onUpdate?.({ content: [{ type: "text", text: "Fetching page..." }], details: undefined });
-			const timeoutSignal = AbortSignal.timeout(timeout * 1000);
-			const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
-
-			try {
-				const accept =
-					format === "markdown"
-						? "text/markdown;q=1.0, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1"
-						: format === "text"
-							? "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1"
-							: "text/html;q=1.0, application/xhtml+xml;q=0.9, */*;q=0.1";
-				const headers = {
-					"User-Agent":
-						"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
-					Accept: accept,
-					"Accept-Language": "en-US,en;q=0.9",
-				};
-				const first = (await fetch(url.toString(), {
-					method: "GET",
-					headers,
-					signal: requestSignal,
-				})) as FetchResponse;
-				let response = first;
-				if (first.status === 403 && first.headers.get("cf-mitigated")?.toLowerCase() === "challenge") {
-					await first.body?.cancel().catch(() => undefined);
-					response = (await fetch(url.toString(), {
-						method: "GET",
-						headers: { ...headers, "User-Agent": "pi" },
-						signal: requestSignal,
-					})) as FetchResponse;
-				}
-				if (!response.ok) throw new Error(`Request failed with status ${response.status}`);
-
-				const declaredLength = response.headers.get("content-length");
-				if (declaredLength !== null) {
-					const bytes = Number.parseInt(declaredLength, 10);
-					if (Number.isFinite(bytes) && bytes > MAX_RESPONSE_BYTES) {
-						await response.body?.cancel().catch(() => undefined);
-						throw new Error("Response too large (limit is 5MB)");
-					}
-				}
-
-				const body = await readResponseBody(response);
-				const mime = (response.headers.get("content-type")?.split(";", 1)[0] ?? "").trim().toLowerCase();
-				const details = { url: url.toString(), format, mime, bytes: body.byteLength } satisfies WebFetchDetails;
-				if (mime.startsWith("image/") && mime !== "image/svg+xml") {
-					return {
-						content: [
-							{ type: "text", text: `Fetched image from ${url.toString()} (${mime})` },
-							{ type: "image", data: Buffer.from(body).toString("base64"), mimeType: mime },
-						],
-						details,
-					};
-				}
-
-				const raw = new TextDecoder().decode(body);
-				const isHtml = mime === "text/html" || mime === "application/xhtml+xml";
-				const output =
-					format === "html" ? raw : isHtml ? (format === "text" ? htmlToText(raw) : htmlToMarkdown(raw)) : raw;
-				const truncated = truncateToolOutput(output);
-				return {
-					content: [{ type: "text", text: truncated.text }],
-					details: {
-						...details,
-						...(truncated.truncation ? { truncation: truncated.truncation } : {}),
-					},
-				};
-			} catch (error) {
-				if (timeoutSignal.aborted && signal?.aborted !== true) {
-					throw new Error(`Web fetch timed out after ${timeout}s`);
-				}
-				throw error;
-			}
+			return executeWebFetch(params, signal, onUpdate);
 		},
 		renderCall(args, theme, context) {
 			rowState.watch(context.toolCallId, context.invalidate);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@shanepadgett/tau-agent",
-	"version": "0.33.1",
+	"version": "0.34.0",
 	"description": "Tau is a custom agentic harness built with pi extensions",
 	"type": "module",
 	"main": "./src/index.ts",
@@ -35,10 +35,10 @@
 	],
 	"dependencies": {
 		"@ast-grep/wasm": "0.45.0",
-		"@shanepadgett/tau-tui": "0.33.1",
+		"@shanepadgett/tau-tui": "0.34.0",
 		"@vscode/tree-sitter-wasm": "0.3.1",
 		"image-size": "2.0.2",
-		"smol-toml": "1.7.0",
+		"smol-toml": "1.7.1",
 		"web-tree-sitter": "0.26.11"
 	},
 	"peerDependencies": {