@bastani/atomic 0.8.31-alpha.5 → 0.9.0-alpha.1

package/dist/builtin/web-access/CHANGELOG.md

@@ -4,9 +4,12 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [0.9.0-alpha.1] - 2026-06-20
+
 ### Changed
 
 - Aligned the web-access extension peer dependency with upstream pi TUI `^0.79.7` so web-access curator and summary UI surfaces consume the latest shared TUI color-scheme, Warp image capability, and compatibility fixes; no web-access extension code changes were made for this metadata sync ([#1413](https://github.com/bastani-inc/atomic/issues/1413)).
+- Changed contributor validation to include the monorepo-wide file-length gate for tracked TS/JS/Rust files in local `prek` hooks and PR CI, with only documented generated/vendored exclusions and no grandfathered baseline allowlist ([#1445](https://github.com/bastani-inc/atomic/issues/1445)).
 
 ## [0.8.30] - 2026-06-17
 

package/dist/builtin/web-access/content-tools.ts

@@ -0,0 +1,348 @@
+import type { ExtensionAPI } from "@bastani/atomic";
+import { Text } from "@mariozechner/pi-tui";
+import { Type } from "typebox";
+import {
+	renderCodeSearchResult,
+	renderFetchContentResult,
+	renderGetSearchContentResult,
+} from "./result-renderers.js";
+import { fetchAllContent, type ExtractedContent } from "./extract.js";
+import { executeCodeSearch } from "./code-search.js";
+import {
+	generateId,
+	getResult,
+	storeResult,
+	type QueryResultData,
+	type StoredSearchData,
+} from "./storage.js";
+
+interface RegisterContentToolsDeps {
+	maxInlineContent: number;
+	stripThumbnails(results: ExtractedContent[]): ExtractedContent[];
+	formatFullResults(queryData: QueryResultData): string;
+}
+
+export function registerContentTools(pi: ExtensionAPI, deps: RegisterContentToolsDeps): void {
+	pi.registerTool({
+		name: "code_search",
+		label: "Code Search",
+		description: "Search for code examples, documentation, and API references. Returns relevant code snippets and docs from GitHub, Stack Overflow, and official documentation. Use for any programming question — API usage, library examples, debugging help.",
+		promptSnippet:
+			"Use for programming/API/library questions to retrieve concrete examples and docs before implementing or debugging code.",
+		parameters: Type.Object({
+			query: Type.String({ description: "Programming question, API, library, or debugging topic to search for" }),
+			maxTokens: Type.Optional(Type.Integer({
+				minimum: 1000,
+				maximum: 50000,
+				description: "Maximum tokens of code/documentation context to return (default: 5000)",
+			})),
+		}),
+
+		async execute(toolCallId, params, signal) {
+			return executeCodeSearch(toolCallId, params, signal);
+		},
+
+		renderCall(args, theme) {
+			const { query } = args as { query?: string };
+			const display = !query
+				? "(no query)"
+				: query.length > 70 ? query.slice(0, 67) + "..." : query;
+			return new Text(theme.fg("toolTitle", theme.bold("code_search ")) + theme.fg("accent", display), 0, 0);
+		},
+
+		renderResult: renderCodeSearchResult,
+	});
+
+	pi.registerTool({
+		name: "fetch_content",
+		label: "Fetch Content",
+		description: "Fetch URL(s) and extract readable content as markdown. Supports YouTube video transcripts (with thumbnail), GitHub repository contents, and local video files (with frame thumbnail). Video frames can be extracted via timestamp/range or sampled across the entire video with frames alone. Falls back to Gemini for pages that block bots or fail Readability extraction. For YouTube and video files: ALWAYS pass the user's specific question via the prompt parameter — this directs the AI to focus on that aspect of the video, producing much better results than a generic extraction. Content is always stored and can be retrieved with get_search_content.",
+		promptSnippet:
+			"Use to extract readable content from URL(s), YouTube, GitHub repos, or local videos. For video questions, pass the user's exact question in prompt.",
+		parameters: Type.Object({
+			url: Type.Optional(Type.String({ description: "Single URL to fetch" })),
+			urls: Type.Optional(Type.Array(Type.String(), { description: "Multiple URLs (parallel)" })),
+			forceClone: Type.Optional(Type.Boolean({
+				description: "Force cloning large GitHub repositories that exceed the size threshold",
+			})),
+			prompt: Type.Optional(Type.String({
+				description: "Question or instruction for video analysis (YouTube and video files). Pass the user's specific question here — e.g. 'describe the book shown at the advice for beginners section'. Without this, a generic transcript extraction is used which may miss what the user is asking about.",
+			})),
+			timestamp: Type.Optional(Type.String({
+				description: "Extract video frame(s) at a timestamp or time range. Single: '1:23:45', '23:45', or '85' (seconds). Range: '23:41-25:00' extracts evenly-spaced frames across that span (default 6). Use frames with ranges to control density; single+frames uses a fixed 5s interval. YouTube requires yt-dlp + ffmpeg; local videos require ffmpeg. Use a range when you know the approximate area but not the exact moment — you'll get a contact sheet to visually identify the right frame.",
+			})),
+			frames: Type.Optional(Type.Integer({
+				minimum: 1,
+				maximum: 12,
+				description: "Number of frames to extract. Use with timestamp range for custom density, with single timestamp to get N frames at 5s intervals, or alone to sample across the entire video. Requires yt-dlp + ffmpeg for YouTube, ffmpeg for local video.",
+			})),
+			model: Type.Optional(Type.String({
+				description: "Override the Gemini model for video/YouTube analysis (e.g. 'gemini-2.5-flash', 'gemini-3-flash-preview'). Defaults to config or gemini-3-flash-preview.",
+			})),
+		}),
+
+		async execute(_toolCallId, params, signal, onUpdate) {
+			const urlList = params.urls ?? (params.url ? [params.url] : []);
+			if (urlList.length === 0) {
+				return {
+					content: [{ type: "text", text: "Error: No URL provided." }],
+					details: { error: "No URL provided" },
+				};
+			}
+
+			onUpdate?.({
+				content: [{ type: "text", text: `Fetching ${urlList.length} URL(s)...` }],
+				details: { phase: "fetch", progress: 0 },
+			});
+
+			const fetchResults = await fetchAllContent(urlList, signal, {
+				forceClone: params.forceClone,
+				prompt: params.prompt,
+				timestamp: params.timestamp,
+				frames: params.frames,
+				model: params.model,
+			});
+			const successful = fetchResults.filter((r) => !r.error).length;
+			const totalChars = fetchResults.reduce((sum, r) => sum + r.content.length, 0);
+
+			const responseId = generateId();
+			const data: StoredSearchData = {
+				id: responseId,
+				type: "fetch",
+				timestamp: Date.now(),
+				urls: deps.stripThumbnails(fetchResults),
+			};
+			storeResult(responseId, data);
+			pi.appendEntry("web-search-results", data);
+
+			if (urlList.length === 1) {
+				const result = fetchResults[0];
+				if (result.error) {
+					return {
+						content: [{ type: "text", text: `Error: ${result.error}` }],
+						details: { urls: urlList, urlCount: 1, successful: 0, error: result.error, responseId, prompt: params.prompt, timestamp: params.timestamp, frames: params.frames },
+					};
+				}
+
+				const fullLength = result.content.length;
+				const truncated = fullLength > deps.maxInlineContent;
+				let output = truncated
+					? result.content.slice(0, deps.maxInlineContent) + "\n\n[Content truncated...]"
+					: result.content;
+
+				if (truncated) {
+					output += `\n\n---\nShowing ${deps.maxInlineContent} of ${fullLength} chars. ` +
+						`Use get_search_content({ responseId: "${responseId}", urlIndex: 0 }) for full content.`;
+				}
+
+				const content: Array<{ type: string; text?: string; data?: string; mimeType?: string }> = [];
+				if (result.frames?.length) {
+					for (const frame of result.frames) {
+						content.push({ type: "image", data: frame.data, mimeType: frame.mimeType });
+						content.push({ type: "text", text: `Frame at ${frame.timestamp}` });
+					}
+				} else if (result.thumbnail) {
+					content.push({ type: "image", data: result.thumbnail.data, mimeType: result.thumbnail.mimeType });
+				}
+				content.push({ type: "text", text: output });
+
+				const imageCount = (result.frames?.length ?? 0) + (result.thumbnail ? 1 : 0);
+				return {
+					content,
+					details: {
+						urls: urlList,
+						urlCount: 1,
+						successful: 1,
+						totalChars: fullLength,
+						title: result.title,
+						responseId,
+						truncated,
+						hasImage: imageCount > 0,
+						imageCount,
+						prompt: params.prompt,
+						timestamp: params.timestamp,
+						frames: params.frames,
+						duration: result.duration,
+					},
+				};
+			}
+
+			let output = "## Fetched URLs\n\n";
+			for (const { url, title, content, error } of fetchResults) {
+				if (error) {
+					output += `- ${url}: Error - ${error}\n`;
+				} else {
+					output += `- ${title || url} (${content.length} chars)\n`;
+				}
+			}
+			output += `\n---\nUse get_search_content({ responseId: "${responseId}", urlIndex: 0 }) to retrieve full content.`;
+
+			return {
+				content: [{ type: "text", text: output }],
+				details: { urls: urlList, urlCount: urlList.length, successful, totalChars, responseId },
+			};
+		},
+
+		renderCall(args, theme) {
+			const { url, urls, prompt, timestamp, frames, model } = args as { url?: string; urls?: string[]; prompt?: string; timestamp?: string; frames?: number; model?: string };
+			const urlList = urls ?? (url ? [url] : []);
+			if (urlList.length === 0) {
+				return new Text(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("error", "(no URL)"), 0, 0);
+			}
+			const lines: string[] = [];
+			if (urlList.length === 1) {
+				const display = urlList[0].length > 60 ? urlList[0].slice(0, 57) + "..." : urlList[0];
+				lines.push(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("accent", display));
+			} else {
+				lines.push(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("accent", `${urlList.length} URLs`));
+				for (const u of urlList.slice(0, 5)) {
+					const display = u.length > 60 ? u.slice(0, 57) + "..." : u;
+					lines.push(theme.fg("muted", "  " + display));
+				}
+				if (urlList.length > 5) {
+					lines.push(theme.fg("muted", `  ... and ${urlList.length - 5} more`));
+				}
+			}
+			if (timestamp) lines.push(theme.fg("dim", "  timestamp: ") + theme.fg("warning", timestamp));
+			if (typeof frames === "number") lines.push(theme.fg("dim", "  frames: ") + theme.fg("warning", String(frames)));
+			if (prompt) {
+				const display = prompt.length > 250 ? prompt.slice(0, 247) + "..." : prompt;
+				lines.push(theme.fg("dim", "  prompt: ") + theme.fg("muted", `"${display}"`));
+			}
+			if (model) lines.push(theme.fg("dim", "  model: ") + theme.fg("warning", model));
+			return new Text(lines.join("\n"), 0, 0);
+		},
+
+		renderResult: renderFetchContentResult,
+	});
+
+	pi.registerTool({
+		name: "get_search_content",
+		label: "Get Search Content",
+		description: "Retrieve full content from a previous web_search or fetch_content call.",
+		promptSnippet:
+			"Use after web_search/fetch_content when full stored content is needed via responseId plus query/url selectors.",
+		parameters: Type.Object({
+			responseId: Type.String({ description: "The responseId from web_search or fetch_content" }),
+			query: Type.Optional(Type.String({ description: "Get content for this query (web_search)" })),
+			queryIndex: Type.Optional(Type.Number({ description: "Get content for query at index" })),
+			url: Type.Optional(Type.String({ description: "Get content for this URL" })),
+			urlIndex: Type.Optional(Type.Number({ description: "Get content for URL at index" })),
+		}),
+
+		async execute(_toolCallId, params) {
+			const data = getResult(params.responseId);
+			if (!data) {
+				return {
+					content: [{ type: "text", text: `Error: No stored results for "${params.responseId}"` }],
+					details: { error: "Not found", responseId: params.responseId },
+				};
+			}
+
+			if (data.type === "search" && data.queries) {
+				let queryData: QueryResultData | undefined;
+
+				if (params.query !== undefined) {
+					queryData = data.queries.find((q) => q.query === params.query);
+					if (!queryData) {
+						const available = data.queries.map((q) => `"${q.query}"`).join(", ");
+						return {
+							content: [{ type: "text", text: `Query "${params.query}" not found. Available: ${available}` }],
+							details: { error: "Query not found" },
+						};
+					}
+				} else if (params.queryIndex !== undefined) {
+					queryData = data.queries[params.queryIndex];
+					if (!queryData) {
+						return {
+							content: [{ type: "text", text: `Index ${params.queryIndex} out of range (0-${data.queries.length - 1})` }],
+							details: { error: "Index out of range" },
+						};
+					}
+				} else {
+					const available = data.queries.map((q, i) => `${i}: "${q.query}"`).join(", ");
+					return {
+						content: [{ type: "text", text: `Specify query or queryIndex. Available: ${available}` }],
+						details: { error: "No query specified" },
+					};
+				}
+
+				if (queryData.error) {
+					return {
+						content: [{ type: "text", text: `Error for "${queryData.query}": ${queryData.error}` }],
+						details: { error: queryData.error, query: queryData.query },
+					};
+				}
+
+				return {
+					content: [{ type: "text", text: deps.formatFullResults(queryData) }],
+					details: { query: queryData.query, resultCount: queryData.results.length },
+				};
+			}
+
+			if (data.type === "fetch" && data.urls) {
+				let urlData: ExtractedContent | undefined;
+
+				if (params.url !== undefined) {
+					urlData = data.urls.find((u) => u.url === params.url);
+					if (!urlData) {
+						const available = data.urls.map((u) => u.url).join("\n  ");
+						return {
+							content: [{ type: "text", text: `URL not found. Available:\n  ${available}` }],
+							details: { error: "URL not found" },
+						};
+					}
+				} else if (params.urlIndex !== undefined) {
+					urlData = data.urls[params.urlIndex];
+					if (!urlData) {
+						return {
+							content: [{ type: "text", text: `Index ${params.urlIndex} out of range (0-${data.urls.length - 1})` }],
+							details: { error: "Index out of range" },
+						};
+					}
+				} else {
+					const available = data.urls.map((u, i) => `${i}: ${u.url}`).join("\n  ");
+					return {
+						content: [{ type: "text", text: `Specify url or urlIndex. Available:\n  ${available}` }],
+						details: { error: "No URL specified" },
+					};
+				}
+
+				if (urlData.error) {
+					return {
+						content: [{ type: "text", text: `Error for ${urlData.url}: ${urlData.error}` }],
+						details: { error: urlData.error, url: urlData.url },
+					};
+				}
+
+				return {
+					content: [{ type: "text", text: `# ${urlData.title}\n\n${urlData.content}` }],
+					details: { url: urlData.url, title: urlData.title, contentLength: urlData.content.length },
+				};
+			}
+
+			return {
+				content: [{ type: "text", text: "Invalid stored data format" }],
+				details: { error: "Invalid data" },
+			};
+		},
+
+		renderCall(args, theme) {
+			const { responseId, query, queryIndex, url, urlIndex } = args as {
+				responseId: string;
+				query?: string;
+				queryIndex?: number;
+				url?: string;
+				urlIndex?: number;
+			};
+			let target = "";
+			if (query) target = `query="${query}"`;
+			else if (queryIndex !== undefined) target = `queryIndex=${queryIndex}`;
+			else if (url) target = url.length > 30 ? url.slice(0, 27) + "..." : url;
+			else if (urlIndex !== undefined) target = `urlIndex=${urlIndex}`;
+			return new Text(theme.fg("toolTitle", theme.bold("get_content ")) + theme.fg("accent", target || responseId.slice(0, 8)), 0, 0);
+		},
+
+		renderResult: renderGetSearchContentResult,
+	});
+}