@bastani/atomic 0.9.16-alpha.2 → 0.9.16-alpha.3

package/dist/builtin/web-access/code-search.ts

@@ -1,107 +0,0 @@
-import { activityMonitor } from "./activity.js";
-import { callExaMcp } from "./exa.js";
-
-const CODE_CONTEXT_TOOL = "get_code_context_exa";
-const WEB_SEARCH_TOOL = "web_search_exa";
-const DEFAULT_MAX_TOKENS = 5000;
-
-let codeContextToolMissing = false;
-
-function isMissingMcpToolError(message: string): boolean {
-	const normalized = message.toLowerCase();
-	return normalized.includes("tool") && normalized.includes("not found");
-}
-
-function buildFallbackQuery(query: string): string {
-	const normalized = query.toLowerCase();
-	const hasCodeTerms = /\b(api|code|docs?|documentation|example|github|implementation|library|source|stackoverflow|stack overflow)\b/.test(normalized);
-	return hasCodeTerms ? query : `${query} code examples documentation GitHub Stack Overflow official docs`;
-}
-
-function maxTokensToResultCount(maxTokens: number): number {
-	return Math.min(20, Math.max(5, Math.ceil(maxTokens / 1000)));
-}
-
-function trimApproxTokens(text: string, maxTokens: number): string {
-	const maxCharacters = Math.max(1000, maxTokens * 4);
-	if (text.length <= maxCharacters) return text;
-	return `${text.slice(0, maxCharacters).trimEnd()}\n\n[Truncated by code_search to approximately ${maxTokens} tokens.]`;
-}
-
-async function executeFallbackSearch(query: string, maxTokens: number, signal?: AbortSignal): Promise<string> {
-	const text = await callExaMcp(
-		WEB_SEARCH_TOOL,
-		{
-			query: buildFallbackQuery(query),
-			numResults: maxTokensToResultCount(maxTokens),
-			livecrawl: "fallback",
-			type: "auto",
-			contextMaxCharacters: Math.min(50000, Math.max(1000, maxTokens * 4)),
-		},
-		signal,
-	);
-	return trimApproxTokens(text, maxTokens);
-}
-
-export async function executeCodeSearch(
-	_toolCallId: string,
-	params: { query: string; maxTokens?: number },
-	signal?: AbortSignal,
-): Promise<{
-	content: Array<{ type: "text"; text: string }>;
-	details: { query: string; maxTokens: number; error?: string; mode?: "code-context" | "web-search-fallback" };
-}> {
-	const query = params.query.trim();
-	if (!query) {
-		return {
-			content: [{ type: "text", text: "Error: No query provided." }],
-			details: { query: "", maxTokens: params.maxTokens ?? DEFAULT_MAX_TOKENS, error: "No query provided" },
-		};
-	}
-
-	const maxTokens = params.maxTokens ?? DEFAULT_MAX_TOKENS;
-	const activityId = activityMonitor.logStart({ type: "api", query });
-
-	try {
-		let mode: "code-context" | "web-search-fallback" = "web-search-fallback";
-		let text: string;
-
-		if (codeContextToolMissing) {
-			text = await executeFallbackSearch(query, maxTokens, signal);
-		} else {
-			try {
-				text = await callExaMcp(
-					CODE_CONTEXT_TOOL,
-					{
-						query,
-						tokensNum: maxTokens,
-					},
-					signal,
-				);
-				mode = "code-context";
-			} catch (err) {
-				const message = err instanceof Error ? err.message : String(err);
-				if (!isMissingMcpToolError(message)) throw err;
-				codeContextToolMissing = true;
-				text = await executeFallbackSearch(query, maxTokens, signal);
-			}
-		}
-
-		activityMonitor.logComplete(activityId, 200);
-		return {
-			content: [{ type: "text", text }],
-			details: { query, maxTokens, mode },
-		};
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		if (message.toLowerCase().includes("abort")) {
-			activityMonitor.logComplete(activityId, 0);
-			throw err;
-		}
-		activityMonitor.logError(activityId, message);
-		return {
-			content: [{ type: "text", text: `Error: ${message}` }],
-			details: { query, maxTokens, error: message },
-		};
-	}
-}

package/dist/builtin/web-access/config-paths.ts

@@ -1,11 +0,0 @@
-import { existsSync } from "node:fs";
-import { getUserConfigPaths } from "@bastani/atomic";
-
-export const WEB_SEARCH_CONFIG_PATHS = getUserConfigPaths("web-search.json");
-export const WEB_SEARCH_CONFIG_PATH = WEB_SEARCH_CONFIG_PATHS[0] ?? "~/.atomic/web-search.json";
-export const EXA_USAGE_PATHS = getUserConfigPaths("exa-usage.json");
-export const EXA_USAGE_PATH = EXA_USAGE_PATHS[0] ?? "~/.atomic/exa-usage.json";
-
-export function findReadableConfigPath(paths = WEB_SEARCH_CONFIG_PATHS): string {
-	return paths.find((path) => existsSync(path)) ?? paths[0] ?? WEB_SEARCH_CONFIG_PATH;
-}

package/dist/builtin/web-access/content-tools.ts

@@ -1,354 +0,0 @@
-import type { ExtensionAPI } from "@bastani/atomic";
-import { Text } from "@earendil-works/pi-tui";
-import { Type } from "typebox";
-import {
-	renderCodeSearchResult,
-	renderFetchContentResult,
-	renderGetSearchContentResult,
-} from "./result-renderers.js";
-import { fetchAllContent, type ExtractedContent } from "./extract.js";
-import { executeCodeSearch } from "./code-search.js";
-import {
-	generateId,
-	getResult,
-	storeResult,
-	type QueryResultData,
-	type StoredSearchData,
-} from "./storage.js";
-
-interface RegisterContentToolsDeps {
-	maxInlineContent: number;
-	stripThumbnails(results: ExtractedContent[]): ExtractedContent[];
-	formatFullResults(queryData: QueryResultData): string;
-}
-
-export function registerContentTools(pi: ExtensionAPI, deps: RegisterContentToolsDeps): void {
-	pi.registerTool({
-		name: "code_search",
-		label: "Code Search",
-		description: "Search for code examples, documentation, and API references. Returns relevant code snippets and docs from GitHub, Stack Overflow, and official documentation. Use for any programming question — API usage, library examples, debugging help.",
-		promptSnippet:
-			"Use for programming/API/library questions to retrieve concrete examples and docs before implementing or debugging code.",
-		parameters: Type.Object({
-			query: Type.String({ description: "Programming question, API, library, or debugging topic to search for" }),
-			maxTokens: Type.Optional(Type.Integer({
-				minimum: 1000,
-				maximum: 50000,
-				description: "Maximum tokens of code/documentation context to return (default: 5000)",
-			})),
-		}),
-
-		async execute(toolCallId, params, signal) {
-			return executeCodeSearch(toolCallId, params, signal);
-		},
-
-		renderCall(args, theme) {
-			const { query } = args as { query?: string };
-			// This status-line preview is built and dropped within the same turn, so a bare slice is sufficient.
-			const display = !query
-				? "(no query)"
-				: query.length > 70 ? query.slice(0, 67) + "..." : query;
-			return new Text(theme.fg("toolTitle", theme.bold("code_search ")) + theme.fg("accent", display), 0, 0);
-		},
-
-		renderResult: renderCodeSearchResult,
-	});
-
-	pi.registerTool({
-		name: "fetch_content",
-		label: "Fetch Content",
-		description: "Fetch URL(s) and extract readable content as markdown. Supports YouTube video transcripts (with thumbnail), GitHub repository contents, and local video files (with frame thumbnail). Video frames can be extracted via timestamp/range or sampled across the entire video with frames alone. Falls back to Gemini for pages that block bots or fail Readability extraction. For YouTube and video files: ALWAYS pass the user's specific question via the prompt parameter — this directs the AI to focus on that aspect of the video, producing much better results than a generic extraction. Content is always stored and can be retrieved with get_search_content.",
-		promptSnippet:
-			"Use to extract readable content from URL(s), YouTube, GitHub repos, or local videos. For video questions, pass the user's exact question in prompt.",
-		parameters: Type.Object({
-			url: Type.Optional(Type.String({ description: "Single URL to fetch" })),
-			urls: Type.Optional(Type.Array(Type.String(), { description: "Multiple URLs (parallel)" })),
-			forceClone: Type.Optional(Type.Boolean({
-				description: "Force cloning large GitHub repositories that exceed the size threshold",
-			})),
-			prompt: Type.Optional(Type.String({
-				description: "Question or instruction for video analysis (YouTube and video files). Pass the user's specific question here — e.g. 'describe the book shown at the advice for beginners section'. Without this, a generic transcript extraction is used which may miss what the user is asking about.",
-			})),
-			timestamp: Type.Optional(Type.String({
-				description: "Extract video frame(s) at a timestamp or time range. Single: '1:23:45', '23:45', or '85' (seconds). Range: '23:41-25:00' extracts evenly-spaced frames across that span (default 6). Use frames with ranges to control density; single+frames uses a fixed 5s interval. YouTube requires yt-dlp + ffmpeg; local videos require ffmpeg. Use a range when you know the approximate area but not the exact moment — you'll get a contact sheet to visually identify the right frame.",
-			})),
-			frames: Type.Optional(Type.Integer({
-				minimum: 1,
-				maximum: 12,
-				description: "Number of frames to extract. Use with timestamp range for custom density, with single timestamp to get N frames at 5s intervals, or alone to sample across the entire video. Requires yt-dlp + ffmpeg for YouTube, ffmpeg for local video.",
-			})),
-			model: Type.Optional(Type.String({
-				description: "Override the Gemini model for video/YouTube analysis (e.g. 'gemini-2.5-flash', 'gemini-3-flash-preview'). Defaults to config or gemini-3-flash-preview.",
-			})),
-		}),
-
-		async execute(_toolCallId, params, signal, onUpdate) {
-			const urlList = params.urls ?? (params.url ? [params.url] : []);
-			if (urlList.length === 0) {
-				return {
-					content: [{ type: "text", text: "Error: No URL provided." }],
-					details: { error: "No URL provided" },
-				};
-			}
-
-			onUpdate?.({
-				content: [{ type: "text", text: `Fetching ${urlList.length} URL(s)...` }],
-				details: { phase: "fetch", progress: 0 },
-			});
-
-			const fetchResults = await fetchAllContent(urlList, signal, {
-				forceClone: params.forceClone,
-				prompt: params.prompt,
-				timestamp: params.timestamp,
-				frames: params.frames,
-				model: params.model,
-			});
-			signal?.throwIfAborted();
-			const successful = fetchResults.filter((r) => !r.error).length;
-			const totalChars = fetchResults.reduce((sum, r) => sum + r.content.length, 0);
-
-			const responseId = generateId();
-			const data: StoredSearchData = {
-				id: responseId,
-				type: "fetch",
-				timestamp: Date.now(),
-				urls: deps.stripThumbnails(fetchResults),
-			};
-			storeResult(responseId, data);
-			pi.appendEntry("web-search-results", data);
-
-			if (urlList.length === 1) {
-				const result = fetchResults[0];
-				if (result.error) {
-					return {
-						content: [{ type: "text", text: `Error: ${result.error}` }],
-						details: { outcome: "all_failed", stage: "fetch", urls: urlList, urlCount: 1, successful: 0, error: result.error, responseId, prompt: params.prompt, timestamp: params.timestamp, frames: params.frames },
-					};
-				}
-
-				const fullLength = result.content.length;
-				const truncated = fullLength > deps.maxInlineContent;
-				let output = truncated
-					? result.content.slice(0, deps.maxInlineContent) + "\n\n[Content truncated...]"
-					: result.content;
-
-				if (truncated) {
-					output += `\n\n---\nShowing ${deps.maxInlineContent} of ${fullLength} chars. ` +
-						`Use get_search_content({ responseId: "${responseId}", urlIndex: 0 }) for full content.`;
-				}
-
-				const content: Array<{ type: string; text?: string; data?: string; mimeType?: string }> = [];
-				if (result.frames?.length) {
-					for (const frame of result.frames) {
-						content.push({ type: "image", data: frame.data, mimeType: frame.mimeType });
-						content.push({ type: "text", text: `Frame at ${frame.timestamp}` });
-					}
-				} else if (result.thumbnail) {
-					content.push({ type: "image", data: result.thumbnail.data, mimeType: result.thumbnail.mimeType });
-				}
-				content.push({ type: "text", text: output });
-
-				const imageCount = (result.frames?.length ?? 0) + (result.thumbnail ? 1 : 0);
-				return {
-					content,
-					details: {
-						urls: urlList,
-						urlCount: 1,
-						successful: 1,
-						totalChars: fullLength,
-						title: result.title,
-						responseId,
-						truncated,
-						hasImage: imageCount > 0,
-						imageCount,
-						prompt: params.prompt,
-						timestamp: params.timestamp,
-						frames: params.frames,
-						duration: result.duration,
-					},
-				};
-			}
-
-			let output = "## Fetched URLs\n\n";
-			for (const { url, title, content, error } of fetchResults) {
-				if (error) {
-					output += `- ${url}: Error - ${error}\n`;
-				} else {
-					output += `- ${title || url} (${content.length} chars)\n`;
-				}
-			}
-			output += `\n---\nUse get_search_content({ responseId: "${responseId}", urlIndex: 0 }) to retrieve full content.`;
-
-			const allFailed = successful === 0;
-			return {
-				content: [{ type: "text", text: output }],
-				details: {
-					...(allFailed ? { outcome: "all_failed", stage: "fetch", error: `All ${urlList.length} URL fetch(es) failed`, failedUrls: urlList.length } : {}),
-					urls: urlList, urlCount: urlList.length, successful, totalChars, responseId,
-				},
-			};
-		},
-
-		renderCall(args, theme) {
-			const { url, urls, prompt, timestamp, frames, model } = args as { url?: string; urls?: string[]; prompt?: string; timestamp?: string; frames?: number; model?: string };
-			const urlList = urls ?? (url ? [url] : []);
-			if (urlList.length === 0) {
-				return new Text(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("error", "(no URL)"), 0, 0);
-			}
-			const lines: string[] = [];
-			if (urlList.length === 1) {
-				const display = urlList[0].length > 60 ? urlList[0].slice(0, 57) + "..." : urlList[0];
-				lines.push(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("accent", display));
-			} else {
-				lines.push(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("accent", `${urlList.length} URLs`));
-				for (const u of urlList.slice(0, 5)) {
-					const display = u.length > 60 ? u.slice(0, 57) + "..." : u;
-					lines.push(theme.fg("muted", "  " + display));
-				}
-				if (urlList.length > 5) {
-					lines.push(theme.fg("muted", `  ... and ${urlList.length - 5} more`));
-				}
-			}
-			if (timestamp) lines.push(theme.fg("dim", "  timestamp: ") + theme.fg("warning", timestamp));
-			if (typeof frames === "number") lines.push(theme.fg("dim", "  frames: ") + theme.fg("warning", String(frames)));
-			if (prompt) {
-				const display = prompt.length > 250 ? prompt.slice(0, 247) + "..." : prompt;
-				lines.push(theme.fg("dim", "  prompt: ") + theme.fg("muted", `"${display}"`));
-			}
-			if (model) lines.push(theme.fg("dim", "  model: ") + theme.fg("warning", model));
-			return new Text(lines.join("\n"), 0, 0);
-		},
-
-		renderResult: renderFetchContentResult,
-	});
-
-	pi.registerTool({
-		name: "get_search_content",
-		label: "Get Search Content",
-		description: "Retrieve full content from a previous web_search or fetch_content call.",
-		promptSnippet:
-			"Use after web_search/fetch_content when full stored content is needed via responseId plus query/url selectors.",
-		parameters: Type.Object({
-			responseId: Type.String({ description: "The responseId from web_search or fetch_content" }),
-			query: Type.Optional(Type.String({ description: "Get content for this query (web_search)" })),
-			queryIndex: Type.Optional(Type.Number({ description: "Get content for query at index" })),
-			url: Type.Optional(Type.String({ description: "Get content for this URL" })),
-			urlIndex: Type.Optional(Type.Number({ description: "Get content for URL at index" })),
-		}),
-
-		async execute(_toolCallId, params) {
-			const data = getResult(params.responseId);
-			if (!data) {
-				return {
-					content: [{ type: "text", text: `Error: No stored results for "${params.responseId}"` }],
-					details: { error: "Not found", responseId: params.responseId },
-				};
-			}
-
-			if (data.type === "search" && data.queries) {
-				let queryData: QueryResultData | undefined;
-
-				if (params.query !== undefined) {
-					queryData = data.queries.find((q) => q.query === params.query);
-					if (!queryData) {
-						const available = data.queries.map((q) => `"${q.query}"`).join(", ");
-						return {
-							content: [{ type: "text", text: `Query "${params.query}" not found. Available: ${available}` }],
-							details: { error: "Query not found" },
-						};
-					}
-				} else if (params.queryIndex !== undefined) {
-					queryData = data.queries[params.queryIndex];
-					if (!queryData) {
-						return {
-							content: [{ type: "text", text: `Index ${params.queryIndex} out of range (0-${data.queries.length - 1})` }],
-							details: { error: "Index out of range" },
-						};
-					}
-				} else {
-					const available = data.queries.map((q, i) => `${i}: "${q.query}"`).join(", ");
-					return {
-						content: [{ type: "text", text: `Specify query or queryIndex. Available: ${available}` }],
-						details: { error: "No query specified" },
-					};
-				}
-
-				if (queryData.error) {
-					return {
-						content: [{ type: "text", text: `Error for "${queryData.query}": ${queryData.error}` }],
-						details: { error: queryData.error, query: queryData.query },
-					};
-				}
-
-				return {
-					content: [{ type: "text", text: deps.formatFullResults(queryData) }],
-					details: { query: queryData.query, resultCount: queryData.results.length },
-				};
-			}
-
-			if (data.type === "fetch" && data.urls) {
-				let urlData: ExtractedContent | undefined;
-
-				if (params.url !== undefined) {
-					urlData = data.urls.find((u) => u.url === params.url);
-					if (!urlData) {
-						const available = data.urls.map((u) => u.url).join("\n  ");
-						return {
-							content: [{ type: "text", text: `URL not found. Available:\n  ${available}` }],
-							details: { error: "URL not found" },
-						};
-					}
-				} else if (params.urlIndex !== undefined) {
-					urlData = data.urls[params.urlIndex];
-					if (!urlData) {
-						return {
-							content: [{ type: "text", text: `Index ${params.urlIndex} out of range (0-${data.urls.length - 1})` }],
-							details: { error: "Index out of range" },
-						};
-					}
-				} else {
-					const available = data.urls.map((u, i) => `${i}: ${u.url}`).join("\n  ");
-					return {
-						content: [{ type: "text", text: `Specify url or urlIndex. Available:\n  ${available}` }],
-						details: { error: "No URL specified" },
-					};
-				}
-
-				if (urlData.error) {
-					return {
-						content: [{ type: "text", text: `Error for ${urlData.url}: ${urlData.error}` }],
-						details: { error: urlData.error, url: urlData.url },
-					};
-				}
-
-				return {
-					content: [{ type: "text", text: `# ${urlData.title}\n\n${urlData.content}` }],
-					details: { url: urlData.url, title: urlData.title, contentLength: urlData.content.length },
-				};
-			}
-
-			return {
-				content: [{ type: "text", text: "Invalid stored data format" }],
-				details: { error: "Invalid data" },
-			};
-		},
-
-		renderCall(args, theme) {
-			const { responseId, query, queryIndex, url, urlIndex } = args as {
-				responseId: string;
-				query?: string;
-				queryIndex?: number;
-				url?: string;
-				urlIndex?: number;
-			};
-			let target = "";
-			if (query) target = `query="${query}"`;
-			else if (queryIndex !== undefined) target = `queryIndex=${queryIndex}`;
-			else if (url) target = url.length > 30 ? url.slice(0, 27) + "..." : url;
-			else if (urlIndex !== undefined) target = `urlIndex=${urlIndex}`;
-			return new Text(theme.fg("toolTitle", theme.bold("get_content ")) + theme.fg("accent", target || responseId.slice(0, 8)), 0, 0);
-		},
-
-		renderResult: renderGetSearchContentResult,
-	});
-}