@bastani/atomic 0.7.17 → 0.8.0

package/dist/builtin/web-access/gemini-api.ts

@@ -0,0 +1,113 @@
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { CONFIG_DIR_NAME } from "@bastani/atomic";
+
+export const API_BASE = "https://generativelanguage.googleapis.com/v1beta";
+const CONFIG_PATH = join(homedir(), CONFIG_DIR_NAME, "web-search.json");
+export const DEFAULT_MODEL = "gemini-3-flash-preview";
+
+interface GeminiApiConfig {
+	geminiApiKey?: unknown;
+}
+
+let cachedConfig: GeminiApiConfig | null = null;
+
+function loadConfig(): GeminiApiConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+
+	const raw = readFileSync(CONFIG_PATH, "utf-8");
+	try {
+		cachedConfig = JSON.parse(raw) as GeminiApiConfig;
+		return cachedConfig;
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+}
+
+function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
+	const timeout = AbortSignal.timeout(timeoutMs);
+	return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+
+function normalizeApiKey(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const normalized = value.trim();
+	return normalized.length > 0 ? normalized : null;
+}
+
+export function getApiKey(): string | null {
+	return normalizeApiKey(process.env.GEMINI_API_KEY) ?? normalizeApiKey(loadConfig().geminiApiKey);
+}
+
+export function isGeminiApiAvailable(): boolean {
+	return getApiKey() !== null;
+}
+
+export interface GeminiApiOptions {
+	model?: string;
+	mimeType?: string;
+	signal?: AbortSignal;
+	timeoutMs?: number;
+}
+
+export async function queryGeminiApiWithVideo(
+	prompt: string,
+	videoUri: string,
+	options: GeminiApiOptions = {},
+): Promise<string> {
+	const apiKey = getApiKey();
+	if (!apiKey) throw new Error("GEMINI_API_KEY not configured");
+
+	const model = options.model ?? DEFAULT_MODEL;
+	const signal = withTimeout(options.signal, options.timeoutMs ?? 120000);
+	const url = `${API_BASE}/models/${model}:generateContent?key=${apiKey}`;
+
+	const fileData: Record<string, string> = { fileUri: videoUri };
+	if (options.mimeType) fileData.mimeType = options.mimeType;
+
+	const body = {
+		contents: [
+			{
+				parts: [
+					{ fileData },
+					{ text: prompt },
+				],
+			},
+		],
+	};
+
+	const res = await fetch(url, {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(body),
+		signal,
+	});
+
+	if (!res.ok) {
+		const errorText = await res.text();
+		throw new Error(`Gemini API error ${res.status}: ${errorText.slice(0, 300)}`);
+	}
+
+	const data = (await res.json()) as GenerateContentResponse;
+	const text = data.candidates?.[0]?.content?.parts
+		?.map((p) => p.text)
+		.filter(Boolean)
+		.join("\n");
+
+	if (!text) throw new Error("Gemini API returned empty response");
+	return text;
+}
+
+interface GenerateContentResponse {
+	candidates?: Array<{
+		content?: {
+			parts?: Array<{ text?: string }>;
+		};
+	}>;
+}

package/dist/builtin/web-access/gemini-search.ts

@@ -0,0 +1,362 @@
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { CONFIG_DIR_NAME } from "@bastani/atomic";
+import { activityMonitor } from "./activity.js";
+import { getApiKey, API_BASE, DEFAULT_MODEL } from "./gemini-api.js";
+import { isGeminiWebAvailable, queryWithCookies } from "./gemini-web.js";
+import { isPerplexityAvailable, searchWithPerplexity, type SearchResult, type SearchResponse, type SearchOptions } from "./perplexity.js";
+import { hasExaApiKey, isExaAvailable, searchWithExa } from "./exa.js";
+
+export type SearchProvider = "auto" | "perplexity" | "gemini" | "exa";
+export type ResolvedSearchProvider = Exclude<SearchProvider, "auto">;
+
+export interface AttributedSearchResponse extends SearchResponse {
+	provider: ResolvedSearchProvider;
+}
+
+const CONFIG_PATH = join(homedir(), CONFIG_DIR_NAME, "web-search.json");
+
+let cachedSearchConfig: { searchProvider: SearchProvider; searchModel?: string } | null = null;
+
+function getSearchConfig(): { searchProvider: SearchProvider; searchModel?: string } {
+	if (cachedSearchConfig) return cachedSearchConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedSearchConfig = { searchProvider: "auto", searchModel: undefined };
+		return cachedSearchConfig;
+	}
+
+	const rawText = readFileSync(CONFIG_PATH, "utf-8");
+	let raw: {
+		searchProvider?: SearchProvider;
+		provider?: SearchProvider;
+		searchModel?: unknown;
+	};
+	try {
+		raw = JSON.parse(rawText) as {
+			searchProvider?: SearchProvider;
+			provider?: SearchProvider;
+			searchModel?: unknown;
+		};
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+
+	cachedSearchConfig = {
+		searchProvider: normalizeSearchProvider(raw.searchProvider ?? raw.provider),
+		searchModel: normalizeSearchModel(raw.searchModel),
+	};
+	return cachedSearchConfig;
+}
+
+function normalizeSearchModel(value: unknown): string | undefined {
+	if (typeof value !== "string") return undefined;
+	const normalized = value.trim();
+	return normalized.length > 0 ? normalized : undefined;
+}
+
+function normalizeSearchProvider(value: unknown): SearchProvider {
+	const normalized = typeof value === "string" ? value.trim().toLowerCase() : "";
+	return normalized === "auto" || normalized === "perplexity" || normalized === "gemini" || normalized === "exa"
+		? normalized
+		: "auto";
+}
+
+export interface FullSearchOptions extends SearchOptions {
+	provider?: SearchProvider;
+	includeContent?: boolean;
+}
+
+function errorMessage(err: unknown): string {
+	return err instanceof Error ? err.message : String(err);
+}
+
+function isAbortError(err: unknown): boolean {
+	return errorMessage(err).toLowerCase().includes("abort");
+}
+
+async function searchWithGemini(
+	query: string,
+	options: SearchOptions,
+	strictErrors: boolean,
+): Promise<SearchResponse | null> {
+	const errors: string[] = [];
+
+	try {
+		const apiResult = await searchWithGeminiApi(query, options);
+		if (apiResult) return apiResult;
+	} catch (err) {
+		if (isAbortError(err)) throw err;
+		errors.push(`Gemini API: ${errorMessage(err)}`);
+	}
+
+	try {
+		const webResult = await searchWithGeminiWeb(query, options);
+		if (webResult) return webResult;
+	} catch (err) {
+		if (isAbortError(err)) throw err;
+		errors.push(`Gemini Web: ${errorMessage(err)}`);
+	}
+
+	if (strictErrors && errors.length > 0) {
+		throw new Error(`Gemini search failed:\n  - ${errors.join("\n  - ")}`);
+	}
+
+	return null;
+}
+
+export async function search(query: string, options: FullSearchOptions = {}): Promise<AttributedSearchResponse> {
+	const config = getSearchConfig();
+	const provider = options.provider ?? config.searchProvider;
+
+	if (provider === "perplexity") {
+		const result = await searchWithPerplexity(query, options);
+		return { ...result, provider: "perplexity" };
+	}
+
+	if (provider === "gemini") {
+		const result = await searchWithGemini(query, options, true);
+		if (result) return { ...result, provider: "gemini" };
+		throw new Error(
+			"Gemini search unavailable. Either:\n" +
+			`  1. Set GEMINI_API_KEY in ~/${CONFIG_DIR_NAME}/web-search.json\n` +
+			"  2. Sign into gemini.google.com in a supported Chromium-based browser"
+		);
+	}
+
+	if (provider === "exa") {
+		const exaApiKeyConfigured = hasExaApiKey();
+		try {
+			const result = await searchWithExa(query, options);
+			if (result && "exhausted" in result) {
+				throw new Error(
+					"Exa monthly free tier exhausted (1,000 requests). Resets next month.\n" +
+					"  Use provider: 'perplexity' or 'gemini', or upgrade at exa.ai/pricing"
+				);
+			}
+			if (result && "answer" in result) return { ...result, provider: "exa" };
+			if (exaApiKeyConfigured) {
+				throw new Error("Exa search returned no results.");
+			}
+		} catch (err) {
+			const message = err instanceof Error ? err.message : String(err);
+			if (message.toLowerCase().includes("abort")) throw err;
+			if (exaApiKeyConfigured) throw err;
+			// No API key: allow provider fallback.
+		}
+	}
+
+	const fallbackErrors: string[] = [];
+
+	if (provider !== "exa" && isExaAvailable()) {
+		try {
+			const result = await searchWithExa(query, options);
+			if (result && "answer" in result) return { ...result, provider: "exa" };
+		} catch (err) {
+			if (isAbortError(err)) throw err;
+			fallbackErrors.push(`Exa: ${errorMessage(err)}`);
+		}
+	}
+
+	if (isPerplexityAvailable()) {
+		try {
+			const result = await searchWithPerplexity(query, options);
+			return { ...result, provider: "perplexity" };
+		} catch (err) {
+			if (isAbortError(err)) throw err;
+			fallbackErrors.push(`Perplexity: ${errorMessage(err)}`);
+		}
+	}
+
+	try {
+		const geminiResult = await searchWithGemini(query, options, false);
+		if (geminiResult) return { ...geminiResult, provider: "gemini" };
+	} catch (err) {
+		if (isAbortError(err)) throw err;
+		fallbackErrors.push(`Gemini: ${errorMessage(err)}`);
+	}
+
+	if (fallbackErrors.length > 0) {
+		throw new Error(`Auto provider search failed:\n  - ${fallbackErrors.join("\n  - ")}`);
+	}
+
+	throw new Error(
+		"No search provider available. Either:\n" +
+		`  1. Set perplexityApiKey in ~/${CONFIG_DIR_NAME}/web-search.json\n` +
+		`  2. Set EXA_API_KEY (or exaApiKey) in ~/${CONFIG_DIR_NAME}/web-search.json\n` +
+		`  3. Set GEMINI_API_KEY in ~/${CONFIG_DIR_NAME}/web-search.json\n` +
+		"  4. Sign into gemini.google.com in a supported Chromium-based browser"
+	);
+}
+
+async function searchWithGeminiApi(query: string, options: SearchOptions = {}): Promise<SearchResponse | null> {
+	const apiKey = getApiKey();
+	if (!apiKey) return null;
+
+	const activityId = activityMonitor.logStart({ type: "api", query });
+
+	try {
+		const model = getSearchConfig().searchModel ?? DEFAULT_MODEL;
+		const body = {
+			contents: [{ parts: [{ text: query }] }],
+			tools: [{ google_search: {} }],
+		};
+
+		const res = await fetch(`${API_BASE}/models/${model}:generateContent?key=${apiKey}`, {
+			method: "POST",
+			headers: { "Content-Type": "application/json" },
+			body: JSON.stringify(body),
+			signal: AbortSignal.any([
+				AbortSignal.timeout(60000),
+				...(options.signal ? [options.signal] : []),
+			]),
+		});
+
+		if (!res.ok) {
+			const errorText = await res.text();
+			throw new Error(`Gemini API error ${res.status}: ${errorText.slice(0, 300)}`);
+		}
+
+		const data = await res.json() as GeminiSearchResponse;
+		activityMonitor.logComplete(activityId, res.status);
+
+		const answer = data.candidates?.[0]?.content?.parts
+			?.map(p => p.text).filter(Boolean).join("\n") ?? "";
+
+		const metadata = data.candidates?.[0]?.groundingMetadata;
+		const results = await resolveGroundingChunks(metadata?.groundingChunks, options.signal);
+
+		if (!answer && results.length === 0) return null;
+		return { answer, results };
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		if (message.toLowerCase().includes("abort")) {
+			activityMonitor.logComplete(activityId, 0);
+		} else {
+			activityMonitor.logError(activityId, message);
+		}
+		throw err;
+	}
+}
+
+async function searchWithGeminiWeb(query: string, options: SearchOptions = {}): Promise<SearchResponse | null> {
+	const cookies = await isGeminiWebAvailable();
+	if (!cookies) return null;
+
+	const prompt = buildSearchPrompt(query, options);
+	const activityId = activityMonitor.logStart({ type: "api", query });
+
+	try {
+		const text = await queryWithCookies(prompt, cookies, {
+			model: "gemini-3-flash-preview",
+			signal: options.signal,
+			timeoutMs: 60000,
+		});
+
+		activityMonitor.logComplete(activityId, 200);
+
+		const results = extractSourceUrls(text);
+		return { answer: text, results };
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		if (message.toLowerCase().includes("abort")) {
+			activityMonitor.logComplete(activityId, 0);
+		} else {
+			activityMonitor.logError(activityId, message);
+		}
+		throw err;
+	}
+}
+
+function buildSearchPrompt(query: string, options: SearchOptions): string {
+	let prompt = `Search the web and answer the following question. Include source URLs for your claims.\nFormat your response as:\n1. A direct answer to the question\n2. Cited sources as markdown links\n\nQuestion: ${query}`;
+
+	if (options.recencyFilter) {
+		const labels: Record<string, string> = {
+			day: "past 24 hours",
+			week: "past week",
+			month: "past month",
+			year: "past year",
+		};
+		prompt += `\n\nOnly include results from the ${labels[options.recencyFilter]}.`;
+	}
+
+	if (options.domainFilter?.length) {
+		const includes = options.domainFilter.filter(d => !d.startsWith("-"));
+		const excludes = options.domainFilter.filter(d => d.startsWith("-")).map(d => d.slice(1));
+		if (includes.length) prompt += `\n\nOnly cite sources from: ${includes.join(", ")}`;
+		if (excludes.length) prompt += `\n\nDo not cite sources from: ${excludes.join(", ")}`;
+	}
+
+	return prompt;
+}
+
+function extractSourceUrls(markdown: string): SearchResult[] {
+	const results: SearchResult[] = [];
+	const seen = new Set<string>();
+	const linkRegex = /\[([^\]]+)\]\((https?:\/\/[^)]+)\)/g;
+	for (const match of markdown.matchAll(linkRegex)) {
+		const url = match[2];
+		if (seen.has(url)) continue;
+		seen.add(url);
+		results.push({ title: match[1], url, snippet: "" });
+	}
+	return results;
+}
+
+async function resolveGroundingChunks(
+	chunks: GroundingChunk[] | undefined,
+	signal?: AbortSignal,
+): Promise<SearchResult[]> {
+	if (!chunks?.length) return [];
+
+	const results: SearchResult[] = [];
+	for (const chunk of chunks) {
+		if (!chunk.web) continue;
+		const title = chunk.web.title || "";
+		let url = chunk.web.uri || "";
+
+		if (url.includes("vertexaisearch.cloud.google.com/grounding-api-redirect")) {
+			const resolved = await resolveRedirect(url, signal);
+			if (resolved) url = resolved;
+		}
+
+		if (url) results.push({ title, url, snippet: "" });
+	}
+	return results;
+}
+
+async function resolveRedirect(proxyUrl: string, signal?: AbortSignal): Promise<string | null> {
+	try {
+		const res = await fetch(proxyUrl, {
+			method: "HEAD",
+			redirect: "manual",
+			signal: AbortSignal.any([
+				AbortSignal.timeout(5000),
+				...(signal ? [signal] : []),
+			]),
+		});
+		return res.headers.get("location") || null;
+	} catch {
+		return null;
+	}
+}
+
+interface GeminiSearchResponse {
+	candidates?: Array<{
+		content?: { parts?: Array<{ text?: string }> };
+		groundingMetadata?: {
+			webSearchQueries?: string[];
+			groundingChunks?: GroundingChunk[];
+			groundingSupports?: Array<{
+				segment?: { startIndex?: number; endIndex?: number; text?: string };
+				groundingChunkIndices?: number[];
+			}>;
+		};
+	}>;
+}
+
+interface GroundingChunk {
+	web?: { uri?: string; title?: string };
+}

package/dist/builtin/web-access/gemini-url-context.ts

@@ -0,0 +1,126 @@
+import { activityMonitor } from "./activity.js";
+import { getApiKey, API_BASE, DEFAULT_MODEL } from "./gemini-api.js";
+import { isGeminiWebAvailable, queryWithCookies } from "./gemini-web.js";
+import { extractHeadingTitle, type ExtractedContent } from "./extract.js";
+
+const EXTRACTION_PROMPT = `Extract the complete readable content from this URL as clean markdown.
+Include the page title, all text content, code blocks, and tables.
+Do not summarize — extract the full content.
+
+URL: `;
+
+function shouldRethrow(err: unknown): boolean {
+	const message = err instanceof Error ? err.message : String(err);
+	return message.startsWith("Failed to parse ");
+}
+
+export async function extractWithUrlContext(
+	url: string,
+	signal?: AbortSignal,
+): Promise<ExtractedContent | null> {
+	const apiKey = getApiKey();
+	if (!apiKey) return null;
+
+	const activityId = activityMonitor.logStart({ type: "api", query: `url_context: ${url}` });
+
+	try {
+		const model = DEFAULT_MODEL;
+		const body = {
+			contents: [{ parts: [{ text: EXTRACTION_PROMPT + url }] }],
+			tools: [{ url_context: {} }],
+		};
+
+		const res = await fetch(`${API_BASE}/models/${model}:generateContent?key=${apiKey}`, {
+			method: "POST",
+			headers: { "Content-Type": "application/json" },
+			body: JSON.stringify(body),
+			signal: AbortSignal.any([
+				AbortSignal.timeout(60000),
+				...(signal ? [signal] : []),
+			]),
+		});
+
+		if (!res.ok) {
+			activityMonitor.logComplete(activityId, res.status);
+			return null;
+		}
+
+		const data = await res.json() as UrlContextResponse;
+		activityMonitor.logComplete(activityId, res.status);
+
+		const metadata = data.candidates?.[0]?.url_context_metadata;
+		if (metadata?.url_metadata?.length) {
+			const status = metadata.url_metadata[0].url_retrieval_status;
+			if (status === "URL_RETRIEVAL_STATUS_UNSAFE" || status === "URL_RETRIEVAL_STATUS_ERROR") {
+				return null;
+			}
+		}
+
+		const content = data.candidates?.[0]?.content?.parts
+			?.map(p => p.text).filter(Boolean).join("\n") ?? "";
+
+		if (!content || content.length < 50) return null;
+
+		const title = extractTitleFromContent(content, url);
+		return { url, title, content, error: null };
+	} catch (err) {
+		if (shouldRethrow(err)) throw err;
+		const message = err instanceof Error ? err.message : String(err);
+		if (message.toLowerCase().includes("abort")) {
+			activityMonitor.logComplete(activityId, 0);
+		} else {
+			activityMonitor.logError(activityId, message);
+		}
+		return null;
+	}
+}
+
+export async function extractWithGeminiWeb(
+	url: string,
+	signal?: AbortSignal,
+): Promise<ExtractedContent | null> {
+	const cookies = await isGeminiWebAvailable();
+	if (!cookies) return null;
+
+	const activityId = activityMonitor.logStart({ type: "api", query: `gemini_web: ${url}` });
+
+	try {
+		const text = await queryWithCookies(EXTRACTION_PROMPT + url, cookies, {
+			model: "gemini-3-flash-preview",
+			signal,
+			timeoutMs: 60000,
+		});
+
+		activityMonitor.logComplete(activityId, 200);
+
+		if (!text || text.length < 50) return null;
+
+		const title = extractTitleFromContent(text, url);
+		return { url, title, content: text, error: null };
+	} catch (err) {
+		if (shouldRethrow(err)) throw err;
+		const message = err instanceof Error ? err.message : String(err);
+		if (message.toLowerCase().includes("abort")) {
+			activityMonitor.logComplete(activityId, 0);
+		} else {
+			activityMonitor.logError(activityId, message);
+		}
+		return null;
+	}
+}
+
+function extractTitleFromContent(text: string, url: string): string {
+	return extractHeadingTitle(text) ?? (new URL(url).pathname.split("/").pop() || url);
+}
+
+interface UrlContextResponse {
+	candidates?: Array<{
+		content?: { parts?: Array<{ text?: string }> };
+		url_context_metadata?: {
+			url_metadata?: Array<{
+				retrieved_url?: string;
+				url_retrieval_status?: string;
+			}>;
+		};
+	}>;
+}

package/dist/builtin/web-access/gemini-web-config.ts

@@ -0,0 +1,54 @@
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { APP_NAME, CONFIG_DIR_NAME } from "@bastani/atomic";
+
+const CONFIG_PATH = join(homedir(), CONFIG_DIR_NAME, "web-search.json");
+const ALLOW_BROWSER_COOKIES_ENV = `${APP_NAME.toUpperCase()}_ALLOW_BROWSER_COOKIES`;
+
+interface GeminiWebConfig {
+	chromeProfile?: string;
+	allowBrowserCookies?: boolean;
+}
+
+let cachedConfig: GeminiWebConfig | null = null;
+
+export function normalizeChromeProfile(value: unknown): string | undefined {
+	if (typeof value !== "string") return undefined;
+	const normalized = value.trim();
+	return normalized.length > 0 ? normalized : undefined;
+}
+
+function loadConfig(): GeminiWebConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+
+	const rawText = readFileSync(CONFIG_PATH, "utf-8");
+	let raw: { chromeProfile?: unknown; allowBrowserCookies?: unknown };
+	try {
+		raw = JSON.parse(rawText) as { chromeProfile?: unknown; allowBrowserCookies?: unknown };
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+
+	cachedConfig = {
+		chromeProfile: normalizeChromeProfile(raw.chromeProfile),
+		allowBrowserCookies: raw.allowBrowserCookies === true,
+	};
+	return cachedConfig;
+}
+
+export function getChromeProfileFromConfig(): string | undefined {
+	return loadConfig().chromeProfile;
+}
+
+export function isBrowserCookieAccessAllowed(): boolean {
+	if (process.env[ALLOW_BROWSER_COOKIES_ENV] === "1") {
+		return true;
+	}
+	return loadConfig().allowBrowserCookies === true;
+}