@bastani/atomic 0.8.31-alpha.5 → 0.9.0-alpha.1

package/dist/builtin/web-access/web-search-curator.ts

@@ -0,0 +1,214 @@
+import type { ExtensionAPI } from "@bastani/atomic";
+import { randomUUID } from "node:crypto";
+import { startCuratorServer, type CuratorServerHandle } from "./curator-server.js";
+import { search } from "./gemini-search.js";
+import {
+	buildCurationCancelledReturn,
+	collectAllResultsAndUrls,
+	extractDomain,
+	filterByQueryIndices,
+} from "./web-search-formatting.js";
+import type { SearchReturnBuilder, SearchReturnOptions } from "./web-search-return.js";
+import { generateSummaryForSelectedIndices, resolveSummaryForSubmit, rewriteSearchQuery } from "./web-search-summary.js";
+import { openCuratorWindow } from "./web-search-browser.js";
+import type { PendingCurate, WebSearchRuntimeState } from "./web-search-types.js";
+import { normalizeProviderInput, saveConfig } from "./web-search-config.js";
+
+interface OpenCuratorBrowserDeps {
+	pi: ExtensionAPI;
+	state: WebSearchRuntimeState;
+	buildSearchReturn: SearchReturnBuilder;
+	closeCurator(): void;
+}
+
+export async function openCuratorBrowser(
+	deps: OpenCuratorBrowserDeps,
+	pc: PendingCurate,
+	searchesComplete = true,
+): Promise<void> {
+	let handle: CuratorServerHandle | null = null;
+	try {
+		pc.phase = "curating";
+
+		const searchAbort = new AbortController();
+		const addSearchSignal = pc.signal
+			? AbortSignal.any([pc.signal, searchAbort.signal])
+			: searchAbort.signal;
+
+		const sessionToken = randomUUID();
+		handle = await startCuratorServer(
+			{
+				queries: pc.queryList,
+				sessionToken,
+				timeout: pc.timeoutSeconds,
+				availableProviders: pc.availableProviders,
+				defaultProvider: pc.defaultProvider,
+				summaryModels: pc.summaryModels,
+				defaultSummaryModel: pc.defaultSummaryModel,
+			},
+			{
+				async onSummarize(selectedQueryIndices, summarizeSignal, model, feedback) {
+					if (deps.state.pendingCurate !== pc) throw new Error("Curator session is no longer active.");
+					pc.onUpdate?.({
+						content: [{ type: "text", text: "Generating summary draft..." }],
+						details: { phase: "generating-summary", progress: 0.9 },
+					});
+					const draft = await generateSummaryForSelectedIndices(
+						selectedQueryIndices,
+						pc.searchResults,
+						pc.summaryContext,
+						summarizeSignal,
+						model,
+						feedback,
+					);
+					if (deps.state.pendingCurate !== pc) throw new Error("Curator session is no longer active.");
+					pc.onUpdate?.({
+						content: [{ type: "text", text: "Summary draft ready — waiting for approval..." }],
+						details: { phase: "waiting-for-approval", progress: 1 },
+					});
+					return draft;
+				},
+				onSubmit(payload) {
+					if (deps.state.pendingCurate !== pc) return;
+					searchAbort.abort();
+					const filtered = payload.selectedQueryIndices.length > 0
+						? filterByQueryIndices(payload.selectedQueryIndices, pc.searchResults)
+						: collectAllResultsAndUrls(pc.searchResults);
+					const filteredInline = pc.allInlineContent.filter(c => filtered.urls.includes(c.url));
+					const base: SearchReturnOptions = {
+						queryList: filtered.results.map(r => r.query),
+						results: filtered.results,
+						urls: filtered.urls,
+						includeContent: pc.includeContent,
+						inlineContent: filteredInline.length > 0 ? filteredInline : undefined,
+						curated: true,
+						curatedFrom: pc.searchResults.size,
+					};
+					if (!payload.rawResults) {
+						const resolvedSummary = resolveSummaryForSubmit(payload, pc.searchResults);
+						base.workflow = pc.workflow;
+						base.approvedSummary = resolvedSummary.approvedSummary;
+						base.summaryMeta = resolvedSummary.summaryMeta;
+					}
+					pc.finish(deps.buildSearchReturn(base));
+					deps.closeCurator();
+				},
+				onCancel(reason) {
+					if (deps.state.pendingCurate !== pc) return;
+					searchAbort.abort();
+					if (reason === "timeout") {
+						const resolvedSummary = resolveSummaryForSubmit({ selectedQueryIndices: [], summary: undefined, summaryMeta: undefined }, pc.searchResults);
+						const all = collectAllResultsAndUrls(pc.searchResults);
+						const filteredInline = pc.allInlineContent.filter(c => all.urls.includes(c.url));
+						pc.finish(deps.buildSearchReturn({
+							queryList: all.results.map(r => r.query),
+							results: all.results,
+							urls: all.urls,
+							includeContent: pc.includeContent,
+							inlineContent: filteredInline.length > 0 ? filteredInline : undefined,
+							curated: true,
+							curatedFrom: pc.searchResults.size,
+							workflow: pc.workflow,
+							approvedSummary: resolvedSummary.approvedSummary,
+							summaryMeta: resolvedSummary.summaryMeta,
+						}));
+					} else {
+						pc.finish(buildCurationCancelledReturn(reason));
+					}
+					deps.closeCurator();
+				},
+				onProviderChange(provider) {
+					if (deps.state.pendingCurate !== pc) return;
+					const normalized = normalizeProviderInput(provider);
+					if (!normalized || normalized === "auto") return;
+					pc.defaultProvider = normalized;
+					try {
+						saveConfig({ provider: normalized });
+					} catch (err) {
+						const message = err instanceof Error ? err.message : String(err);
+						console.error(`Failed to persist default provider: ${message}`);
+					}
+				},
+				async onAddSearch(query, queryIndex, provider) {
+					if (deps.state.pendingCurate !== pc) throw new Error("Curator session is no longer active.");
+					const normalizedProvider = normalizeProviderInput(provider);
+					const requestedProvider = !normalizedProvider || normalizedProvider === "auto"
+						? pc.defaultProvider
+						: normalizedProvider;
+					try {
+						const { answer, results, inlineContent, provider: actualProvider } = await search(query, {
+							provider: requestedProvider,
+							numResults: pc.numResults,
+							recencyFilter: pc.recencyFilter,
+							domainFilter: pc.domainFilter,
+							includeContent: pc.includeContent,
+							signal: addSearchSignal,
+						});
+						if (deps.state.pendingCurate !== pc) throw new Error("Curator session is no longer active.");
+						pc.searchResults.set(queryIndex, { query, answer, results, error: null, provider: actualProvider });
+						if (inlineContent) pc.allInlineContent.push(...inlineContent);
+						return {
+							answer,
+							results: results.map(r => ({ title: r.title, url: r.url, domain: extractDomain(r.url) })),
+							provider: actualProvider,
+						};
+					} catch (err) {
+						const message = err instanceof Error ? err.message : String(err);
+						if (deps.state.pendingCurate === pc) {
+							pc.searchResults.set(queryIndex, { query, answer: "", results: [], error: message, provider: requestedProvider });
+						}
+						throw err;
+					}
+				},
+				async onRewriteQuery(query, rewriteSignal) {
+					if (deps.state.pendingCurate !== pc) throw new Error("Curator session is no longer active.");
+					return rewriteSearchQuery(query, pc.summaryContext, rewriteSignal);
+				},
+			},
+		);
+
+		if (deps.state.pendingCurate !== pc) {
+			handle.close();
+			return;
+		}
+
+		deps.state.activeCurator = handle;
+
+		for (const [qi, data] of pc.searchResults) {
+			if (data.error) {
+				handle.pushError(qi, data.error, data.provider);
+			} else {
+				handle.pushResult(qi, {
+					answer: data.answer,
+					results: data.results.map(r => ({ title: r.title, url: r.url, domain: extractDomain(r.url) })),
+					provider: data.provider || pc.defaultProvider,
+				});
+			}
+		}
+		if (searchesComplete) handle.searchesDone();
+
+		pc.onUpdate?.({
+			content: [{ type: "text", text: searchesComplete ? "Waiting for summary approval in browser..." : "Searches streaming to browser..." }],
+			details: { phase: "curating", progress: searchesComplete ? 1 : 0.5 },
+		});
+
+		await openCuratorWindow(
+			deps.pi,
+			handle.url,
+			"Search Curator",
+			win => { deps.state.glimpseWin = win; },
+			win => {
+				if (deps.state.glimpseWin === win) {
+					deps.state.glimpseWin = null;
+					deps.closeCurator();
+				}
+			},
+		);
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		console.error(`Failed to open curator UI: ${message}`);
+		if (deps.state.pendingCurate === pc || (handle && deps.state.activeCurator === handle)) {
+			deps.closeCurator();
+		}
+	}
+}

package/dist/builtin/web-access/web-search-features.ts

@@ -0,0 +1,165 @@
+import type { ExtensionAPI, ExtensionContext } from "@bastani/atomic";
+import { fetchAllContent } from "./extract.js";
+import { clearCloneCache } from "./github-extract.js";
+import { registerContentTools } from "./content-tools.js";
+import { clearResults, generateId, restoreFromSession, storeResult, type StoredSearchData } from "./storage.js";
+import {
+	createActivityWidgetState,
+	refreshActivityForSession,
+	shutdownActivityWidget,
+	toggleActivityWidget,
+} from "./web-search-activity.js";
+import { openCuratorBrowser as openCuratorBrowserForSearch } from "./web-search-curator.js";
+import { MAX_INLINE_CONTENT, formatFullResults, stripThumbnails } from "./web-search-formatting.js";
+import { buildSearchReturn, type SearchReturnBuilder } from "./web-search-return.js";
+import { cancelPendingCurate, type PendingCurate, type WebSearchRuntimeState } from "./web-search-types.js";
+import { DEFAULT_SHORTCUTS, type WebSearchConfig } from "./web-search-config.js";
+import { registerWebSearchCommand } from "./web-search-command.js";
+import { registerWebSearchTool } from "./web-search-tool.js";
+
+const pendingFetches = new Map<string, AbortController>();
+
+const runtimeState: WebSearchRuntimeState = {
+	sessionActive: false,
+	pendingCurate: null,
+	activeCurator: null,
+	glimpseWin: null,
+};
+
+const activityState = createActivityWidgetState();
+
+function abortPendingFetches(): void {
+	for (const controller of pendingFetches.values()) {
+		controller.abort();
+	}
+	pendingFetches.clear();
+}
+
+function closeCurator(): void {
+	const win = runtimeState.glimpseWin;
+	runtimeState.glimpseWin = null;
+	try { win?.close(); } catch {}
+	cancelPendingCurate(runtimeState);
+	if (runtimeState.activeCurator) {
+		runtimeState.activeCurator.close();
+		runtimeState.activeCurator = null;
+	}
+}
+
+function handleSessionChange(ctx: ExtensionContext): void {
+	abortPendingFetches();
+	closeCurator();
+	clearCloneCache();
+	runtimeState.sessionActive = true;
+	restoreFromSession(ctx);
+	refreshActivityForSession(activityState, ctx);
+}
+
+export function registerWebSearchFeatures(pi: ExtensionAPI, initConfig: WebSearchConfig): void {
+	const curateKey = initConfig.shortcuts?.curate || DEFAULT_SHORTCUTS.curate;
+	const activityKey = initConfig.shortcuts?.activity || DEFAULT_SHORTCUTS.activity;
+
+	function startBackgroundFetch(urls: string[]): string | null {
+		if (urls.length === 0) return null;
+		const fetchId = generateId();
+		const controller = new AbortController();
+		pendingFetches.set(fetchId, controller);
+		fetchAllContent(urls, controller.signal)
+			.then((fetched) => {
+				if (!runtimeState.sessionActive || !pendingFetches.has(fetchId)) return;
+				const data: StoredSearchData = {
+					id: fetchId,
+					type: "fetch",
+					timestamp: Date.now(),
+					urls: stripThumbnails(fetched),
+				};
+				storeResult(fetchId, data);
+				pi.appendEntry("web-search-results", data);
+				const ok = fetched.filter(f => !f.error).length;
+				pi.sendMessage(
+					{
+						customType: "web-search-content-ready",
+						content: `Content fetched for ${ok}/${fetched.length} URLs [${fetchId}]. Full page content now available.`,
+						display: true,
+					},
+					{ triggerTurn: true },
+				);
+			})
+			.catch((err) => {
+				if (!runtimeState.sessionActive || !pendingFetches.has(fetchId)) return;
+				const message = err instanceof Error ? err.message : String(err);
+				const isAbort = (err instanceof Error && err.name === "AbortError") || message.toLowerCase().includes("abort");
+				if (!isAbort) {
+					pi.sendMessage(
+						{
+							customType: "web-search-error",
+							content: `Content fetch failed [${fetchId}]: ${message}`,
+							display: true,
+						},
+						{ triggerTurn: false },
+					);
+				}
+			})
+			.finally(() => { pendingFetches.delete(fetchId); });
+		return fetchId;
+	}
+
+	const buildReturn: SearchReturnBuilder = (opts) => buildSearchReturn(opts, { pi, startBackgroundFetch });
+	const openCuratorBrowser = (pc: PendingCurate, searchesComplete = true) => openCuratorBrowserForSearch(
+		{ pi, state: runtimeState, buildSearchReturn: buildReturn, closeCurator },
+		pc,
+		searchesComplete,
+	);
+
+	pi.registerShortcut(curateKey, {
+		description: "Review search results",
+		handler: async (ctx) => {
+			const pc = runtimeState.pendingCurate;
+			if (!pc) return;
+
+			if (pc.phase === "searching") {
+				pc.browserPromise = openCuratorBrowser(pc, false);
+				ctx.ui.notify("Opening curator — remaining searches will stream in", "info");
+				return;
+			}
+		},
+	});
+
+	pi.registerShortcut(activityKey, {
+		description: "Toggle web search activity",
+		handler: async (ctx) => {
+			toggleActivityWidget(activityState, ctx);
+		},
+	});
+
+	pi.on("session_start", async (_event, ctx) => handleSessionChange(ctx));
+	pi.on("session_tree", async (_event, ctx) => handleSessionChange(ctx));
+
+	pi.on("session_shutdown", () => {
+		runtimeState.sessionActive = false;
+		abortPendingFetches();
+		closeCurator();
+		clearCloneCache();
+		clearResults();
+		shutdownActivityWidget(activityState);
+	});
+
+	registerWebSearchTool(pi, {
+		state: runtimeState,
+		closeCurator,
+		openCuratorBrowser,
+		buildSearchReturn: buildReturn,
+	});
+
+	registerContentTools(pi, {
+		maxInlineContent: MAX_INLINE_CONTENT,
+		stripThumbnails,
+		formatFullResults,
+	});
+
+	registerWebSearchCommand(pi, {
+		state: runtimeState,
+		closeCurator,
+		buildSearchReturn: buildReturn,
+	});
+}

package/dist/builtin/web-access/web-search-formatting.ts

@@ -0,0 +1,117 @@
+import type { ExtractedContent } from "./extract.js";
+import type { SearchResult } from "./perplexity.js";
+import type { QueryResultData } from "./storage.js";
+import type { SummaryMeta } from "./summary-review.js";
+
+export const MAX_INLINE_CONTENT = 30000;
+
+export function stripThumbnails(results: ExtractedContent[]): ExtractedContent[] {
+	return results.map(({ thumbnail, frames, ...rest }) => rest);
+}
+
+export function formatSearchSummary(results: SearchResult[], answer: string): string {
+	let output = answer ? `${answer}\n\n---\n\n**Sources:**\n` : "";
+	output += results.map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}`).join("\n\n");
+	return output;
+}
+
+export function duplicateQuerySet(results: QueryResultData[]): Set<string> {
+	const counts = new Map<string, number>();
+	for (const result of results) {
+		counts.set(result.query, (counts.get(result.query) ?? 0) + 1);
+	}
+	const duplicates = new Set<string>();
+	for (const [query, count] of counts) {
+		if (count > 1) duplicates.add(query);
+	}
+	return duplicates;
+}
+
+export function formatQueryHeader(query: string, provider: string | undefined, duplicateQueries: Set<string>): string {
+	const suffix = duplicateQueries.has(query) && provider ? ` (${provider})` : "";
+	return `## Query: "${query}"${suffix}\n\n`;
+}
+
+export function hasFullInlineCoverage(urls: string[], inlineContent: ExtractedContent[] | undefined): boolean {
+	if (!inlineContent || inlineContent.length === 0) return false;
+	const coveredUrls = new Set(inlineContent.map(c => c.url));
+	return urls.every(url => coveredUrls.has(url));
+}
+
+export function formatFullResults(queryData: QueryResultData): string {
+	let output = `## Results for: "${queryData.query}"\n\n`;
+	if (queryData.answer) {
+		output += `${queryData.answer}\n\n---\n\n`;
+	}
+	for (const r of queryData.results) {
+		output += `### ${r.title}\n${r.url}\n\n`;
+	}
+	return output;
+}
+
+export function normalizeSummaryMeta(meta: SummaryMeta | undefined, summaryText: string): SummaryMeta {
+	const normalizedText = summaryText.trim();
+	if (!meta) {
+		return {
+			model: null,
+			durationMs: 0,
+			tokenEstimate: normalizedText.length > 0 ? Math.max(1, Math.ceil(normalizedText.length / 4)) : 0,
+			fallbackUsed: false,
+			edited: false,
+		};
+	}
+
+	return {
+		model: meta.model,
+		durationMs: Number.isFinite(meta.durationMs) && meta.durationMs >= 0 ? meta.durationMs : 0,
+		tokenEstimate: Number.isFinite(meta.tokenEstimate) && meta.tokenEstimate >= 0
+			? meta.tokenEstimate
+			: (normalizedText.length > 0 ? Math.max(1, Math.ceil(normalizedText.length / 4)) : 0),
+		fallbackUsed: meta.fallbackUsed === true,
+		fallbackReason: meta.fallbackReason,
+		edited: meta.edited === true,
+	};
+}
+
+export function buildCurationCancelledReturn(reason: "user" | "stale") {
+	const message = `Search curation cancelled (${reason}).`;
+	return {
+		content: [{ type: "text", text: message }],
+		details: {
+			error: message,
+			cancelled: true,
+			cancelReason: reason,
+		},
+	};
+}
+
+export function filterByQueryIndices(selectedQueryIndices: number[], results: Map<number, QueryResultData>) {
+	const filteredResults: QueryResultData[] = [];
+	const filteredUrls: string[] = [];
+	for (const qi of selectedQueryIndices) {
+		const r = results.get(qi);
+		if (r) {
+			filteredResults.push(r);
+			for (const res of r.results) {
+				if (!filteredUrls.includes(res.url)) filteredUrls.push(res.url);
+			}
+		}
+	}
+	return { results: filteredResults, urls: filteredUrls };
+}
+
+export function collectAllResultsAndUrls(resultsByIndex: Map<number, QueryResultData>) {
+	const results = [...resultsByIndex.values()];
+	const urls: string[] = [];
+	for (const result of results) {
+		for (const source of result.results) {
+			if (!urls.includes(source.url)) urls.push(source.url);
+		}
+	}
+	return { results, urls };
+}
+
+export function extractDomain(url: string): string {
+	try { return new URL(url).hostname; }
+	catch { return url; }
+}

package/dist/builtin/web-access/web-search-return.ts

@@ -0,0 +1,136 @@
+import type { ExtensionAPI } from "@bastani/atomic";
+import type { ExtractedContent } from "./extract.js";
+import {
+	duplicateQuerySet,
+	formatQueryHeader,
+	formatSearchSummary,
+	hasFullInlineCoverage,
+} from "./web-search-formatting.js";
+import { generateId, storeResult, type QueryResultData, type StoredSearchData } from "./storage.js";
+import type { SummaryMeta } from "./summary-review.js";
+import type { CuratorWorkflow } from "./web-search-config.js";
+
+export interface SearchReturnOptions {
+	queryList: string[];
+	results: QueryResultData[];
+	urls: string[];
+	includeContent: boolean;
+	inlineContent?: ExtractedContent[];
+	curated?: boolean;
+	curatedFrom?: number;
+	workflow?: CuratorWorkflow;
+	approvedSummary?: string;
+	summaryMeta?: SummaryMeta;
+}
+
+export interface SearchReturnPayload {
+	content: Array<{ type: string; text: string }>;
+	details: Record<string, unknown>;
+}
+
+export type SearchReturnBuilder = (opts: SearchReturnOptions) => SearchReturnPayload;
+
+interface BuildSearchReturnDeps {
+	pi: ExtensionAPI;
+	startBackgroundFetch(urls: string[]): string | null;
+}
+
+function storeAndPublishSearch(pi: ExtensionAPI, results: QueryResultData[]): string {
+	const id = generateId();
+	const data: StoredSearchData = {
+		id, type: "search", timestamp: Date.now(), queries: results,
+	};
+	storeResult(id, data);
+	pi.appendEntry("web-search-results", data);
+	return id;
+}
+
+export function buildSearchReturn(opts: SearchReturnOptions, deps: BuildSearchReturnDeps): SearchReturnPayload {
+	const sc = opts.results.filter(r => !r.error).length;
+	const tr = opts.results.reduce((sum, r) => sum + r.results.length, 0);
+
+	const hasApprovedSummary = typeof opts.approvedSummary === "string" && opts.approvedSummary.trim().length > 0;
+	let output = "";
+	if (hasApprovedSummary) {
+		output = opts.approvedSummary!.trim();
+	} else {
+		if (opts.curated) {
+			output += "[These results were manually curated by the user in the browser. Use them as-is — do not re-search or discard.]\n\n";
+		}
+		const duplicateQueries = opts.curated ? duplicateQuerySet(opts.results) : new Set<string>();
+		for (const { query, answer, results, error, provider } of opts.results) {
+			if (opts.queryList.length > 1) {
+				output += opts.curated
+					? formatQueryHeader(query, provider, duplicateQueries)
+					: `## Query: "${query}"\n\n`;
+			}
+			if (error) output += `Error: ${error}\n\n`;
+			else if (results.length === 0) output += "No results found.\n\n";
+			else output += formatSearchSummary(results, answer) + "\n\n";
+		}
+	}
+
+	const hasInlineReady = hasFullInlineCoverage(opts.urls, opts.inlineContent);
+	let fetchId: string | null = null;
+	if (hasInlineReady && opts.inlineContent) {
+		fetchId = generateId();
+		const data: StoredSearchData = {
+			id: fetchId,
+			type: "fetch",
+			timestamp: Date.now(),
+			urls: opts.inlineContent,
+		};
+		storeResult(fetchId, data);
+		deps.pi.appendEntry("web-search-results", data);
+		if (!hasApprovedSummary) {
+			output += `---\nFull content for ${opts.inlineContent.length} sources available [${fetchId}].`;
+		}
+	} else if (opts.includeContent) {
+		fetchId = deps.startBackgroundFetch(opts.urls);
+		if (fetchId && !hasApprovedSummary) {
+			output += `---\nContent fetching in background [${fetchId}]. Will notify when ready.`;
+		}
+	}
+
+	const searchId = storeAndPublishSearch(deps.pi, opts.results);
+	const isBackgroundFetch = fetchId !== null && !hasInlineReady;
+
+	return {
+		content: [{ type: "text", text: output.trim() }],
+		details: {
+			queries: opts.queryList,
+			queryCount: opts.queryList.length,
+			successfulQueries: sc,
+			totalResults: tr,
+			includeContent: opts.includeContent,
+			fetchId,
+			fetchUrls: isBackgroundFetch ? opts.urls : undefined,
+			searchId,
+			...(opts.curated ? {
+				curated: true,
+				curatedFrom: opts.curatedFrom,
+				curatedQueries: opts.results.map(r => ({
+					query: r.query,
+					provider: r.provider || null,
+					answer: r.answer || null,
+					sources: r.results.map(s => ({ title: s.title, url: s.url })),
+					error: r.error,
+				})),
+			} : {}),
+			...((opts.workflow && hasApprovedSummary)
+				? {
+					summary: {
+						text: opts.approvedSummary!.trim(),
+						workflow: opts.workflow,
+						model: opts.summaryMeta?.model ?? null,
+						durationMs: opts.summaryMeta?.durationMs ?? 0,
+						tokenEstimate: opts.summaryMeta?.tokenEstimate ?? 0,
+						fallbackUsed: opts.summaryMeta?.fallbackUsed === true,
+						fallbackReason: opts.summaryMeta?.fallbackReason,
+						edited: opts.summaryMeta?.edited === true,
+					},
+				}
+				: {}),
+		},
+	};
+}