@bastani/atomic 0.9.16-alpha.2 → 0.9.16-alpha.3

package/dist/builtin/web-access/curator-server.ts

@@ -1,494 +0,0 @@
-import http, { type ServerResponse } from "node:http";
-import { generateCuratorPage } from "./curator-page.js";
-import type { SummaryMeta } from "./summary-review.js";
-import { normalizeSelectedIndices, normalizeSummaryMeta, parseBodyOrSend, sendJson } from "./curator-server-helpers.js";
-
-const STALE_THRESHOLD_MS = 30000;
-const WATCHDOG_INTERVAL_MS = 5000;
-type ServerState = "SEARCHING" | "RESULT_SELECTION" | "COMPLETED";
-
-export interface CuratorServerOptions {
-	queries: string[];
-	sessionToken: string;
-	timeout: number;
-	availableProviders: { perplexity: boolean; exa: boolean; gemini: boolean };
-	defaultProvider: string;
-	summaryModels: Array<{ value: string; label: string }>;
-	defaultSummaryModel: string | null;
-}
-
-export interface CuratorServerCallbacks {
-	onSubmit: (payload: { selectedQueryIndices: number[]; summary?: string; summaryMeta?: SummaryMeta; rawResults?: boolean }) => void;
-	onCancel: (reason: "user" | "timeout" | "stale") => void;
-	onProviderChange: (provider: string) => void;
-	onAddSearch: (query: string, queryIndex: number, provider?: string) => Promise<{
-		answer: string;
-		results: Array<{ title: string; url: string; domain: string }>;
-		provider: string;
-	}>;
-	onSummarize: (
-		selectedQueryIndices: number[],
-		signal: AbortSignal,
-		model?: string,
-		feedback?: string,
-	) => Promise<{ summary: string; meta: SummaryMeta }>;
-	onRewriteQuery: (query: string, signal: AbortSignal) => Promise<string>;
-}
-
-export interface CuratorServerHandle {
-	server: http.Server;
-	url: string;
-	close: () => void;
-	pushResult: (queryIndex: number, data: { answer: string; results: Array<{ title: string; url: string; domain: string }>; provider: string }) => void;
-	pushError: (queryIndex: number, error: string, provider?: string) => void;
-	searchesDone: () => void;
-}
-
-
-export function startCuratorServer(
-	options: CuratorServerOptions,
-	callbacks: CuratorServerCallbacks,
-): Promise<CuratorServerHandle> {
-	const {
-		queries,
-		sessionToken,
-		timeout,
-		availableProviders,
-		defaultProvider,
-		summaryModels,
-		defaultSummaryModel,
-	} = options;
-	let browserConnected = false;
-	let lastHeartbeatAt = Date.now();
-	let completed = false;
-	let watchdog: NodeJS.Timeout | null = null;
-	let state: ServerState = "SEARCHING";
-	let sseResponse: ServerResponse | null = null;
-	const sseBuffer: string[] = [];
-	let nextQueryIndex = queries.length;
-	let summarizeAbortController: AbortController | null = null;
-	let summarizeRequestSeq = 0;
-
-	let sseKeepalive: NodeJS.Timeout | null = null;
-
-	const abortInFlightSummarize = (): void => {
-		if (!summarizeAbortController) return;
-		summarizeAbortController.abort();
-		summarizeAbortController = null;
-	};
-
-	const markCompleted = (): boolean => {
-		if (completed) return false;
-		completed = true;
-		state = "COMPLETED";
-		if (watchdog) {
-			clearInterval(watchdog);
-			watchdog = null;
-		}
-		if (sseKeepalive) {
-			clearInterval(sseKeepalive);
-			sseKeepalive = null;
-		}
-		abortInFlightSummarize();
-		if (sseResponse) {
-			try { sseResponse.end(); } catch {}
-			sseResponse = null;
-		}
-		return true;
-	};
-
-	const touchHeartbeat = (): void => {
-		lastHeartbeatAt = Date.now();
-		browserConnected = true;
-	};
-
-	function validateToken(body: unknown, res: ServerResponse): boolean {
-		if (!body || typeof body !== "object") {
-			sendJson(res, 400, { ok: false, error: "Invalid body" });
-			return false;
-		}
-		if ((body as { token?: string }).token !== sessionToken) {
-			sendJson(res, 403, { ok: false, error: "Invalid session" });
-			return false;
-		}
-		return true;
-	}
-
-	function isAvailableProvider(provider: string): boolean {
-		if (provider === "perplexity") return availableProviders.perplexity;
-		if (provider === "exa") return availableProviders.exa;
-		if (provider === "gemini") return availableProviders.gemini;
-		return false;
-	}
-
-	function sendSSE(event: string, data: unknown): void {
-		const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
-		const res = sseResponse;
-		if (res && !res.writableEnded && res.socket && !res.socket.destroyed) {
-			try { res.write(payload); return; } catch {}
-		}
-		sseBuffer.push(payload);
-	}
-
-	const pageHtml = generateCuratorPage(
-		queries,
-		sessionToken,
-		timeout,
-		availableProviders,
-		defaultProvider,
-		summaryModels,
-		defaultSummaryModel,
-	);
-
-	const server = http.createServer(async (req, res) => {
-		try {
-			const method = req.method || "GET";
-			const url = new URL(req.url || "/", `http://${req.headers.host || "127.0.0.1"}`);
-
-			if (method === "GET" && url.pathname === "/") {
-				const token = url.searchParams.get("session");
-				if (token !== sessionToken) {
-					res.writeHead(403, { "Content-Type": "text/plain" });
-					res.end("Invalid session");
-					return;
-				}
-				touchHeartbeat();
-				res.writeHead(200, {
-					"Content-Type": "text/html; charset=utf-8",
-					"Cache-Control": "no-store",
-				});
-				res.end(pageHtml);
-				return;
-			}
-
-			if (method === "GET" && url.pathname === "/events") {
-				const token = url.searchParams.get("session");
-				if (token !== sessionToken) {
-					res.writeHead(403, { "Content-Type": "text/plain" });
-					res.end("Invalid session");
-					return;
-				}
-				if (state === "COMPLETED") {
-					sendJson(res, 409, { ok: false, error: "No events available" });
-					return;
-				}
-				if (sseResponse) {
-					try { sseResponse.end(); } catch {}
-				}
-				res.writeHead(200, {
-					"Content-Type": "text/event-stream",
-					"Cache-Control": "no-cache",
-					Connection: "keep-alive",
-					"X-Accel-Buffering": "no",
-				});
-				res.flushHeaders();
-				if (res.socket) res.socket.setNoDelay(true);
-				sseResponse = res;
-				if (sseBuffer.length > 0) {
-					const pending = sseBuffer.splice(0, sseBuffer.length);
-					for (let i = 0; i < pending.length; i++) {
-						const msg = pending[i];
-						try {
-							res.write(msg);
-						} catch {
-							sseBuffer.unshift(...pending.slice(i));
-							break;
-						}
-					}
-				}
-				if (sseKeepalive) clearInterval(sseKeepalive);
-				sseKeepalive = setInterval(() => {
-					if (sseResponse) {
-						try { sseResponse.write(":keepalive\n\n"); } catch {}
-					}
-				}, 15000);
-				req.on("close", () => {
-					if (sseResponse === res) sseResponse = null;
-				});
-				return;
-			}
-
-			if (method === "POST" && url.pathname === "/heartbeat") {
-				const body = await parseBodyOrSend(req, res);
-				if (!body) return;
-				if (!validateToken(body, res)) return;
-				touchHeartbeat();
-				sendJson(res, 200, { ok: true });
-				return;
-			}
-
-			if (method === "POST" && url.pathname === "/provider") {
-				const body = await parseBodyOrSend(req, res);
-				if (!body) return;
-				if (!validateToken(body, res)) return;
-				const { provider } = body as { provider?: string };
-				if (typeof provider !== "string" || provider.length === 0) {
-					sendJson(res, 400, { ok: false, error: "Invalid provider" });
-					return;
-				}
-				if (!isAvailableProvider(provider)) {
-					sendJson(res, 400, { ok: false, error: `Provider unavailable: ${provider}` });
-					return;
-				}
-				setImmediate(() => callbacks.onProviderChange(provider));
-				sendJson(res, 200, { ok: true });
-				return;
-			}
-
-			if (method === "POST" && url.pathname === "/search") {
-				const body = await parseBodyOrSend(req, res);
-				if (!body) return;
-				if (!validateToken(body, res)) return;
-				if (state === "COMPLETED") {
-					sendJson(res, 409, { ok: false, error: "Session closed" });
-					return;
-				}
-				const { query, provider } = body as { query?: string; provider?: string };
-				if (typeof query !== "string" || query.trim().length === 0) {
-					sendJson(res, 400, { ok: false, error: "Invalid query" });
-					return;
-				}
-				if (provider !== undefined) {
-					if (typeof provider !== "string" || provider.length === 0) {
-						sendJson(res, 400, { ok: false, error: "Invalid provider" });
-						return;
-					}
-					if (!isAvailableProvider(provider)) {
-						sendJson(res, 400, { ok: false, error: `Provider unavailable: ${provider}` });
-						return;
-					}
-				}
-				const qi = nextQueryIndex++;
-				touchHeartbeat();
-				try {
-					const result = await callbacks.onAddSearch(query.trim(), qi, provider);
-					sendJson(res, 200, {
-						ok: true,
-						queryIndex: qi,
-						answer: result.answer,
-						results: result.results,
-						provider: result.provider,
-					});
-				} catch (err) {
-					const message = err instanceof Error ? err.message : "Search failed";
-					sendJson(res, 200, {
-						ok: true,
-						queryIndex: qi,
-						error: message,
-						provider: typeof provider === "string" && provider.length > 0 ? provider : undefined,
-					});
-				}
-				return;
-			}
-
-			if (method === "POST" && url.pathname === "/summarize") {
-				const body = await parseBodyOrSend(req, res);
-				if (!body) return;
-				if (!validateToken(body, res)) return;
-				if (state === "COMPLETED") {
-					sendJson(res, 409, { ok: false, error: "Session closed" });
-					return;
-				}
-
-				const parsed = normalizeSelectedIndices((body as { selected?: unknown }).selected, {
-					allowEmpty: false,
-					maxExclusive: nextQueryIndex,
-				});
-				if (!parsed.ok) {
-					sendJson(res, 400, { ok: false, error: parsed.error });
-					return;
-				}
-
-				let model: string | undefined;
-				const bodyModel = (body as { model?: unknown }).model;
-				if (bodyModel !== undefined) {
-					if (typeof bodyModel !== "string") {
-						sendJson(res, 400, { ok: false, error: "Invalid model" });
-						return;
-					}
-					const trimmedModel = bodyModel.trim();
-					model = trimmedModel.length > 0 ? trimmedModel : undefined;
-				}
-
-				const bodyFeedback = (body as { feedback?: unknown }).feedback;
-				const feedback = typeof bodyFeedback === "string" && bodyFeedback.trim().length > 0
-					? bodyFeedback.trim()
-					: undefined;
-
-				abortInFlightSummarize();
-				const controller = new AbortController();
-				summarizeAbortController = controller;
-				const requestId = ++summarizeRequestSeq;
-
-				try {
-					const result = await callbacks.onSummarize(parsed.indices, controller.signal, model, feedback);
-					if (requestId !== summarizeRequestSeq || state === "COMPLETED") {
-						sendJson(res, 409, { ok: false, error: "Summarize request superseded" });
-						return;
-					}
-					sendJson(res, 200, {
-						ok: true,
-						summary: result.summary,
-						meta: result.meta,
-					});
-				} catch (err) {
-					const message = err instanceof Error ? err.message : "Summary generation failed";
-					const status = controller.signal.aborted ? 409 : 500;
-					sendJson(res, status, { ok: false, error: message });
-				} finally {
-					if (summarizeAbortController === controller) {
-						summarizeAbortController = null;
-					}
-				}
-				return;
-			}
-
-			if (method === "POST" && url.pathname === "/rewrite") {
-				const body = await parseBodyOrSend(req, res);
-				if (!body) return;
-				if (!validateToken(body, res)) return;
-				if (state === "COMPLETED") {
-					sendJson(res, 409, { ok: false, error: "Session closed" });
-					return;
-				}
-				const { query } = body as { query?: unknown };
-				if (typeof query !== "string" || query.trim().length === 0) {
-					sendJson(res, 400, { ok: false, error: "Invalid query" });
-					return;
-				}
-				const controller = new AbortController();
-				req.on("close", () => controller.abort());
-				touchHeartbeat();
-				try {
-					const rewritten = await callbacks.onRewriteQuery(query.trim(), controller.signal);
-					sendJson(res, 200, { ok: true, query: rewritten });
-				} catch (err) {
-					const message = err instanceof Error ? err.message : "Rewrite failed";
-					const status = controller.signal.aborted ? 409 : 500;
-					sendJson(res, status, { ok: false, error: message });
-				}
-				return;
-			}
-
-			if (method === "POST" && url.pathname === "/submit") {
-				const body = await parseBodyOrSend(req, res);
-				if (!body) return;
-				if (!validateToken(body, res)) return;
-
-				const parsed = normalizeSelectedIndices((body as { selected?: unknown }).selected, {
-					allowEmpty: true,
-					maxExclusive: nextQueryIndex,
-				});
-				if (!parsed.ok) {
-					sendJson(res, 400, { ok: false, error: parsed.error });
-					return;
-				}
-
-				let summary: string | undefined;
-				const bodySummary = (body as { summary?: unknown }).summary;
-				if (bodySummary !== undefined) {
-					if (typeof bodySummary !== "string") {
-						sendJson(res, 400, { ok: false, error: "Invalid summary" });
-						return;
-					}
-					const trimmedSummary = bodySummary.trim();
-					summary = trimmedSummary.length > 0 ? trimmedSummary : undefined;
-				}
-
-				let summaryMeta: SummaryMeta | undefined;
-				const bodySummaryMeta = (body as { summaryMeta?: unknown }).summaryMeta;
-				if (bodySummaryMeta !== undefined) {
-					const parsedSummaryMeta = normalizeSummaryMeta(bodySummaryMeta);
-					if (!parsedSummaryMeta) {
-						sendJson(res, 400, { ok: false, error: "Invalid summaryMeta" });
-						return;
-					}
-					summaryMeta = parsedSummaryMeta;
-				}
-
-				if (state !== "SEARCHING" && state !== "RESULT_SELECTION") {
-					sendJson(res, 409, { ok: false, error: "Cannot submit in current state" });
-					return;
-				}
-				if (!markCompleted()) {
-					sendJson(res, 409, { ok: false, error: "Session closed" });
-					return;
-				}
-				const rawResults = (body as { rawResults?: unknown }).rawResults === true;
-				sendJson(res, 200, { ok: true });
-				setImmediate(() => callbacks.onSubmit({ selectedQueryIndices: parsed.indices, summary, summaryMeta, rawResults }));
-				return;
-			}
-
-			if (method === "POST" && url.pathname === "/cancel") {
-				const body = await parseBodyOrSend(req, res);
-				if (!body) return;
-				if (!validateToken(body, res)) return;
-				if (!markCompleted()) {
-					sendJson(res, 200, { ok: true });
-					return;
-				}
-				const { reason } = body as { reason?: string };
-				sendJson(res, 200, { ok: true });
-				const cancelReason = reason === "timeout" ? "timeout" : "user";
-				setImmediate(() => callbacks.onCancel(cancelReason));
-				return;
-			}
-
-			res.writeHead(404, { "Content-Type": "text/plain" });
-			res.end("Not found");
-		} catch (err) {
-			const message = err instanceof Error ? err.message : "Server error";
-			sendJson(res, 500, { ok: false, error: message });
-		}
-	});
-
-	return new Promise((resolve, reject) => {
-		const onError = (err: Error) => {
-			reject(new Error(`Curator server failed to start: ${err.message}`));
-		};
-
-		server.once("error", onError);
-		server.listen(0, "127.0.0.1", () => {
-			server.off("error", onError);
-			const addr = server.address();
-			if (!addr || typeof addr === "string") {
-				reject(new Error("Curator server: invalid address"));
-				return;
-			}
-			const url = `http://localhost:${addr.port}/?session=${sessionToken}`;
-
-			watchdog = setInterval(() => {
-				if (completed || !browserConnected) return;
-				if (Date.now() - lastHeartbeatAt <= STALE_THRESHOLD_MS) return;
-				if (!markCompleted()) return;
-				setImmediate(() => callbacks.onCancel("stale"));
-			}, WATCHDOG_INTERVAL_MS);
-
-			resolve({
-				server,
-				url,
-				close: () => {
-					const wasOpen = markCompleted();
-					try { server.close(); } catch {}
-					if (wasOpen) {
-						setImmediate(() => callbacks.onCancel("stale"));
-					}
-				},
-				pushResult: (queryIndex, data) => {
-					if (completed) return;
-					sendSSE("result", { queryIndex, query: queries[queryIndex] ?? "", ...data });
-				},
-				pushError: (queryIndex, error, provider) => {
-					if (completed) return;
-					sendSSE("search-error", { queryIndex, query: queries[queryIndex] ?? "", error, provider });
-				},
-				searchesDone: () => {
-					if (completed) return;
-					sendSSE("done", {});
-					state = "RESULT_SELECTION";
-				},
-			});
-		});
-	});
-}

package/dist/builtin/web-access/exa-mcp.ts

@@ -1,218 +0,0 @@
-import { activityMonitor } from "./activity.js";
-import type { ExtractedContent } from "./extract.js";
-import type { SearchResponse } from "./perplexity.js";
-import type { ExaSearchOptions } from "./exa.js";
-
-const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
-
-interface ExaMcpRpcResponse {
-	result?: {
-		content?: Array<{ type?: string; text?: string }>;
-		isError?: boolean;
-	};
-	error?: {
-		code?: number;
-		message?: string;
-	};
-}
-
-type McpParsedResult = { title: string; url: string; content: string };
-
-function requestSignal(signal?: AbortSignal): AbortSignal {
-	const timeout = AbortSignal.timeout(60000);
-	return signal ? AbortSignal.any([signal, timeout]) : timeout;
-}
-
-export async function callExaMcp(
-	toolName: string,
-	args: Record<string, unknown>,
-	signal?: AbortSignal,
-): Promise<string> {
-	const response = await fetch(EXA_MCP_URL, {
-		method: "POST",
-		headers: {
-			"Content-Type": "application/json",
-			"Accept": "application/json, text/event-stream",
-		},
-		body: JSON.stringify({
-			jsonrpc: "2.0",
-			id: 1,
-			method: "tools/call",
-			params: {
-				name: toolName,
-				arguments: args,
-			},
-		}),
-		signal: requestSignal(signal),
-	});
-
-	if (!response.ok) {
-		const errorText = await response.text();
-		throw new Error(`Exa MCP error ${response.status}: ${errorText.slice(0, 300)}`);
-	}
-
-	const body = await response.text();
-	const dataLines = body.split("\n").filter(line => line.startsWith("data:"));
-
-	let parsed: ExaMcpRpcResponse | null = null;
-	for (const line of dataLines) {
-		const payload = line.slice(5).trim();
-		if (!payload) continue;
-		try {
-			const candidate = JSON.parse(payload) as ExaMcpRpcResponse;
-			if (candidate?.result || candidate?.error) {
-				parsed = candidate;
-				break;
-			}
-		} catch {
-		}
-	}
-
-	if (!parsed) {
-		try {
-			const candidate = JSON.parse(body) as ExaMcpRpcResponse;
-			if (candidate?.result || candidate?.error) {
-				parsed = candidate;
-			}
-		} catch {
-		}
-	}
-
-	if (!parsed) {
-		throw new Error("Exa MCP returned an empty response");
-	}
-
-	if (parsed.error) {
-		const code = typeof parsed.error.code === "number" ? ` ${parsed.error.code}` : "";
-		const message = parsed.error.message || "Unknown error";
-		throw new Error(`Exa MCP error${code}: ${message}`);
-	}
-
-	if (parsed.result?.isError) {
-		const message = parsed.result.content
-			?.find(item => item.type === "text" && typeof item.text === "string")
-			?.text?.trim();
-		throw new Error(message || "Exa MCP returned an error");
-	}
-
-	const text = parsed.result?.content
-		?.find(item => item.type === "text" && typeof item.text === "string" && item.text.trim().length > 0)
-		?.text;
-
-	if (!text) {
-		throw new Error("Exa MCP returned empty content");
-	}
-
-	return text;
-}
-
-function parseMcpResults(text: string): McpParsedResult[] | null {
-	const blocks = text.split(/(?=^Title: )/m).filter(block => block.trim().length > 0);
-	const parsed = blocks.map(block => {
-		const title = block.match(/^Title: (.+)/m)?.[1]?.trim() ?? "";
-		const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? "";
-		let content = "";
-		const textStart = block.indexOf("\nText: ");
-		if (textStart >= 0) {
-			content = block.slice(textStart + 7).trim();
-		} else {
-			const hlMatch = block.match(/\nHighlights:\s*\n/);
-			if (hlMatch?.index != null) {
-				content = block.slice(hlMatch.index + hlMatch[0].length).trim();
-			}
-		}
-		content = content.replace(/\n---\s*$/, "").trim();
-		return { title, url, content };
-	}).filter(result => result.url.length > 0);
-	return parsed.length > 0 ? parsed : null;
-}
-
-function buildAnswerFromMcpResults(results: McpParsedResult[]): string {
-	if (results.length === 0) return "";
-	const parts: string[] = [];
-	for (let i = 0; i < results.length; i++) {
-		const result = results[i];
-		const snippet = result.content.replace(/\s+/g, " ").trim().slice(0, 500);
-		if (!snippet) continue;
-		const sourceTitle = result.title || `Source ${i + 1}`;
-		parts.push(`${snippet}\nSource: ${sourceTitle} (${result.url})`);
-	}
-	return parts.join("\n\n");
-}
-
-function mapMcpInlineContent(results: McpParsedResult[]): ExtractedContent[] {
-	return results
-		.filter(result => result.content.length > 0)
-		.map(result => ({
-			url: result.url,
-			title: result.title,
-			content: result.content,
-			error: null,
-		}));
-}
-
-function buildMcpQuery(query: string, options: ExaSearchOptions): string {
-	const parts = [query];
-	if (options.domainFilter?.length) {
-		for (const d of options.domainFilter) {
-			parts.push(d.startsWith("-") ? `-site:${d.slice(1)}` : `site:${d}`);
-		}
-	}
-	if (options.recencyFilter) {
-		const now = new Date();
-		switch (options.recencyFilter) {
-			case "day": parts.push("past 24 hours"); break;
-			case "week": parts.push("past week"); break;
-			case "month": parts.push(`${now.toLocaleString("en", { month: "long" })} ${now.getFullYear()}`); break;
-			case "year": parts.push(String(now.getFullYear())); break;
-		}
-	}
-	return parts.join(" ");
-}
-
-export async function searchWithExaMcp(query: string, options: ExaSearchOptions = {}): Promise<SearchResponse | null> {
-	const enrichedQuery = buildMcpQuery(query, options);
-	const activityId = activityMonitor.logStart({ type: "api", query: enrichedQuery });
-
-	try {
-		const text = await callExaMcp(
-			"web_search_exa",
-			{
-				query: enrichedQuery,
-				numResults: options.numResults ?? 5,
-				livecrawl: "fallback",
-				type: "auto",
-				contextMaxCharacters: options.includeContent ? 50000 : 3000,
-			},
-			options.signal,
-		);
-		const parsedResults = parseMcpResults(text);
-		activityMonitor.logComplete(activityId, 200);
-
-		if (!parsedResults) return null;
-
-		const response: SearchResponse = {
-			answer: buildAnswerFromMcpResults(parsedResults),
-			results: parsedResults.map((result, index) => ({
-				title: result.title || `Source ${index + 1}`,
-				url: result.url,
-				snippet: "",
-			})),
-		};
-
-		if (options.includeContent) {
-			const inlineContent = mapMcpInlineContent(parsedResults);
-			if (inlineContent.length > 0) response.inlineContent = inlineContent;
-		}
-
-		return response;
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		if (message.toLowerCase().includes("abort")) {
-			activityMonitor.logComplete(activityId, 0);
-		} else {
-			activityMonitor.logError(activityId, message);
-		}
-		throw err;
-	}
-}