@bastani/atomic 0.9.16-alpha.2 → 0.9.16-alpha.4

package/dist/builtin/web-access/extract.ts

@@ -1,500 +0,0 @@
-import { Readability } from "@mozilla/readability";
-import { parseHTML } from "linkedom";
-import TurndownService from "turndown";
-import pLimit from "p-limit";
-import { CONFIG_DIR_NAME } from "@bastani/atomic";
-import { activityMonitor } from "./activity.js";
-import { extractRSCContent } from "./rsc-extract.js";
-import { extractPDFToMarkdown, isPDF } from "./pdf-extract.js";
-import { extractGitHub } from "./github-extract.js";
-import { isYouTubeURL, isYouTubeEnabled, extractYouTube } from "./youtube-extract.js";
-import { extractWithUrlContext, extractWithGeminiWeb } from "./gemini-url-context.js";
-import { isVideoFile, extractVideo } from "./video-extract.js";
-import { extractRequestedFrames } from "./extract-frames.js";
-
-const DEFAULT_TIMEOUT_MS = 30000;
-const CONCURRENT_LIMIT = 3;
-
-const NON_RECOVERABLE_ERRORS = ["Unsupported content type", "Response too large"];
-const MIN_USEFUL_CONTENT = 500;
-
-function errorMessage(err: unknown): string {
-	return err instanceof Error ? err.message : String(err);
-}
-
-function isConfigParseError(err: unknown): boolean {
-	return errorMessage(err).startsWith("Failed to parse ");
-}
-
-function isAbortError(err: unknown): boolean {
-	return errorMessage(err).toLowerCase().includes("abort");
-}
-
-function abortedResult(url: string): ExtractedContent {
-	return { url, title: "", content: "", error: "Aborted" };
-}
-
-function safeVideoInfo(url: string): { info: ReturnType<typeof isVideoFile>; error?: string } {
-	try {
-		return { info: isVideoFile(url) };
-	} catch (err) {
-		return { info: null, error: errorMessage(err) };
-	}
-}
-
-const turndown = new TurndownService({
-	headingStyle: "atx",
-	codeBlockStyle: "fenced",
-});
-
-const fetchLimit = pLimit(CONCURRENT_LIMIT);
-
-export interface VideoFrame {
-	data: string;
-	mimeType: string;
-	timestamp: string;
-}
-
-export type FrameData = { data: string; mimeType: string };
-export type FrameResult = FrameData | { error: string };
-
-export interface ExtractedContent {
-	url: string;
-	title: string;
-	content: string;
-	error: string | null;
-	thumbnail?: { data: string; mimeType: string };
-	frames?: VideoFrame[];
-	duration?: number;
-}
-
-export interface ExtractOptions {
-	timeoutMs?: number;
-	forceClone?: boolean;
-	prompt?: string;
-	timestamp?: string;
-	frames?: number;
-	model?: string;
-}
-
-const JINA_READER_BASE = "https://r.jina.ai/";
-const JINA_TIMEOUT_MS = 30000;
-
-async function extractWithJinaReader(
-	url: string,
-	signal?: AbortSignal,
-): Promise<ExtractedContent | null> {
-	const jinaUrl = JINA_READER_BASE + url;
-
-	const activityId = activityMonitor.logStart({ type: "api", query: `jina: ${url}` });
-
-	try {
-		const res = await fetch(jinaUrl, {
-			headers: {
-				"Accept": "text/markdown",
-				"X-No-Cache": "true",
-			},
-			signal: AbortSignal.any([
-				AbortSignal.timeout(JINA_TIMEOUT_MS),
-				...(signal ? [signal] : []),
-			]),
-		});
-
-		if (!res.ok) {
-			activityMonitor.logComplete(activityId, res.status);
-			return null;
-		}
-
-		const content = await res.text();
-		activityMonitor.logComplete(activityId, res.status);
-
-		const contentStart = content.indexOf("Markdown Content:");
-		if (contentStart < 0) {
-			return null;
-		}
-
-		const markdownPart = content.slice(contentStart + 17).trim(); // 17 = "Markdown Content:".length
-
-		// Check for failed JS rendering or minimal content
-		if (markdownPart.length < 100 ||
-			markdownPart.startsWith("Loading...") ||
-			markdownPart.startsWith("Please enable JavaScript")) {
-			return null;
-		}
-
-		const title = extractHeadingTitle(markdownPart) ?? (new URL(url).pathname.split("/").pop() || url);
-		return { url, title, content: markdownPart, error: null };
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		if (message.toLowerCase().includes("abort")) {
-			activityMonitor.logComplete(activityId, 0);
-		} else {
-			activityMonitor.logError(activityId, message);
-		}
-		return null;
-	}
-}
-
-
-export async function extractContent(
-	url: string,
-	signal?: AbortSignal,
-	options?: ExtractOptions,
-): Promise<ExtractedContent> {
-	if (signal?.aborted) {
-		return { url, title: "", content: "", error: "Aborted" };
-	}
-
-	const frameResult = await extractRequestedFrames(url, options, signal);
-	if (frameResult) return frameResult;
-
-	const localVideo = safeVideoInfo(url);
-	if (localVideo.error) {
-		return { url, title: "", content: "", error: localVideo.error };
-	}
-	if (localVideo.info) {
-		try {
-			const result = await extractVideo(localVideo.info, signal, options);
-			if (signal?.aborted) return abortedResult(url);
-			return result ?? { url, title: "", content: "", error: `Video analysis requires Gemini access. Either:\n  1. Sign into gemini.google.com in Chrome (free, uses cookies)\n  2. Set GEMINI_API_KEY in ~/${CONFIG_DIR_NAME}/web-search.json` };
-		} catch (err) {
-			if (isAbortError(err)) return abortedResult(url);
-			return { url, title: "", content: "", error: errorMessage(err) };
-		}
-	}
-
-	try {
-		new URL(url);
-	} catch {
-		return { url, title: "", content: "", error: "Invalid URL" };
-	}
-
-	try {
-		const ghResult = await extractGitHub(url, signal, options?.forceClone);
-		if (ghResult) return ghResult;
-		if (signal?.aborted) return abortedResult(url);
-	} catch (err) {
-		const message = errorMessage(err);
-		if (isAbortError(err)) return abortedResult(url);
-		if (isConfigParseError(err)) {
-			return { url, title: "", content: "", error: message };
-		}
-	}
-
-	const ytInfo = isYouTubeURL(url);
-	let youtubeEnabled = false;
-	try {
-		youtubeEnabled = isYouTubeEnabled();
-	} catch (err) {
-		return { url, title: "", content: "", error: errorMessage(err) };
-	}
-	if (ytInfo.isYouTube && youtubeEnabled) {
-		try {
-			const ytResult = await extractYouTube(url, signal, options?.prompt, options?.model);
-			if (ytResult) return ytResult;
-			if (signal?.aborted) return abortedResult(url);
-		} catch (err) {
-			const message = errorMessage(err);
-			if (isAbortError(err)) return abortedResult(url);
-			if (isConfigParseError(err)) {
-				return { url, title: "", content: "", error: message };
-			}
-		}
-		return {
-			url,
-			title: "",
-			content: "",
-			error: "Could not extract YouTube video content. Sign into Google in Chrome for automatic access, or set GEMINI_API_KEY.",
-		};
-	}
-
-	if (signal?.aborted) return abortedResult(url);
-
-	const httpResult = await extractViaHttp(url, signal, options);
-
-	if (signal?.aborted) return abortedResult(url);
-	if (!httpResult.error) return httpResult;
-	if (NON_RECOVERABLE_ERRORS.some(prefix => httpResult.error!.startsWith(prefix))) return httpResult;
-
-	const jinaResult = await extractWithJinaReader(url, signal);
-	if (jinaResult) return jinaResult;
-	if (signal?.aborted) return abortedResult(url);
-
-	let geminiResult: ExtractedContent | null = null;
-	try {
-		geminiResult = await extractWithUrlContext(url, signal)
-			?? await extractWithGeminiWeb(url, signal);
-	} catch (err) {
-		if (isAbortError(err)) return abortedResult(url);
-		if (isConfigParseError(err)) {
-			return { ...httpResult, error: errorMessage(err) };
-		}
-	}
-
-	if (geminiResult) return geminiResult;
-	if (signal?.aborted) return abortedResult(url);
-
-	const guidance = [
-		httpResult.error,
-		"",
-		"Fallback options:",
-		`  \u2022 Set GEMINI_API_KEY in ~/${CONFIG_DIR_NAME}/web-search.json`,
-		"  \u2022 Sign into gemini.google.com in Chrome",
-		"  \u2022 Use web_search to find content about this topic",
-	].join("\n");
-	return { ...httpResult, error: guidance };
-}
-
-function stripElementBlocks(html: string, tagName: "script" | "style"): string {
-	let output = "";
-	let cursor = 0;
-	const lower = html.toLowerCase();
-	const startNeedle = `<${tagName}`;
-	const endNeedle = `</${tagName}`;
-
-	while (cursor < html.length) {
-		const start = lower.indexOf(startNeedle, cursor);
-		if (start === -1) {
-			output += html.slice(cursor);
-			break;
-		}
-
-		output += html.slice(cursor, start);
-		const end = lower.indexOf(endNeedle, start + startNeedle.length);
-		if (end === -1) {
-			break;
-		}
-
-		const endClose = lower.indexOf(">", end + endNeedle.length);
-		if (endClose === -1) {
-			break;
-		}
-		cursor = endClose + 1;
-	}
-
-	return output;
-}
-
-function stripTags(html: string): string {
-	let output = "";
-	let insideTag = false;
-	for (const char of html) {
-		if (char === "<") {
-			insideTag = true;
-			continue;
-		}
-		if (char === ">") {
-			insideTag = false;
-			continue;
-		}
-		if (!insideTag) output += char;
-	}
-	return output;
-}
-
-function collapseWhitespace(text: string): string {
-	let output = "";
-	let pendingSpace = false;
-	for (const char of text) {
-		if (/\s/.test(char)) {
-			pendingSpace = output.length > 0;
-			continue;
-		}
-		if (pendingSpace) {
-			output += " ";
-			pendingSpace = false;
-		}
-		output += char;
-	}
-	return output.trim();
-}
-
-function isLikelyJSRendered(html: string): boolean {
-	// Extract body content
-	const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
-	if (!bodyMatch) return false;
-
-	const bodyHtml = bodyMatch[1];
-
-	// Strip tags to get text content
-	const textContent = collapseWhitespace(stripTags(stripElementBlocks(stripElementBlocks(bodyHtml, "script"), "style")));
-
-	// Count scripts
-	const scriptCount = (html.match(/<script\b/gi) || []).length;
-
-	// Heuristic: little text content but many scripts suggests JS rendering
-	return textContent.length < 500 && scriptCount > 3;
-}
-
-async function extractViaHttp(
-	url: string,
-	signal?: AbortSignal,
-	options?: ExtractOptions,
-): Promise<ExtractedContent> {
-	const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-	const activityId = activityMonitor.logStart({ type: "fetch", url });
-
-	const controller = new AbortController();
-	const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-
-	const onAbort = () => controller.abort();
-	signal?.addEventListener("abort", onAbort);
-
-	try {
-		const response = await fetch(url, {
-			signal: controller.signal,
-			headers: {
-				"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
-				"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
-				"Accept-Language": "en-US,en;q=0.9",
-				"Cache-Control": "no-cache",
-				"Sec-Fetch-Dest": "document",
-				"Sec-Fetch-Mode": "navigate",
-				"Sec-Fetch-Site": "none",
-				"Sec-Fetch-User": "?1",
-				"Upgrade-Insecure-Requests": "1",
-			},
-		});
-
-		if (!response.ok) {
-			activityMonitor.logComplete(activityId, response.status);
-			return {
-				url,
-				title: "",
-				content: "",
-				error: `HTTP ${response.status}: ${response.statusText}`,
-			};
-		}
-
-		const contentLengthHeader = response.headers.get("content-length");
-		const contentType = response.headers.get("content-type") || "";
-		const isPDFContent = isPDF(url, contentType);
-		const maxResponseSize = isPDFContent ? 20 * 1024 * 1024 : 5 * 1024 * 1024;
-		if (contentLengthHeader) {
-			const contentLength = parseInt(contentLengthHeader, 10);
-			if (contentLength > maxResponseSize) {
-				activityMonitor.logComplete(activityId, response.status);
-				return {
-					url,
-					title: "",
-					content: "",
-					error: `Response too large (${Math.round(contentLength / 1024 / 1024)}MB)`,
-				};
-			}
-		}
-
-		if (isPDFContent) {
-			try {
-				const buffer = await response.arrayBuffer();
-				const result = await extractPDFToMarkdown(buffer, url);
-				activityMonitor.logComplete(activityId, response.status);
-				return {
-					url,
-					title: result.title,
-					content: `PDF extracted and saved to: ${result.outputPath}\n\nPages: ${result.pages}\nCharacters: ${result.chars}`,
-					error: null,
-				};
-			} catch (err) {
-				const message = err instanceof Error ? err.message : String(err);
-				activityMonitor.logError(activityId, message);
-				return { url, title: "", content: "", error: `PDF extraction failed: ${message}` };
-			}
-		}
-
-		if (contentType.includes("application/octet-stream") ||
-			contentType.includes("image/") ||
-			contentType.includes("audio/") ||
-			contentType.includes("video/") ||
-			contentType.includes("application/zip")) {
-			activityMonitor.logComplete(activityId, response.status);
-			return {
-				url,
-				title: "",
-				content: "",
-				error: `Unsupported content type: ${contentType.split(";")[0]}`,
-			};
-		}
-
-		const text = await response.text();
-		const isHTML = contentType.includes("text/html") || contentType.includes("application/xhtml+xml");
-
-		if (!isHTML) {
-			activityMonitor.logComplete(activityId, response.status);
-			const title = extractTextTitle(text, url);
-			return { url, title, content: text, error: null };
-		}
-
-		const { document } = parseHTML(text);
-		const reader = new Readability(document as unknown as Document);
-		const article = reader.parse();
-
-		if (!article) {
-			const rscResult = extractRSCContent(text);
-			if (rscResult) {
-				activityMonitor.logComplete(activityId, response.status);
-				return { url, title: rscResult.title, content: rscResult.content, error: null };
-			}
-
-			activityMonitor.logComplete(activityId, response.status);
-
-			// Provide more specific error message
-			const jsRendered = isLikelyJSRendered(text);
-			const errorMsg = jsRendered
-				? "Page appears to be JavaScript-rendered (content loads dynamically)"
-				: "Could not extract readable content from HTML structure";
-
-			return {
-				url,
-				title: "",
-				content: "",
-				error: errorMsg,
-			};
-		}
-
-		const markdown = turndown.turndown(article.content ?? "");
-		activityMonitor.logComplete(activityId, response.status);
-
-		if (markdown.length < MIN_USEFUL_CONTENT) {
-			return {
-				url,
-				title: article.title || "",
-				content: markdown,
-				error: isLikelyJSRendered(text)
-					? "Page appears to be JavaScript-rendered (content loads dynamically)"
-					: "Extracted content appears incomplete",
-			};
-		}
-
-		return { url, title: article.title || "", content: markdown, error: null };
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		if (message.toLowerCase().includes("abort")) {
-			activityMonitor.logComplete(activityId, 0);
-		} else {
-			activityMonitor.logError(activityId, message);
-		}
-		return { url, title: "", content: "", error: message };
-	} finally {
-		clearTimeout(timeoutId);
-		signal?.removeEventListener("abort", onAbort);
-	}
-}
-
-export function extractHeadingTitle(text: string): string | null {
-	const match = text.match(/^#{1,2}\s+(.+)/m);
-	if (!match) return null;
-	const cleaned = match[1].replace(/\*+/g, "").trim();
-	return cleaned || null;
-}
-
-function extractTextTitle(text: string, url: string): string {
-	return extractHeadingTitle(text) ?? (new URL(url).pathname.split("/").pop() || url);
-}
-
-export async function fetchAllContent(
-	urls: string[],
-	signal?: AbortSignal,
-	options?: ExtractOptions,
-): Promise<ExtractedContent[]> {
-	return Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));
-}

package/dist/builtin/web-access/flat-string.ts

@@ -1,37 +0,0 @@
-/**
- * Truncation that actually releases the untruncated source.
- *
- * This local web-access copy mirrors the original helper in
- * `packages/workflows/src/shared/flat-string.ts`; web-access is a standalone package and
- * cannot import that workflows module.
- *
- * V8 does not copy characters for `String.prototype.slice`. For a flat parent of
- * 13 characters or more it allocates a **SlicedString**, which is a length, an
- * offset, and a pointer to the parent. Template interpolation then wraps that in
- * a **ConsString**, which keeps the same pointer. So the obvious way to bound a
- * value for storage:
- *
- * ```ts
- * `${JSON.stringify(value).slice(0, 237)}...`
- * ```
- *
- * produces a 240-character string that reports `length === 240` while holding
- * the entire serialized payload alive. The bound is on what you can read, not on
- * what you retain. An 8 MiB tool result summarized to 240 characters still costs
- * 8 MiB for as long as anything keeps the summary.
- *
- * That is invisible in every ordinary test: the value is correct, its length is
- * correct, and equality holds. It only shows up as heap that never comes back.
- *
- * `split("").join("")` copies the code units into a fresh flat string with no
- * parent pointer. It is code-unit exact, including lone surrogates, because
- * `split("")` splits on UTF-16 code units rather than code points.
- *
- * Use this at any truncation whose result outlives the value it came from —
- * memoized projections, durable checkpoints, catalogs, persisted summaries.
- * A truncation that is consumed and dropped within the same turn does not need
- * it, and paying for a copy there is waste.
- */
-export function flattenTruncatedString(value: string): string {
-	return value.split("").join("");
-}

package/dist/builtin/web-access/gemini-api.ts

@@ -1,111 +0,0 @@
-import { existsSync, readFileSync } from "node:fs";
-import { findReadableConfigPath } from "./config-paths.ts";
-
-export const API_BASE = "https://generativelanguage.googleapis.com/v1beta";
-const CONFIG_PATH = findReadableConfigPath();
-export const DEFAULT_MODEL = "gemini-3-flash-preview";
-
-interface GeminiApiConfig {
-	geminiApiKey?: unknown;
-}
-
-let cachedConfig: GeminiApiConfig | null = null;
-
-function loadConfig(): GeminiApiConfig {
-	if (cachedConfig) return cachedConfig;
-	if (!existsSync(CONFIG_PATH)) {
-		cachedConfig = {};
-		return cachedConfig;
-	}
-
-	const raw = readFileSync(CONFIG_PATH, "utf-8");
-	try {
-		cachedConfig = JSON.parse(raw) as GeminiApiConfig;
-		return cachedConfig;
-	} catch (err) {
-		const message = err instanceof Error ? err.message : String(err);
-		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
-	}
-}
-
-function withTimeout(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {
-	const timeout = AbortSignal.timeout(timeoutMs);
-	return signal ? AbortSignal.any([signal, timeout]) : timeout;
-}
-
-function normalizeApiKey(value: unknown): string | null {
-	if (typeof value !== "string") return null;
-	const normalized = value.trim();
-	return normalized.length > 0 ? normalized : null;
-}
-
-export function getApiKey(): string | null {
-	return normalizeApiKey(process.env.GEMINI_API_KEY) ?? normalizeApiKey(loadConfig().geminiApiKey);
-}
-
-export function isGeminiApiAvailable(): boolean {
-	return getApiKey() !== null;
-}
-
-export interface GeminiApiOptions {
-	model?: string;
-	mimeType?: string;
-	signal?: AbortSignal;
-	timeoutMs?: number;
-}
-
-export async function queryGeminiApiWithVideo(
-	prompt: string,
-	videoUri: string,
-	options: GeminiApiOptions = {},
-): Promise<string> {
-	const apiKey = getApiKey();
-	if (!apiKey) throw new Error("GEMINI_API_KEY not configured");
-
-	const model = options.model ?? DEFAULT_MODEL;
-	const signal = withTimeout(options.signal, options.timeoutMs ?? 120000);
-	const url = `${API_BASE}/models/${model}:generateContent?key=${apiKey}`;
-
-	const fileData: Record<string, string> = { fileUri: videoUri };
-	if (options.mimeType) fileData.mimeType = options.mimeType;
-
-	const body = {
-		contents: [
-			{
-				parts: [
-					{ fileData },
-					{ text: prompt },
-				],
-			},
-		],
-	};
-
-	const res = await fetch(url, {
-		method: "POST",
-		headers: { "Content-Type": "application/json" },
-		body: JSON.stringify(body),
-		signal,
-	});
-
-	if (!res.ok) {
-		const errorText = await res.text();
-		throw new Error(`Gemini API error ${res.status}: ${errorText.slice(0, 300)}`);
-	}
-
-	const data = (await res.json()) as GenerateContentResponse;
-	const text = data.candidates?.[0]?.content?.parts
-		?.map((p) => p.text)
-		.filter(Boolean)
-		.join("\n");
-
-	if (!text) throw new Error("Gemini API returned empty response");
-	return text;
-}
-
-interface GenerateContentResponse {
-	candidates?: Array<{
-		content?: {
-			parts?: Array<{ text?: string }>;
-		};
-	}>;
-}