@aria-cli/tools 1.0.12 → 1.0.14

package/dist/executors/utils.js

@@ -1,33 +0,0 @@
-/**
- * @aria/tools - Shared executor utilities
- *
- * Common helper functions used across all executor modules.
- */
-import * as nodePath from "node:path";
-export { getErrorMessage } from "@aria-cli/types";
-/**
- * Creates a successful result.
- */
-export function success(message, data) {
-    return { success: true, message, data };
-}
-/**
- * Creates a failure result.
- */
-export function fail(message, data) {
-    return { success: false, message, data };
-}
-/**
- * Checks if a resolved path is within the allowed base directory.
- * Prevents path traversal attacks.
- */
-export function isPathWithinBase(resolvedPath, baseDir) {
-    const normalizedPath = nodePath.normalize(resolvedPath);
-    const normalizedBase = nodePath.normalize(baseDir);
-    // normalize() strips trailing separators except for root "/".
-    // Guard against root-as-base which would allow every absolute path.
-    const baseWithSep = normalizedBase.endsWith(nodePath.sep)
-        ? normalizedBase
-        : normalizedBase + nodePath.sep;
-    return normalizedPath === normalizedBase || normalizedPath.startsWith(baseWithSep);
-}

package/dist/executors/web.js

@@ -1,542 +0,0 @@
-/**
- * @aria/tools - Web tool executors
- *
- * Implementation of web operations for ARIA tool system.
- */
-import { createHash } from "node:crypto";
-import { log } from "@aria-cli/types";
-import { z } from "zod";
-import { success, fail, getErrorMessage } from "./utils.js";
-import { safeParseJson } from "../utils/safe-parse-json.js";
-import { validateUrlStructure, followRedirects, discardResponseBody } from "../security/ssrf.js";
-import { SearchProviderRouter } from "../providers/router.js";
-import { TavilySearchProvider, DuckDuckGoSearchProvider, BraveSearchProvider, JinaSearchProvider, ExaSearchProvider, FirecrawlSearchProvider, } from "../providers/index.js";
-import { resolveSearchProviderEnv } from "../providers/search-provider.js";
-import { searchCache, fetchCache, browseCache, } from "../cache/web-cache.js";
-import { fetchWithSsrf } from "../utils/retry.js";
-import { wrapExternalContent } from "../security/external-content.js";
-import { extractFromResponse } from "../extraction/content-extraction.js";
-import { normalizeGitHubUrl, getUserAgent } from "../utils/url.js";
-function buildSearchEnv(ctx) {
-    const env = ctx.env ?? {};
-    return {
-        ARIA_SEARCH_PROVIDER: env.ARIA_SEARCH_PROVIDER ?? process.env.ARIA_SEARCH_PROVIDER,
-        BRAVE_API_KEY: env.BRAVE_API_KEY ?? process.env.BRAVE_API_KEY,
-        FIRECRAWL_API_KEY: env.FIRECRAWL_API_KEY ?? process.env.FIRECRAWL_API_KEY,
-        EXA_API_KEY: env.EXA_API_KEY ?? process.env.EXA_API_KEY,
-        TAVILY_API_KEY: env.TAVILY_API_KEY ?? process.env.TAVILY_API_KEY,
-        JINA_API_KEY: env.JINA_API_KEY ?? process.env.JINA_API_KEY,
-    };
-}
-function createSearchRouter(env) {
-    return new SearchProviderRouter([
-        new BraveSearchProvider(env),
-        new TavilySearchProvider(env),
-        new JinaSearchProvider(env),
-        new ExaSearchProvider(env),
-        new FirecrawlSearchProvider(env),
-        new DuckDuckGoSearchProvider(),
-    ], env);
-}
-/** Default timeout for fetch requests in milliseconds */
-const DEFAULT_TIMEOUT_MS = 30_000;
-/** Default maximum response size in bytes for fetch (1MB) */
-const FETCH_MAX_SIZE_BYTES = 1 * 1024 * 1024;
-/** Default maximum response size in bytes for browse (2MB) */
-const BROWSE_MAX_SIZE_BYTES = 2 * 1024 * 1024;
-const JsonFetchContentSchema = z.union([
-    z.record(z.string(), z.unknown()),
-    z.array(z.unknown()),
-    z.string(),
-    z.number(),
-    z.boolean(),
-    z.null(),
-]);
-/**
- * Creates an AbortController with a timeout.
- */
-function createTimeoutController(timeoutMs) {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => {
-        if (!controller.signal.aborted) {
-            controller.abort();
-        }
-    }, timeoutMs);
-    return { controller, timeoutId };
-}
-/**
- * Links a parent AbortSignal to a child AbortController, forwarding abort.
- * Returns a cleanup function that MUST be called when the operation completes
- * to prevent the listener from firing after the try/catch scope has exited.
- */
-function linkAbortSignal(parentSignal, childController) {
-    if (!parentSignal)
-        return () => { };
-    const onAbort = () => {
-        if (!childController.signal.aborted) {
-            childController.abort();
-        }
-    };
-    parentSignal.addEventListener("abort", onAbort, { once: true });
-    return () => {
-        parentSignal.removeEventListener("abort", onAbort);
-    };
-}
-/**
- * Reads a response body with a streaming byte limit.
- * Unlike checking Content-Length alone, this protects against chunked/streaming
- * responses that lack a Content-Length header.
- *
- * When `truncate` is false (default), throws if the limit is exceeded.
- * When `truncate` is true, returns whatever was read up to the limit.
- */
-async function readResponseWithLimit(response, maxBytes, options) {
-    const body = response.body;
-    if (!body) {
-        return { text: "", truncated: false, contentBytes: 0 };
-    }
-    const truncateMode = options?.truncate ?? false;
-    const reader = body.getReader();
-    const decoder = new TextDecoder();
-    const chunks = [];
-    let totalBytes = 0;
-    try {
-        for (;;) {
-            const { done, value } = await reader.read();
-            if (done)
-                break;
-            const chunkBytes = value.byteLength;
-            if (totalBytes + chunkBytes > maxBytes) {
-                if (truncateMode) {
-                    // Keep what we have so far (exclude the chunk that exceeded the limit)
-                    reader.cancel();
-                    chunks.push(decoder.decode());
-                    return {
-                        text: chunks.join(""),
-                        truncated: true,
-                        contentBytes: totalBytes,
-                    };
-                }
-                reader.cancel();
-                throw new Error(`Response body exceeds maximum size of ${maxBytes} bytes`);
-            }
-            totalBytes += chunkBytes;
-            chunks.push(decoder.decode(value, { stream: true }));
-        }
-        // Flush the decoder
-        chunks.push(decoder.decode());
-        return { text: chunks.join(""), truncated: false, contentBytes: totalBytes };
-    }
-    catch (err) {
-        reader.cancel();
-        throw err;
-    }
-}
-function ensureWrappedExternalContent(content, source) {
-    return wrapExternalContent(content, source).content;
-}
-function wrapWebSearchOutput(output) {
-    return {
-        query: output.query,
-        results: output.results.map((result) => ({
-            ...result,
-            content: ensureWrappedExternalContent(result.content, "web_search"),
-        })),
-    };
-}
-function hasAdvancedSearchOptions(input) {
-    return (input.limit !== undefined ||
-        input.topic !== undefined ||
-        (input.domains?.length ?? 0) > 0 ||
-        (input.excludeDomains?.length ?? 0) > 0 ||
-        input.timeRange !== undefined);
-}
-function buildSearchCachePayload(query, options) {
-    return JSON.stringify({ query, ...options });
-}
-function buildRouterSearchCacheKey(env, query, options) {
-    const resolved = resolveSearchProviderEnv(env);
-    const providerOverride = resolved.ARIA_SEARCH_PROVIDER ?? "auto";
-    const providerAvailability = [
-        resolved.BRAVE_API_KEY ? "brave=1" : "brave=0",
-        resolved.TAVILY_API_KEY ? "tavily=1" : "tavily=0",
-        resolved.JINA_API_KEY ? "jina=1" : "jina=0",
-        resolved.EXA_API_KEY ? "exa=1" : "exa=0",
-        resolved.FIRECRAWL_API_KEY ? "firecrawl=1" : "firecrawl=0",
-    ].join(",");
-    return `search:router:${providerOverride}:${providerAvailability}:${buildSearchCachePayload(query, options)}`;
-}
-function buildNativeSearchCacheKey(providerName, query, options) {
-    return `search:native:${providerName}:${buildSearchCachePayload(query, options)}`;
-}
-/**
- * Searches the web using the SearchProviderRouter with automatic fallback.
- * Providers are selected based on available API keys (Brave, Tavily, Jina, Exa,
- * Firecrawl, DuckDuckGo). DuckDuckGo is always available as a last resort.
- *
- * Results are cached via LRU+TTL cache for 15 minutes.
- */
-export async function executeWebSearch(input, ctx) {
-    const limit = input.limit ?? 10;
-    const searchOptions = {
-        limit,
-        ...(input.topic ? { topic: input.topic } : {}),
-        ...(input.domains ? { domains: input.domains } : {}),
-        ...(input.excludeDomains ? { excludeDomains: input.excludeDomains } : {}),
-        ...(input.timeRange ? { timeRange: input.timeRange } : {}),
-    };
-    const searchEnv = buildSearchEnv(ctx);
-    const advancedOptionsProvided = hasAdvancedSearchOptions(input);
-    const nativeSearchAvailable = Boolean(ctx.nativeSearchAdapter && ctx.providerContext?.capabilities?.nativeSearch);
-    const nativeProviderName = ctx.providerContext?.name ?? "unknown";
-    const routerCacheKey = buildRouterSearchCacheKey(searchEnv, input.query, searchOptions);
-    const nativeCacheKey = nativeSearchAvailable && !advancedOptionsProvided
-        ? buildNativeSearchCacheKey(nativeProviderName, input.query, searchOptions)
-        : undefined;
-    if (nativeCacheKey) {
-        const cachedNative = searchCache.get(nativeCacheKey);
-        if (cachedNative) {
-            return success(`Found ${cachedNative.results.length} cached results for "${input.query}"`, wrapWebSearchOutput(cachedNative));
-        }
-    }
-    else {
-        const cachedRouter = searchCache.get(routerCacheKey);
-        if (cachedRouter) {
-            return success(`Found ${cachedRouter.results.length} cached results for "${input.query}"`, wrapWebSearchOutput(cachedRouter));
-        }
-    }
-    if (advancedOptionsProvided && nativeSearchAvailable) {
-        log.debug("[web_search] advanced options provided; bypassing native adapter");
-    }
-    // Native search routing: If provider supports native search and adapter is wired, use it
-    // Phase 1: Gemini isolated adapter (router-based callback pattern)
-    if (nativeSearchAvailable && ctx.nativeSearchAdapter && !advancedOptionsProvided) {
-        try {
-            const rawResults = await ctx.nativeSearchAdapter(input.query);
-            if (rawResults.length === 0) {
-                throw new Error("Native search returned no results");
-            }
-            const results = rawResults.map((r) => ({
-                title: r.title,
-                url: r.url,
-                content: r.content,
-                score: r.score,
-            }));
-            const output = { query: input.query, results };
-            searchCache.set(nativeCacheKey, output);
-            return success(`Found ${results.length} results for "${input.query}" (native search)`, wrapWebSearchOutput(output));
-        }
-        catch (err) {
-            // Fall through to SearchProviderRouter on native search failure
-            log.debug(`[web_search] native adapter failed, falling back: ${getErrorMessage(err)}`);
-            const cachedRouterFallback = searchCache.get(routerCacheKey);
-            if (cachedRouterFallback) {
-                return success(`Found ${cachedRouterFallback.results.length} cached results for "${input.query}"`, wrapWebSearchOutput(cachedRouterFallback));
-            }
-        }
-    }
-    const searchRouter = createSearchRouter(searchEnv);
-    try {
-        // Race search against context abort signal for cancellation support
-        let results;
-        if (ctx.abortSignal) {
-            const abortPromise = new Promise((_resolve, reject) => {
-                if (ctx.abortSignal.aborted) {
-                    reject(new DOMException("The operation was aborted", "AbortError"));
-                    return;
-                }
-                ctx.abortSignal.addEventListener("abort", () => reject(new DOMException("The operation was aborted", "AbortError")), { once: true });
-            });
-            results = await Promise.race([searchRouter.search(input.query, searchOptions), abortPromise]);
-        }
-        else {
-            results = await searchRouter.search(input.query, searchOptions);
-        }
-        // Normalize results to WebSearchResult shape (cache raw; wrap at return-time)
-        const webResults = results.map((r) => ({
-            title: r.title,
-            url: r.url,
-            content: r.content,
-            score: r.score,
-        }));
-        const output = { query: input.query, results: webResults };
-        searchCache.set(routerCacheKey, output);
-        return success(`Found ${webResults.length} results for "${input.query}"`, wrapWebSearchOutput(output));
-    }
-    catch (err) {
-        if (ctx.abortSignal?.aborted) {
-            return fail("Web search cancelled");
-        }
-        return fail(`Web search failed: ${getErrorMessage(err)}`);
-    }
-}
-function wrapWebFetchOutput(output) {
-    if (typeof output.content !== "string") {
-        return output;
-    }
-    return {
-        ...output,
-        content: ensureWrappedExternalContent(output.content, "web_fetch"),
-    };
-}
-function markFetchOutputFromCache(cached) {
-    return {
-        ...cached,
-        fromCache: true,
-    };
-}
-function normalizeRequestHeaders(headers) {
-    if (!headers) {
-        return [];
-    }
-    return Object.entries(headers)
-        .map(([name, value]) => [name.trim().toLowerCase(), value.trim()])
-        .sort(([left], [right]) => left.localeCompare(right));
-}
-function hashRequestHeaders(headers) {
-    const normalizedHeaders = normalizeRequestHeaders(headers);
-    if (normalizedHeaders.length === 0) {
-        return "none";
-    }
-    return createHash("sha256").update(JSON.stringify(normalizedHeaders)).digest("hex");
-}
-function createFetchCacheKey({ url, format, headers, timeoutMs, maxSizeBytes, }) {
-    return `fetch:${url}:${format}:headers=${hashRequestHeaders(headers)}:maxSizeBytes=${maxSizeBytes}:timeoutMs=${timeoutMs}`;
-}
-/**
- * Fetches content from a URL with retry resilience, caching, and security wrapping.
- * Uses fetchWithRetry for automatic retry on transient failures (429, 5xx, ECONNRESET).
- * Results are cached via LRU+TTL cache for 15 minutes.
- */
-export async function executeWebFetch(input, ctx) {
-    // Normalize GitHub blob URLs to raw content URLs before fetching
-    const url = normalizeGitHubUrl(input.url);
-    // Validate URL structure. DNS/SSRF safety is enforced by fetchWithSsrf
-    // so validation and fetch share one trust boundary.
-    const urlError = validateUrlStructure(url);
-    if (urlError) {
-        return fail(urlError);
-    }
-    const format = input.format ?? "text";
-    const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-    const maxSizeBytes = input.maxSizeBytes ?? FETCH_MAX_SIZE_BYTES;
-    // Check cache first (use normalized URL + request variants to avoid collisions)
-    const cacheKey = createFetchCacheKey({
-        url,
-        format,
-        headers: input.headers,
-        timeoutMs,
-        maxSizeBytes,
-    });
-    const cached = fetchCache.get(cacheKey);
-    if (cached) {
-        return success(`Fetched ${input.url} (cached)`, wrapWebFetchOutput(markFetchOutputFromCache(cached)));
-    }
-    const { controller, timeoutId } = createTimeoutController(timeoutMs);
-    // Forward context abort signal to our controller
-    const unlinkAbort = linkAbortSignal(ctx.abortSignal, controller);
-    try {
-        const fetchOptions = {
-            headers: input.headers ?? {},
-            signal: controller.signal,
-            redirect: "manual",
-        };
-        // Use DNS-pinned SSRF fetch wrapper with transient retry support.
-        const initialResponse = await fetchWithSsrf(url, fetchOptions, {
-            maxAttempts: 2,
-        });
-        // Follow redirects manually and keep each hop on the same DNS-pinned path.
-        const response = await followRedirects(initialResponse, fetchOptions, {
-            baseUrl: url,
-            fetchFn: (redirectUrl, redirectInit) => fetchWithSsrf(redirectUrl, redirectInit, { maxAttempts: 2 }),
-            validateRedirectUrl: validateUrlStructure,
-        });
-        clearTimeout(timeoutId);
-        if (!response.ok) {
-            await discardResponseBody(response);
-            unlinkAbort();
-            return fail(`HTTP error: ${response.status} ${response.statusText}`, {
-                status: response.status,
-                statusText: response.statusText,
-            });
-        }
-        // Check Content-Length header for size limit
-        const contentLength = response.headers.get("Content-Length");
-        if (contentLength) {
-            const size = parseInt(contentLength, 10);
-            if (!isNaN(size) && size > maxSizeBytes) {
-                await discardResponseBody(response);
-                unlinkAbort();
-                return fail(`Response too large: ${size} bytes exceeds maximum of ${maxSizeBytes} bytes`);
-            }
-        }
-        const contentType = response.headers.get("Content-Type") ?? undefined;
-        let content;
-        // Use streaming reader to enforce size limit even for chunked responses
-        let text;
-        let contentBytes = 0;
-        try {
-            const result = await readResponseWithLimit(response, maxSizeBytes);
-            text = result.text;
-            contentBytes = result.contentBytes;
-            if (result.truncated) {
-                unlinkAbort();
-                return fail(`Response body exceeds maximum size of ${maxSizeBytes} bytes`);
-            }
-        }
-        catch (sizeErr) {
-            unlinkAbort();
-            return fail(getErrorMessage(sizeErr));
-        }
-        if (format === "json") {
-            const parsed = safeParseJson(text, JsonFetchContentSchema);
-            if (!parsed.ok) {
-                unlinkAbort();
-                return fail(`Failed to parse JSON response (${parsed.reason})`);
-            }
-            content = parsed.data;
-        }
-        else {
-            content = text;
-        }
-        const output = {
-            content,
-            status: response.status,
-            contentType,
-            fromCache: false,
-            fetchedAt: new Date().toISOString(),
-            finalUrl: response.url || url,
-            contentBytes,
-            truncated: false,
-        };
-        fetchCache.set(cacheKey, output);
-        unlinkAbort();
-        return success(`Fetched ${input.url} (${response.status})`, wrapWebFetchOutput(output));
-    }
-    catch (err) {
-        clearTimeout(timeoutId);
-        unlinkAbort();
-        if (err instanceof Error &&
-            (err.name === "AbortError" || (err instanceof DOMException && err.name === "AbortError"))) {
-            if (ctx.abortSignal?.aborted) {
-                return fail("Request cancelled");
-            }
-            return fail(`Request timed out after ${timeoutMs}ms`);
-        }
-        return fail(getErrorMessage(err));
-    }
-}
-// ============================================================================
-// Browse
-// ============================================================================
-/** Default timeout for browsing in milliseconds */
-const BROWSE_TIMEOUT_MS = 30_000;
-/** Maximum content length returned by browse (characters) — increased from 10K to 50K */
-const BROWSE_MAX_CONTENT_LENGTH = 50_000;
-function wrapBrowseOutput(output) {
-    return {
-        ...output,
-        content: ensureWrappedExternalContent(output.content, "browse"),
-    };
-}
-function markBrowseOutputFromCache(cached) {
-    return {
-        ...cached,
-        fromCache: true,
-    };
-}
-/**
- * Browses a URL by fetching its HTML content and extracting Markdown.
- * Uses Readability.js + Turndown for article extraction with three-tier fallback.
- * Supports GitHub URL normalization, retry on transient failures, LRU+TTL caching,
- * and nonce-based external content wrapping.
- */
-export async function executeBrowse(input, ctx) {
-    // Validate URL
-    if (!input.url) {
-        return fail("URL is required for browse");
-    }
-    // Normalize GitHub blob URLs to raw content URLs
-    const url = normalizeGitHubUrl(input.url);
-    // Check cache (using normalized URL as key)
-    const cached = browseCache.get(url);
-    if (cached) {
-        return success(`Browsed ${input.url} (cached)`, wrapBrowseOutput(markBrowseOutputFromCache(cached)));
-    }
-    const urlError = validateUrlStructure(url);
-    if (urlError) {
-        return fail(urlError);
-    }
-    const timeoutMs = input.timeoutMs ?? BROWSE_TIMEOUT_MS;
-    const { controller, timeoutId } = createTimeoutController(timeoutMs);
-    // Forward context abort signal to our controller
-    const unlinkAbort = linkAbortSignal(ctx.abortSignal, controller);
-    try {
-        const fetchOptions = {
-            headers: {
-                "User-Agent": getUserAgent(),
-                Accept: "text/html, application/xhtml+xml, */*",
-            },
-            signal: controller.signal,
-            redirect: "manual",
-        };
-        // Use DNS-pinned SSRF fetch wrapper with transient retry support.
-        const initialResponse = await fetchWithSsrf(url, fetchOptions, {
-            maxAttempts: 2,
-        });
-        // Follow redirects manually and keep each hop on the same DNS-pinned path.
-        const response = await followRedirects(initialResponse, fetchOptions, {
-            baseUrl: url,
-            fetchFn: (redirectUrl, redirectInit) => fetchWithSsrf(redirectUrl, redirectInit, { maxAttempts: 2 }),
-            validateRedirectUrl: validateUrlStructure,
-        });
-        clearTimeout(timeoutId);
-        if (!response.ok) {
-            await discardResponseBody(response);
-            unlinkAbort();
-            return fail(`HTTP error fetching ${input.url}: ${response.status} ${response.statusText}`);
-        }
-        // Truncate large pages gracefully — content gets reduced to 50K chars by
-        // Readability+Turndown anyway, so the first 2MB of HTML is plenty.
-        const { text: html, truncated: responseTruncated, contentBytes, } = await readResponseWithLimit(response, BROWSE_MAX_SIZE_BYTES, {
-            truncate: true,
-        });
-        // Extract content using Content-Type-aware extraction pipeline.
-        const { title, content: extractedContent } = await extractFromResponse(html, url, response.headers.get("Content-Type"));
-        // Truncate to 50K character limit
-        // extractContent() internally truncates to MAX_CONTENT_LENGTH (50K),
-        // so content that was originally longer arrives here at exactly 50K.
-        // Use >= to detect content that hit the internal limit.
-        const contentLimitReached = extractedContent.length >= BROWSE_MAX_CONTENT_LENGTH;
-        const truncated = contentLimitReached || responseTruncated;
-        const truncatedContent = contentLimitReached
-            ? extractedContent.slice(0, BROWSE_MAX_CONTENT_LENGTH) + "\n\n[Content truncated]"
-            : responseTruncated
-                ? extractedContent + "\n\n[Content truncated]"
-                : extractedContent;
-        const output = {
-            url: input.url,
-            title,
-            content: truncatedContent,
-            fromCache: false,
-            fetchedAt: new Date().toISOString(),
-            finalUrl: response.url || url,
-            contentBytes,
-            truncated,
-        };
-        browseCache.set(url, output);
-        unlinkAbort();
-        return success(`Browsed ${input.url}`, wrapBrowseOutput(output));
-    }
-    catch (err) {
-        clearTimeout(timeoutId);
-        unlinkAbort();
-        if (err instanceof Error && err.name === "AbortError") {
-            if (ctx.abortSignal?.aborted) {
-                return fail("Browse cancelled");
-            }
-            return fail(`Browse timed out after ${timeoutMs}ms: ${input.url}`);
-        }
-        return fail(`Browse failed: ${getErrorMessage(err)}`);
-    }
-}