@qearlyao/familiar 0.2.5 → 0.4.0

package/dist/web-tools/fetch-providers.js

@@ -0,0 +1,119 @@
+import { isRecord } from "../util/guards.js";
+import { fetchJson, fetchText } from "./http.js";
+import { FETCH_TIMEOUT_MS, MAX_RESPONSE_BYTES, ProviderError } from "./types.js";
+export function createJinaProvider(apiKey) {
+    return {
+        name: "jina",
+        async fetch(url, signal) {
+            const target = `https://r.jina.ai/${url}`;
+            const headers = buildJinaHeaders(apiKey, "application/json");
+            try {
+                const jsonContent = await fetchJinaContent(target, headers, signal, true);
+                if (jsonContent)
+                    return jsonContent;
+            }
+            catch (error) {
+                if (!shouldFallbackToText(error)) {
+                    throw error;
+                }
+            }
+            const textContent = await fetchJinaContent(target, buildJinaHeaders(apiKey, "text/plain"), signal, false);
+            if (textContent)
+                return textContent;
+            throw new ProviderError("jina", "jina returned an empty response.", false);
+        },
+    };
+}
+export function createTinyfishProvider(apiKey) {
+    const trimmed = apiKey.trim();
+    return {
+        name: "tinyfish",
+        async fetch(url, signal) {
+            const response = await fetchJson("tinyfish", "https://api.fetch.tinyfish.ai", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "X-API-Key": trimmed,
+                },
+                body: JSON.stringify({
+                    urls: [url],
+                    format: "markdown",
+                }),
+                signal,
+                timeoutMs: FETCH_TIMEOUT_MS,
+                maxBytes: MAX_RESPONSE_BYTES.fetch,
+                validate: parseTinyfishResponse,
+            });
+            return response.content;
+        },
+    };
+}
+export function parseTinyfishResponse(value) {
+    if (!isRecord(value)) {
+        throw new ProviderError("tinyfish", "TinyFish returned unexpected response shape.", false);
+    }
+    const results = value.results;
+    if (Array.isArray(results)) {
+        const first = results[0];
+        if (isRecord(first)) {
+            const content = typeof first.content === "string"
+                ? first.content
+                : typeof first.markdown === "string"
+                    ? first.markdown
+                    : typeof first.text === "string"
+                        ? first.text
+                        : "";
+            if (content.trim())
+                return { content: content.replaceAll(/\r\n/g, "\n").trim() };
+        }
+    }
+    const errors = Array.isArray(value.errors) ? value.errors : undefined;
+    const firstError = errors?.find((entry) => isRecord(entry));
+    if (isRecord(firstError)) {
+        const message = typeof firstError.message === "string"
+            ? firstError.message
+            : typeof firstError.error === "string"
+                ? firstError.error
+                : "TinyFish failed to fetch the page.";
+        throw new ProviderError("tinyfish", message, false);
+    }
+    throw new ProviderError("tinyfish", "TinyFish returned no page content.", false);
+}
+export function buildJinaHeaders(apiKey, accept) {
+    const headers = {
+        Accept: accept,
+        "X-Retain-Images": "none",
+    };
+    if (apiKey?.trim())
+        headers.Authorization = `Bearer ${apiKey.trim()}`;
+    return headers;
+}
+export async function fetchJinaContent(targetUrl, headers, signal, preferJson) {
+    const responseText = await fetchText("jina", targetUrl, {
+        headers,
+        signal,
+        timeoutMs: FETCH_TIMEOUT_MS,
+        maxBytes: MAX_RESPONSE_BYTES.fetch,
+    });
+    if (preferJson) {
+        try {
+            const parsed = JSON.parse(responseText);
+            if (isRecord(parsed) && isRecord(parsed.data)) {
+                if (typeof parsed.data.content === "string" && parsed.data.content.trim()) {
+                    return parsed.data.content.replaceAll(/\r\n/g, "\n").trim();
+                }
+                if (typeof parsed.data.markdown === "string" && parsed.data.markdown.trim()) {
+                    return parsed.data.markdown.replaceAll(/\r\n/g, "\n").trim();
+                }
+            }
+        }
+        catch {
+            return undefined;
+        }
+        return undefined;
+    }
+    return responseText.replaceAll(/\r\n/g, "\n").trim() || undefined;
+}
+export function shouldFallbackToText(error) {
+    return error instanceof ProviderError && (error.status === 406 || error.status === 415);
+}

package/dist/web-tools/format.js

@@ -0,0 +1,88 @@
+import { FETCH_DEFAULT_MAX_CHARS, SEARCH_OUTPUT_BUDGET, WEB_UNTRUSTED_PREFIX, } from "./types.js";
+import { normalizeIsoDate } from "./util.js";
+export function collectSearchNotes(requested, served, notes = []) {
+    if (requested !== served) {
+        notes.push(`Depth: requested ${requested}, served ${served}`);
+    }
+    return [...new Set(notes)];
+}
+export function buildSearchDocument(args) {
+    const lines = [`## Search Results (via ${args.provider}, ${args.depth})`];
+    if (args.notes?.length) {
+        lines.push("", ...args.notes);
+    }
+    if (args.freshness || args.domains?.length || args.appliedFilters) {
+        const notes = [];
+        if (args.freshness)
+            notes.push(`Freshness: ${args.freshness}`);
+        if (args.domains?.length)
+            notes.push(`Domains: ${args.domains.join(", ")}`);
+        lines.push("", ...notes);
+    }
+    for (const [index, result] of args.results.entries()) {
+        lines.push("", `### ${index + 1}. ${result.title}`, `URL: ${result.url}`);
+        const published = normalizeIsoDate(result.publishedAt);
+        if (published)
+            lines.push(`Published: ${published.slice(0, 10)}`);
+        lines.push(`Snippet: ${result.snippet || "[No snippet available]"}`);
+        if (result.content) {
+            lines.push("", "Content:", result.content);
+        }
+    }
+    return lines.join("\n");
+}
+export function formatFetchContent(url, provider, chunk) {
+    const header = `## Content from ${url} (via ${provider})`;
+    if (chunk.offset >= chunk.totalChars) {
+        return prefixUntrustedWebContent([
+            header,
+            "",
+            `[Offset ${chunk.offset} is beyond the end of the document. Total content length: ${chunk.totalChars} characters.]`,
+        ].join("\n"));
+    }
+    const lines = [
+        header,
+        "",
+        `[Showing chars ${chunk.offset}-${chunk.offset + chunk.returnedChars - 1} of ${chunk.totalChars}]`,
+        "",
+        chunk.text,
+    ];
+    if (chunk.hasMore && chunk.nextOffset !== undefined) {
+        lines.push("", `[More content available. Next chunk: fetch_web(url="${url}", offset=${chunk.nextOffset})]`);
+    }
+    return prefixUntrustedWebContent(lines.join("\n"));
+}
+export function prefixUntrustedWebContent(text) {
+    return `${WEB_UNTRUSTED_PREFIX}\n\n${text}`;
+}
+export function paginateContent(content, offset, maxChars = FETCH_DEFAULT_MAX_CHARS) {
+    const totalChars = content.length;
+    if (offset >= totalChars) {
+        return { text: "", offset, returnedChars: 0, totalChars, hasMore: false };
+    }
+    const safeMaxChars = Math.max(1, Math.min(maxChars, 20_000));
+    const end = Math.min(offset + safeMaxChars, totalChars);
+    const text = content.slice(offset, end).trim();
+    return {
+        text,
+        offset,
+        returnedChars: text.length,
+        totalChars,
+        nextOffset: end < totalChars ? end : undefined,
+        hasMore: end < totalChars,
+    };
+}
+export function formatSearchResults(args) {
+    const notes = collectSearchNotes(args.requestedDepth, args.servedDepth, [...(args.notes ?? [])]);
+    const document = buildSearchDocument({
+        provider: args.provider,
+        depth: args.servedDepth,
+        freshness: args.freshness,
+        domains: args.domains,
+        results: args.results,
+        appliedFilters: args.appliedFilters,
+        notes,
+    });
+    const output = prefixUntrustedWebContent(document);
+    return output.length > SEARCH_OUTPUT_BUDGET ? `${output.slice(0, SEARCH_OUTPUT_BUDGET - 3).trimEnd()}...` : output;
+}

package/dist/web-tools/http.js

@@ -0,0 +1,81 @@
+import { ProviderError } from "./types.js";
+export function buildRequestSignal(signal, timeoutMs) {
+    return AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]);
+}
+export async function readBoundedBody(response, maxBytes) {
+    if (!response.body)
+        return "";
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    const chunks = [];
+    let totalBytes = 0;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        if (!value)
+            continue;
+        totalBytes += value.byteLength;
+        if (totalBytes > maxBytes) {
+            throw new Error(`Response exceeded size limit of ${maxBytes} bytes.`);
+        }
+        chunks.push(decoder.decode(value, { stream: true }));
+    }
+    chunks.push(decoder.decode());
+    return chunks.join("");
+}
+export function createHttpError(provider, response) {
+    return new ProviderError(provider, `${provider} request failed: ${response.status} ${response.statusText}`.trim(), response.status >= 500 || response.status === 408 || response.status === 429, response.status);
+}
+// Pass through aborts and already-typed ProviderErrors raw; wrap everything else
+// (network failures, JSON.parse/validate throws) as a retryable ProviderError.
+function rethrowAsProviderError(provider, error, signal) {
+    if (error instanceof ProviderError)
+        throw error;
+    if (signal.aborted)
+        throw error;
+    throw new ProviderError(provider, error instanceof Error ? `${provider} request failed: ${error.message}` : `${provider} request failed.`, true, undefined, error);
+}
+async function performFetch(provider, url, options) {
+    try {
+        const response = await fetch(url, {
+            method: options.method ?? "GET",
+            headers: options.headers,
+            body: options.body,
+            redirect: "error",
+            signal: buildRequestSignal(options.signal, options.timeoutMs),
+        });
+        if (!response.ok)
+            throw createHttpError(provider, response);
+        return await readBoundedBody(response, options.maxBytes);
+    }
+    catch (error) {
+        rethrowAsProviderError(provider, error, options.signal);
+    }
+}
+export async function fetchJson(provider, url, options) {
+    try {
+        const body = await performFetch(provider, url, {
+            method: options.method,
+            headers: options.headers,
+            body: options.body,
+            signal: options.signal,
+            timeoutMs: options.timeoutMs,
+            maxBytes: options.maxBytes,
+        });
+        const parsed = body ? JSON.parse(body) : null;
+        return options.validate(parsed);
+    }
+    catch (error) {
+        rethrowAsProviderError(provider, error, options.signal);
+    }
+}
+export async function fetchText(provider, url, options) {
+    return await performFetch(provider, url, {
+        method: "GET",
+        headers: options.headers,
+        signal: options.signal,
+        timeoutMs: options.timeoutMs,
+        maxBytes: options.maxBytes,
+    });
+}

package/dist/web-tools/index.js

@@ -0,0 +1,152 @@
+import { PageCache } from "./cache.js";
+import { loadWebConfig } from "./config.js";
+import { createJinaProvider, createTinyfishProvider } from "./fetch-providers.js";
+import { formatFetchContent, formatSearchResults, paginateContent } from "./format.js";
+import { resolveSearchProviders } from "./routing.js";
+import { isTransientProviderError, validateFetchUrl } from "./safety.js";
+import { createBraveProvider, createExaProvider, createTavilyProvider, normalizeDomains } from "./search-providers.js";
+import { FETCH_DEFAULT_MAX_CHARS, WEB_UNTRUSTED_PREFIX, webFetchSchema, webSearchSchema, } from "./types.js";
+const pageCache = new PageCache();
+function makeSearchTool(config) {
+    const providers = {};
+    if (config.apiKeys.BRAVE_API_KEY)
+        providers.brave = createBraveProvider(config.apiKeys.BRAVE_API_KEY);
+    if (config.apiKeys.TAVILY_API_KEY)
+        providers.tavily = createTavilyProvider(config.apiKeys.TAVILY_API_KEY);
+    if (config.apiKeys.EXA_API_KEY)
+        providers.exa = createExaProvider(config.apiKeys.EXA_API_KEY);
+    return {
+        name: "search_web",
+        label: "Web Search",
+        description: "look something up on the open web. returns titles, urls, snippets, and dates when present. depth=thorough swaps brevity for inline excerpts.",
+        parameters: webSearchSchema,
+        async execute(_toolCallId, params, signal, onUpdate) {
+            const activeSignal = signal ?? new AbortController().signal;
+            if (Object.keys(providers).length === 0) {
+                throw new Error("No search provider configured. Set BRAVE_API_KEY, TAVILY_API_KEY, or EXA_API_KEY.");
+            }
+            const domains = normalizeDomains(params.domains);
+            const depth = params.depth ?? "basic";
+            const providersInOrder = resolveSearchProviders({ depth, freshness: params.freshness, domains }, providers);
+            if (providersInOrder.length === 0) {
+                throw new Error("No search provider available for this request.");
+            }
+            let lastError;
+            for (const provider of providersInOrder) {
+                if (activeSignal.aborted)
+                    throw new Error("Search aborted.");
+                onUpdate?.({ content: [{ type: "text", text: `Searching via ${provider.name}...` }], details: undefined });
+                try {
+                    const response = await provider.search({
+                        query: params.query,
+                        maxResults: params.maxResults ?? 5,
+                        includeContent: depth === "thorough",
+                        freshness: params.freshness,
+                        domains,
+                        signal: activeSignal,
+                    });
+                    return {
+                        content: [
+                            {
+                                type: "text",
+                                text: formatSearchResults({
+                                    results: response.results,
+                                    provider: provider.name,
+                                    requestedDepth: depth,
+                                    servedDepth: depth,
+                                    freshness: params.freshness,
+                                    domains,
+                                    appliedFilters: response.appliedFilters,
+                                    notes: response.notes,
+                                }),
+                            },
+                        ],
+                        details: {
+                            provider: provider.name,
+                            requestedDepth: depth,
+                            servedDepth: depth,
+                            degraded: false,
+                            freshness: params.freshness ?? null,
+                            domains: domains ?? [],
+                            resultCount: response.results.length,
+                        },
+                    };
+                }
+                catch (error) {
+                    lastError = error instanceof Error ? error : new Error(String(error));
+                    if (!isTransientProviderError(lastError))
+                        throw lastError;
+                }
+            }
+            throw new Error(`All search providers failed for this request. ${lastError?.message ?? ""}`.trim());
+        },
+    };
+}
+function makeFetchTool(config) {
+    const providers = createFetchProviders(config);
+    return {
+        name: "fetch_web",
+        label: "Web Fetch",
+        description: "pull a webpage down as clean markdown.",
+        parameters: webFetchSchema,
+        async execute(_toolCallId, params, signal) {
+            const activeSignal = signal ?? new AbortController().signal;
+            const url = validateFetchUrl(params.url);
+            const offset = params.offset ?? 0;
+            const maxChars = params.maxChars ?? FETCH_DEFAULT_MAX_CHARS;
+            const cached = pageCache.get(url);
+            let providerName = cached?.provider ?? providers[0]?.name ?? "jina";
+            let content = cached?.content;
+            if (!content) {
+                let lastError;
+                for (const provider of providers) {
+                    try {
+                        content = await provider.fetch(url, activeSignal);
+                        providerName = provider.name;
+                        pageCache.set(url, content, provider.name);
+                        break;
+                    }
+                    catch (error) {
+                        lastError = error instanceof Error ? error : new Error(String(error));
+                        if (!isTransientProviderError(lastError))
+                            throw lastError;
+                    }
+                }
+                if (!content)
+                    throw new Error(`All fetch providers failed for this request. ${lastError?.message ?? ""}`.trim());
+            }
+            const chunk = paginateContent(content, offset, maxChars);
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: formatFetchContent(url, providerName, chunk),
+                    },
+                ],
+                details: {
+                    provider: providerName,
+                    url,
+                    totalChars: content.length,
+                    offset: chunk.offset,
+                    returnedChars: chunk.returnedChars,
+                    nextOffset: chunk.nextOffset,
+                    hasMore: chunk.hasMore,
+                },
+            };
+        },
+    };
+}
+export function createFetchProviders(config) {
+    const providers = [];
+    if (config.apiKeys.TINYFISH_API_KEY)
+        providers.push(createTinyfishProvider(config.apiKeys.TINYFISH_API_KEY));
+    providers.push(createJinaProvider(config.apiKeys.JINA_API_KEY));
+    return providers;
+}
+export function webContentWarning() {
+    return WEB_UNTRUSTED_PREFIX;
+}
+export function createWebTools(_config) {
+    const loaded = loadWebConfig();
+    return [makeSearchTool(loaded), makeFetchTool(loaded)];
+}

package/dist/web-tools/routing.js

@@ -0,0 +1,29 @@
+export function searchProviderOrder(depth, args) {
+    if (depth === "thorough")
+        return ["tavily", "exa", "brave"];
+    if (args.domains?.length)
+        return ["tavily", "exa", "brave"];
+    return ["brave", "tavily", "exa"];
+}
+export function canServe(provider, depth) {
+    if (depth === "thorough")
+        return provider.capabilities.has("search") && provider.capabilities.has("content");
+    return provider.capabilities.has("search");
+}
+export function canServeSearchArgs(provider, args) {
+    if (!canServe(provider, args.depth))
+        return false;
+    if ((args.domains?.length ?? 0) > 1 && !provider.capabilities.has("domainFilter"))
+        return false;
+    return true;
+}
+export function resolveSearchProviders(args, searchProviders) {
+    const providers = [];
+    for (const name of searchProviderOrder(args.depth, args)) {
+        const candidate = searchProviders[name];
+        if (candidate && canServeSearchArgs(candidate, args) && !providers.includes(candidate)) {
+            providers.push(candidate);
+        }
+    }
+    return providers;
+}

package/dist/web-tools/safety.js

@@ -0,0 +1,73 @@
+import net from "node:net";
+import { ProviderError } from "./types.js";
+export function isBlockedHostname(hostname) {
+    if (hostname === "localhost" ||
+        hostname === "metadata.google.internal" ||
+        hostname === "metadata" ||
+        hostname === "169.254.169.254" ||
+        hostname === "169.254.169.250" ||
+        hostname === "100.100.100.200" ||
+        hostname.endsWith(".local") ||
+        hostname.endsWith(".localhost") ||
+        hostname.endsWith(".internal") ||
+        hostname.endsWith(".home")) {
+        return true;
+    }
+    const ipVersion = net.isIP(hostname);
+    if (ipVersion === 4) {
+        const octets = hostname.split(".").map((part) => Number.parseInt(part, 10));
+        const [a, b] = octets;
+        if (a === 0 || a === 10 || a === 127)
+            return true;
+        if (a === 169 && b === 254)
+            return true;
+        if (a === 172 && b >= 16 && b <= 31)
+            return true;
+        if (a === 192 && b === 168)
+            return true;
+        if (a === 100 && b >= 64 && b <= 127)
+            return true;
+        if (a >= 224)
+            return true;
+        return false;
+    }
+    if (ipVersion === 6) {
+        const normalized = hostname.toLowerCase();
+        return (normalized === "::1" ||
+            normalized.startsWith("fc") ||
+            normalized.startsWith("fd") ||
+            normalized.startsWith("fe8") ||
+            normalized.startsWith("fe9") ||
+            normalized.startsWith("fea") ||
+            normalized.startsWith("feb") ||
+            normalized.startsWith("ff"));
+    }
+    return false;
+}
+export function validateFetchUrl(input) {
+    const trimmed = input.trim();
+    if (!trimmed)
+        throw new Error("Invalid URL: URL is required.");
+    let parsed;
+    try {
+        parsed = new URL(trimmed);
+    }
+    catch {
+        throw new Error("Invalid URL: malformed URL.");
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        throw new Error("Invalid URL: only http and https URLs are allowed.");
+    }
+    if (parsed.username || parsed.password) {
+        throw new Error("Invalid URL: embedded credentials are not allowed.");
+    }
+    const hostname = parsed.hostname.replaceAll(/^\[|\]$/g, "").toLowerCase();
+    if (!hostname)
+        throw new Error("Invalid URL: hostname is required.");
+    if (isBlockedHostname(hostname))
+        throw new Error("Blocked URL: target host is not allowed.");
+    return parsed.toString();
+}
+export function isTransientProviderError(error) {
+    return error instanceof ProviderError ? error.transient : false;
+}