@humain/terminal 0.0.2 → 0.0.3-beta.0

package/dist/humain/web-access/vendor/tavily.js

@@ -0,0 +1,194 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const TAVILY_API_URL = "https://api.tavily.com/search";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60_000;
+let cachedConfig = null;
+function loadConfig() {
+    if (cachedConfig)
+        return cachedConfig;
+    if (!existsSync(CONFIG_PATH)) {
+        cachedConfig = {};
+        return cachedConfig;
+    }
+    const raw = readFileSync(CONFIG_PATH, "utf-8");
+    try {
+        cachedConfig = JSON.parse(raw);
+        return cachedConfig;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+    }
+}
+async function getApiKey(signal) {
+    return resolveCredential({
+        provider: "Tavily",
+        configuredValue: loadConfig().tavilyApiKey,
+        environmentValue: process.env.TAVILY_API_KEY,
+        signal,
+    });
+}
+async function requireApiKey(signal) {
+    const apiKey = await getApiKey(signal);
+    if (!apiKey) {
+        throw new Error("Tavily API key not found. Either:\n" +
+            `  1. Create ${CONFIG_PATH} with { "tavilyApiKey": "your-key" }\n` +
+            "  2. Set TAVILY_API_KEY environment variable\n" +
+            "Get a key at https://app.tavily.com/");
+    }
+    return apiKey;
+}
+function normalizeCount(value) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return 5;
+    return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function normalizeDomain(value) {
+    let input = value.trim().toLowerCase();
+    if (!input)
+        return null;
+    if (input.startsWith("-"))
+        input = input.slice(1).trim();
+    if (!input)
+        return null;
+    try {
+        const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+        input = parsed.hostname;
+    }
+    catch {
+        input = input.split("/")[0]?.split(":")[0] ?? "";
+    }
+    input = input.replace(/^\.+|\.+$/g, "");
+    return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function mapDomainFilter(domainFilter) {
+    if (!domainFilter?.length)
+        return {};
+    const include_domains = [];
+    const exclude_domains = [];
+    for (const raw of domainFilter) {
+        const domain = normalizeDomain(raw);
+        if (!domain)
+            continue;
+        const target = raw.trim().startsWith("-") ? exclude_domains : include_domains;
+        if (!target.includes(domain))
+            target.push(domain);
+    }
+    return {
+        ...(include_domains.length > 0 ? { include_domains } : {}),
+        ...(exclude_domains.length > 0 ? { exclude_domains } : {}),
+    };
+}
+function requestSignal(signal) {
+    const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+    return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function mapResults(results, numResults) {
+    if (!Array.isArray(results))
+        return [];
+    const mapped = [];
+    for (const item of results) {
+        if (!item?.url)
+            continue;
+        mapped.push({
+            title: item.title || `Source ${mapped.length + 1}`,
+            url: item.url,
+            snippet: typeof item.content === "string" ? item.content.replace(/\s+/g, " ").trim() : "",
+        });
+        if (mapped.length >= numResults)
+            break;
+    }
+    return mapped;
+}
+function mapInlineContent(results) {
+    if (!Array.isArray(results))
+        return [];
+    return results.flatMap((item) => {
+        if (!item?.url || typeof item.raw_content !== "string" || item.raw_content.trim().length === 0)
+            return [];
+        return [{
+                url: item.url,
+                title: item.title || "",
+                content: item.raw_content,
+                error: null,
+            }];
+    });
+}
+export function isTavilyAvailable() {
+    return hasCredentialSource({
+        provider: "Tavily",
+        configuredValue: loadConfig().tavilyApiKey,
+        environmentValue: process.env.TAVILY_API_KEY,
+    });
+}
+export async function searchWithTavily(query, options = {}) {
+    const apiKey = await requireApiKey(options.signal);
+    const numResults = normalizeCount(options.numResults);
+    const body = {
+        query,
+        search_depth: "basic",
+        max_results: numResults,
+        include_answer: "basic",
+        include_raw_content: options.includeContent ? "markdown" : false,
+        ...(options.recencyFilter ? { time_range: options.recencyFilter } : {}),
+        ...mapDomainFilter(options.domainFilter),
+    };
+    const activityId = activityMonitor.logStart({ type: "api", query });
+    let response;
+    try {
+        response = await fetch(TAVILY_API_URL, {
+            method: "POST",
+            headers: {
+                "Authorization": `Bearer ${apiKey}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(body),
+            signal: requestSignal(options.signal),
+        });
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        const redactedMessage = redactCredential(message, apiKey);
+        if (redactedMessage.toLowerCase().includes("abort"))
+            activityMonitor.logComplete(activityId, 0);
+        else
+            activityMonitor.logError(activityId, redactedMessage);
+        if (redactedMessage === message)
+            throw err;
+        const redactedError = new Error(redactedMessage);
+        if (err instanceof Error)
+            redactedError.name = err.name;
+        throw redactedError;
+    }
+    if (!response.ok) {
+        activityMonitor.logComplete(activityId, response.status);
+        const errorText = redactCredential(await response.text(), apiKey);
+        throw new Error(`Tavily API error ${response.status}: ${errorText.slice(0, 300)}`);
+    }
+    let data;
+    try {
+        data = await response.json();
+    }
+    catch (err) {
+        activityMonitor.logComplete(activityId, response.status);
+        throw new Error(`Tavily API returned invalid JSON: ${errorMessage(err)}`);
+    }
+    activityMonitor.logComplete(activityId, response.status);
+    const result = {
+        answer: typeof data.answer === "string" ? data.answer : "",
+        results: mapResults(data.results, numResults),
+    };
+    if (options.includeContent) {
+        const inlineContent = mapInlineContent(data.results);
+        if (inlineContent.length > 0)
+            result.inlineContent = inlineContent;
+    }
+    return result;
+}
+//# sourceMappingURL=tavily.js.map

package/dist/humain/web-access/vendor/tavily.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tavily.js","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/tavily.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGhD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAClG,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,cAAc,GAAG,+BAA+B,CAAC;AACvD,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAC7C,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAsBjC,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,SAAS,UAAU,GAAoB;IACtC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,YAAY,GAAG,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC;IACrB,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,CAAC;QACJ,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;QAClD,OAAO,YAAY,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;AAAA,CACD;AAED,KAAK,UAAU,SAAS,CAAC,MAAoB,EAA0B;IACtE,OAAO,iBAAiB,CAAC;QACxB,QAAQ,EAAE,QAAQ;QAClB,eAAe,EAAE,UAAU,EAAE,CAAC,YAAY;QAC1C,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;QAC5C,MAAM;KACN,CAAC,CAAC;AAAA,CACH;AAED,KAAK,UAAU,aAAa,CAAC,MAAoB,EAAmB;IACnE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACd,qCAAqC;YACrC,eAAe,WAAW,wCAAwC;YAClE,gDAAgD;YAChD,sCAAsC,CACtC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,SAAS,cAAc,CAAC,KAAyB,EAAU;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,SAAS,eAAe,CAAC,KAAa,EAAiB;IACtD,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;QACpF,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IACD,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,mCAAmC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAAA,CACtE;AAED,SAAS,eAAe,CAAC,YAAkC,EAA8D;IACxH,IAAI,CAAC,YAAY,EAAE,MAAM;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC;QAC9E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,OAAO;QACN,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;AAAA,CACF;AAED,SAAS,aAAa,CAAC,MAAoB,EAAe;IACzD,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACvD,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAAA,CAC7D;AAED,SAAS,YAAY,CAAC,GAAY,EAAU;IAC3C,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,UAAU,CAAC,OAAmC,EAAE,UAAkB,EAA6B;IACvG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,MAAM,MAAM,GAA8B,EAAE,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,IAAI,EAAE,GAAG;YAAE,SAAS;QACzB,MAAM,CAAC,IAAI,CAAC;YACX,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,UAAU,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YAClD,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;SACzF,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,SAAS,gBAAgB,CAAC,OAAmC,EAAsB;IAClF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC1G,OAAO,CAAC;gBACP,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;gBACvB,OAAO,EAAE,IAAI,CAAC,WAAW;gBACzB,KAAK,EAAE,IAAI;aACX,CAAC,CAAC;IAAA,CACH,CAAC,CAAC;AAAA,CACH;AAED,MAAM,UAAU,iBAAiB,GAAY;IAC5C,OAAO,mBAAmB,CAAC;QAC1B,QAAQ,EAAE,QAAQ;QAClB,eAAe,EAAE,UAAU,EAAE,CAAC,YAAY;QAC1C,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc;KAC5C,CAAC,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,OAAO,GAAwB,EAAE,EAA2B;IACjH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACtD,MAAM,IAAI,GAA4B;QACrC,KAAK;QACL,YAAY,EAAE,OAAO;QACrB,WAAW,EAAE,UAAU;QACvB,cAAc,EAAE,OAAO;QACvB,mBAAmB,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK;QAChE,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,eAAe,CAAC,OAAO,CAAC,YAAY,CAAC;KACxC,CAAC;IAEF,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACJ,QAAQ,GAAG,MAAM,KAAK,CAAC,cAAc,EAAE;YACtC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACR,eAAe,EAAE,UAAU,MAAM,EAAE;gBACnC,cAAc,EAAE,kBAAkB;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1D,IAAI,eAAe,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;;YAC3F,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAC3D,IAAI,eAAe,KAAK,OAAO;YAAE,MAAM,GAAG,CAAC;QAC3C,MAAM,aAAa,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;QACjD,IAAI,GAAG,YAAY,KAAK;YAAE,aAAa,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACxD,MAAM,aAAa,CAAC;IACrB,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;QAClE,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,IAAI,IAAoB,CAAC;IACzB,IAAI,CAAC;QACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAoB,CAAC;IAChD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,qCAAqC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,MAAM,GAAmB;QAC9B,MAAM,EAAE,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QAC1D,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC;KAC7C,CAAC;IACF,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,aAAa,GAAG,aAAa,CAAC;IACpE,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport type { ExtractedContent } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { hasCredentialSource, redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst TAVILY_API_URL = \"https://api.tavily.com/search\";\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst SEARCH_TIMEOUT_MS = 60_000;\n\ninterface WebSearchConfig {\n\ttavilyApiKey?: unknown;\n}\n\ninterface TavilyResult {\n\ttitle?: string;\n\turl?: string;\n\tcontent?: string;\n\traw_content?: string | null;\n}\n\ninterface TavilyResponse {\n\tanswer?: string;\n\tresults?: TavilyResult[];\n}\n\ninterface TavilySearchOptions extends SearchOptions {\n\tincludeContent?: boolean;\n}\n\nlet cachedConfig: WebSearchConfig | null = null;\n\nfunction loadConfig(): WebSearchConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst raw = readFileSync(CONFIG_PATH, \"utf-8\");\n\ttry {\n\t\tcachedConfig = JSON.parse(raw) as WebSearchConfig;\n\t\treturn cachedConfig;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string | null> {\n\treturn resolveCredential({\n\t\tprovider: \"Tavily\",\n\t\tconfiguredValue: loadConfig().tavilyApiKey,\n\t\tenvironmentValue: process.env.TAVILY_API_KEY,\n\t\tsignal,\n\t});\n}\n\nasync function requireApiKey(signal?: AbortSignal): Promise<string> {\n\tconst apiKey = await getApiKey(signal);\n\tif (!apiKey) {\n\t\tthrow new Error(\n\t\t\t\"Tavily API key not found. Either:\\n\" +\n\t\t\t`  1. Create ${CONFIG_PATH} with { \"tavilyApiKey\": \"your-key\" }\\n` +\n\t\t\t\"  2. Set TAVILY_API_KEY environment variable\\n\" +\n\t\t\t\"Get a key at https://app.tavily.com/\",\n\t\t);\n\t}\n\treturn apiKey;\n}\n\nfunction normalizeCount(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return 5;\n\treturn Math.max(1, Math.min(Math.floor(value), 20));\n}\n\nfunction normalizeDomain(value: string): string | null {\n\tlet input = value.trim().toLowerCase();\n\tif (!input) return null;\n\tif (input.startsWith(\"-\")) input = input.slice(1).trim();\n\tif (!input) return null;\n\ttry {\n\t\tconst parsed = input.includes(\"://\") ? new URL(input) : new URL(`https://${input}`);\n\t\tinput = parsed.hostname;\n\t} catch {\n\t\tinput = input.split(\"/\")[0]?.split(\":\")[0] ?? \"\";\n\t}\n\tinput = input.replace(/^\\.+|\\.+$/g, \"\");\n\treturn /^[a-z0-9][a-z0-9.-]*\\.[a-z]{2,}$/i.test(input) ? input : null;\n}\n\nfunction mapDomainFilter(domainFilter: string[] | undefined): { include_domains?: string[]; exclude_domains?: string[] } {\n\tif (!domainFilter?.length) return {};\n\tconst include_domains: string[] = [];\n\tconst exclude_domains: string[] = [];\n\tfor (const raw of domainFilter) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? exclude_domains : include_domains;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn {\n\t\t...(include_domains.length > 0 ? { include_domains } : {}),\n\t\t...(exclude_domains.length > 0 ? { exclude_domains } : {}),\n\t};\n}\n\nfunction requestSignal(signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);\n\treturn signal ? AbortSignal.any([signal, timeout]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction mapResults(results: TavilyResult[] | undefined, numResults: number): SearchResponse[\"results\"] {\n\tif (!Array.isArray(results)) return [];\n\tconst mapped: SearchResponse[\"results\"] = [];\n\tfor (const item of results) {\n\t\tif (!item?.url) continue;\n\t\tmapped.push({\n\t\t\ttitle: item.title || `Source ${mapped.length + 1}`,\n\t\t\turl: item.url,\n\t\t\tsnippet: typeof item.content === \"string\" ? item.content.replace(/\\s+/g, \" \").trim() : \"\",\n\t\t});\n\t\tif (mapped.length >= numResults) break;\n\t}\n\treturn mapped;\n}\n\nfunction mapInlineContent(results: TavilyResult[] | undefined): ExtractedContent[] {\n\tif (!Array.isArray(results)) return [];\n\treturn results.flatMap((item) => {\n\t\tif (!item?.url || typeof item.raw_content !== \"string\" || item.raw_content.trim().length === 0) return [];\n\t\treturn [{\n\t\t\turl: item.url,\n\t\t\ttitle: item.title || \"\",\n\t\t\tcontent: item.raw_content,\n\t\t\terror: null,\n\t\t}];\n\t});\n}\n\nexport function isTavilyAvailable(): boolean {\n\treturn hasCredentialSource({\n\t\tprovider: \"Tavily\",\n\t\tconfiguredValue: loadConfig().tavilyApiKey,\n\t\tenvironmentValue: process.env.TAVILY_API_KEY,\n\t});\n}\n\nexport async function searchWithTavily(query: string, options: TavilySearchOptions = {}): Promise<SearchResponse> {\n\tconst apiKey = await requireApiKey(options.signal);\n\tconst numResults = normalizeCount(options.numResults);\n\tconst body: Record<string, unknown> = {\n\t\tquery,\n\t\tsearch_depth: \"basic\",\n\t\tmax_results: numResults,\n\t\tinclude_answer: \"basic\",\n\t\tinclude_raw_content: options.includeContent ? \"markdown\" : false,\n\t\t...(options.recencyFilter ? { time_range: options.recencyFilter } : {}),\n\t\t...mapDomainFilter(options.domainFilter),\n\t};\n\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query });\n\tlet response: Response;\n\ttry {\n\t\tresponse = await fetch(TAVILY_API_URL, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders: {\n\t\t\t\t\"Authorization\": `Bearer ${apiKey}`,\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(options.signal),\n\t\t});\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (redactedMessage.toLowerCase().includes(\"abort\")) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, redactedMessage);\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n\n\tif (!response.ok) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst errorText = redactCredential(await response.text(), apiKey);\n\t\tthrow new Error(`Tavily API error ${response.status}: ${errorText.slice(0, 300)}`);\n\t}\n\n\tlet data: TavilyResponse;\n\ttry {\n\t\tdata = await response.json() as TavilyResponse;\n\t} catch (err) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tthrow new Error(`Tavily API returned invalid JSON: ${errorMessage(err)}`);\n\t}\n\n\tactivityMonitor.logComplete(activityId, response.status);\n\tconst result: SearchResponse = {\n\t\tanswer: typeof data.answer === \"string\" ? data.answer : \"\",\n\t\tresults: mapResults(data.results, numResults),\n\t};\n\tif (options.includeContent) {\n\t\tconst inlineContent = mapInlineContent(data.results);\n\t\tif (inlineContent.length > 0) result.inlineContent = inlineContent;\n\t}\n\treturn result;\n}\n"]}

package/dist/humain/web-access/vendor/tinyfish.d.ts

@@ -0,0 +1,11 @@
+import type { ExtractedContent, ExtractOptions } from "./extract.ts";
+import type { SearchOptions, SearchResponse } from "./perplexity.ts";
+interface TinyFishSearchOptions extends SearchOptions {
+    includeContent?: boolean;
+}
+export declare function clearTinyFishConfigCache(): void;
+export declare function isTinyFishAvailable(): boolean;
+export declare function searchWithTinyFish(query: string, options?: TinyFishSearchOptions): Promise<SearchResponse>;
+export declare function extractWithTinyFish(url: string, signal?: AbortSignal, options?: ExtractOptions): Promise<ExtractedContent | null>;
+export {};
+//# sourceMappingURL=tinyfish.d.ts.map

package/dist/humain/web-access/vendor/tinyfish.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tinyfish.d.ts","sourceRoot":"","sources":["../../../../src/humain/web-access/vendor/tinyfish.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAoDrE,UAAU,qBAAsB,SAAQ,aAAa;IACpD,cAAc,CAAC,EAAE,OAAO,CAAC;CACzB;AAqBD,wBAAgB,wBAAwB,IAAI,IAAI,CAE/C;AAoBD,wBAAgB,mBAAmB,IAAI,OAAO,CAM7C;AAsND,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,cAAc,CAAC,CAuCpH;AAED,wBAAsB,mBAAmB,CACxC,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,GAAE,cAAmB,GAC1B,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CA+BlC","sourcesContent":["import { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { hasCredentialSource, redactCredential, resolveCredential } from \"./credential-source.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst TINYFISH_SEARCH_URL = \"https://api.search.tinyfish.ai\";\nconst TINYFISH_FETCH_URL = \"https://api.fetch.tinyfish.ai\";\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst SEARCH_TIMEOUT_MS = 60_000;\nconst FETCH_TIMEOUT_MS = 150_000;\nconst MAX_FETCH_URLS = 10;\nconst MAX_FETCH_PER_URL_TIMEOUT_MS = 110_000;\n\ninterface WebSearchConfig {\n\ttinyfishApiKey?: unknown;\n}\n\ninterface TinyFishSearchResult {\n\tposition?: number;\n\tsite_name?: string | null;\n\ttitle?: string | null;\n\tsnippet?: string | null;\n\turl?: string | null;\n\tdate?: string | null;\n\tpublisher?: string | null;\n}\n\ninterface TinyFishSearchResponse {\n\tquery?: string;\n\tresults?: TinyFishSearchResult[];\n\ttotal_results?: number;\n\tpage?: number;\n}\n\ninterface TinyFishFetchResult {\n\turl?: string;\n\tfinal_url?: string;\n\ttitle?: string | null;\n\ttext?: string | Record<string, unknown> | null;\n\tformat?: string;\n}\n\ninterface TinyFishFetchError {\n\turl?: string;\n\terror?: string;\n\tstatus?: number;\n}\n\ninterface TinyFishFetchResponse {\n\tresults?: TinyFishFetchResult[];\n\terrors?: TinyFishFetchError[];\n}\n\ninterface TinyFishSearchOptions extends SearchOptions {\n\tincludeContent?: boolean;\n}\n\nlet cachedConfig: WebSearchConfig | null = null;\n\nfunction loadConfig(): WebSearchConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst raw = readFileSync(CONFIG_PATH, \"utf-8\");\n\ttry {\n\t\tcachedConfig = JSON.parse(raw) as WebSearchConfig;\n\t\treturn cachedConfig;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n}\n\nexport function clearTinyFishConfigCache(): void {\n\tcachedConfig = null;\n}\n\nasync function getApiKey(signal?: AbortSignal): Promise<string> {\n\tconst key = await resolveCredential({\n\t\tprovider: \"TinyFish\",\n\t\tconfiguredValue: loadConfig().tinyfishApiKey,\n\t\tenvironmentValue: process.env.TINYFISH_API_KEY,\n\t\tsignal,\n\t});\n\tif (!key) {\n\t\tthrow new Error(\n\t\t\t\"TinyFish API key not found. Either:\\n\" +\n\t\t\t`  1. Create ${CONFIG_PATH} with { \"tinyfishApiKey\": \"your-key\" }\\n` +\n\t\t\t\"  2. Set TINYFISH_API_KEY environment variable\\n\" +\n\t\t\t\"Get a key at https://agent.tinyfish.ai/api-keys\",\n\t\t);\n\t}\n\treturn key;\n}\n\nexport function isTinyFishAvailable(): boolean {\n\treturn hasCredentialSource({\n\t\tprovider: \"TinyFish\",\n\t\tconfiguredValue: loadConfig().tinyfishApiKey,\n\t\tenvironmentValue: process.env.TINYFISH_API_KEY,\n\t});\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction requestSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal {\n\tconst timeout = AbortSignal.timeout(timeoutMs);\n\treturn signal ? AbortSignal.any([signal, timeout]) : timeout;\n}\n\nfunction normalizeDomain(value: string): string | null {\n\tlet input = value.trim().toLowerCase();\n\tif (!input) return null;\n\tif (input.startsWith(\"-\")) input = input.slice(1).trim();\n\tif (!input) return null;\n\ttry {\n\t\tconst parsed = input.includes(\"://\") ? new URL(input) : new URL(`https://${input}`);\n\t\tinput = parsed.hostname;\n\t} catch {\n\t\tinput = input.split(\"/\")[0]?.split(\":\")[0] ?? \"\";\n\t}\n\tinput = input.replace(/^\\.+|\\.+$/g, \"\");\n\treturn /^[a-z0-9][a-z0-9.-]*\\.[a-z]{2,}$/i.test(input) ? input : null;\n}\n\nfunction mapDomainFilter(domainFilter: string[] | undefined): { includeDomains: string[]; excludeDomains: string[] } {\n\tconst includeDomains: string[] = [];\n\tconst excludeDomains: string[] = [];\n\tfor (const raw of domainFilter ?? []) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? excludeDomains : includeDomains;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn { includeDomains, excludeDomains };\n}\n\nfunction recencyMinutes(filter: SearchOptions[\"recencyFilter\"]): number | undefined {\n\tif (!filter) return undefined;\n\tconst minutes: Record<NonNullable<SearchOptions[\"recencyFilter\"]>, number> = {\n\t\tday: 1_440,\n\t\tweek: 10_080,\n\t\tmonth: 43_200,\n\t\tyear: 525_600,\n\t};\n\treturn minutes[filter];\n}\n\nfunction normalizeNumResults(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return 5;\n\treturn Math.max(1, Math.min(Math.floor(value), 20));\n}\n\nfunction buildSearchUrl(query: string, options: SearchOptions, page: number): string {\n\tconst params = new URLSearchParams({ query });\n\tconst { includeDomains, excludeDomains } = mapDomainFilter(options.domainFilter);\n\tif (includeDomains.length > 0) params.set(\"include_domains\", includeDomains.join(\",\"));\n\tif (excludeDomains.length > 0) params.set(\"exclude_domains\", excludeDomains.join(\",\"));\n\tconst recency = recencyMinutes(options.recencyFilter);\n\tif (recency !== undefined) params.set(\"recency_minutes\", String(recency));\n\tif (page > 0) params.set(\"page\", String(page));\n\treturn `${TINYFISH_SEARCH_URL}?${params.toString()}`;\n}\n\nasync function tinyFishJsonRequest<T>(\n\tlabel: \"Search\" | \"Fetch\",\n\turl: string,\n\tapiKey: string,\n\tinit: RequestInit,\n\ttimeoutMs: number,\n\tsignal?: AbortSignal,\n): Promise<T> {\n\tlet response: Response;\n\ttry {\n\t\tresponse = await fetch(url, {\n\t\t\t...init,\n\t\t\theaders: {\n\t\t\t\t\"X-API-Key\": apiKey,\n\t\t\t\t...(init.body ? { \"Content-Type\": \"application/json\" } : {}),\n\t\t\t\t...init.headers,\n\t\t\t},\n\t\t\tsignal: requestSignal(signal, timeoutMs),\n\t\t});\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n\n\tconst raw = await response.text();\n\tif (!response.ok) {\n\t\tthrow new Error(`TinyFish ${label} API error ${response.status}: ${redactCredential(raw, apiKey).slice(0, 300)}`);\n\t}\n\ttry {\n\t\treturn JSON.parse(raw) as T;\n\t} catch (err) {\n\t\tthrow new Error(`TinyFish ${label} API returned invalid JSON: ${errorMessage(err)}`);\n\t}\n}\n\nfunction mapSearchResults(results: TinyFishSearchResult[] | undefined): SearchResponse[\"results\"] {\n\tif (!Array.isArray(results)) return [];\n\treturn results.flatMap((item) => {\n\t\tif (!item || typeof item.url !== \"string\" || item.url.trim().length === 0) return [];\n\t\tconst url = item.url.trim();\n\t\treturn [{\n\t\t\ttitle: typeof item.title === \"string\" && item.title.trim() ? item.title.trim() : url,\n\t\t\turl,\n\t\t\tsnippet: typeof item.snippet === \"string\" ? item.snippet.replace(/\\s+/g, \" \").trim() : \"\",\n\t\t}];\n\t});\n}\n\nfunction deduplicateResults(results: SearchResponse[\"results\"], limit: number): SearchResponse[\"results\"] {\n\tconst seen = new Set<string>();\n\tconst unique: SearchResponse[\"results\"] = [];\n\tfor (const result of results) {\n\t\tif (seen.has(result.url)) continue;\n\t\tseen.add(result.url);\n\t\tunique.push(result);\n\t\tif (unique.length >= limit) break;\n\t}\n\treturn unique;\n}\n\nfunction buildAnswer(results: SearchResponse[\"results\"]): string {\n\treturn results.map((result) => {\n\t\tif (result.snippet) return `${result.snippet}\\nSource: ${result.title} (${result.url})`;\n\t\treturn `Source: ${result.title} (${result.url})`;\n\t}).join(\"\\n\\n\");\n}\n\nfunction fetchPerUrlTimeout(value: number | undefined): number {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) return MAX_FETCH_PER_URL_TIMEOUT_MS;\n\treturn Math.max(1, Math.min(Math.floor(value), MAX_FETCH_PER_URL_TIMEOUT_MS));\n}\n\nfunction fetchBody(urls: string[], options: ExtractOptions = {}): Record<string, unknown> {\n\tconst body: Record<string, unknown> = {\n\t\turls,\n\t\tformat: \"markdown\",\n\t\tper_url_timeout_ms: fetchPerUrlTimeout(options.timeoutMs),\n\t};\n\tconst purpose = options.prompt?.trim();\n\tif (purpose) body.purpose = purpose.slice(0, 2_000);\n\treturn body;\n}\n\nfunction findFetchError(errors: TinyFishFetchError[] | undefined, url: string): TinyFishFetchError | undefined {\n\tif (!Array.isArray(errors)) return undefined;\n\treturn errors.find(item => item?.url === url) ?? errors[0];\n}\n\nfunction fetchResultContent(result: TinyFishFetchResult): string {\n\tif (typeof result.text === \"string\") return result.text.trim();\n\tif (result.text && typeof result.text === \"object\") return JSON.stringify(result.text, null, 2);\n\treturn \"\";\n}\n\nfunction mapFetchResult(result: TinyFishFetchResult | undefined, requestedUrl: string): ExtractedContent | null {\n\tif (!result) return null;\n\tconst content = fetchResultContent(result);\n\tif (!content) return null;\n\treturn {\n\t\turl: requestedUrl,\n\t\ttitle: typeof result.title === \"string\" ? result.title.trim() : \"\",\n\t\tcontent,\n\t\terror: null,\n\t};\n}\n\nasync function fetchBatch(\n\turls: string[],\n\tapiKey: string,\n\tsignal?: AbortSignal,\n\toptions: ExtractOptions = {},\n): Promise<TinyFishFetchResponse> {\n\treturn tinyFishJsonRequest<TinyFishFetchResponse>(\n\t\t\"Fetch\",\n\t\tTINYFISH_FETCH_URL,\n\t\tapiKey,\n\t\t{ method: \"POST\", body: JSON.stringify(fetchBody(urls, options)) },\n\t\tFETCH_TIMEOUT_MS,\n\t\tsignal,\n\t);\n}\n\nasync function fetchInlineContent(\n\turls: string[],\n\tapiKey: string,\n\tsignal?: AbortSignal,\n): Promise<ExtractedContent[]> {\n\tconst content: ExtractedContent[] = [];\n\tfor (let offset = 0; offset < urls.length; offset += MAX_FETCH_URLS) {\n\t\tconst batch = urls.slice(offset, offset + MAX_FETCH_URLS);\n\t\tconst data = await fetchBatch(batch, apiKey, signal);\n\t\tif (!Array.isArray(data.results) || !Array.isArray(data.errors)) {\n\t\t\tthrow new Error(\"TinyFish Fetch API returned an unexpected response shape\");\n\t\t}\n\t\tfor (const url of batch) {\n\t\t\tconst result = Array.isArray(data.results)\n\t\t\t\t? data.results.find(item => item?.url === url || item?.final_url === url)\n\t\t\t\t: undefined;\n\t\t\tconst mapped = mapFetchResult(result, url);\n\t\t\tif (mapped) content.push(mapped);\n\t\t}\n\t}\n\treturn content;\n}\n\nexport async function searchWithTinyFish(query: string, options: TinyFishSearchOptions = {}): Promise<SearchResponse> {\n\tconst apiKey = await getApiKey(options.signal);\n\tconst numResults = normalizeNumResults(options.numResults);\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query });\n\ttry {\n\t\tconst combined: SearchResponse[\"results\"] = [];\n\t\tconst pages = numResults > 10 ? 2 : 1;\n\t\tfor (let page = 0; page < pages; page++) {\n\t\t\tconst data = await tinyFishJsonRequest<TinyFishSearchResponse>(\n\t\t\t\t\"Search\",\n\t\t\t\tbuildSearchUrl(query, options, page),\n\t\t\t\tapiKey,\n\t\t\t\t{ method: \"GET\" },\n\t\t\t\tSEARCH_TIMEOUT_MS,\n\t\t\t\toptions.signal,\n\t\t\t);\n\t\t\tif (!Array.isArray(data.results)) throw new Error(\"TinyFish Search API returned an unexpected response shape\");\n\t\t\tcombined.push(...mapSearchResults(data.results));\n\t\t\tif (data.results.length < 10) break;\n\t\t}\n\n\t\tconst results = deduplicateResults(combined, numResults);\n\t\tconst response: SearchResponse = { answer: buildAnswer(results), results };\n\t\tif (options.includeContent && results.length > 0) {\n\t\t\tconst inlineContent = await fetchInlineContent(results.map(result => result.url), apiKey, options.signal);\n\t\t\tif (inlineContent.length > 0) response.inlineContent = inlineContent;\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, 200);\n\t\treturn response;\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (redactedMessage.toLowerCase().includes(\"abort\")) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, redactedMessage);\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n}\n\nexport async function extractWithTinyFish(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions: ExtractOptions = {},\n): Promise<ExtractedContent | null> {\n\tconst apiKey = await getApiKey(signal);\n\tconst activityId = activityMonitor.logStart({ type: \"fetch\", url });\n\ttry {\n\t\tconst data = await fetchBatch([url], apiKey, signal, options);\n\t\tif (!Array.isArray(data.results) || !Array.isArray(data.errors)) {\n\t\t\tthrow new Error(\"TinyFish Fetch API returned an unexpected response shape\");\n\t\t}\n\t\tconst result = data.results.find(item => item?.url === url || item?.final_url === url) ?? data.results[0];\n\t\tconst mapped = mapFetchResult(result, url);\n\t\tif (mapped) {\n\t\t\tactivityMonitor.logComplete(activityId, 200);\n\t\t\treturn mapped;\n\t\t}\n\t\tconst fetchError = findFetchError(data.errors, url);\n\t\tif (fetchError) {\n\t\t\tconst status = typeof fetchError.status === \"number\" ? ` (HTTP ${fetchError.status})` : \"\";\n\t\t\tthrow new Error(`TinyFish Fetch failed for ${url}: ${fetchError.error || \"unknown error\"}${status}`);\n\t\t}\n\t\tactivityMonitor.logComplete(activityId, 200);\n\t\treturn null;\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tconst redactedMessage = redactCredential(message, apiKey);\n\t\tif (redactedMessage.toLowerCase().includes(\"abort\")) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, redactedMessage);\n\t\tif (redactedMessage === message) throw err;\n\t\tconst redactedError = new Error(redactedMessage);\n\t\tif (err instanceof Error) redactedError.name = err.name;\n\t\tthrow redactedError;\n\t}\n}\n"]}

package/dist/humain/web-access/vendor/tinyfish.js

@@ -0,0 +1,331 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const TINYFISH_SEARCH_URL = "https://api.search.tinyfish.ai";
+const TINYFISH_FETCH_URL = "https://api.fetch.tinyfish.ai";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60_000;
+const FETCH_TIMEOUT_MS = 150_000;
+const MAX_FETCH_URLS = 10;
+const MAX_FETCH_PER_URL_TIMEOUT_MS = 110_000;
+let cachedConfig = null;
+function loadConfig() {
+    if (cachedConfig)
+        return cachedConfig;
+    if (!existsSync(CONFIG_PATH)) {
+        cachedConfig = {};
+        return cachedConfig;
+    }
+    const raw = readFileSync(CONFIG_PATH, "utf-8");
+    try {
+        cachedConfig = JSON.parse(raw);
+        return cachedConfig;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+    }
+}
+export function clearTinyFishConfigCache() {
+    cachedConfig = null;
+}
+async function getApiKey(signal) {
+    const key = await resolveCredential({
+        provider: "TinyFish",
+        configuredValue: loadConfig().tinyfishApiKey,
+        environmentValue: process.env.TINYFISH_API_KEY,
+        signal,
+    });
+    if (!key) {
+        throw new Error("TinyFish API key not found. Either:\n" +
+            `  1. Create ${CONFIG_PATH} with { "tinyfishApiKey": "your-key" }\n` +
+            "  2. Set TINYFISH_API_KEY environment variable\n" +
+            "Get a key at https://agent.tinyfish.ai/api-keys");
+    }
+    return key;
+}
+export function isTinyFishAvailable() {
+    return hasCredentialSource({
+        provider: "TinyFish",
+        configuredValue: loadConfig().tinyfishApiKey,
+        environmentValue: process.env.TINYFISH_API_KEY,
+    });
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function requestSignal(signal, timeoutMs) {
+    const timeout = AbortSignal.timeout(timeoutMs);
+    return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+function normalizeDomain(value) {
+    let input = value.trim().toLowerCase();
+    if (!input)
+        return null;
+    if (input.startsWith("-"))
+        input = input.slice(1).trim();
+    if (!input)
+        return null;
+    try {
+        const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+        input = parsed.hostname;
+    }
+    catch {
+        input = input.split("/")[0]?.split(":")[0] ?? "";
+    }
+    input = input.replace(/^\.+|\.+$/g, "");
+    return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function mapDomainFilter(domainFilter) {
+    const includeDomains = [];
+    const excludeDomains = [];
+    for (const raw of domainFilter ?? []) {
+        const domain = normalizeDomain(raw);
+        if (!domain)
+            continue;
+        const target = raw.trim().startsWith("-") ? excludeDomains : includeDomains;
+        if (!target.includes(domain))
+            target.push(domain);
+    }
+    return { includeDomains, excludeDomains };
+}
+function recencyMinutes(filter) {
+    if (!filter)
+        return undefined;
+    const minutes = {
+        day: 1_440,
+        week: 10_080,
+        month: 43_200,
+        year: 525_600,
+    };
+    return minutes[filter];
+}
+function normalizeNumResults(value) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return 5;
+    return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function buildSearchUrl(query, options, page) {
+    const params = new URLSearchParams({ query });
+    const { includeDomains, excludeDomains } = mapDomainFilter(options.domainFilter);
+    if (includeDomains.length > 0)
+        params.set("include_domains", includeDomains.join(","));
+    if (excludeDomains.length > 0)
+        params.set("exclude_domains", excludeDomains.join(","));
+    const recency = recencyMinutes(options.recencyFilter);
+    if (recency !== undefined)
+        params.set("recency_minutes", String(recency));
+    if (page > 0)
+        params.set("page", String(page));
+    return `${TINYFISH_SEARCH_URL}?${params.toString()}`;
+}
+async function tinyFishJsonRequest(label, url, apiKey, init, timeoutMs, signal) {
+    let response;
+    try {
+        response = await fetch(url, {
+            ...init,
+            headers: {
+                "X-API-Key": apiKey,
+                ...(init.body ? { "Content-Type": "application/json" } : {}),
+                ...init.headers,
+            },
+            signal: requestSignal(signal, timeoutMs),
+        });
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        const redactedMessage = redactCredential(message, apiKey);
+        if (redactedMessage === message)
+            throw err;
+        const redactedError = new Error(redactedMessage);
+        if (err instanceof Error)
+            redactedError.name = err.name;
+        throw redactedError;
+    }
+    const raw = await response.text();
+    if (!response.ok) {
+        throw new Error(`TinyFish ${label} API error ${response.status}: ${redactCredential(raw, apiKey).slice(0, 300)}`);
+    }
+    try {
+        return JSON.parse(raw);
+    }
+    catch (err) {
+        throw new Error(`TinyFish ${label} API returned invalid JSON: ${errorMessage(err)}`);
+    }
+}
+function mapSearchResults(results) {
+    if (!Array.isArray(results))
+        return [];
+    return results.flatMap((item) => {
+        if (!item || typeof item.url !== "string" || item.url.trim().length === 0)
+            return [];
+        const url = item.url.trim();
+        return [{
+                title: typeof item.title === "string" && item.title.trim() ? item.title.trim() : url,
+                url,
+                snippet: typeof item.snippet === "string" ? item.snippet.replace(/\s+/g, " ").trim() : "",
+            }];
+    });
+}
+function deduplicateResults(results, limit) {
+    const seen = new Set();
+    const unique = [];
+    for (const result of results) {
+        if (seen.has(result.url))
+            continue;
+        seen.add(result.url);
+        unique.push(result);
+        if (unique.length >= limit)
+            break;
+    }
+    return unique;
+}
+function buildAnswer(results) {
+    return results.map((result) => {
+        if (result.snippet)
+            return `${result.snippet}\nSource: ${result.title} (${result.url})`;
+        return `Source: ${result.title} (${result.url})`;
+    }).join("\n\n");
+}
+function fetchPerUrlTimeout(value) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return MAX_FETCH_PER_URL_TIMEOUT_MS;
+    return Math.max(1, Math.min(Math.floor(value), MAX_FETCH_PER_URL_TIMEOUT_MS));
+}
+function fetchBody(urls, options = {}) {
+    const body = {
+        urls,
+        format: "markdown",
+        per_url_timeout_ms: fetchPerUrlTimeout(options.timeoutMs),
+    };
+    const purpose = options.prompt?.trim();
+    if (purpose)
+        body.purpose = purpose.slice(0, 2_000);
+    return body;
+}
+function findFetchError(errors, url) {
+    if (!Array.isArray(errors))
+        return undefined;
+    return errors.find(item => item?.url === url) ?? errors[0];
+}
+function fetchResultContent(result) {
+    if (typeof result.text === "string")
+        return result.text.trim();
+    if (result.text && typeof result.text === "object")
+        return JSON.stringify(result.text, null, 2);
+    return "";
+}
+function mapFetchResult(result, requestedUrl) {
+    if (!result)
+        return null;
+    const content = fetchResultContent(result);
+    if (!content)
+        return null;
+    return {
+        url: requestedUrl,
+        title: typeof result.title === "string" ? result.title.trim() : "",
+        content,
+        error: null,
+    };
+}
+async function fetchBatch(urls, apiKey, signal, options = {}) {
+    return tinyFishJsonRequest("Fetch", TINYFISH_FETCH_URL, apiKey, { method: "POST", body: JSON.stringify(fetchBody(urls, options)) }, FETCH_TIMEOUT_MS, signal);
+}
+async function fetchInlineContent(urls, apiKey, signal) {
+    const content = [];
+    for (let offset = 0; offset < urls.length; offset += MAX_FETCH_URLS) {
+        const batch = urls.slice(offset, offset + MAX_FETCH_URLS);
+        const data = await fetchBatch(batch, apiKey, signal);
+        if (!Array.isArray(data.results) || !Array.isArray(data.errors)) {
+            throw new Error("TinyFish Fetch API returned an unexpected response shape");
+        }
+        for (const url of batch) {
+            const result = Array.isArray(data.results)
+                ? data.results.find(item => item?.url === url || item?.final_url === url)
+                : undefined;
+            const mapped = mapFetchResult(result, url);
+            if (mapped)
+                content.push(mapped);
+        }
+    }
+    return content;
+}
+export async function searchWithTinyFish(query, options = {}) {
+    const apiKey = await getApiKey(options.signal);
+    const numResults = normalizeNumResults(options.numResults);
+    const activityId = activityMonitor.logStart({ type: "api", query });
+    try {
+        const combined = [];
+        const pages = numResults > 10 ? 2 : 1;
+        for (let page = 0; page < pages; page++) {
+            const data = await tinyFishJsonRequest("Search", buildSearchUrl(query, options, page), apiKey, { method: "GET" }, SEARCH_TIMEOUT_MS, options.signal);
+            if (!Array.isArray(data.results))
+                throw new Error("TinyFish Search API returned an unexpected response shape");
+            combined.push(...mapSearchResults(data.results));
+            if (data.results.length < 10)
+                break;
+        }
+        const results = deduplicateResults(combined, numResults);
+        const response = { answer: buildAnswer(results), results };
+        if (options.includeContent && results.length > 0) {
+            const inlineContent = await fetchInlineContent(results.map(result => result.url), apiKey, options.signal);
+            if (inlineContent.length > 0)
+                response.inlineContent = inlineContent;
+        }
+        activityMonitor.logComplete(activityId, 200);
+        return response;
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        const redactedMessage = redactCredential(message, apiKey);
+        if (redactedMessage.toLowerCase().includes("abort"))
+            activityMonitor.logComplete(activityId, 0);
+        else
+            activityMonitor.logError(activityId, redactedMessage);
+        if (redactedMessage === message)
+            throw err;
+        const redactedError = new Error(redactedMessage);
+        if (err instanceof Error)
+            redactedError.name = err.name;
+        throw redactedError;
+    }
+}
+export async function extractWithTinyFish(url, signal, options = {}) {
+    const apiKey = await getApiKey(signal);
+    const activityId = activityMonitor.logStart({ type: "fetch", url });
+    try {
+        const data = await fetchBatch([url], apiKey, signal, options);
+        if (!Array.isArray(data.results) || !Array.isArray(data.errors)) {
+            throw new Error("TinyFish Fetch API returned an unexpected response shape");
+        }
+        const result = data.results.find(item => item?.url === url || item?.final_url === url) ?? data.results[0];
+        const mapped = mapFetchResult(result, url);
+        if (mapped) {
+            activityMonitor.logComplete(activityId, 200);
+            return mapped;
+        }
+        const fetchError = findFetchError(data.errors, url);
+        if (fetchError) {
+            const status = typeof fetchError.status === "number" ? ` (HTTP ${fetchError.status})` : "";
+            throw new Error(`TinyFish Fetch failed for ${url}: ${fetchError.error || "unknown error"}${status}`);
+        }
+        activityMonitor.logComplete(activityId, 200);
+        return null;
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        const redactedMessage = redactCredential(message, apiKey);
+        if (redactedMessage.toLowerCase().includes("abort"))
+            activityMonitor.logComplete(activityId, 0);
+        else
+            activityMonitor.logError(activityId, redactedMessage);
+        if (redactedMessage === message)
+            throw err;
+        const redactedError = new Error(redactedMessage);
+        if (err instanceof Error)
+            redactedError.name = err.name;
+        throw redactedError;
+    }
+}
+//# sourceMappingURL=tinyfish.js.map