@ashx-j/lunr 0.1.0

package/dist/builtin-extensions/pi-web-access/parallel.js

@@ -0,0 +1,316 @@
+// @ts-nocheck
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const PARALLEL_SEARCH_URL = "https://api.parallel.ai/v1/search";
+const PARALLEL_EXTRACT_URL = "https://api.parallel.ai/v1/extract";
+const CONFIG_PATH = getWebSearchConfigPath();
+const MIN_PARALLEL_API_KEY_LENGTH = 8;
+const MIN_USEFUL_CONTENT = 500;
+const SEARCH_TIMEOUT_MS = 60_000;
+const PLACEHOLDER_API_KEY_DENYLIST = new Set([
+    "replace_with_your_parallel_api_key",
+    "parallel_api_key",
+    "your-key",
+    "your-key-here",
+    "your-api-key-here",
+    "dummy",
+    "placeholder",
+    "changeme",
+    "insert-your-key",
+    "insert-your-key-here",
+    "api-key",
+    "xxx",
+]);
+let cachedConfig = null;
+function loadConfig() {
+    if (cachedConfig)
+        return cachedConfig;
+    if (!existsSync(CONFIG_PATH)) {
+        cachedConfig = {};
+        return cachedConfig;
+    }
+    const raw = readFileSync(CONFIG_PATH, "utf-8");
+    try {
+        cachedConfig = JSON.parse(raw);
+        return cachedConfig;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+    }
+}
+export function clearParallelConfigCache() {
+    cachedConfig = null;
+}
+function normalizeApiKey(value) {
+    if (typeof value !== "string")
+        return null;
+    const normalized = value.trim();
+    return normalized.length > 0 ? normalized : null;
+}
+function isPlaceholderApiKey(key) {
+    const normalized = key.trim();
+    return normalized.length < MIN_PARALLEL_API_KEY_LENGTH || PLACEHOLDER_API_KEY_DENYLIST.has(normalized.toLowerCase());
+}
+function resolveApiKey() {
+    const envKey = normalizeApiKey(process.env.PARALLEL_API_KEY);
+    if (envKey && !isPlaceholderApiKey(envKey))
+        return envKey;
+    const configKey = normalizeApiKey(loadConfig().parallelApiKey);
+    if (configKey && !isPlaceholderApiKey(configKey))
+        return configKey;
+    return null;
+}
+function getApiKey() {
+    const key = resolveApiKey();
+    if (!key) {
+        throw new Error("Parallel API key not found. Either:\n" +
+            `  1. Create ${CONFIG_PATH} with { "parallelApiKey": "your-key" }\n` +
+            "  2. Set PARALLEL_API_KEY environment variable\n" +
+            "Get a key at https://platform.parallel.ai");
+    }
+    return key;
+}
+export function hasParallelApiKey() {
+    return !!resolveApiKey();
+}
+export function isParallelAvailable() {
+    return hasParallelApiKey();
+}
+function requestSignal(signal) {
+    const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+    return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function activityContext(url, body) {
+    if (typeof body.objective === "string" && body.objective.trim().length > 0) {
+        return { type: "api", query: body.objective };
+    }
+    const searchQueries = body.search_queries;
+    if (Array.isArray(searchQueries) && typeof searchQueries[0] === "string") {
+        return { type: "api", query: searchQueries[0] };
+    }
+    const urls = body.urls;
+    if (Array.isArray(urls) && typeof urls[0] === "string") {
+        return { type: "fetch", url: urls[0] };
+    }
+    return url.includes("/search") ? { type: "api", query: "Parallel search" } : { type: "fetch", url };
+}
+function recencyToAfterDate(filter) {
+    const now = new Date();
+    const offsets = { day: 1, week: 7, month: 30, year: 365 };
+    const days = offsets[filter] ?? 0;
+    return new Date(now.getTime() - days * 86_400_000).toISOString().slice(0, 10);
+}
+function normalizeDomain(value) {
+    let input = value.trim().toLowerCase();
+    if (!input)
+        return null;
+    if (input.startsWith("-"))
+        input = input.slice(1).trim();
+    if (!input)
+        return null;
+    try {
+        const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+        input = parsed.hostname;
+    }
+    catch {
+        input = input.split("/")[0]?.split(":")[0] ?? "";
+    }
+    input = input.replace(/^\.+|\.+$/g, "");
+    return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function mapDomainFilter(domainFilter) {
+    if (!domainFilter?.length)
+        return {};
+    const include_domains = [];
+    const exclude_domains = [];
+    for (const raw of domainFilter) {
+        const domain = normalizeDomain(raw);
+        if (!domain)
+            continue;
+        const target = raw.trim().startsWith("-") ? exclude_domains : include_domains;
+        if (!target.includes(domain))
+            target.push(domain);
+    }
+    return {
+        ...(include_domains.length ? { include_domains } : {}),
+        ...(exclude_domains.length ? { exclude_domains } : {}),
+    };
+}
+function buildSearchRequestBody(query, options = {}) {
+    const numResults = Math.max(1, Math.min(Math.floor(options.numResults ?? 5), 20));
+    const sourcePolicy = {
+        ...mapDomainFilter(options.domainFilter),
+        ...(options.recencyFilter ? { after_date: recencyToAfterDate(options.recencyFilter) } : {}),
+    };
+    return {
+        objective: query,
+        search_queries: [query],
+        advanced_settings: {
+            max_results: numResults,
+            ...(Object.keys(sourcePolicy).length > 0 ? { source_policy: sourcePolicy } : {}),
+        },
+    };
+}
+function normalizeExcerpts(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter((item) => typeof item === "string" && item.trim().length > 0);
+}
+function mapSearchResults(results) {
+    if (!Array.isArray(results))
+        return [];
+    const mapped = [];
+    for (let i = 0; i < results.length; i++) {
+        const item = results[i];
+        if (!item?.url)
+            continue;
+        const excerpts = normalizeExcerpts(item.excerpts);
+        mapped.push({
+            title: item.title || `Source ${i + 1}`,
+            url: item.url,
+            snippet: excerpts.length > 0 ? excerpts[0].replace(/\s+/g, " ").trim().slice(0, 200) : "",
+        });
+    }
+    return mapped;
+}
+function buildAnswerFromExcerpts(results) {
+    if (!Array.isArray(results))
+        return "";
+    const parts = [];
+    for (let i = 0; i < results.length; i++) {
+        const item = results[i];
+        if (!item?.url)
+            continue;
+        const excerpts = normalizeExcerpts(item.excerpts);
+        if (excerpts.length === 0)
+            continue;
+        parts.push(`${excerpts.join(" ")}\nSource: ${item.title || `Source ${i + 1}`} (${item.url})`);
+    }
+    return parts.join("\n\n");
+}
+function mapInlineContent(results) {
+    if (!Array.isArray(results))
+        return [];
+    return results.flatMap((result) => {
+        if (!result?.url)
+            return [];
+        const excerpts = normalizeExcerpts(result.excerpts);
+        if (excerpts.length === 0)
+            return [];
+        return [{ url: result.url, title: result.title || "", content: excerpts.join("\n\n"), error: null }];
+    });
+}
+function resolveExtractContent(result) {
+    const fullContent = typeof result.full_content === "string" ? result.full_content.trim() : "";
+    return fullContent.length > 0 ? fullContent : normalizeExcerpts(result.excerpts).join("\n\n");
+}
+function mapExtractResult(result) {
+    if (!result?.url)
+        return null;
+    const content = resolveExtractContent(result);
+    if (content.length < MIN_USEFUL_CONTENT)
+        return null;
+    return {
+        url: result.url,
+        title: typeof result.title === "string" ? result.title.trim() : "",
+        content,
+        error: null,
+    };
+}
+function buildExtractRequestBody(url, options = {}, fullContent = false) {
+    const body = { urls: [url] };
+    const prompt = options.prompt?.trim();
+    if (prompt)
+        body.objective = prompt;
+    if (fullContent)
+        body.advanced_settings = { full_content: true };
+    return body;
+}
+function findExtractResult(results, url) {
+    if (!Array.isArray(results))
+        return undefined;
+    return results.find(item => item?.url === url) ?? results[0];
+}
+function hasExtractUrlError(errors, url) {
+    if (!Array.isArray(errors))
+        return false;
+    return errors.some((entry) => {
+        if (typeof entry === "string")
+            return entry === url;
+        return typeof entry === "object" && entry !== null && entry.url === url;
+    });
+}
+async function fetchAndMapExtractResult(url, body, signal) {
+    const data = await parallelFetch(PARALLEL_EXTRACT_URL, body, signal);
+    if (hasExtractUrlError(data.errors, url))
+        return { mapped: null, result: undefined };
+    const result = findExtractResult(data.results, url);
+    return { mapped: mapExtractResult(result), result };
+}
+export async function searchWithParallel(query, options = {}) {
+    const data = await parallelFetch(PARALLEL_SEARCH_URL, buildSearchRequestBody(query, options), options.signal);
+    const results = data.results;
+    const response = {
+        answer: buildAnswerFromExcerpts(results),
+        results: mapSearchResults(results),
+    };
+    if (options.includeContent) {
+        const inlineContent = mapInlineContent(results);
+        if (inlineContent.length > 0)
+            response.inlineContent = inlineContent;
+    }
+    return response;
+}
+export async function extractWithParallel(url, signal, options = {}) {
+    const initial = await fetchAndMapExtractResult(url, buildExtractRequestBody(url, options), signal);
+    if (initial.mapped)
+        return initial.mapped;
+    if (!initial.result || resolveExtractContent(initial.result).length >= MIN_USEFUL_CONTENT)
+        return null;
+    const retry = await fetchAndMapExtractResult(url, buildExtractRequestBody(url, options, true), signal);
+    return retry.mapped;
+}
+async function parallelFetch(url, body, signal) {
+    const apiKey = getApiKey();
+    const activityId = activityMonitor.logStart(activityContext(url, body));
+    let response;
+    try {
+        response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "x-api-key": apiKey,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(body),
+            signal: requestSignal(signal),
+        });
+    }
+    catch (err) {
+        const message = errorMessage(err);
+        if (message.toLowerCase().includes("abort"))
+            activityMonitor.logComplete(activityId, 0);
+        else
+            activityMonitor.logError(activityId, message);
+        throw err;
+    }
+    if (!response.ok) {
+        activityMonitor.logComplete(activityId, response.status);
+        const errorText = await response.text();
+        throw new Error(`Parallel API error ${response.status}: ${errorText.slice(0, 300)}`);
+    }
+    try {
+        const data = await response.json();
+        activityMonitor.logComplete(activityId, response.status);
+        return data;
+    }
+    catch (err) {
+        activityMonitor.logComplete(activityId, response.status);
+        throw new Error(`Parallel API returned invalid JSON: ${errorMessage(err)}`);
+    }
+}
+//# sourceMappingURL=parallel.js.map

package/dist/builtin-extensions/pi-web-access/parallel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parallel.js","sourceRoot":"","sources":["../../../src/builtin-extensions/pi-web-access/parallel.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGhD,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEpD,MAAM,mBAAmB,GAAG,mCAAmC,CAAC;AAChE,MAAM,oBAAoB,GAAG,oCAAoC,CAAC;AAClE,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAC;AAC7C,MAAM,2BAA2B,GAAG,CAAC,CAAC;AACtC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAEjC,MAAM,4BAA4B,GAAG,IAAI,GAAG,CAAC;IAC5C,oCAAoC;IACpC,kBAAkB;IAClB,UAAU;IACV,eAAe;IACf,mBAAmB;IACnB,OAAO;IACP,aAAa;IACb,UAAU;IACV,iBAAiB;IACjB,sBAAsB;IACtB,SAAS;IACT,KAAK;CACL,CAAC,CAAC;AAyBH,IAAI,YAAY,GAA2B,IAAI,CAAC;AAEhD,SAAS,UAAU,GAAoB;IACtC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9B,YAAY,GAAG,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC;IACrB,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,CAAC;QACJ,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;QAClD,OAAO,YAAY,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;IAC/D,CAAC;AAAA,CACD;AAED,MAAM,UAAU,wBAAwB,GAAS;IAChD,YAAY,GAAG,IAAI,CAAC;AAAA,CACpB;AAED,SAAS,eAAe,CAAC,KAAc,EAAiB;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;AAAA,CACjD;AAED,SAAS,mBAAmB,CAAC,GAAW,EAAW;IAClD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,OAAO,UAAU,CAAC,MAAM,GAAG,2BAA2B,IAAI,4BAA4B,CAAC,GAAG,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;AAAA,CACrH;AAED,SAAS,aAAa,GAAkB;IACvC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAE1D,MAAM,SAAS,GAAG,eAAe,CAAC,UAAU,EAAE,CAAC,cAAc,CAAC,CAAC;IAC/D,IAAI,SAAS,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAEnE,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,SAAS,GAAW;IAC5B,MAAM,GAAG,GAAG,aAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CACd,uCAAuC;YACvC,eAAe,WAAW,0CAA0C;YACpE,kDAAkD;YAClD,2CAA2C,CAC3C,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AAAA,CACX;AAED,MAAM,UAAU,iBAAiB,GAAY;IAC5C,OAAO,CAAC,CAAC,aAAa,EAAE,CAAC;AAAA,CACzB;AAED,MAAM,UAAU,mBAAmB,GAAY;IAC9C,OAAO,iBAAiB,EAAE,CAAC;AAAA,CAC3B;AAED,SAAS,aAAa,CAAC,MAAoB,EAAe;IACzD,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACvD,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAAA,CAC7D;AAED,SAAS,YAAY,CAAC,GAAY,EAAU;IAC3C,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAAA,CACxD;AAED,SAAS,eAAe,CACvB,GAAW,EACX,IAA6B,EAC6B;IAC1D,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5E,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC;IAC/C,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,aAAa,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC1E,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACjD,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACvB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QACxD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAED,OAAO,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AAAA,CACpG;AAED,SAAS,kBAAkB,CAAC,MAAc,EAAU;IACnD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,OAAO,GAA2B,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IAClF,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAAA,CAC9E;AAED,SAAS,eAAe,CAAC,KAAa,EAAiB;IACtD,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;QACpF,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,CAAC;IACD,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,mCAAmC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAAA,CACtE;AAED,SAAS,eAAe,CAAC,YAAkC,EAA8D;IACxH,IAAI,CAAC,YAAY,EAAE,MAAM;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC;QAC9E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,OAAO;QACN,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC;AAAA,CACF;AAED,SAAS,sBAAsB,CAAC,KAAa,EAAE,OAAO,GAA0B,EAAE,EAA2B;IAC5G,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,YAAY,GAAG;QACpB,GAAG,eAAe,CAAC,OAAO,CAAC,YAAY,CAAC;QACxC,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,kBAAkB,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3F,CAAC;IACF,OAAO;QACN,SAAS,EAAE,KAAK;QAChB,cAAc,EAAE,CAAC,KAAK,CAAC;QACvB,iBAAiB,EAAE;YAClB,WAAW,EAAE,UAAU;YACvB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChF;KACD,CAAC;AAAA,CACF;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAY;IACpD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAAA,CAClG;AAED,SAAS,gBAAgB,CAAC,OAAwC,EAA6B;IAC9F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,MAAM,MAAM,GAA8B,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,IAAI,EAAE,GAAG;YAAE,SAAS;QACzB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC;YACX,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE;YACtC,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE;SACzF,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,SAAS,uBAAuB,CAAC,OAAwC,EAAU;IAClF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,IAAI,EAAE,GAAG;YAAE,SAAS;QACzB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAAA,CAC1B;AAED,SAAS,gBAAgB,CAAC,OAAwC,EAAsB;IACvF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,EAAE,GAAG;YAAE,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACrC,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAAA,CACrG,CAAC,CAAC;AAAA,CACH;AAED,SAAS,qBAAqB,CAAC,MAAuB,EAAU;IAC/D,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9F,OAAO,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAAA,CAC9F;AAED,SAAS,gBAAgB,CAAC,MAA0C,EAA2B;IAC9F,IAAI,CAAC,MAAM,EAAE,GAAG;QAAE,OAAO,IAAI,CAAC;IAC9B,MAAM,OAAO,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB;QAAE,OAAO,IAAI,CAAC;IACrD,OAAO;QACN,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;QAClE,OAAO;QACP,KAAK,EAAE,IAAI;KACX,CAAC;AAAA,CACF;AAED,SAAS,uBAAuB,CAAC,GAAW,EAAE,OAAO,GAAmB,EAAE,EAAE,WAAW,GAAG,KAAK,EAA2B;IACzH,MAAM,IAAI,GAA4B,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;IACtC,IAAI,MAAM;QAAE,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC;IACpC,IAAI,WAAW;QAAE,IAAI,CAAC,iBAAiB,GAAG,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;IACjE,OAAO,IAAI,CAAC;AAAA,CACZ;AAED,SAAS,iBAAiB,CAAC,OAAsC,EAAE,GAAW,EAA+B;IAC5G,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;AAAA,CAC7D;AAED,SAAS,kBAAkB,CAAC,MAAe,EAAE,GAAW,EAAW;IAClE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IACzC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,KAAK,GAAG,CAAC;QACpD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAK,KAA2B,CAAC,GAAG,KAAK,GAAG,CAAC;IAAA,CAC/F,CAAC,CAAC;AAAA,CACH;AAED,KAAK,UAAU,wBAAwB,CACtC,GAAW,EACX,IAA6B,EAC7B,MAAoB,EACgE;IACpF,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,oBAAoB,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACrE,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACrF,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAwC,EAAE,GAAG,CAAC,CAAC;IACrF,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;AAAA,CACpD;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAa,EAAE,OAAO,GAA0B,EAAE,EAA2B;IACrH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9G,MAAM,OAAO,GAAG,IAAI,CAAC,OAA0C,CAAC;IAChE,MAAM,QAAQ,GAAmB;QAChC,MAAM,EAAE,uBAAuB,CAAC,OAAO,CAAC;QACxC,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC;KAClC,CAAC;IACF,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,aAAa,GAAG,aAAa,CAAC;IACtE,CAAC;IACD,OAAO,QAAQ,CAAC;AAAA,CAChB;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,GAAW,EACX,MAAoB,EACpB,OAAO,GAAmB,EAAE,EACO;IACnC,MAAM,OAAO,GAAG,MAAM,wBAAwB,CAAC,GAAG,EAAE,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;IACnG,IAAI,OAAO,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC,MAAM,CAAC;IAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAEvG,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,GAAG,EAAE,uBAAuB,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACvG,OAAO,KAAK,CAAC,MAAM,CAAC;AAAA,CACpB;AAED,KAAK,UAAU,aAAa,CAC3B,GAAW,EACX,IAA6B,EAC7B,MAAoB,EACe;IACnC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,UAAU,GAAG,eAAe,CAAC,QAAQ,CAAC,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IACxE,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACJ,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACR,WAAW,EAAE,MAAM;gBACnB,cAAc,EAAE,kBAAkB;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAC1B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC;SAC7B,CAAC,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;;YACnF,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,GAAG,CAAC;IACX,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAClB,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA6B,CAAC;QAC9D,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,eAAe,CAAC,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,uCAAuC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;AAAA,CACD","sourcesContent":["// @ts-nocheck\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport type { ExtractedContent, ExtractOptions } from \"./extract.ts\";\nimport type { SearchOptions, SearchResponse } from \"./perplexity.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst PARALLEL_SEARCH_URL = \"https://api.parallel.ai/v1/search\";\nconst PARALLEL_EXTRACT_URL = \"https://api.parallel.ai/v1/extract\";\nconst CONFIG_PATH = getWebSearchConfigPath();\nconst MIN_PARALLEL_API_KEY_LENGTH = 8;\nconst MIN_USEFUL_CONTENT = 500;\nconst SEARCH_TIMEOUT_MS = 60_000;\n\nconst PLACEHOLDER_API_KEY_DENYLIST = new Set([\n\t\"replace_with_your_parallel_api_key\",\n\t\"parallel_api_key\",\n\t\"your-key\",\n\t\"your-key-here\",\n\t\"your-api-key-here\",\n\t\"dummy\",\n\t\"placeholder\",\n\t\"changeme\",\n\t\"insert-your-key\",\n\t\"insert-your-key-here\",\n\t\"api-key\",\n\t\"xxx\",\n]);\n\ninterface WebSearchConfig {\n\tparallelApiKey?: unknown;\n}\n\ninterface V1WebSearchResult {\n\turl: string;\n\ttitle?: string | null;\n\tpublish_date?: string | null;\n\texcerpts?: string[];\n}\n\ninterface V1ExtractResult {\n\turl: string;\n\ttitle?: string | null;\n\tpublish_date?: string | null;\n\texcerpts?: string[];\n\tfull_content?: string | null;\n}\n\ninterface ParallelSearchOptions extends SearchOptions {\n\tincludeContent?: boolean;\n}\n\nlet cachedConfig: WebSearchConfig | null = null;\n\nfunction loadConfig(): WebSearchConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst raw = readFileSync(CONFIG_PATH, \"utf-8\");\n\ttry {\n\t\tcachedConfig = JSON.parse(raw) as WebSearchConfig;\n\t\treturn cachedConfig;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n}\n\nexport function clearParallelConfigCache(): void {\n\tcachedConfig = null;\n}\n\nfunction normalizeApiKey(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst normalized = value.trim();\n\treturn normalized.length > 0 ? normalized : null;\n}\n\nfunction isPlaceholderApiKey(key: string): boolean {\n\tconst normalized = key.trim();\n\treturn normalized.length < MIN_PARALLEL_API_KEY_LENGTH || PLACEHOLDER_API_KEY_DENYLIST.has(normalized.toLowerCase());\n}\n\nfunction resolveApiKey(): string | null {\n\tconst envKey = normalizeApiKey(process.env.PARALLEL_API_KEY);\n\tif (envKey && !isPlaceholderApiKey(envKey)) return envKey;\n\n\tconst configKey = normalizeApiKey(loadConfig().parallelApiKey);\n\tif (configKey && !isPlaceholderApiKey(configKey)) return configKey;\n\n\treturn null;\n}\n\nfunction getApiKey(): string {\n\tconst key = resolveApiKey();\n\tif (!key) {\n\t\tthrow new Error(\n\t\t\t\"Parallel API key not found. Either:\\n\" +\n\t\t\t`  1. Create ${CONFIG_PATH} with { \"parallelApiKey\": \"your-key\" }\\n` +\n\t\t\t\"  2. Set PARALLEL_API_KEY environment variable\\n\" +\n\t\t\t\"Get a key at https://platform.parallel.ai\",\n\t\t);\n\t}\n\treturn key;\n}\n\nexport function hasParallelApiKey(): boolean {\n\treturn !!resolveApiKey();\n}\n\nexport function isParallelAvailable(): boolean {\n\treturn hasParallelApiKey();\n}\n\nfunction requestSignal(signal?: AbortSignal): AbortSignal {\n\tconst timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);\n\treturn signal ? AbortSignal.any([signal, timeout]) : timeout;\n}\n\nfunction errorMessage(err: unknown): string {\n\treturn err instanceof Error ? err.message : String(err);\n}\n\nfunction activityContext(\n\turl: string,\n\tbody: Record<string, unknown>,\n): { type: \"api\" | \"fetch\"; query?: string; url?: string } {\n\tif (typeof body.objective === \"string\" && body.objective.trim().length > 0) {\n\t\treturn { type: \"api\", query: body.objective };\n\t}\n\n\tconst searchQueries = body.search_queries;\n\tif (Array.isArray(searchQueries) && typeof searchQueries[0] === \"string\") {\n\t\treturn { type: \"api\", query: searchQueries[0] };\n\t}\n\n\tconst urls = body.urls;\n\tif (Array.isArray(urls) && typeof urls[0] === \"string\") {\n\t\treturn { type: \"fetch\", url: urls[0] };\n\t}\n\n\treturn url.includes(\"/search\") ? { type: \"api\", query: \"Parallel search\" } : { type: \"fetch\", url };\n}\n\nfunction recencyToAfterDate(filter: string): string {\n\tconst now = new Date();\n\tconst offsets: Record<string, number> = { day: 1, week: 7, month: 30, year: 365 };\n\tconst days = offsets[filter] ?? 0;\n\treturn new Date(now.getTime() - days * 86_400_000).toISOString().slice(0, 10);\n}\n\nfunction normalizeDomain(value: string): string | null {\n\tlet input = value.trim().toLowerCase();\n\tif (!input) return null;\n\tif (input.startsWith(\"-\")) input = input.slice(1).trim();\n\tif (!input) return null;\n\ttry {\n\t\tconst parsed = input.includes(\"://\") ? new URL(input) : new URL(`https://${input}`);\n\t\tinput = parsed.hostname;\n\t} catch {\n\t\tinput = input.split(\"/\")[0]?.split(\":\")[0] ?? \"\";\n\t}\n\tinput = input.replace(/^\\.+|\\.+$/g, \"\");\n\treturn /^[a-z0-9][a-z0-9.-]*\\.[a-z]{2,}$/i.test(input) ? input : null;\n}\n\nfunction mapDomainFilter(domainFilter: string[] | undefined): { include_domains?: string[]; exclude_domains?: string[] } {\n\tif (!domainFilter?.length) return {};\n\tconst include_domains: string[] = [];\n\tconst exclude_domains: string[] = [];\n\tfor (const raw of domainFilter) {\n\t\tconst domain = normalizeDomain(raw);\n\t\tif (!domain) continue;\n\t\tconst target = raw.trim().startsWith(\"-\") ? exclude_domains : include_domains;\n\t\tif (!target.includes(domain)) target.push(domain);\n\t}\n\treturn {\n\t\t...(include_domains.length ? { include_domains } : {}),\n\t\t...(exclude_domains.length ? { exclude_domains } : {}),\n\t};\n}\n\nfunction buildSearchRequestBody(query: string, options: ParallelSearchOptions = {}): Record<string, unknown> {\n\tconst numResults = Math.max(1, Math.min(Math.floor(options.numResults ?? 5), 20));\n\tconst sourcePolicy = {\n\t\t...mapDomainFilter(options.domainFilter),\n\t\t...(options.recencyFilter ? { after_date: recencyToAfterDate(options.recencyFilter) } : {}),\n\t};\n\treturn {\n\t\tobjective: query,\n\t\tsearch_queries: [query],\n\t\tadvanced_settings: {\n\t\t\tmax_results: numResults,\n\t\t\t...(Object.keys(sourcePolicy).length > 0 ? { source_policy: sourcePolicy } : {}),\n\t\t},\n\t};\n}\n\nfunction normalizeExcerpts(value: unknown): string[] {\n\tif (!Array.isArray(value)) return [];\n\treturn value.filter((item): item is string => typeof item === \"string\" && item.trim().length > 0);\n}\n\nfunction mapSearchResults(results: V1WebSearchResult[] | undefined): SearchResponse[\"results\"] {\n\tif (!Array.isArray(results)) return [];\n\tconst mapped: SearchResponse[\"results\"] = [];\n\tfor (let i = 0; i < results.length; i++) {\n\t\tconst item = results[i];\n\t\tif (!item?.url) continue;\n\t\tconst excerpts = normalizeExcerpts(item.excerpts);\n\t\tmapped.push({\n\t\t\ttitle: item.title || `Source ${i + 1}`,\n\t\t\turl: item.url,\n\t\t\tsnippet: excerpts.length > 0 ? excerpts[0].replace(/\\s+/g, \" \").trim().slice(0, 200) : \"\",\n\t\t});\n\t}\n\treturn mapped;\n}\n\nfunction buildAnswerFromExcerpts(results: V1WebSearchResult[] | undefined): string {\n\tif (!Array.isArray(results)) return \"\";\n\tconst parts: string[] = [];\n\tfor (let i = 0; i < results.length; i++) {\n\t\tconst item = results[i];\n\t\tif (!item?.url) continue;\n\t\tconst excerpts = normalizeExcerpts(item.excerpts);\n\t\tif (excerpts.length === 0) continue;\n\t\tparts.push(`${excerpts.join(\" \")}\\nSource: ${item.title || `Source ${i + 1}`} (${item.url})`);\n\t}\n\treturn parts.join(\"\\n\\n\");\n}\n\nfunction mapInlineContent(results: V1WebSearchResult[] | undefined): ExtractedContent[] {\n\tif (!Array.isArray(results)) return [];\n\treturn results.flatMap((result) => {\n\t\tif (!result?.url) return [];\n\t\tconst excerpts = normalizeExcerpts(result.excerpts);\n\t\tif (excerpts.length === 0) return [];\n\t\treturn [{ url: result.url, title: result.title || \"\", content: excerpts.join(\"\\n\\n\"), error: null }];\n\t});\n}\n\nfunction resolveExtractContent(result: V1ExtractResult): string {\n\tconst fullContent = typeof result.full_content === \"string\" ? result.full_content.trim() : \"\";\n\treturn fullContent.length > 0 ? fullContent : normalizeExcerpts(result.excerpts).join(\"\\n\\n\");\n}\n\nfunction mapExtractResult(result: V1ExtractResult | undefined | null): ExtractedContent | null {\n\tif (!result?.url) return null;\n\tconst content = resolveExtractContent(result);\n\tif (content.length < MIN_USEFUL_CONTENT) return null;\n\treturn {\n\t\turl: result.url,\n\t\ttitle: typeof result.title === \"string\" ? result.title.trim() : \"\",\n\t\tcontent,\n\t\terror: null,\n\t};\n}\n\nfunction buildExtractRequestBody(url: string, options: ExtractOptions = {}, fullContent = false): Record<string, unknown> {\n\tconst body: Record<string, unknown> = { urls: [url] };\n\tconst prompt = options.prompt?.trim();\n\tif (prompt) body.objective = prompt;\n\tif (fullContent) body.advanced_settings = { full_content: true };\n\treturn body;\n}\n\nfunction findExtractResult(results: V1ExtractResult[] | undefined, url: string): V1ExtractResult | undefined {\n\tif (!Array.isArray(results)) return undefined;\n\treturn results.find(item => item?.url === url) ?? results[0];\n}\n\nfunction hasExtractUrlError(errors: unknown, url: string): boolean {\n\tif (!Array.isArray(errors)) return false;\n\treturn errors.some((entry) => {\n\t\tif (typeof entry === \"string\") return entry === url;\n\t\treturn typeof entry === \"object\" && entry !== null && (entry as { url?: unknown }).url === url;\n\t});\n}\n\nasync function fetchAndMapExtractResult(\n\turl: string,\n\tbody: Record<string, unknown>,\n\tsignal?: AbortSignal,\n): Promise<{ mapped: ExtractedContent | null; result: V1ExtractResult | undefined }> {\n\tconst data = await parallelFetch(PARALLEL_EXTRACT_URL, body, signal);\n\tif (hasExtractUrlError(data.errors, url)) return { mapped: null, result: undefined };\n\tconst result = findExtractResult(data.results as V1ExtractResult[] | undefined, url);\n\treturn { mapped: mapExtractResult(result), result };\n}\n\nexport async function searchWithParallel(query: string, options: ParallelSearchOptions = {}): Promise<SearchResponse> {\n\tconst data = await parallelFetch(PARALLEL_SEARCH_URL, buildSearchRequestBody(query, options), options.signal);\n\tconst results = data.results as V1WebSearchResult[] | undefined;\n\tconst response: SearchResponse = {\n\t\tanswer: buildAnswerFromExcerpts(results),\n\t\tresults: mapSearchResults(results),\n\t};\n\tif (options.includeContent) {\n\t\tconst inlineContent = mapInlineContent(results);\n\t\tif (inlineContent.length > 0) response.inlineContent = inlineContent;\n\t}\n\treturn response;\n}\n\nexport async function extractWithParallel(\n\turl: string,\n\tsignal?: AbortSignal,\n\toptions: ExtractOptions = {},\n): Promise<ExtractedContent | null> {\n\tconst initial = await fetchAndMapExtractResult(url, buildExtractRequestBody(url, options), signal);\n\tif (initial.mapped) return initial.mapped;\n\tif (!initial.result || resolveExtractContent(initial.result).length >= MIN_USEFUL_CONTENT) return null;\n\n\tconst retry = await fetchAndMapExtractResult(url, buildExtractRequestBody(url, options, true), signal);\n\treturn retry.mapped;\n}\n\nasync function parallelFetch(\n\turl: string,\n\tbody: Record<string, unknown>,\n\tsignal?: AbortSignal,\n): Promise<Record<string, unknown>> {\n\tconst apiKey = getApiKey();\n\tconst activityId = activityMonitor.logStart(activityContext(url, body));\n\tlet response: Response;\n\ttry {\n\t\tresponse = await fetch(url, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders: {\n\t\t\t\t\"x-api-key\": apiKey,\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tbody: JSON.stringify(body),\n\t\t\tsignal: requestSignal(signal),\n\t\t});\n\t} catch (err) {\n\t\tconst message = errorMessage(err);\n\t\tif (message.toLowerCase().includes(\"abort\")) activityMonitor.logComplete(activityId, 0);\n\t\telse activityMonitor.logError(activityId, message);\n\t\tthrow err;\n\t}\n\n\tif (!response.ok) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst errorText = await response.text();\n\t\tthrow new Error(`Parallel API error ${response.status}: ${errorText.slice(0, 300)}`);\n\t}\n\n\ttry {\n\t\tconst data = await response.json() as Record<string, unknown>;\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\treturn data;\n\t} catch (err) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tthrow new Error(`Parallel API returned invalid JSON: ${errorMessage(err)}`);\n\t}\n}\n"]}

package/dist/builtin-extensions/pi-web-access/pdf-extract.d.ts

@@ -0,0 +1,26 @@
+/**
+ * PDF Content Extractor
+ *
+ * Extracts text from PDF files and saves to markdown.
+ * Uses unpdf (pdfjs-dist wrapper) for text extraction.
+ */
+export interface PDFExtractResult {
+    title: string;
+    pages: number;
+    chars: number;
+    outputPath: string;
+}
+export interface PDFExtractOptions {
+    maxPages?: number;
+    outputDir?: string;
+    filename?: string;
+}
+/**
+ * Extract text from a PDF buffer and save to markdown file
+ */
+export declare function extractPDFToMarkdown(buffer: ArrayBuffer, url: string, options?: PDFExtractOptions): Promise<PDFExtractResult>;
+/**
+ * Check if URL or content-type indicates a PDF
+ */
+export declare function isPDF(url: string, contentType?: string): boolean;
+//# sourceMappingURL=pdf-extract.d.ts.map

package/dist/builtin-extensions/pi-web-access/pdf-extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.d.ts","sourceRoot":"","sources":["../../../src/builtin-extensions/pi-web-access/pdf-extract.ts"],"names":[],"mappings":"AACA;;;;;GAKG;AAOH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAKD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,WAAW,EACnB,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAgG3B;AA+CD;;GAEG;AACH,wBAAgB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAUhE","sourcesContent":["// @ts-nocheck\n/**\n * PDF Content Extractor\n * \n * Extracts text from PDF files and saves to markdown.\n * Uses unpdf (pdfjs-dist wrapper) for text extraction.\n */\n\nimport { getDocumentProxy } from \"unpdf\";\nimport { writeFile, mkdir } from \"node:fs/promises\";\nimport { join, basename } from \"node:path\";\nimport { homedir } from \"node:os\";\n\nexport interface PDFExtractResult {\n  title: string;\n  pages: number;\n  chars: number;\n  outputPath: string;\n}\n\nexport interface PDFExtractOptions {\n  maxPages?: number;\n  outputDir?: string;\n  filename?: string;\n}\n\nconst DEFAULT_MAX_PAGES = 100;\nconst DEFAULT_OUTPUT_DIR = join(homedir(), \"Downloads\");\n\n/**\n * Extract text from a PDF buffer and save to markdown file\n */\nexport async function extractPDFToMarkdown(\n  buffer: ArrayBuffer,\n  url: string,\n  options: PDFExtractOptions = {}\n): Promise<PDFExtractResult> {\n  const {\n    maxPages = DEFAULT_MAX_PAGES,\n    outputDir = DEFAULT_OUTPUT_DIR,\n    filename,\n  } = options;\n\n  const safeMaxPages = Number.isFinite(maxPages)\n    ? Math.max(1, Math.floor(maxPages))\n    : DEFAULT_MAX_PAGES;\n\n  const pdf = await getDocumentProxy(new Uint8Array(buffer));\n  const metadata = await pdf.getMetadata();\n  const metadataInfo = metadata.info && typeof metadata.info === \"object\"\n    ? metadata.info as Record<string, unknown>\n    : null;\n\n  // Extract title from metadata or URL\n  const metaTitle = typeof metadataInfo?.Title === \"string\" ? metadataInfo.Title : undefined;\n  const metaAuthor = typeof metadataInfo?.Author === \"string\" ? metadataInfo.Author : undefined;\n  const urlTitle = extractTitleFromURL(url);\n  const title = metaTitle?.trim() || urlTitle;\n\n  // Determine pages to extract\n  const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);\n  const truncated = pdf.numPages > safeMaxPages;\n\n  // Extract text page by page for better structure\n  const pages: { pageNum: number; text: string }[] = [];\n  for (let i = 1; i <= pagesToExtract; i++) {\n    const page = await pdf.getPage(i);\n    const textContent = await page.getTextContent();\n    const pageText = textContent.items\n      .map((item: unknown) => {\n        const textItem = item as { str?: string };\n        return textItem.str || \"\";\n      })\n      .join(\" \")\n      .replace(/\\s+/g, \" \")\n      .trim();\n    \n    if (pageText) {\n      pages.push({ pageNum: i, text: pageText });\n    }\n  }\n\n  // Build markdown content\n  const lines: string[] = [];\n  \n  // Header with metadata\n  lines.push(`# ${title}`);\n  lines.push(\"\");\n  lines.push(`> Source: ${url}`);\n  lines.push(`> Pages: ${pdf.numPages}${truncated ? ` (extracted first ${pagesToExtract})` : \"\"}`);\n  if (metaAuthor) {\n    lines.push(`> Author: ${metaAuthor}`);\n  }\n  lines.push(\"\");\n  lines.push(\"---\");\n  lines.push(\"\");\n\n  // Content with page markers\n  for (let i = 0; i < pages.length; i++) {\n    if (i > 0) {\n      lines.push(\"\");\n      lines.push(`<!-- Page ${pages[i].pageNum} -->`);\n      lines.push(\"\");\n    }\n    lines.push(pages[i].text);\n  }\n\n  if (truncated) {\n    lines.push(\"\");\n    lines.push(\"---\");\n    lines.push(\"\");\n    lines.push(`*[Truncated: Only first ${pagesToExtract} of ${pdf.numPages} pages extracted]*`);\n  }\n\n  const content = lines.join(\"\\n\");\n\n  // Generate output filename\n  const outputFilename = filename || sanitizeFilename(title) + \".md\";\n  const outputPath = join(outputDir, outputFilename);\n\n  // Ensure output directory exists\n  await mkdir(outputDir, { recursive: true });\n\n  // Write file\n  await writeFile(outputPath, content, \"utf-8\");\n\n  return {\n    title,\n    pages: pdf.numPages,\n    chars: content.length,\n    outputPath,\n  };\n}\n\n/**\n * Extract a reasonable title from URL\n */\nfunction extractTitleFromURL(url: string): string {\n  try {\n    const urlObj = new URL(url);\n    const pathname = urlObj.pathname;\n    \n    // Get filename without extension\n    let filename = basename(pathname, \".pdf\");\n    \n    // Handle arxiv URLs: /pdf/1706.03762 → \"arxiv-1706.03762\"\n    if (urlObj.hostname.includes(\"arxiv.org\")) {\n      const match = pathname.match(/\\/(?:pdf|abs)\\/(\\d+\\.\\d+)/);\n      if (match) {\n        filename = `arxiv-${match[1]}`;\n      }\n    }\n    \n    // Clean up filename\n    filename = filename\n      .replace(/[_-]+/g, \" \")\n      .replace(/\\s+/g, \" \")\n      .trim();\n    \n    return filename || \"document\";\n  } catch {\n    return \"document\";\n  }\n}\n\n/**\n * Sanitize string for use as filename\n */\nfunction sanitizeFilename(name: string): string {\n  return name\n    .toLowerCase()\n    .replace(/[^a-z0-9\\s-]/g, \"\")\n    .replace(/\\s+/g, \"-\")\n    .replace(/-+/g, \"-\")\n    .slice(0, 100)\n    .replace(/^-|-$/g, \"\")\n    || \"document\";\n}\n\n/**\n * Check if URL or content-type indicates a PDF\n */\nexport function isPDF(url: string, contentType?: string): boolean {\n  if (contentType?.includes(\"application/pdf\")) {\n    return true;\n  }\n  try {\n    const urlObj = new URL(url);\n    return urlObj.pathname.toLowerCase().endsWith(\".pdf\");\n  } catch {\n    return false;\n  }\n}\n"]}

package/dist/builtin-extensions/pi-web-access/pdf-extract.js

@@ -0,0 +1,150 @@
+// @ts-nocheck
+/**
+ * PDF Content Extractor
+ *
+ * Extracts text from PDF files and saves to markdown.
+ * Uses unpdf (pdfjs-dist wrapper) for text extraction.
+ */
+import { getDocumentProxy } from "unpdf";
+import { writeFile, mkdir } from "node:fs/promises";
+import { join, basename } from "node:path";
+import { homedir } from "node:os";
+const DEFAULT_MAX_PAGES = 100;
+const DEFAULT_OUTPUT_DIR = join(homedir(), "Downloads");
+/**
+ * Extract text from a PDF buffer and save to markdown file
+ */
+export async function extractPDFToMarkdown(buffer, url, options = {}) {
+    const { maxPages = DEFAULT_MAX_PAGES, outputDir = DEFAULT_OUTPUT_DIR, filename, } = options;
+    const safeMaxPages = Number.isFinite(maxPages)
+        ? Math.max(1, Math.floor(maxPages))
+        : DEFAULT_MAX_PAGES;
+    const pdf = await getDocumentProxy(new Uint8Array(buffer));
+    const metadata = await pdf.getMetadata();
+    const metadataInfo = metadata.info && typeof metadata.info === "object"
+        ? metadata.info
+        : null;
+    // Extract title from metadata or URL
+    const metaTitle = typeof metadataInfo?.Title === "string" ? metadataInfo.Title : undefined;
+    const metaAuthor = typeof metadataInfo?.Author === "string" ? metadataInfo.Author : undefined;
+    const urlTitle = extractTitleFromURL(url);
+    const title = metaTitle?.trim() || urlTitle;
+    // Determine pages to extract
+    const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);
+    const truncated = pdf.numPages > safeMaxPages;
+    // Extract text page by page for better structure
+    const pages = [];
+    for (let i = 1; i <= pagesToExtract; i++) {
+        const page = await pdf.getPage(i);
+        const textContent = await page.getTextContent();
+        const pageText = textContent.items
+            .map((item) => {
+            const textItem = item;
+            return textItem.str || "";
+        })
+            .join(" ")
+            .replace(/\s+/g, " ")
+            .trim();
+        if (pageText) {
+            pages.push({ pageNum: i, text: pageText });
+        }
+    }
+    // Build markdown content
+    const lines = [];
+    // Header with metadata
+    lines.push(`# ${title}`);
+    lines.push("");
+    lines.push(`> Source: ${url}`);
+    lines.push(`> Pages: ${pdf.numPages}${truncated ? ` (extracted first ${pagesToExtract})` : ""}`);
+    if (metaAuthor) {
+        lines.push(`> Author: ${metaAuthor}`);
+    }
+    lines.push("");
+    lines.push("---");
+    lines.push("");
+    // Content with page markers
+    for (let i = 0; i < pages.length; i++) {
+        if (i > 0) {
+            lines.push("");
+            lines.push(`<!-- Page ${pages[i].pageNum} -->`);
+            lines.push("");
+        }
+        lines.push(pages[i].text);
+    }
+    if (truncated) {
+        lines.push("");
+        lines.push("---");
+        lines.push("");
+        lines.push(`*[Truncated: Only first ${pagesToExtract} of ${pdf.numPages} pages extracted]*`);
+    }
+    const content = lines.join("\n");
+    // Generate output filename
+    const outputFilename = filename || sanitizeFilename(title) + ".md";
+    const outputPath = join(outputDir, outputFilename);
+    // Ensure output directory exists
+    await mkdir(outputDir, { recursive: true });
+    // Write file
+    await writeFile(outputPath, content, "utf-8");
+    return {
+        title,
+        pages: pdf.numPages,
+        chars: content.length,
+        outputPath,
+    };
+}
+/**
+ * Extract a reasonable title from URL
+ */
+function extractTitleFromURL(url) {
+    try {
+        const urlObj = new URL(url);
+        const pathname = urlObj.pathname;
+        // Get filename without extension
+        let filename = basename(pathname, ".pdf");
+        // Handle arxiv URLs: /pdf/1706.03762 → "arxiv-1706.03762"
+        if (urlObj.hostname.includes("arxiv.org")) {
+            const match = pathname.match(/\/(?:pdf|abs)\/(\d+\.\d+)/);
+            if (match) {
+                filename = `arxiv-${match[1]}`;
+            }
+        }
+        // Clean up filename
+        filename = filename
+            .replace(/[_-]+/g, " ")
+            .replace(/\s+/g, " ")
+            .trim();
+        return filename || "document";
+    }
+    catch {
+        return "document";
+    }
+}
+/**
+ * Sanitize string for use as filename
+ */
+function sanitizeFilename(name) {
+    return name
+        .toLowerCase()
+        .replace(/[^a-z0-9\s-]/g, "")
+        .replace(/\s+/g, "-")
+        .replace(/-+/g, "-")
+        .slice(0, 100)
+        .replace(/^-|-$/g, "")
+        || "document";
+}
+/**
+ * Check if URL or content-type indicates a PDF
+ */
+export function isPDF(url, contentType) {
+    if (contentType?.includes("application/pdf")) {
+        return true;
+    }
+    try {
+        const urlObj = new URL(url);
+        return urlObj.pathname.toLowerCase().endsWith(".pdf");
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=pdf-extract.js.map

package/dist/builtin-extensions/pi-web-access/pdf-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.js","sourceRoot":"","sources":["../../../src/builtin-extensions/pi-web-access/pdf-extract.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,OAAO,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAelC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC,CAAC;AAExD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAmB,EACnB,GAAW,EACX,OAAO,GAAsB,EAAE,EACJ;IAC3B,MAAM,EACJ,QAAQ,GAAG,iBAAiB,EAC5B,SAAS,GAAG,kBAAkB,EAC9B,QAAQ,GACT,GAAG,OAAO,CAAC;IAEZ,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC5C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC,CAAC,iBAAiB,CAAC;IAEtB,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ;QACrE,CAAC,CAAC,QAAQ,CAAC,IAA+B;QAC1C,CAAC,CAAC,IAAI,CAAC;IAET,qCAAqC;IACrC,MAAM,SAAS,GAAG,OAAO,YAAY,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3F,MAAM,UAAU,GAAG,OAAO,YAAY,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9F,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC;IAE5C,6BAA6B;IAC7B,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,GAAG,YAAY,CAAC;IAE9C,iDAAiD;IACjD,MAAM,KAAK,GAAwC,EAAE,CAAC;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK;aAC/B,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,IAAwB,CAAC;YAC1C,OAAO,QAAQ,CAAC,GAAG,IAAI,EAAE,CAAC;QAAA,CAC3B,CAAC;aACD,IAAI,CAAC,GAAG,CAAC;aACT,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,IAAI,EAAE,CAAC;QAEV,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,uBAAuB;IACvB,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC;IAC/B,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,qBAAqB,cAAc,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjG,IAAI,UAAU,EAAE,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,aAAa,UAAU,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,4BAA4B;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,2BAA2B,cAAc,OAAO,GAAG,CAAC,QAAQ,oBAAoB,CAAC,CAAC;IAC/F,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEjC,2BAA2B;IAC3B,MAAM,cAAc,GAAG,QAAQ,IAAI,gBAAgB,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAEnD,iCAAiC;IACjC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE5C,aAAa;IACb,MAAM,SAAS,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAE9C,OAAO;QACL,KAAK;QACL,KAAK,EAAE,GAAG,CAAC,QAAQ;QACnB,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,UAAU;KACX,CAAC;AAAA,CACH;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,GAAW,EAAU;IAChD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAEjC,iCAAiC;QACjC,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE1C,4DAA0D;QAC1D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC1D,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,QAAQ,GAAG,QAAQ;aAChB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,IAAI,EAAE,CAAC;QAEV,OAAO,QAAQ,IAAI,UAAU,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,UAAU,CAAC;IACpB,CAAC;AAAA,CACF;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAY,EAAU;IAC9C,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;SAC5B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SACb,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;WACnB,UAAU,CAAC;AAAA,CACjB;AAED;;GAEG;AACH,MAAM,UAAU,KAAK,CAAC,GAAW,EAAE,WAAoB,EAAW;IAChE,IAAI,WAAW,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AAAA,CACF","sourcesContent":["// @ts-nocheck\n/**\n * PDF Content Extractor\n * \n * Extracts text from PDF files and saves to markdown.\n * Uses unpdf (pdfjs-dist wrapper) for text extraction.\n */\n\nimport { getDocumentProxy } from \"unpdf\";\nimport { writeFile, mkdir } from \"node:fs/promises\";\nimport { join, basename } from \"node:path\";\nimport { homedir } from \"node:os\";\n\nexport interface PDFExtractResult {\n  title: string;\n  pages: number;\n  chars: number;\n  outputPath: string;\n}\n\nexport interface PDFExtractOptions {\n  maxPages?: number;\n  outputDir?: string;\n  filename?: string;\n}\n\nconst DEFAULT_MAX_PAGES = 100;\nconst DEFAULT_OUTPUT_DIR = join(homedir(), \"Downloads\");\n\n/**\n * Extract text from a PDF buffer and save to markdown file\n */\nexport async function extractPDFToMarkdown(\n  buffer: ArrayBuffer,\n  url: string,\n  options: PDFExtractOptions = {}\n): Promise<PDFExtractResult> {\n  const {\n    maxPages = DEFAULT_MAX_PAGES,\n    outputDir = DEFAULT_OUTPUT_DIR,\n    filename,\n  } = options;\n\n  const safeMaxPages = Number.isFinite(maxPages)\n    ? Math.max(1, Math.floor(maxPages))\n    : DEFAULT_MAX_PAGES;\n\n  const pdf = await getDocumentProxy(new Uint8Array(buffer));\n  const metadata = await pdf.getMetadata();\n  const metadataInfo = metadata.info && typeof metadata.info === \"object\"\n    ? metadata.info as Record<string, unknown>\n    : null;\n\n  // Extract title from metadata or URL\n  const metaTitle = typeof metadataInfo?.Title === \"string\" ? metadataInfo.Title : undefined;\n  const metaAuthor = typeof metadataInfo?.Author === \"string\" ? metadataInfo.Author : undefined;\n  const urlTitle = extractTitleFromURL(url);\n  const title = metaTitle?.trim() || urlTitle;\n\n  // Determine pages to extract\n  const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);\n  const truncated = pdf.numPages > safeMaxPages;\n\n  // Extract text page by page for better structure\n  const pages: { pageNum: number; text: string }[] = [];\n  for (let i = 1; i <= pagesToExtract; i++) {\n    const page = await pdf.getPage(i);\n    const textContent = await page.getTextContent();\n    const pageText = textContent.items\n      .map((item: unknown) => {\n        const textItem = item as { str?: string };\n        return textItem.str || \"\";\n      })\n      .join(\" \")\n      .replace(/\\s+/g, \" \")\n      .trim();\n    \n    if (pageText) {\n      pages.push({ pageNum: i, text: pageText });\n    }\n  }\n\n  // Build markdown content\n  const lines: string[] = [];\n  \n  // Header with metadata\n  lines.push(`# ${title}`);\n  lines.push(\"\");\n  lines.push(`> Source: ${url}`);\n  lines.push(`> Pages: ${pdf.numPages}${truncated ? ` (extracted first ${pagesToExtract})` : \"\"}`);\n  if (metaAuthor) {\n    lines.push(`> Author: ${metaAuthor}`);\n  }\n  lines.push(\"\");\n  lines.push(\"---\");\n  lines.push(\"\");\n\n  // Content with page markers\n  for (let i = 0; i < pages.length; i++) {\n    if (i > 0) {\n      lines.push(\"\");\n      lines.push(`<!-- Page ${pages[i].pageNum} -->`);\n      lines.push(\"\");\n    }\n    lines.push(pages[i].text);\n  }\n\n  if (truncated) {\n    lines.push(\"\");\n    lines.push(\"---\");\n    lines.push(\"\");\n    lines.push(`*[Truncated: Only first ${pagesToExtract} of ${pdf.numPages} pages extracted]*`);\n  }\n\n  const content = lines.join(\"\\n\");\n\n  // Generate output filename\n  const outputFilename = filename || sanitizeFilename(title) + \".md\";\n  const outputPath = join(outputDir, outputFilename);\n\n  // Ensure output directory exists\n  await mkdir(outputDir, { recursive: true });\n\n  // Write file\n  await writeFile(outputPath, content, \"utf-8\");\n\n  return {\n    title,\n    pages: pdf.numPages,\n    chars: content.length,\n    outputPath,\n  };\n}\n\n/**\n * Extract a reasonable title from URL\n */\nfunction extractTitleFromURL(url: string): string {\n  try {\n    const urlObj = new URL(url);\n    const pathname = urlObj.pathname;\n    \n    // Get filename without extension\n    let filename = basename(pathname, \".pdf\");\n    \n    // Handle arxiv URLs: /pdf/1706.03762 → \"arxiv-1706.03762\"\n    if (urlObj.hostname.includes(\"arxiv.org\")) {\n      const match = pathname.match(/\\/(?:pdf|abs)\\/(\\d+\\.\\d+)/);\n      if (match) {\n        filename = `arxiv-${match[1]}`;\n      }\n    }\n    \n    // Clean up filename\n    filename = filename\n      .replace(/[_-]+/g, \" \")\n      .replace(/\\s+/g, \" \")\n      .trim();\n    \n    return filename || \"document\";\n  } catch {\n    return \"document\";\n  }\n}\n\n/**\n * Sanitize string for use as filename\n */\nfunction sanitizeFilename(name: string): string {\n  return name\n    .toLowerCase()\n    .replace(/[^a-z0-9\\s-]/g, \"\")\n    .replace(/\\s+/g, \"-\")\n    .replace(/-+/g, \"-\")\n    .slice(0, 100)\n    .replace(/^-|-$/g, \"\")\n    || \"document\";\n}\n\n/**\n * Check if URL or content-type indicates a PDF\n */\nexport function isPDF(url: string, contentType?: string): boolean {\n  if (contentType?.includes(\"application/pdf\")) {\n    return true;\n  }\n  try {\n    const urlObj = new URL(url);\n    return urlObj.pathname.toLowerCase().endsWith(\".pdf\");\n  } catch {\n    return false;\n  }\n}\n"]}

package/dist/builtin-extensions/pi-web-access/perplexity.d.ts

@@ -0,0 +1,20 @@
+import type { ExtractedContent } from "./extract.ts";
+export interface SearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+}
+export interface SearchResponse {
+    answer: string;
+    results: SearchResult[];
+    inlineContent?: ExtractedContent[];
+}
+export interface SearchOptions {
+    numResults?: number;
+    recencyFilter?: "day" | "week" | "month" | "year";
+    domainFilter?: string[];
+    signal?: AbortSignal;
+}
+export declare function isPerplexityAvailable(): boolean;
+export declare function searchWithPerplexity(query: string, options?: SearchOptions): Promise<SearchResponse>;
+//# sourceMappingURL=perplexity.d.ts.map

package/dist/builtin-extensions/pi-web-access/perplexity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"perplexity.d.ts","sourceRoot":"","sources":["../../../src/builtin-extensions/pi-web-access/perplexity.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAarD,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,aAAa,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,aAAa;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAClD,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB;AAoED,wBAAgB,qBAAqB,IAAI,OAAO,CAG/C;AAED,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,CAAC,CAwF9G","sourcesContent":["// @ts-nocheck\nimport { existsSync, readFileSync } from \"node:fs\";\nimport { activityMonitor } from \"./activity.ts\";\nimport type { ExtractedContent } from \"./extract.ts\";\nimport { getWebSearchConfigPath } from \"./utils.ts\";\n\nconst PERPLEXITY_API_URL = \"https://api.perplexity.ai/chat/completions\";\nconst CONFIG_PATH = getWebSearchConfigPath();\n\nconst RATE_LIMIT = {\n\tmaxRequests: 10,\n\twindowMs: 60 * 1000,\n};\n\nconst requestTimestamps: number[] = [];\n\nexport interface SearchResult {\n\ttitle: string;\n\turl: string;\n\tsnippet: string;\n}\n\nexport interface SearchResponse {\n\tanswer: string;\n\tresults: SearchResult[];\n\tinlineContent?: ExtractedContent[];\n}\n\nexport interface SearchOptions {\n\tnumResults?: number;\n\trecencyFilter?: \"day\" | \"week\" | \"month\" | \"year\";\n\tdomainFilter?: string[];\n\tsignal?: AbortSignal;\n}\n\ninterface WebSearchConfig {\n\tperplexityApiKey?: unknown;\n}\n\nlet cachedConfig: WebSearchConfig | null = null;\n\nfunction loadConfig(): WebSearchConfig {\n\tif (cachedConfig) return cachedConfig;\n\tif (!existsSync(CONFIG_PATH)) {\n\t\tcachedConfig = {};\n\t\treturn cachedConfig;\n\t}\n\n\tconst content = readFileSync(CONFIG_PATH, \"utf-8\");\n\ttry {\n\t\tcachedConfig = JSON.parse(content) as WebSearchConfig;\n\t\treturn cachedConfig;\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);\n\t}\n}\n\nfunction normalizeApiKey(value: unknown): string | null {\n\tif (typeof value !== \"string\") return null;\n\tconst normalized = value.trim();\n\treturn normalized.length > 0 ? normalized : null;\n}\n\nfunction getApiKey(): string {\n\tconst config = loadConfig();\n\tconst key = normalizeApiKey(process.env.PERPLEXITY_API_KEY) ?? normalizeApiKey(config.perplexityApiKey);\n\tif (!key) {\n\t\tthrow new Error(\n\t\t\t\"Perplexity API key not found. Either:\\n\" +\n\t\t\t`  1. Create ${CONFIG_PATH} with { \"perplexityApiKey\": \"your-key\" }\\n` +\n\t\t\t\"  2. Set PERPLEXITY_API_KEY environment variable\\n\" +\n\t\t\t\"Get a key at https://perplexity.ai/settings/api\"\n\t\t);\n\t}\n\treturn key;\n}\n\nfunction checkRateLimit(): void {\n\tconst now = Date.now();\n\tconst windowStart = now - RATE_LIMIT.windowMs;\n\n\twhile (requestTimestamps.length > 0 && requestTimestamps[0] < windowStart) {\n\t\trequestTimestamps.shift();\n\t}\n\n\tif (requestTimestamps.length >= RATE_LIMIT.maxRequests) {\n\t\tconst waitMs = requestTimestamps[0] + RATE_LIMIT.windowMs - now;\n\t\tthrow new Error(`Rate limited. Try again in ${Math.ceil(waitMs / 1000)}s`);\n\t}\n\n\trequestTimestamps.push(now);\n}\n\nfunction validateDomainFilter(domains: string[]): string[] {\n\treturn domains.filter((d) => {\n\t\tconst domain = d.startsWith(\"-\") ? d.slice(1) : d;\n\t\treturn /^[a-zA-Z0-9][a-zA-Z0-9-_.]*\\.[a-zA-Z]{2,}$/.test(domain);\n\t});\n}\n\nexport function isPerplexityAvailable(): boolean {\n\tconst config = loadConfig();\n\treturn !!(normalizeApiKey(process.env.PERPLEXITY_API_KEY) ?? normalizeApiKey(config.perplexityApiKey));\n}\n\nexport async function searchWithPerplexity(query: string, options: SearchOptions = {}): Promise<SearchResponse> {\n\tcheckRateLimit();\n\n\tconst activityId = activityMonitor.logStart({ type: \"api\", query });\n\n\tactivityMonitor.updateRateLimit({\n\t\tused: requestTimestamps.length,\n\t\tmax: RATE_LIMIT.maxRequests,\n\t\toldestTimestamp: requestTimestamps[0] ?? null,\n\t\twindowMs: RATE_LIMIT.windowMs,\n\t});\n\n\tconst apiKey = getApiKey();\n\tconst numResults = Math.min(options.numResults ?? 5, 20);\n\n\tconst requestBody: Record<string, unknown> = {\n\t\tmodel: \"sonar\",\n\t\tmessages: [{ role: \"user\", content: query }],\n\t\tmax_tokens: 1024,\n\t\treturn_related_questions: false,\n\t};\n\n\tif (options.recencyFilter) {\n\t\trequestBody.search_recency_filter = options.recencyFilter;\n\t}\n\n\tif (options.domainFilter && options.domainFilter.length > 0) {\n\t\tconst validated = validateDomainFilter(options.domainFilter);\n\t\tif (validated.length > 0) {\n\t\t\trequestBody.search_domain_filter = validated;\n\t\t}\n\t}\n\n\tlet response: Response;\n\ttry {\n\t\tresponse = await fetch(PERPLEXITY_API_URL, {\n\t\t\tmethod: \"POST\",\n\t\t\theaders: {\n\t\t\t\tAuthorization: `Bearer ${apiKey}`,\n\t\t\t\t\"Content-Type\": \"application/json\",\n\t\t\t},\n\t\t\tbody: JSON.stringify(requestBody),\n\t\t\tsignal: options.signal,\n\t\t});\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tif (message.toLowerCase().includes(\"abort\")) {\n\t\t\tactivityMonitor.logComplete(activityId, 0);\n\t\t} else {\n\t\t\tactivityMonitor.logError(activityId, message);\n\t\t}\n\t\tthrow err;\n\t}\n\n\tif (!response.ok) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst errorText = await response.text();\n\t\tthrow new Error(`Perplexity API error ${response.status}: ${errorText}`);\n\t}\n\n\tlet data: Record<string, unknown>;\n\ttry {\n\t\tdata = await response.json();\n\t} catch (err) {\n\t\tactivityMonitor.logComplete(activityId, response.status);\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`Perplexity API returned invalid JSON: ${message}`);\n\t}\n\n\tconst answer = (data.choices as Array<{ message?: { content?: string } }>)?.[0]?.message?.content || \"\";\n\tconst citations = Array.isArray(data.citations) ? data.citations : [];\n\n\tconst results: SearchResult[] = [];\n\tfor (let i = 0; i < Math.min(citations.length, numResults); i++) {\n\t\tconst citation = citations[i];\n\t\tif (typeof citation === \"string\") {\n\t\t\tresults.push({ title: `Source ${i + 1}`, url: citation, snippet: \"\" });\n\t\t} else if (citation && typeof citation === \"object\" && typeof citation.url === \"string\") {\n\t\t\tresults.push({\n\t\t\t\ttitle: citation.title || `Source ${i + 1}`,\n\t\t\t\turl: citation.url,\n\t\t\t\tsnippet: \"\",\n\t\t\t});\n\t\t}\n\t}\n\n\tactivityMonitor.logComplete(activityId, response.status);\n\treturn { answer, results };\n}\n"]}