@librechat/agents 3.2.32 → 3.2.34

package/dist/cjs/tools/search/tavily-scraper.cjs

@@ -1,189 +1,137 @@
-'use strict';
-
-var axios = require('axios');
-var utils = require('./utils.cjs');
-
-const DEFAULT_BASIC_TIMEOUT = 15000;
-const DEFAULT_ADVANCED_TIMEOUT = 30000;
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_utils = require("./utils.cjs");
+let axios = require("axios");
+axios = require_runtime.__toESM(axios, 1);
+//#region src/tools/search/tavily-scraper.ts
+const DEFAULT_BASIC_TIMEOUT = 15e3;
+const DEFAULT_ADVANCED_TIMEOUT = 3e4;
 const MAX_BATCH_SIZE = 20;
-const getDefaultTimeout = (extractDepth) => extractDepth === 'advanced'
-    ? DEFAULT_ADVANCED_TIMEOUT
-    : DEFAULT_BASIC_TIMEOUT;
+const getDefaultTimeout = (extractDepth) => extractDepth === "advanced" ? DEFAULT_ADVANCED_TIMEOUT : DEFAULT_BASIC_TIMEOUT;
 const normalizeUrlKey = (url) => {
-    try {
-        const parsedUrl = new URL(url);
-        parsedUrl.hash = '';
-        if (parsedUrl.pathname.length > 1) {
-            parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '');
-        }
-        return parsedUrl.toString();
-    }
-    catch {
-        return url;
-    }
+	try {
+		const parsedUrl = new URL(url);
+		parsedUrl.hash = "";
+		if (parsedUrl.pathname.length > 1) parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, "");
+		return parsedUrl.toString();
+	} catch {
+		return url;
+	}
 };
 const setUrlResult = (map, result) => {
-    map.set(result.url, result);
-    const normalizedUrl = normalizeUrlKey(result.url);
-    if (!map.has(normalizedUrl)) {
-        map.set(normalizedUrl, result);
-    }
+	map.set(result.url, result);
+	const normalizedUrl = normalizeUrlKey(result.url);
+	if (!map.has(normalizedUrl)) map.set(normalizedUrl, result);
+};
+var TavilyScraper = class {
+	apiKey;
+	apiUrl;
+	timeout;
+	payloadTimeout;
+	logger;
+	extractDepth;
+	includeImages;
+	includeFavicon;
+	format;
+	constructor(config = {}) {
+		this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? "";
+		this.apiUrl = config.apiUrl ?? process.env.TAVILY_EXTRACT_URL ?? "https://api.tavily.com/extract";
+		this.payloadTimeout = config.timeout;
+		this.extractDepth = config.extractDepth ?? "basic";
+		this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);
+		this.includeImages = config.includeImages ?? false;
+		this.includeFavicon = config.includeFavicon ?? false;
+		this.format = config.format;
+		this.logger = config.logger || require_utils.createDefaultLogger();
+		if (!this.apiKey) this.logger.warn("TAVILY_API_KEY is not set. Scraping will not work.");
+	}
+	async scrapeUrl(url, options = {}) {
+		return (await this.scrapeUrls([url], options))[0];
+	}
+	async scrapeUrls(urls, options = {}) {
+		if (!this.apiKey) return urls.map((url) => [url, {
+			success: false,
+			error: "TAVILY_API_KEY is not set"
+		}]);
+		const batches = [];
+		for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) batches.push(urls.slice(i, i + MAX_BATCH_SIZE));
+		const allResults = [];
+		for (const batch of batches) {
+			const batchResults = await this.extractBatch(batch, options);
+			allResults.push(...batchResults);
+		}
+		return allResults;
+	}
+	async extractBatch(urls, options = {}) {
+		try {
+			const includeFavicon = options.includeFavicon ?? this.includeFavicon;
+			const format = options.format ?? this.format;
+			const extractDepth = options.extractDepth ?? this.extractDepth;
+			const payload = {
+				urls,
+				extract_depth: extractDepth,
+				include_images: options.includeImages ?? this.includeImages
+			};
+			if (includeFavicon) payload.include_favicon = true;
+			if (format != null) payload.format = format;
+			const effectiveTimeout = options.timeout ?? this.payloadTimeout ?? (options.extractDepth != null ? getDefaultTimeout(extractDepth) : this.timeout);
+			const payloadTimeout = options.timeout ?? this.payloadTimeout;
+			if (payloadTimeout != null) payload.timeout = Math.min(Math.max(payloadTimeout / 1e3, 1), 60);
+			const data = (await axios.default.post(this.apiUrl, payload, {
+				headers: {
+					Authorization: `Bearer ${this.apiKey}`,
+					"Content-Type": "application/json"
+				},
+				timeout: effectiveTimeout
+			})).data;
+			const successMap = /* @__PURE__ */ new Map();
+			const failedMap = /* @__PURE__ */ new Map();
+			for (const result of data.results ?? []) setUrlResult(successMap, result);
+			for (const result of data.failed_results ?? []) setUrlResult(failedMap, result);
+			return urls.map((url) => {
+				const success = successMap.get(url) ?? successMap.get(normalizeUrlKey(url));
+				if (success && success.error == null) return [url, {
+					success: true,
+					data: {
+						rawContent: success.raw_content ?? "",
+						images: success.images ?? [],
+						favicon: success.favicon
+					}
+				}];
+				const failed = failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));
+				return [url, {
+					success: false,
+					error: success?.error ?? failed?.error ?? "URL not found in Tavily Extract response"
+				}];
+			});
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : String(error);
+			return urls.map((url) => [url, {
+				success: false,
+				error: `Tavily Extract API request failed: ${errorMessage}`
+			}]);
+		}
+	}
+	extractContent(response) {
+		if (!response.success || !response.data) return ["", void 0];
+		const content = response.data.rawContent ?? "";
+		const images = response.data.images ?? [];
+		return [content, images.length > 0 ? {
+			links: [],
+			images: images.map((imageUrl) => ({ originalUrl: imageUrl })),
+			videos: []
+		} : void 0];
+	}
+	extractMetadata(response) {
+		if (!response.success || !response.data) return {};
+		const metadata = { images_count: response.data.images?.length ?? 0 };
+		if (response.data.favicon != null) metadata.favicon = response.data.favicon;
+		return metadata;
+	}
 };
-class TavilyScraper {
-    apiKey;
-    apiUrl;
-    timeout;
-    payloadTimeout;
-    logger;
-    extractDepth;
-    includeImages;
-    includeFavicon;
-    format;
-    constructor(config = {}) {
-        this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';
-        this.apiUrl =
-            config.apiUrl ??
-                process.env.TAVILY_EXTRACT_URL ??
-                'https://api.tavily.com/extract';
-        this.payloadTimeout = config.timeout;
-        this.extractDepth = config.extractDepth ?? 'basic';
-        this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);
-        this.includeImages = config.includeImages ?? false;
-        this.includeFavicon = config.includeFavicon ?? false;
-        this.format = config.format;
-        this.logger = config.logger || utils.createDefaultLogger();
-        if (!this.apiKey) {
-            this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');
-        }
-    }
-    async scrapeUrl(url, options = {}) {
-        const results = await this.scrapeUrls([url], options);
-        return results[0];
-    }
-    async scrapeUrls(urls, options = {}) {
-        if (!this.apiKey) {
-            return urls.map((url) => [
-                url,
-                { success: false, error: 'TAVILY_API_KEY is not set' },
-            ]);
-        }
-        const batches = [];
-        for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {
-            batches.push(urls.slice(i, i + MAX_BATCH_SIZE));
-        }
-        const allResults = [];
-        for (const batch of batches) {
-            const batchResults = await this.extractBatch(batch, options);
-            allResults.push(...batchResults);
-        }
-        return allResults;
-    }
-    async extractBatch(urls, options = {}) {
-        try {
-            const includeFavicon = options.includeFavicon ?? this.includeFavicon;
-            const format = options.format ?? this.format;
-            const extractDepth = options.extractDepth ?? this.extractDepth;
-            const payload = {
-                urls,
-                extract_depth: extractDepth,
-                include_images: options.includeImages ?? this.includeImages,
-            };
-            if (includeFavicon) {
-                payload.include_favicon = true;
-            }
-            if (format != null) {
-                payload.format = format;
-            }
-            const effectiveTimeout = options.timeout ??
-                this.payloadTimeout ??
-                (options.extractDepth != null
-                    ? getDefaultTimeout(extractDepth)
-                    : this.timeout);
-            const payloadTimeout = options.timeout ?? this.payloadTimeout;
-            if (payloadTimeout != null) {
-                payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);
-            }
-            const response = await axios.post(this.apiUrl, payload, {
-                headers: {
-                    Authorization: `Bearer ${this.apiKey}`,
-                    'Content-Type': 'application/json',
-                },
-                timeout: effectiveTimeout,
-            });
-            const data = response.data;
-            const successMap = new Map();
-            const failedMap = new Map();
-            for (const result of data.results ?? []) {
-                setUrlResult(successMap, result);
-            }
-            for (const result of data.failed_results ?? []) {
-                setUrlResult(failedMap, result);
-            }
-            return urls.map((url) => {
-                const success = successMap.get(url) ?? successMap.get(normalizeUrlKey(url));
-                if (success && success.error == null) {
-                    return [
-                        url,
-                        {
-                            success: true,
-                            data: {
-                                rawContent: success.raw_content ?? '',
-                                images: success.images ?? [],
-                                favicon: success.favicon,
-                            },
-                        },
-                    ];
-                }
-                const failed = failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));
-                const error = success?.error ??
-                    failed?.error ??
-                    'URL not found in Tavily Extract response';
-                return [url, { success: false, error }];
-            });
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return urls.map((url) => [
-                url,
-                {
-                    success: false,
-                    error: `Tavily Extract API request failed: ${errorMessage}`,
-                },
-            ]);
-        }
-    }
-    extractContent(response) {
-        if (!response.success || !response.data) {
-            return ['', undefined];
-        }
-        const content = response.data.rawContent ?? '';
-        const images = response.data.images ?? [];
-        const references = images.length > 0
-            ? {
-                links: [],
-                images: images.map((imageUrl) => ({ originalUrl: imageUrl })),
-                videos: [],
-            }
-            : undefined;
-        return [content, references];
-    }
-    extractMetadata(response) {
-        if (!response.success || !response.data) {
-            return {};
-        }
-        const metadata = {
-            images_count: response.data.images?.length ?? 0,
-        };
-        if (response.data.favicon != null) {
-            metadata.favicon = response.data.favicon;
-        }
-        return metadata;
-    }
-}
 const createTavilyScraper = (config = {}) => {
-    return new TavilyScraper(config);
+	return new TavilyScraper(config);
 };
-
-exports.TavilyScraper = TavilyScraper;
+//#endregion
 exports.createTavilyScraper = createTavilyScraper;
-//# sourceMappingURL=tavily-scraper.cjs.map
+
+//# sourceMappingURL=tavily-scraper.cjs.map

package/dist/cjs/tools/search/tavily-scraper.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"tavily-scraper.cjs","sources":["../../../../src/tools/search/tavily-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_BASIC_TIMEOUT = 15000;\nconst DEFAULT_ADVANCED_TIMEOUT = 30000;\nconst MAX_BATCH_SIZE = 20;\n\nconst getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>\n  extractDepth === 'advanced'\n    ? DEFAULT_ADVANCED_TIMEOUT\n    : DEFAULT_BASIC_TIMEOUT;\n\nconst normalizeUrlKey = (url: string): string => {\n  try {\n    const parsedUrl = new URL(url);\n    parsedUrl.hash = '';\n    if (parsedUrl.pathname.length > 1) {\n      parsedUrl.pathname = parsedUrl.pathname.replace(/\\/+$/, '');\n    }\n    return parsedUrl.toString();\n  } catch {\n    return url;\n  }\n};\n\nconst setUrlResult = (\n  map: Map<string, t.TavilyExtractResult>,\n  result: t.TavilyExtractResult\n): void => {\n  map.set(result.url, result);\n  const normalizedUrl = normalizeUrlKey(result.url);\n  if (!map.has(normalizedUrl)) {\n    map.set(normalizedUrl, result);\n  }\n};\n\nexport class TavilyScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private payloadTimeout: number | undefined;\n  private logger: t.Logger;\n  private extractDepth: 'basic' | 'advanced';\n  private includeImages: boolean;\n  private includeFavicon: boolean;\n  private format: 'markdown' | 'text' | undefined;\n\n  constructor(config: t.TavilyScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.TAVILY_EXTRACT_URL ??\n      'https://api.tavily.com/extract';\n    this.payloadTimeout = config.timeout;\n    this.extractDepth = config.extractDepth ?? 'basic';\n    this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);\n    this.includeImages = config.includeImages ?? false;\n    this.includeFavicon = config.includeFavicon ?? false;\n    this.format = config.format;\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');\n    }\n  }\n\n  async scrapeUrl(\n    url: string,\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<[string, t.TavilyScrapeResponse]> {\n    const results = await this.scrapeUrls([url], options);\n    return results[0];\n  }\n\n  async scrapeUrls(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    if (!this.apiKey) {\n      return urls.map((url) => [\n        url,\n        { success: false, error: 'TAVILY_API_KEY is not set' },\n      ]);\n    }\n\n    const batches: string[][] = [];\n    for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {\n      batches.push(urls.slice(i, i + MAX_BATCH_SIZE));\n    }\n\n    const allResults: Array<[string, t.TavilyScrapeResponse]> = [];\n\n    for (const batch of batches) {\n      const batchResults = await this.extractBatch(batch, options);\n      allResults.push(...batchResults);\n    }\n\n    return allResults;\n  }\n\n  private async extractBatch(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    try {\n      const includeFavicon = options.includeFavicon ?? this.includeFavicon;\n      const format = options.format ?? this.format;\n      const extractDepth = options.extractDepth ?? this.extractDepth;\n      const payload: t.TavilyExtractPayload = {\n        urls,\n        extract_depth: extractDepth,\n        include_images: options.includeImages ?? this.includeImages,\n      };\n\n      if (includeFavicon) {\n        payload.include_favicon = true;\n      }\n      if (format != null) {\n        payload.format = format;\n      }\n\n      const effectiveTimeout =\n        options.timeout ??\n        this.payloadTimeout ??\n        (options.extractDepth != null\n          ? getDefaultTimeout(extractDepth)\n          : this.timeout);\n      const payloadTimeout = options.timeout ?? this.payloadTimeout;\n      if (payloadTimeout != null) {\n        payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);\n      }\n\n      const response = await axios.post<{\n        results?: t.TavilyExtractResult[];\n        failed_results?: t.TavilyExtractResult[];\n      }>(this.apiUrl, payload, {\n        headers: {\n          Authorization: `Bearer ${this.apiKey}`,\n          'Content-Type': 'application/json',\n        },\n        timeout: effectiveTimeout,\n      });\n\n      const data = response.data;\n      const successMap = new Map<string, t.TavilyExtractResult>();\n      const failedMap = new Map<string, t.TavilyExtractResult>();\n\n      for (const result of data.results ?? []) {\n        setUrlResult(successMap, result);\n      }\n      for (const result of data.failed_results ?? []) {\n        setUrlResult(failedMap, result);\n      }\n\n      return urls.map((url): [string, t.TavilyScrapeResponse] => {\n        const success =\n          successMap.get(url) ?? successMap.get(normalizeUrlKey(url));\n        if (success && success.error == null) {\n          return [\n            url,\n            {\n              success: true,\n              data: {\n                rawContent: success.raw_content ?? '',\n                images: success.images ?? [],\n                favicon: success.favicon,\n              },\n            },\n          ];\n        }\n\n        const failed =\n          failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));\n        const error =\n          success?.error ??\n          failed?.error ??\n          'URL not found in Tavily Extract response';\n        return [url, { success: false, error }];\n      });\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return urls.map((url) => [\n        url,\n        {\n          success: false,\n          error: `Tavily Extract API request failed: ${errorMessage}`,\n        },\n      ]);\n    }\n  }\n\n  extractContent(\n    response: t.TavilyScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    const content = response.data.rawContent ?? '';\n    const images = response.data.images ?? [];\n\n    const references: t.References | undefined =\n      images.length > 0\n        ? {\n          links: [],\n          images: images.map((imageUrl) => ({ originalUrl: imageUrl })),\n          videos: [],\n        }\n        : undefined;\n\n    return [content, references];\n  }\n\n  extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {\n    if (!response.success || !response.data) {\n      return {};\n    }\n\n    const metadata: t.GenericScrapeMetadata = {\n      images_count: response.data.images?.length ?? 0,\n    };\n    if (response.data.favicon != null) {\n      metadata.favicon = response.data.favicon;\n    }\n    return metadata;\n  }\n}\n\nexport const createTavilyScraper = (\n  config: t.TavilyScraperConfig = {}\n): TavilyScraper => {\n  return new TavilyScraper(config);\n};\n"],"names":["createDefaultLogger"],"mappings":";;;;;AAIA,MAAM,qBAAqB,GAAG,KAAK;AACnC,MAAM,wBAAwB,GAAG,KAAK;AACtC,MAAM,cAAc,GAAG,EAAE;AAEzB,MAAM,iBAAiB,GAAG,CAAC,YAAkC,KAC3D,YAAY,KAAK;AACf,MAAE;MACA,qBAAqB;AAE3B,MAAM,eAAe,GAAG,CAAC,GAAW,KAAY;AAC9C,IAAA,IAAI;AACF,QAAA,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC9B,QAAA,SAAS,CAAC,IAAI,GAAG,EAAE;QACnB,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,YAAA,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QAC7D;AACA,QAAA,OAAO,SAAS,CAAC,QAAQ,EAAE;IAC7B;AAAE,IAAA,MAAM;AACN,QAAA,OAAO,GAAG;IACZ;AACF,CAAC;AAED,MAAM,YAAY,GAAG,CACnB,GAAuC,EACvC,MAA6B,KACrB;IACR,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC;IAC3B,MAAM,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;IACjD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE;AAC3B,QAAA,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC;IAChC;AACF,CAAC;MAEY,aAAa,CAAA;AAChB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,cAAc;AACd,IAAA,MAAM;AACN,IAAA,YAAY;AACZ,IAAA,aAAa;AACb,IAAA,cAAc;AACd,IAAA,MAAM;AAEd,IAAA,WAAA,CAAY,SAAgC,EAAE,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;AAC/D,QAAA,IAAI,CAAC,MAAM;AACT,YAAA,MAAM,CAAC,MAAM;gBACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,gBAAA,gCAAgC;AAClC,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO;QACpC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,OAAO;AAClD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;QACrE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,KAAK;QAClD,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,KAAK;AACpD,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC;QACxE;IACF;AAEA,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAiC,EAAE,EAAA;AAEnC,QAAA,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;AACrD,QAAA,OAAO,OAAO,CAAC,CAAC,CAAC;IACnB;AAEA,IAAA,MAAM,UAAU,CACd,IAAc,EACd,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK;gBACvB,GAAG;AACH,gBAAA,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,2BAA2B,EAAE;AACvD,aAAA,CAAC;QACJ;QAEA,MAAM,OAAO,GAAe,EAAE;AAC9B,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE;AACpD,YAAA,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;QACjD;QAEA,MAAM,UAAU,GAA4C,EAAE;AAE9D,QAAA,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;YAC3B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC;AAC5D,YAAA,UAAU,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC;QAClC;AAEA,QAAA,OAAO,UAAU;IACnB;AAEQ,IAAA,MAAM,YAAY,CACxB,IAAc,EACd,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI;YACF,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc;YACpE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;YAC5C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AAC9D,YAAA,MAAM,OAAO,GAA2B;gBACtC,IAAI;AACJ,gBAAA,aAAa,EAAE,YAAY;AAC3B,gBAAA,cAAc,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa;aAC5D;YAED,IAAI,cAAc,EAAE;AAClB,gBAAA,OAAO,CAAC,eAAe,GAAG,IAAI;YAChC;AACA,YAAA,IAAI,MAAM,IAAI,IAAI,EAAE;AAClB,gBAAA,OAAO,CAAC,MAAM,GAAG,MAAM;YACzB;AAEA,YAAA,MAAM,gBAAgB,GACpB,OAAO,CAAC,OAAO;AACf,gBAAA,IAAI,CAAC,cAAc;AACnB,iBAAC,OAAO,CAAC,YAAY,IAAI;AACvB,sBAAE,iBAAiB,CAAC,YAAY;AAChC,sBAAE,IAAI,CAAC,OAAO,CAAC;YACnB,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC7D,YAAA,IAAI,cAAc,IAAI,IAAI,EAAE;gBAC1B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;YACpE;AAEA,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAG9B,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACvB,gBAAA,OAAO,EAAE;AACP,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAA,CAAE;AACtC,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;AACD,gBAAA,OAAO,EAAE,gBAAgB;AAC1B,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,UAAU,GAAG,IAAI,GAAG,EAAiC;AAC3D,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAiC;YAE1D,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE;AACvC,gBAAA,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC;YAClC;YACA,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,cAAc,IAAI,EAAE,EAAE;AAC9C,gBAAA,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC;YACjC;AAEA,YAAA,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAsC;AACxD,gBAAA,MAAM,OAAO,GACX,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBAC7D,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE;oBACpC,OAAO;wBACL,GAAG;AACH,wBAAA;AACE,4BAAA,OAAO,EAAE,IAAI;AACb,4BAAA,IAAI,EAAE;AACJ,gCAAA,UAAU,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;AACrC,gCAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE;gCAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;AACzB,6BAAA;AACF,yBAAA;qBACF;gBACH;AAEA,gBAAA,MAAM,MAAM,GACV,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;AAC3D,gBAAA,MAAM,KAAK,GACT,OAAO,EAAE,KAAK;AACd,oBAAA,MAAM,EAAE,KAAK;AACb,oBAAA,0CAA0C;gBAC5C,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACzC,YAAA,CAAC,CAAC;QACJ;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK;gBACvB,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAA,mCAAA,EAAsC,YAAY,CAAA,CAAE;AAC5D,iBAAA;AACF,aAAA,CAAC;QACJ;IACF;AAEA,IAAA,cAAc,CACZ,QAAgC,EAAA;QAEhC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;QACxB;QAEA,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;QAC9C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAEzC,QAAA,MAAM,UAAU,GACd,MAAM,CAAC,MAAM,GAAG;AACd,cAAE;AACA,gBAAA,KAAK,EAAE,EAAE;AACT,gBAAA,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC7D,gBAAA,MAAM,EAAE,EAAE;AACX;cACC,SAAS;AAEf,QAAA,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC;IAC9B;AAEA,IAAA,eAAe,CAAC,QAAgC,EAAA;QAC9C,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,EAAE;QACX;AAEA,QAAA,MAAM,QAAQ,GAA4B;YACxC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC;SAChD;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO;QAC1C;AACA,QAAA,OAAO,QAAQ;IACjB;AACD;MAEY,mBAAmB,GAAG,CACjC,MAAA,GAAgC,EAAE,KACjB;AACjB,IAAA,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC;AAClC;;;;;"}
+{"version":3,"file":"tavily-scraper.cjs","names":["createDefaultLogger"],"sources":["../../../../src/tools/search/tavily-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_BASIC_TIMEOUT = 15000;\nconst DEFAULT_ADVANCED_TIMEOUT = 30000;\nconst MAX_BATCH_SIZE = 20;\n\nconst getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>\n  extractDepth === 'advanced'\n    ? DEFAULT_ADVANCED_TIMEOUT\n    : DEFAULT_BASIC_TIMEOUT;\n\nconst normalizeUrlKey = (url: string): string => {\n  try {\n    const parsedUrl = new URL(url);\n    parsedUrl.hash = '';\n    if (parsedUrl.pathname.length > 1) {\n      parsedUrl.pathname = parsedUrl.pathname.replace(/\\/+$/, '');\n    }\n    return parsedUrl.toString();\n  } catch {\n    return url;\n  }\n};\n\nconst setUrlResult = (\n  map: Map<string, t.TavilyExtractResult>,\n  result: t.TavilyExtractResult\n): void => {\n  map.set(result.url, result);\n  const normalizedUrl = normalizeUrlKey(result.url);\n  if (!map.has(normalizedUrl)) {\n    map.set(normalizedUrl, result);\n  }\n};\n\nexport class TavilyScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private payloadTimeout: number | undefined;\n  private logger: t.Logger;\n  private extractDepth: 'basic' | 'advanced';\n  private includeImages: boolean;\n  private includeFavicon: boolean;\n  private format: 'markdown' | 'text' | undefined;\n\n  constructor(config: t.TavilyScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.TAVILY_EXTRACT_URL ??\n      'https://api.tavily.com/extract';\n    this.payloadTimeout = config.timeout;\n    this.extractDepth = config.extractDepth ?? 'basic';\n    this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);\n    this.includeImages = config.includeImages ?? false;\n    this.includeFavicon = config.includeFavicon ?? false;\n    this.format = config.format;\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');\n    }\n  }\n\n  async scrapeUrl(\n    url: string,\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<[string, t.TavilyScrapeResponse]> {\n    const results = await this.scrapeUrls([url], options);\n    return results[0];\n  }\n\n  async scrapeUrls(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    if (!this.apiKey) {\n      return urls.map((url) => [\n        url,\n        { success: false, error: 'TAVILY_API_KEY is not set' },\n      ]);\n    }\n\n    const batches: string[][] = [];\n    for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {\n      batches.push(urls.slice(i, i + MAX_BATCH_SIZE));\n    }\n\n    const allResults: Array<[string, t.TavilyScrapeResponse]> = [];\n\n    for (const batch of batches) {\n      const batchResults = await this.extractBatch(batch, options);\n      allResults.push(...batchResults);\n    }\n\n    return allResults;\n  }\n\n  private async extractBatch(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    try {\n      const includeFavicon = options.includeFavicon ?? this.includeFavicon;\n      const format = options.format ?? this.format;\n      const extractDepth = options.extractDepth ?? this.extractDepth;\n      const payload: t.TavilyExtractPayload = {\n        urls,\n        extract_depth: extractDepth,\n        include_images: options.includeImages ?? this.includeImages,\n      };\n\n      if (includeFavicon) {\n        payload.include_favicon = true;\n      }\n      if (format != null) {\n        payload.format = format;\n      }\n\n      const effectiveTimeout =\n        options.timeout ??\n        this.payloadTimeout ??\n        (options.extractDepth != null\n          ? getDefaultTimeout(extractDepth)\n          : this.timeout);\n      const payloadTimeout = options.timeout ?? this.payloadTimeout;\n      if (payloadTimeout != null) {\n        payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);\n      }\n\n      const response = await axios.post<{\n        results?: t.TavilyExtractResult[];\n        failed_results?: t.TavilyExtractResult[];\n      }>(this.apiUrl, payload, {\n        headers: {\n          Authorization: `Bearer ${this.apiKey}`,\n          'Content-Type': 'application/json',\n        },\n        timeout: effectiveTimeout,\n      });\n\n      const data = response.data;\n      const successMap = new Map<string, t.TavilyExtractResult>();\n      const failedMap = new Map<string, t.TavilyExtractResult>();\n\n      for (const result of data.results ?? []) {\n        setUrlResult(successMap, result);\n      }\n      for (const result of data.failed_results ?? []) {\n        setUrlResult(failedMap, result);\n      }\n\n      return urls.map((url): [string, t.TavilyScrapeResponse] => {\n        const success =\n          successMap.get(url) ?? successMap.get(normalizeUrlKey(url));\n        if (success && success.error == null) {\n          return [\n            url,\n            {\n              success: true,\n              data: {\n                rawContent: success.raw_content ?? '',\n                images: success.images ?? [],\n                favicon: success.favicon,\n              },\n            },\n          ];\n        }\n\n        const failed =\n          failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));\n        const error =\n          success?.error ??\n          failed?.error ??\n          'URL not found in Tavily Extract response';\n        return [url, { success: false, error }];\n      });\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return urls.map((url) => [\n        url,\n        {\n          success: false,\n          error: `Tavily Extract API request failed: ${errorMessage}`,\n        },\n      ]);\n    }\n  }\n\n  extractContent(\n    response: t.TavilyScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    const content = response.data.rawContent ?? '';\n    const images = response.data.images ?? [];\n\n    const references: t.References | undefined =\n      images.length > 0\n        ? {\n          links: [],\n          images: images.map((imageUrl) => ({ originalUrl: imageUrl })),\n          videos: [],\n        }\n        : undefined;\n\n    return [content, references];\n  }\n\n  extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {\n    if (!response.success || !response.data) {\n      return {};\n    }\n\n    const metadata: t.GenericScrapeMetadata = {\n      images_count: response.data.images?.length ?? 0,\n    };\n    if (response.data.favicon != null) {\n      metadata.favicon = response.data.favicon;\n    }\n    return metadata;\n  }\n}\n\nexport const createTavilyScraper = (\n  config: t.TavilyScraperConfig = {}\n): TavilyScraper => {\n  return new TavilyScraper(config);\n};\n"],"mappings":";;;;;AAIA,MAAM,wBAAwB;AAC9B,MAAM,2BAA2B;AACjC,MAAM,iBAAiB;AAEvB,MAAM,qBAAqB,iBACzB,iBAAiB,aACb,2BACA;AAEN,MAAM,mBAAmB,QAAwB;CAC/C,IAAI;EACF,MAAM,YAAY,IAAI,IAAI,GAAG;EAC7B,UAAU,OAAO;EACjB,IAAI,UAAU,SAAS,SAAS,GAC9B,UAAU,WAAW,UAAU,SAAS,QAAQ,QAAQ,EAAE;EAE5D,OAAO,UAAU,SAAS;CAC5B,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,gBACJ,KACA,WACS;CACT,IAAI,IAAI,OAAO,KAAK,MAAM;CAC1B,MAAM,gBAAgB,gBAAgB,OAAO,GAAG;CAChD,IAAI,CAAC,IAAI,IAAI,aAAa,GACxB,IAAI,IAAI,eAAe,MAAM;AAEjC;AAEA,IAAa,gBAAb,MAAoD;CAClD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAgC,CAAC,GAAG;EAC9C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,kBAAkB;EAC7D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,iBAAiB,OAAO;EAC7B,KAAK,eAAe,OAAO,gBAAgB;EAC3C,KAAK,UAAU,OAAO,WAAW,kBAAkB,KAAK,YAAY;EACpE,KAAK,gBAAgB,OAAO,iBAAiB;EAC7C,KAAK,iBAAiB,OAAO,kBAAkB;EAC/C,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO,UAAUA,cAAAA,oBAAoB;EAEnD,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,oDAAoD;CAEzE;CAEA,MAAM,UACJ,KACA,UAAiC,CAAC,GACS;EAE3C,QAAO,MADe,KAAK,WAAW,CAAC,GAAG,GAAG,OAAO,EAAA,CACrC;CACjB;CAEA,MAAM,WACJ,MACA,UAAiC,CAAC,GACgB;EAClD,IAAI,CAAC,KAAK,QACR,OAAO,KAAK,KAAK,QAAQ,CACvB,KACA;GAAE,SAAS;GAAO,OAAO;EAA4B,CACvD,CAAC;EAGH,MAAM,UAAsB,CAAC;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,gBACpC,QAAQ,KAAK,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC;EAGhD,MAAM,aAAsD,CAAC;EAE7D,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,eAAe,MAAM,KAAK,aAAa,OAAO,OAAO;GAC3D,WAAW,KAAK,GAAG,YAAY;EACjC;EAEA,OAAO;CACT;CAEA,MAAc,aACZ,MACA,UAAiC,CAAC,GACgB;EAClD,IAAI;GACF,MAAM,iBAAiB,QAAQ,kBAAkB,KAAK;GACtD,MAAM,SAAS,QAAQ,UAAU,KAAK;GACtC,MAAM,eAAe,QAAQ,gBAAgB,KAAK;GAClD,MAAM,UAAkC;IACtC;IACA,eAAe;IACf,gBAAgB,QAAQ,iBAAiB,KAAK;GAChD;GAEA,IAAI,gBACF,QAAQ,kBAAkB;GAE5B,IAAI,UAAU,MACZ,QAAQ,SAAS;GAGnB,MAAM,mBACJ,QAAQ,WACR,KAAK,mBACJ,QAAQ,gBAAgB,OACrB,kBAAkB,YAAY,IAC9B,KAAK;GACX,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,IAAI,kBAAkB,MACpB,QAAQ,UAAU,KAAK,IAAI,KAAK,IAAI,iBAAiB,KAAM,CAAC,GAAG,EAAE;GAcnE,MAAM,QAAO,MAXU,MAAA,QAAM,KAG1B,KAAK,QAAQ,SAAS;IACvB,SAAS;KACP,eAAe,UAAU,KAAK;KAC9B,gBAAgB;IAClB;IACA,SAAS;GACX,CAAC,EAAA,CAEqB;GACtB,MAAM,6BAAa,IAAI,IAAmC;GAC1D,MAAM,4BAAY,IAAI,IAAmC;GAEzD,KAAK,MAAM,UAAU,KAAK,WAAW,CAAC,GACpC,aAAa,YAAY,MAAM;GAEjC,KAAK,MAAM,UAAU,KAAK,kBAAkB,CAAC,GAC3C,aAAa,WAAW,MAAM;GAGhC,OAAO,KAAK,KAAK,QAA0C;IACzD,MAAM,UACJ,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,gBAAgB,GAAG,CAAC;IAC5D,IAAI,WAAW,QAAQ,SAAS,MAC9B,OAAO,CACL,KACA;KACE,SAAS;KACT,MAAM;MACJ,YAAY,QAAQ,eAAe;MACnC,QAAQ,QAAQ,UAAU,CAAC;MAC3B,SAAS,QAAQ;KACnB;IACF,CACF;IAGF,MAAM,SACJ,UAAU,IAAI,GAAG,KAAK,UAAU,IAAI,gBAAgB,GAAG,CAAC;IAK1D,OAAO,CAAC,KAAK;KAAE,SAAS;KAAO,OAH7B,SAAS,SACT,QAAQ,SACR;IACmC,CAAC;GACxC,CAAC;EACH,SAAS,OAAO;GACd,MAAM,eACJ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACvD,OAAO,KAAK,KAAK,QAAQ,CACvB,KACA;IACE,SAAS;IACT,OAAO,sCAAsC;GAC/C,CACF,CAAC;EACH;CACF;CAEA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,UAAU,SAAS,KAAK,cAAc;EAC5C,MAAM,SAAS,SAAS,KAAK,UAAU,CAAC;EAWxC,OAAO,CAAC,SARN,OAAO,SAAS,IACZ;GACA,OAAO,CAAC;GACR,QAAQ,OAAO,KAAK,cAAc,EAAE,aAAa,SAAS,EAAE;GAC5D,QAAQ,CAAC;EACX,IACE,KAAA,CAEqB;CAC7B;CAEA,gBAAgB,UAA2D;EACzE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC;EAGV,MAAM,WAAoC,EACxC,cAAc,SAAS,KAAK,QAAQ,UAAU,EAChD;EACA,IAAI,SAAS,KAAK,WAAW,MAC3B,SAAS,UAAU,SAAS,KAAK;EAEnC,OAAO;CACT;AACF;AAEA,MAAa,uBACX,SAAgC,CAAC,MACf;CAClB,OAAO,IAAI,cAAc,MAAM;AACjC"}