@librechat/agents 3.2.32 → 3.2.34

package/dist/esm/tools/search/firecrawl.mjs

@@ -1,176 +1,146 @@
-import axios from 'axios';
-import { createDefaultLogger } from './utils.mjs';
-import { processContent } from './content.mjs';
-
+import { createDefaultLogger } from "./utils.mjs";
+import { processContent } from "./content.mjs";
+import axios from "axios";
+//#region src/tools/search/firecrawl.ts
 /**
- * Firecrawl scraper implementation
- * Uses the Firecrawl API to scrape web pages
- */
-class FirecrawlScraper {
-    apiKey;
-    apiUrl;
-    version;
-    defaultFormats;
-    timeout;
-    logger;
-    includeTags;
-    excludeTags;
-    waitFor;
-    maxAge;
-    mobile;
-    skipTlsVerification;
-    blockAds;
-    removeBase64Images;
-    parsePDF;
-    storeInCache;
-    zeroDataRetention;
-    headers;
-    location;
-    onlyMainContent;
-    changeTrackingOptions;
-    constructor(config = {}) {
-        this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
-        this.version = config.version ?? 'v2';
-        const baseUrl = config.apiUrl ??
-            process.env.FIRECRAWL_BASE_URL ??
-            'https://api.firecrawl.dev';
-        this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/${this.version}/scrape`;
-        this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
-        this.timeout = config.timeout ?? 7500;
-        this.logger = config.logger || createDefaultLogger();
-        this.includeTags = config.includeTags;
-        this.excludeTags = config.excludeTags;
-        this.waitFor = config.waitFor;
-        this.maxAge = config.maxAge;
-        this.mobile = config.mobile;
-        this.skipTlsVerification = config.skipTlsVerification;
-        this.blockAds = config.blockAds;
-        this.removeBase64Images = config.removeBase64Images;
-        this.parsePDF = config.parsePDF;
-        this.storeInCache = config.storeInCache;
-        this.zeroDataRetention = config.zeroDataRetention;
-        this.headers = config.headers;
-        this.location = config.location;
-        this.onlyMainContent = config.onlyMainContent;
-        this.changeTrackingOptions = config.changeTrackingOptions;
-        if (!this.apiKey) {
-            this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
-        }
-        this.logger.debug(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
-    }
-    /**
-     * Scrape a single URL
-     * @param url URL to scrape
-     * @param options Scrape options
-     * @returns Scrape response
-     */
-    async scrapeUrl(url, options = {}) {
-        if (!this.apiKey) {
-            return [
-                url,
-                {
-                    success: false,
-                    error: 'FIRECRAWL_API_KEY is not set',
-                },
-            ];
-        }
-        try {
-            const payload = omitUndefined({
-                url,
-                formats: options.formats ?? this.defaultFormats,
-                includeTags: options.includeTags ?? this.includeTags,
-                excludeTags: options.excludeTags ?? this.excludeTags,
-                headers: options.headers ?? this.headers,
-                waitFor: options.waitFor ?? this.waitFor,
-                timeout: options.timeout ?? this.timeout,
-                onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
-                maxAge: options.maxAge ?? this.maxAge,
-                mobile: options.mobile ?? this.mobile,
-                skipTlsVerification: options.skipTlsVerification ?? this.skipTlsVerification,
-                parsePDF: options.parsePDF ?? this.parsePDF,
-                location: options.location ?? this.location,
-                removeBase64Images: options.removeBase64Images ?? this.removeBase64Images,
-                blockAds: options.blockAds ?? this.blockAds,
-                storeInCache: options.storeInCache ?? this.storeInCache,
-                zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,
-                changeTrackingOptions: options.changeTrackingOptions ?? this.changeTrackingOptions,
-            });
-            const response = await axios.post(this.apiUrl, payload, {
-                headers: {
-                    'Content-Type': 'application/json',
-                    Authorization: `Bearer ${this.apiKey}`,
-                },
-                timeout: this.timeout,
-            });
-            return [url, response.data];
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return [
-                url,
-                {
-                    success: false,
-                    error: `Firecrawl API request failed: ${errorMessage}`,
-                },
-            ];
-        }
-    }
-    /**
-     * Extract content from scrape response
-     * @param response Scrape response
-     * @returns Extracted content or empty string if not available
-     */
-    extractContent(response) {
-        if (!response.success || !response.data) {
-            return ['', undefined];
-        }
-        if (response.data.markdown != null && response.data.html != null) {
-            try {
-                const { markdown, ...rest } = processContent(response.data.html, response.data.markdown);
-                return [markdown, rest];
-            }
-            catch (error) {
-                this.logger.error('Error processing content:', error);
-                return [response.data.markdown, undefined];
-            }
-        }
-        else if (response.data.markdown != null) {
-            return [response.data.markdown, undefined];
-        }
-        // Fall back to HTML content
-        if (response.data.html != null) {
-            return [response.data.html, undefined];
-        }
-        // Fall back to raw HTML content
-        if (response.data.rawHtml != null) {
-            return [response.data.rawHtml, undefined];
-        }
-        return ['', undefined];
-    }
-    /**
-     * Extract metadata from scrape response
-     * @param response Scrape response
-     * @returns Metadata object
-     */
-    extractMetadata(response) {
-        if (!response.success || !response.data || !response.data.metadata) {
-            return {};
-        }
-        return response.data.metadata;
-    }
-}
+* Firecrawl scraper implementation
+* Uses the Firecrawl API to scrape web pages
+*/
+var FirecrawlScraper = class {
+	apiKey;
+	apiUrl;
+	version;
+	defaultFormats;
+	timeout;
+	logger;
+	includeTags;
+	excludeTags;
+	waitFor;
+	maxAge;
+	mobile;
+	skipTlsVerification;
+	blockAds;
+	removeBase64Images;
+	parsePDF;
+	storeInCache;
+	zeroDataRetention;
+	headers;
+	location;
+	onlyMainContent;
+	changeTrackingOptions;
+	constructor(config = {}) {
+		this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";
+		this.version = config.version ?? "v2";
+		const baseUrl = config.apiUrl ?? process.env.FIRECRAWL_BASE_URL ?? "https://api.firecrawl.dev";
+		this.apiUrl = `${baseUrl.replace(/\/+$/, "")}/${this.version}/scrape`;
+		this.defaultFormats = config.formats ?? ["markdown", "rawHtml"];
+		this.timeout = config.timeout ?? 7500;
+		this.logger = config.logger || createDefaultLogger();
+		this.includeTags = config.includeTags;
+		this.excludeTags = config.excludeTags;
+		this.waitFor = config.waitFor;
+		this.maxAge = config.maxAge;
+		this.mobile = config.mobile;
+		this.skipTlsVerification = config.skipTlsVerification;
+		this.blockAds = config.blockAds;
+		this.removeBase64Images = config.removeBase64Images;
+		this.parsePDF = config.parsePDF;
+		this.storeInCache = config.storeInCache;
+		this.zeroDataRetention = config.zeroDataRetention;
+		this.headers = config.headers;
+		this.location = config.location;
+		this.onlyMainContent = config.onlyMainContent;
+		this.changeTrackingOptions = config.changeTrackingOptions;
+		if (!this.apiKey) this.logger.warn("FIRECRAWL_API_KEY is not set. Scraping will not work.");
+		this.logger.debug(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
+	}
+	/**
+	* Scrape a single URL
+	* @param url URL to scrape
+	* @param options Scrape options
+	* @returns Scrape response
+	*/
+	async scrapeUrl(url, options = {}) {
+		if (!this.apiKey) return [url, {
+			success: false,
+			error: "FIRECRAWL_API_KEY is not set"
+		}];
+		try {
+			const payload = omitUndefined({
+				url,
+				formats: options.formats ?? this.defaultFormats,
+				includeTags: options.includeTags ?? this.includeTags,
+				excludeTags: options.excludeTags ?? this.excludeTags,
+				headers: options.headers ?? this.headers,
+				waitFor: options.waitFor ?? this.waitFor,
+				timeout: options.timeout ?? this.timeout,
+				onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
+				maxAge: options.maxAge ?? this.maxAge,
+				mobile: options.mobile ?? this.mobile,
+				skipTlsVerification: options.skipTlsVerification ?? this.skipTlsVerification,
+				parsePDF: options.parsePDF ?? this.parsePDF,
+				location: options.location ?? this.location,
+				removeBase64Images: options.removeBase64Images ?? this.removeBase64Images,
+				blockAds: options.blockAds ?? this.blockAds,
+				storeInCache: options.storeInCache ?? this.storeInCache,
+				zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,
+				changeTrackingOptions: options.changeTrackingOptions ?? this.changeTrackingOptions
+			});
+			return [url, (await axios.post(this.apiUrl, payload, {
+				headers: {
+					"Content-Type": "application/json",
+					Authorization: `Bearer ${this.apiKey}`
+				},
+				timeout: this.timeout
+			})).data];
+		} catch (error) {
+			return [url, {
+				success: false,
+				error: `Firecrawl API request failed: ${error instanceof Error ? error.message : String(error)}`
+			}];
+		}
+	}
+	/**
+	* Extract content from scrape response
+	* @param response Scrape response
+	* @returns Extracted content or empty string if not available
+	*/
+	extractContent(response) {
+		if (!response.success || !response.data) return ["", void 0];
+		if (response.data.markdown != null && response.data.html != null) try {
+			const { markdown, ...rest } = processContent(response.data.html, response.data.markdown);
+			return [markdown, rest];
+		} catch (error) {
+			this.logger.error("Error processing content:", error);
+			return [response.data.markdown, void 0];
+		}
+		else if (response.data.markdown != null) return [response.data.markdown, void 0];
+		if (response.data.html != null) return [response.data.html, void 0];
+		if (response.data.rawHtml != null) return [response.data.rawHtml, void 0];
+		return ["", void 0];
+	}
+	/**
+	* Extract metadata from scrape response
+	* @param response Scrape response
+	* @returns Metadata object
+	*/
+	extractMetadata(response) {
+		if (!response.success || !response.data || !response.data.metadata) return {};
+		return response.data.metadata;
+	}
+};
 /**
- * Create a Firecrawl scraper instance
- * @param config Scraper configuration
- * @returns Firecrawl scraper instance
- */
+* Create a Firecrawl scraper instance
+* @param config Scraper configuration
+* @returns Firecrawl scraper instance
+*/
 const createFirecrawlScraper = (config = {}) => {
-    return new FirecrawlScraper(config);
+	return new FirecrawlScraper(config);
 };
-// Helper function to clean up payload for firecrawl
 function omitUndefined(obj) {
-    return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined));
+	return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== void 0));
 }
+//#endregion
+export { createFirecrawlScraper };
 
-export { FirecrawlScraper, createFirecrawlScraper };
-//# sourceMappingURL=firecrawl.mjs.map
+//# sourceMappingURL=firecrawl.mjs.map

package/dist/esm/tools/search/firecrawl.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.mjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private version: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private maxAge?: number;\n  private mobile?: boolean;\n  private skipTlsVerification?: boolean;\n  private blockAds?: boolean;\n  private removeBase64Images?: boolean;\n  private parsePDF?: boolean;\n  private storeInCache?: boolean;\n  private zeroDataRetention?: boolean;\n  private headers?: Record<string, string>;\n  private location?: { country?: string; languages?: string[] };\n  private onlyMainContent?: boolean;\n  private changeTrackingOptions?: object;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    this.version = config.version ?? 'v2';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.maxAge = config.maxAge;\n    this.mobile = config.mobile;\n    this.skipTlsVerification = config.skipTlsVerification;\n    this.blockAds = config.blockAds;\n    this.removeBase64Images = config.removeBase64Images;\n    this.parsePDF = config.parsePDF;\n    this.storeInCache = config.storeInCache;\n    this.zeroDataRetention = config.zeroDataRetention;\n    this.headers = config.headers;\n    this.location = config.location;\n    this.onlyMainContent = config.onlyMainContent;\n    this.changeTrackingOptions = config.changeTrackingOptions;\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        headers: options.headers ?? this.headers,\n        waitFor: options.waitFor ?? this.waitFor,\n        timeout: options.timeout ?? this.timeout,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        maxAge: options.maxAge ?? this.maxAge,\n        mobile: options.mobile ?? this.mobile,\n        skipTlsVerification:\n          options.skipTlsVerification ?? this.skipTlsVerification,\n        parsePDF: options.parsePDF ?? this.parsePDF,\n        location: options.location ?? this.location,\n        removeBase64Images:\n          options.removeBase64Images ?? this.removeBase64Images,\n        blockAds: options.blockAds ?? this.blockAds,\n        storeInCache: options.storeInCache ?? this.storeInCache,\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n        changeTrackingOptions:\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\n      });\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'Content-Type': 'application/json',\n          Authorization: `Bearer ${this.apiKey}`,\n        },\n        timeout: this.timeout,\n      });\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"names":[],"mappings":";;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,WAAW;AACX,IAAA,WAAW;AACX,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,mBAAmB;AACnB,IAAA,QAAQ;AACR,IAAA,kBAAkB;AAClB,IAAA,QAAQ;AACR,IAAA,YAAY;AACZ,IAAA,iBAAiB;AACjB,IAAA,OAAO;AACP,IAAA,QAAQ;AACR,IAAA,eAAe;AACf,IAAA,qBAAqB;AAE7B,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;QAElE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;AAErC,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAA,EAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA,CAAA,EAAI,IAAI,CAAC,OAAO,SAAS;AAErE,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB;AACrD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB;AACnD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY;AACvC,QAAA,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB;AACjD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe;AAC7C,QAAA,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB;AAEzD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;QAC3E;QAEA,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAA,4CAAA,EAA+C,IAAI,CAAC,MAAM,CAAA,CAAE,CAC7D;IACH;AAEA;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;QACH;AAEA,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC/C,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;AAChE,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,mBAAmB,EACjB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB;AACzD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB;AACvD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AACvD,gBAAA,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;AACtE,gBAAA,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;AAC9D,aAAA,CAAC;AACF,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAA,CAAE;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;QAC7B;QAAE,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAA,8BAAA,EAAiC,YAAY,CAAA,CAAE;AACvD,iBAAA;aACF;QACH;IACF;AAEA;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;QACxB;AAEA,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,cAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;YACzB;YAAE,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;YAC5C;QACF;aAAO,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;QAC5C;;QAGA,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;QACxC;;QAGA,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;QAC3C;AAEA,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;IACxB;AAEA;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;QACX;AAEA,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;IAC/B;AACD;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAA,GAAmC,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;AAEA;AACA,SAAS,aAAa,CAAmB,GAAM,EAAA;IAC7C,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CACzC;AACjB;;;;"}
+{"version":3,"file":"firecrawl.mjs","names":[],"sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private version: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private maxAge?: number;\n  private mobile?: boolean;\n  private skipTlsVerification?: boolean;\n  private blockAds?: boolean;\n  private removeBase64Images?: boolean;\n  private parsePDF?: boolean;\n  private storeInCache?: boolean;\n  private zeroDataRetention?: boolean;\n  private headers?: Record<string, string>;\n  private location?: { country?: string; languages?: string[] };\n  private onlyMainContent?: boolean;\n  private changeTrackingOptions?: object;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    this.version = config.version ?? 'v2';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.maxAge = config.maxAge;\n    this.mobile = config.mobile;\n    this.skipTlsVerification = config.skipTlsVerification;\n    this.blockAds = config.blockAds;\n    this.removeBase64Images = config.removeBase64Images;\n    this.parsePDF = config.parsePDF;\n    this.storeInCache = config.storeInCache;\n    this.zeroDataRetention = config.zeroDataRetention;\n    this.headers = config.headers;\n    this.location = config.location;\n    this.onlyMainContent = config.onlyMainContent;\n    this.changeTrackingOptions = config.changeTrackingOptions;\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        headers: options.headers ?? this.headers,\n        waitFor: options.waitFor ?? this.waitFor,\n        timeout: options.timeout ?? this.timeout,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        maxAge: options.maxAge ?? this.maxAge,\n        mobile: options.mobile ?? this.mobile,\n        skipTlsVerification:\n          options.skipTlsVerification ?? this.skipTlsVerification,\n        parsePDF: options.parsePDF ?? this.parsePDF,\n        location: options.location ?? this.location,\n        removeBase64Images:\n          options.removeBase64Images ?? this.removeBase64Images,\n        blockAds: options.blockAds ?? this.blockAds,\n        storeInCache: options.storeInCache ?? this.storeInCache,\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n        changeTrackingOptions:\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\n      });\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'Content-Type': 'application/json',\n          Authorization: `Bearer ${this.apiKey}`,\n        },\n        timeout: this.timeout,\n      });\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"mappings":";;;;;;;;AASA,IAAa,mBAAb,MAAuD;CACrD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAmC,CAAC,GAAG;EACjD,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,qBAAqB;EAEhE,KAAK,UAAU,OAAO,WAAW;EAEjC,MAAM,UACJ,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE,GAAG,KAAK,QAAQ;EAE7D,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,SAAS;EAC9D,KAAK,UAAU,OAAO,WAAW;EAEjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO;EACrB,KAAK,sBAAsB,OAAO;EAClC,KAAK,WAAW,OAAO;EACvB,KAAK,qBAAqB,OAAO;EACjC,KAAK,WAAW,OAAO;EACvB,KAAK,eAAe,OAAO;EAC3B,KAAK,oBAAoB,OAAO;EAChC,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,kBAAkB,OAAO;EAC9B,KAAK,wBAAwB,OAAO;EAEpC,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,uDAAuD;EAG1E,KAAK,OAAO,MACV,+CAA+C,KAAK,QACtD;CACF;;;;;;;CAQA,MAAM,UACJ,KACA,UAAoC,CAAC,GACS;EAC9C,IAAI,CAAC,KAAK,QACR,OAAO,CACL,KACA;GACE,SAAS;GACT,OAAO;EACT,CACF;EAGF,IAAI;GACF,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,QAAQ,QAAQ,UAAU,KAAK;IAC/B,QAAQ,QAAQ,UAAU,KAAK;IAC/B,qBACE,QAAQ,uBAAuB,KAAK;IACtC,UAAU,QAAQ,YAAY,KAAK;IACnC,UAAU,QAAQ,YAAY,KAAK;IACnC,oBACE,QAAQ,sBAAsB,KAAK;IACrC,UAAU,QAAQ,YAAY,KAAK;IACnC,cAAc,QAAQ,gBAAgB,KAAK;IAC3C,mBAAmB,QAAQ,qBAAqB,KAAK;IACrD,uBACE,QAAQ,yBAAyB,KAAK;GAC1C,CAAC;GASD,OAAO,CAAC,MAAK,MARU,MAAM,KAAK,KAAK,QAAQ,SAAS;IACtD,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CAAC,EAAA,CAEqB,IAAI;EAC5B,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,iCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;CAOA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,IAAI,SAAS,KAAK,YAAY,QAAQ,SAAS,KAAK,QAAQ,MAC1D,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,SAAS,KAAK,MACd,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAG1C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;;;;;;CAOA,gBAAgB,UAAuD;EACrE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,0BACX,SAAmC,CAAC,MACf;CACrB,OAAO,IAAI,iBAAiB,MAAM;AACpC;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}