illuma-agents 1.0.2

package/dist/esm/tools/search/serper-scraper.mjs

@@ -0,0 +1,129 @@
+import axios from 'axios';
+import { createDefaultLogger } from './utils.mjs';
+
+/**
+ * Serper scraper implementation
+ * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages
+ *
+ * Features:
+ * - Simple API with single endpoint
+ * - Returns both text and markdown content
+ * - Includes metadata from scraped pages
+ * - Credits-based pricing model
+ *
+ * @example
+ * ```typescript
+ * const scraper = createSerperScraper({
+ *   apiKey: 'your-serper-api-key',
+ *   includeMarkdown: true,
+ *   timeout: 10000
+ * });
+ *
+ * const [url, response] = await scraper.scrapeUrl('https://example.com');
+ * if (response.success) {
+ *   const [content] = scraper.extractContent(response);
+ *   console.log(content);
+ * }
+ * ```
+ */
+class SerperScraper {
+    apiKey;
+    apiUrl;
+    timeout;
+    logger;
+    includeMarkdown;
+    constructor(config = {}) {
+        this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';
+        this.apiUrl =
+            config.apiUrl ??
+                process.env.SERPER_SCRAPE_URL ??
+                'https://scrape.serper.dev';
+        this.timeout = config.timeout ?? 7500;
+        this.includeMarkdown = config.includeMarkdown ?? true;
+        this.logger = config.logger || createDefaultLogger();
+        if (!this.apiKey) {
+            this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');
+        }
+        this.logger.debug(`Serper scraper initialized with API URL: ${this.apiUrl}`);
+    }
+    /**
+     * Scrape a single URL
+     * @param url URL to scrape
+     * @param options Scrape options
+     * @returns Scrape response
+     */
+    async scrapeUrl(url, options = {}) {
+        if (!this.apiKey) {
+            return [
+                url,
+                {
+                    success: false,
+                    error: 'SERPER_API_KEY is not set',
+                },
+            ];
+        }
+        try {
+            const payload = {
+                url,
+                includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,
+            };
+            const response = await axios.post(this.apiUrl, payload, {
+                headers: {
+                    'X-API-KEY': this.apiKey,
+                    'Content-Type': 'application/json',
+                },
+                timeout: options.timeout ?? this.timeout,
+            });
+            return [url, { success: true, data: response.data }];
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return [
+                url,
+                {
+                    success: false,
+                    error: `Serper Scrape API request failed: ${errorMessage}`,
+                },
+            ];
+        }
+    }
+    /**
+     * Extract content from scrape response
+     * @param response Scrape response
+     * @returns Extracted content or empty string if not available
+     */
+    extractContent(response) {
+        if (!response.success || !response.data) {
+            return ['', undefined];
+        }
+        if (response.data.markdown != null) {
+            return [response.data.markdown, undefined];
+        }
+        if (response.data.text != null) {
+            return [response.data.text, undefined];
+        }
+        return ['', undefined];
+    }
+    /**
+     * Extract metadata from scrape response
+     * @param response Scrape response
+     * @returns Metadata object
+     */
+    extractMetadata(response) {
+        if (!response.success || !response.data || !response.data.metadata) {
+            return {};
+        }
+        return response.data.metadata;
+    }
+}
+/**
+ * Create a Serper scraper instance
+ * @param config Scraper configuration
+ * @returns Serper scraper instance
+ */
+const createSerperScraper = (config = {}) => {
+    return new SerperScraper(config);
+};
+
+export { SerperScraper, createSerperScraper };
+//# sourceMappingURL=serper-scraper.mjs.map

package/dist/esm/tools/search/serper-scraper.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"serper-scraper.mjs","sources":["../../../../src/tools/search/serper-scraper.ts"],"sourcesContent":["import axios from 'axios';\r\nimport type * as t from './types';\r\nimport { createDefaultLogger } from './utils';\r\n\r\n/**\r\n * Serper scraper implementation\r\n * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages\r\n *\r\n * Features:\r\n * - Simple API with single endpoint\r\n * - Returns both text and markdown content\r\n * - Includes metadata from scraped pages\r\n * - Credits-based pricing model\r\n *\r\n * @example\r\n * ```typescript\r\n * const scraper = createSerperScraper({\r\n *   apiKey: 'your-serper-api-key',\r\n *   includeMarkdown: true,\r\n *   timeout: 10000\r\n * });\r\n *\r\n * const [url, response] = await scraper.scrapeUrl('https://example.com');\r\n * if (response.success) {\r\n *   const [content] = scraper.extractContent(response);\r\n *   console.log(content);\r\n * }\r\n * ```\r\n */\r\nexport class SerperScraper implements t.BaseScraper {\r\n  private apiKey: string;\r\n  private apiUrl: string;\r\n  private timeout: number;\r\n  private logger: t.Logger;\r\n  private includeMarkdown: boolean;\r\n\r\n  constructor(config: t.SerperScraperConfig = {}) {\r\n    this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';\r\n\r\n    this.apiUrl =\r\n      config.apiUrl ??\r\n      process.env.SERPER_SCRAPE_URL ??\r\n      'https://scrape.serper.dev';\r\n\r\n    this.timeout = config.timeout ?? 7500;\r\n    this.includeMarkdown = config.includeMarkdown ?? true;\r\n\r\n    this.logger = config.logger || createDefaultLogger();\r\n\r\n    if (!this.apiKey) {\r\n      this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');\r\n    }\r\n\r\n    this.logger.debug(\r\n      `Serper scraper initialized with API URL: ${this.apiUrl}`\r\n    );\r\n  }\r\n\r\n  /**\r\n   * Scrape a single URL\r\n   * @param url URL to scrape\r\n   * @param options Scrape options\r\n   * @returns Scrape response\r\n   */\r\n  async scrapeUrl(\r\n    url: string,\r\n    options: t.SerperScrapeOptions = {}\r\n  ): Promise<[string, t.SerperScrapeResponse]> {\r\n    if (!this.apiKey) {\r\n      return [\r\n        url,\r\n        {\r\n          success: false,\r\n          error: 'SERPER_API_KEY is not set',\r\n        },\r\n      ];\r\n    }\r\n\r\n    try {\r\n      const payload = {\r\n        url,\r\n        includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,\r\n      };\r\n\r\n      const response = await axios.post(this.apiUrl, payload, {\r\n        headers: {\r\n          'X-API-KEY': this.apiKey,\r\n          'Content-Type': 'application/json',\r\n        },\r\n        timeout: options.timeout ?? this.timeout,\r\n      });\r\n\r\n      return [url, { success: true, data: response.data }];\r\n    } catch (error) {\r\n      const errorMessage =\r\n        error instanceof Error ? error.message : String(error);\r\n      return [\r\n        url,\r\n        {\r\n          success: false,\r\n          error: `Serper Scrape API request failed: ${errorMessage}`,\r\n        },\r\n      ];\r\n    }\r\n  }\r\n\r\n  /**\r\n   * Extract content from scrape response\r\n   * @param response Scrape response\r\n   * @returns Extracted content or empty string if not available\r\n   */\r\n  extractContent(\r\n    response: t.SerperScrapeResponse\r\n  ): [string, undefined | t.References] {\r\n    if (!response.success || !response.data) {\r\n      return ['', undefined];\r\n    }\r\n\r\n    if (response.data.markdown != null) {\r\n      return [response.data.markdown, undefined];\r\n    }\r\n\r\n    if (response.data.text != null) {\r\n      return [response.data.text, undefined];\r\n    }\r\n\r\n    return ['', undefined];\r\n  }\r\n\r\n  /**\r\n   * Extract metadata from scrape response\r\n   * @param response Scrape response\r\n   * @returns Metadata object\r\n   */\r\n  extractMetadata(\r\n    response: t.SerperScrapeResponse\r\n  ): Record<string, string | number | boolean | null | undefined> {\r\n    if (!response.success || !response.data || !response.data.metadata) {\r\n      return {};\r\n    }\r\n\r\n    return response.data.metadata;\r\n  }\r\n}\r\n\r\n/**\r\n * Create a Serper scraper instance\r\n * @param config Scraper configuration\r\n * @returns Serper scraper instance\r\n */\r\nexport const createSerperScraper = (\r\n  config: t.SerperScraperConfig = {}\r\n): SerperScraper => {\r\n  return new SerperScraper(config);\r\n};\r\n"],"names":[],"mappings":";;;AAIA;;;;;;;;;;;;;;;;;;;;;;;;AAwBG;MACU,aAAa,CAAA;AAChB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,eAAe;AAEvB,IAAA,WAAA,CAAY,SAAgC,EAAE,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;AAE/D,QAAA,IAAI,CAAC,MAAM;AACT,YAAA,MAAM,CAAC,MAAM;gBACb,OAAO,CAAC,GAAG,CAAC,iBAAiB;AAC7B,gBAAA,2BAA2B;QAE7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC;;QAGxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA4C,yCAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC1D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,2BAA2B;AACnC,iBAAA;aACF;;AAGH,QAAA,IAAI;AACF,YAAA,MAAM,OAAO,GAAG;gBACd,GAAG;AACH,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;aACjE;AAED,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,IAAI,CAAC,MAAM;AACxB,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;AACD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACzC,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;;QACpD,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAqC,kCAAA,EAAA,YAAY,CAAE,CAAA;AAC3D,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAgC,EAAA;QAEhC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;QAGxB,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;QAG5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;AAGxC,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CACb,QAAgC,EAAA;AAEhC,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,mBAAmB,GAAG,CACjC,MAAgC,GAAA,EAAE,KACjB;AACjB,IAAA,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC;AAClC;;;;"}

package/dist/esm/tools/search/tool.mjs

@@ -0,0 +1,329 @@
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { newsSchema, videosSchema, imagesSchema, dateSchema, querySchema, countrySchema } from './schema.mjs';
+import { createSearchAPI, createSourceProcessor } from './search.mjs';
+import { createSerperScraper } from './serper-scraper.mjs';
+import { createFirecrawlScraper } from './firecrawl.mjs';
+import { expandHighlights } from './highlights.mjs';
+import { formatResultsForLLM } from './format.mjs';
+import { createDefaultLogger } from './utils.mjs';
+import { createReranker } from './rerankers.mjs';
+import { Constants } from '../../common/enum.mjs';
+
+/**
+ * Executes parallel searches and merges the results
+ */
+async function executeParallelSearches({ searchAPI, query, date, country, safeSearch, images, videos, news, logger, }) {
+    // Prepare all search tasks to run in parallel
+    const searchTasks = [
+        // Main search
+        searchAPI.getSources({
+            query,
+            date,
+            country,
+            safeSearch,
+        }),
+    ];
+    if (images) {
+        searchTasks.push(searchAPI
+            .getSources({
+            query,
+            date,
+            country,
+            safeSearch,
+            type: 'images',
+        })
+            .catch((error) => {
+            logger.error('Error fetching images:', error);
+            return {
+                success: false,
+                error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,
+            };
+        }));
+    }
+    if (videos) {
+        searchTasks.push(searchAPI
+            .getSources({
+            query,
+            date,
+            country,
+            safeSearch,
+            type: 'videos',
+        })
+            .catch((error) => {
+            logger.error('Error fetching videos:', error);
+            return {
+                success: false,
+                error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,
+            };
+        }));
+    }
+    if (news) {
+        searchTasks.push(searchAPI
+            .getSources({
+            query,
+            date,
+            country,
+            safeSearch,
+            type: 'news',
+        })
+            .catch((error) => {
+            logger.error('Error fetching news:', error);
+            return {
+                success: false,
+                error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,
+            };
+        }));
+    }
+    // Run all searches in parallel
+    const results = await Promise.all(searchTasks);
+    // Get the main search result (first result)
+    const mainResult = results[0];
+    if (!mainResult.success) {
+        throw new Error(mainResult.error ?? 'Search failed');
+    }
+    // Merge additional results with the main results
+    const mergedResults = { ...mainResult.data };
+    // Convert existing news to topStories if present
+    if (mergedResults.news !== undefined && mergedResults.news.length > 0) {
+        const existingNewsAsTopStories = mergedResults.news
+            .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')
+            .map((newsItem) => ({
+            title: newsItem.title ?? '',
+            link: newsItem.link ?? '',
+            source: newsItem.source ?? '',
+            date: newsItem.date ?? '',
+            imageUrl: newsItem.imageUrl ?? '',
+            processed: false,
+        }));
+        mergedResults.topStories = [
+            ...(mergedResults.topStories ?? []),
+            ...existingNewsAsTopStories,
+        ];
+        delete mergedResults.news;
+    }
+    results.slice(1).forEach((result) => {
+        if (result.success && result.data !== undefined) {
+            if (result.data.images !== undefined && result.data.images.length > 0) {
+                mergedResults.images = [
+                    ...(mergedResults.images ?? []),
+                    ...result.data.images,
+                ];
+            }
+            if (result.data.videos !== undefined && result.data.videos.length > 0) {
+                mergedResults.videos = [
+                    ...(mergedResults.videos ?? []),
+                    ...result.data.videos,
+                ];
+            }
+            if (result.data.news !== undefined && result.data.news.length > 0) {
+                const newsAsTopStories = result.data.news.map((newsItem) => ({
+                    ...newsItem,
+                    link: newsItem.link ?? '',
+                }));
+                mergedResults.topStories = [
+                    ...(mergedResults.topStories ?? []),
+                    ...newsAsTopStories,
+                ];
+            }
+        }
+    });
+    return { success: true, data: mergedResults };
+}
+function createSearchProcessor({ searchAPI, safeSearch, sourceProcessor, onGetHighlights, logger, }) {
+    return async function ({ query, date, country, proMode = true, maxSources = 5, onSearchResults, images = false, videos = false, news = false, }) {
+        try {
+            // Execute parallel searches and merge results
+            const searchResult = await executeParallelSearches({
+                searchAPI,
+                query,
+                date,
+                country,
+                safeSearch,
+                images,
+                videos,
+                news,
+                logger,
+            });
+            onSearchResults?.(searchResult);
+            const processedSources = await sourceProcessor.processSources({
+                query,
+                news,
+                result: searchResult,
+                proMode,
+                onGetHighlights,
+                numElements: maxSources,
+            });
+            return expandHighlights(processedSources);
+        }
+        catch (error) {
+            logger.error('Error in search:', error);
+            return {
+                organic: [],
+                topStories: [],
+                images: [],
+                videos: [],
+                news: [],
+                relatedSearches: [],
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    };
+}
+function createOnSearchResults({ runnableConfig, onSearchResults, }) {
+    return function (results) {
+        if (!onSearchResults) {
+            return;
+        }
+        onSearchResults(results, runnableConfig);
+    };
+}
+function createTool({ schema, search, onSearchResults: _onSearchResults, }) {
+    return tool(async (params, runnableConfig) => {
+        const { query, date, country: _c, images, videos, news } = params;
+        const country = typeof _c === 'string' && _c ? _c : undefined;
+        const searchResult = await search({
+            query,
+            date,
+            country,
+            images,
+            videos,
+            news,
+            onSearchResults: createOnSearchResults({
+                runnableConfig,
+                onSearchResults: _onSearchResults,
+            }),
+        });
+        const turn = runnableConfig.toolCall?.turn ?? 0;
+        const { output, references } = formatResultsForLLM(turn, searchResult);
+        const data = { turn, ...searchResult, references };
+        return [output, { [Constants.WEB_SEARCH]: data }];
+    }, {
+        name: Constants.WEB_SEARCH,
+        description: `Real-time search. Results have required citation anchors.
+
+Note: Use ONCE per reply unless instructed otherwise.
+
+Anchors:
+- \\ue202turnXtypeY
+- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx
+
+Special Markers:
+- \\ue203...\\ue204 — highlight start/end of cited text (for Standalone or Group citations)
+- \\ue200...\\ue201 — group block (e.g. \\ue200\\ue202turn0search1\\ue202turn0news2\\ue201)
+
+**CITE EVERY NON-OBVIOUS FACT/QUOTE:**
+Use anchor marker(s) immediately after the statement:
+- Standalone: "Pure functions produce same output. \\ue202turn0search0"
+- Standalone (multiple): "Today's News \\ue202turn0search0\\ue202turn0news0"
+- Highlight: "\\ue203Highlight text.\\ue204\\ue202turn0news1"
+- Group: "Sources. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
+- Group Highlight: "\\ue203Highlight for group.\\ue204 \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
+- Image: "See photo \\ue202turn0image0."
+
+**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**
+`.trim(),
+        schema: schema,
+        responseFormat: Constants.CONTENT_AND_ARTIFACT,
+    });
+}
+/**
+ * Creates a search tool with a schema that dynamically includes the country field
+ * only when the searchProvider is 'serper'.
+ *
+ * Supports multiple scraper providers:
+ * - Firecrawl (default): Full-featured web scraping with multiple formats
+ * - Serper: Lightweight scraping using Serper's scrape API
+ *
+ * @example
+ * ```typescript
+ * // Using Firecrawl scraper (default)
+ * const searchTool = createSearchTool({
+ *   searchProvider: 'serper',
+ *   scraperProvider: 'firecrawl',
+ *   firecrawlApiKey: 'your-firecrawl-key'
+ * });
+ *
+ * // Using Serper scraper
+ * const searchTool = createSearchTool({
+ *   searchProvider: 'serper',
+ *   scraperProvider: 'serper',
+ *   serperApiKey: 'your-serper-key'
+ * });
+ * ```
+ *
+ * @param config - The search tool configuration
+ * @returns A DynamicStructuredTool with a schema that depends on the searchProvider
+ */
+const createSearchTool = (config = {}) => {
+    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, rerankerType = 'cohere', topResults = 5, strategies = ['no_extraction'], filterContent = true, safeSearch = 1, scraperProvider = 'firecrawl', firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights, } = config;
+    const logger = config.logger || createDefaultLogger();
+    const schemaObject = {
+        query: querySchema,
+        date: dateSchema,
+        images: imagesSchema,
+        videos: videosSchema,
+        news: newsSchema,
+    };
+    if (searchProvider === 'serper') {
+        schemaObject.country = countrySchema;
+    }
+    const toolSchema = z.object(schemaObject);
+    const searchAPI = createSearchAPI({
+        searchProvider,
+        serperApiKey,
+        searxngInstanceUrl,
+        searxngApiKey,
+    });
+    /** Create scraper based on scraperProvider */
+    let scraperInstance;
+    if (scraperProvider === 'serper') {
+        scraperInstance = createSerperScraper({
+            ...serperScraperOptions,
+            apiKey: serperApiKey,
+            timeout: scraperTimeout ?? serperScraperOptions?.timeout,
+            logger,
+        });
+    }
+    else {
+        scraperInstance = createFirecrawlScraper({
+            ...firecrawlOptions,
+            apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
+            apiUrl: firecrawlApiUrl,
+            version: firecrawlVersion,
+            timeout: scraperTimeout ?? firecrawlOptions?.timeout,
+            formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],
+            logger,
+        });
+    }
+    const selectedReranker = createReranker({
+        rerankerType,
+        jinaApiKey,
+        jinaApiUrl,
+        cohereApiKey,
+        logger,
+    });
+    if (!selectedReranker) {
+        logger.warn('No reranker selected. Using default ranking.');
+    }
+    const sourceProcessor = createSourceProcessor({
+        reranker: selectedReranker,
+        topResults,
+        logger,
+    }, scraperInstance);
+    const search = createSearchProcessor({
+        searchAPI,
+        safeSearch,
+        sourceProcessor,
+        onGetHighlights,
+        logger,
+    });
+    return createTool({
+        search,
+        schema: toolSchema,
+        onSearchResults: _onSearchResults,
+    });
+};
+
+export { createSearchTool };
+//# sourceMappingURL=tool.mjs.map

package/dist/esm/tools/search/tool.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool.mjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\nimport type { RunnableConfig } from '@langchain/core/runnables';\r\nimport type * as t from './types';\r\nimport {\r\n  DATE_RANGE,\r\n  querySchema,\r\n  dateSchema,\r\n  countrySchema,\r\n  imagesSchema,\r\n  videosSchema,\r\n  newsSchema,\r\n} from './schema';\r\nimport { createSearchAPI, createSourceProcessor } from './search';\r\nimport { createSerperScraper } from './serper-scraper';\r\nimport { createFirecrawlScraper } from './firecrawl';\r\nimport { expandHighlights } from './highlights';\r\nimport { formatResultsForLLM } from './format';\r\nimport { createDefaultLogger } from './utils';\r\nimport { createReranker } from './rerankers';\r\nimport { Constants } from '@/common';\r\n\r\n/**\r\n * Executes parallel searches and merges the results\r\n */\r\nasync function executeParallelSearches({\r\n  searchAPI,\r\n  query,\r\n  date,\r\n  country,\r\n  safeSearch,\r\n  images,\r\n  videos,\r\n  news,\r\n  logger,\r\n}: {\r\n  searchAPI: ReturnType<typeof createSearchAPI>;\r\n  query: string;\r\n  date?: DATE_RANGE;\r\n  country?: string;\r\n  safeSearch: t.SearchToolConfig['safeSearch'];\r\n  images: boolean;\r\n  videos: boolean;\r\n  news: boolean;\r\n  logger: t.Logger;\r\n}): Promise<t.SearchResult> {\r\n  // Prepare all search tasks to run in parallel\r\n  const searchTasks: Promise<t.SearchResult>[] = [\r\n    // Main search\r\n    searchAPI.getSources({\r\n      query,\r\n      date,\r\n      country,\r\n      safeSearch,\r\n    }),\r\n  ];\r\n\r\n  if (images) {\r\n    searchTasks.push(\r\n      searchAPI\r\n        .getSources({\r\n          query,\r\n          date,\r\n          country,\r\n          safeSearch,\r\n          type: 'images',\r\n        })\r\n        .catch((error) => {\r\n          logger.error('Error fetching images:', error);\r\n          return {\r\n            success: false,\r\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\r\n          };\r\n        })\r\n    );\r\n  }\r\n  if (videos) {\r\n    searchTasks.push(\r\n      searchAPI\r\n        .getSources({\r\n          query,\r\n          date,\r\n          country,\r\n          safeSearch,\r\n          type: 'videos',\r\n        })\r\n        .catch((error) => {\r\n          logger.error('Error fetching videos:', error);\r\n          return {\r\n            success: false,\r\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\r\n          };\r\n        })\r\n    );\r\n  }\r\n  if (news) {\r\n    searchTasks.push(\r\n      searchAPI\r\n        .getSources({\r\n          query,\r\n          date,\r\n          country,\r\n          safeSearch,\r\n          type: 'news',\r\n        })\r\n        .catch((error) => {\r\n          logger.error('Error fetching news:', error);\r\n          return {\r\n            success: false,\r\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\r\n          };\r\n        })\r\n    );\r\n  }\r\n\r\n  // Run all searches in parallel\r\n  const results = await Promise.all(searchTasks);\r\n\r\n  // Get the main search result (first result)\r\n  const mainResult = results[0];\r\n  if (!mainResult.success) {\r\n    throw new Error(mainResult.error ?? 'Search failed');\r\n  }\r\n\r\n  // Merge additional results with the main results\r\n  const mergedResults = { ...mainResult.data };\r\n\r\n  // Convert existing news to topStories if present\r\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\r\n    const existingNewsAsTopStories = mergedResults.news\r\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\r\n      .map((newsItem) => ({\r\n        title: newsItem.title ?? '',\r\n        link: newsItem.link ?? '',\r\n        source: newsItem.source ?? '',\r\n        date: newsItem.date ?? '',\r\n        imageUrl: newsItem.imageUrl ?? '',\r\n        processed: false,\r\n      }));\r\n    mergedResults.topStories = [\r\n      ...(mergedResults.topStories ?? []),\r\n      ...existingNewsAsTopStories,\r\n    ];\r\n    delete mergedResults.news;\r\n  }\r\n\r\n  results.slice(1).forEach((result) => {\r\n    if (result.success && result.data !== undefined) {\r\n      if (result.data.images !== undefined && result.data.images.length > 0) {\r\n        mergedResults.images = [\r\n          ...(mergedResults.images ?? []),\r\n          ...result.data.images,\r\n        ];\r\n      }\r\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\r\n        mergedResults.videos = [\r\n          ...(mergedResults.videos ?? []),\r\n          ...result.data.videos,\r\n        ];\r\n      }\r\n      if (result.data.news !== undefined && result.data.news.length > 0) {\r\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\r\n          ...newsItem,\r\n          link: newsItem.link ?? '',\r\n        }));\r\n        mergedResults.topStories = [\r\n          ...(mergedResults.topStories ?? []),\r\n          ...newsAsTopStories,\r\n        ];\r\n      }\r\n    }\r\n  });\r\n\r\n  return { success: true, data: mergedResults };\r\n}\r\n\r\nfunction createSearchProcessor({\r\n  searchAPI,\r\n  safeSearch,\r\n  sourceProcessor,\r\n  onGetHighlights,\r\n  logger,\r\n}: {\r\n  safeSearch: t.SearchToolConfig['safeSearch'];\r\n  searchAPI: ReturnType<typeof createSearchAPI>;\r\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\r\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\r\n  logger: t.Logger;\r\n}) {\r\n  return async function ({\r\n    query,\r\n    date,\r\n    country,\r\n    proMode = true,\r\n    maxSources = 5,\r\n    onSearchResults,\r\n    images = false,\r\n    videos = false,\r\n    news = false,\r\n  }: {\r\n    query: string;\r\n    country?: string;\r\n    date?: DATE_RANGE;\r\n    proMode?: boolean;\r\n    maxSources?: number;\r\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\r\n    images?: boolean;\r\n    videos?: boolean;\r\n    news?: boolean;\r\n  }): Promise<t.SearchResultData> {\r\n    try {\r\n      // Execute parallel searches and merge results\r\n      const searchResult = await executeParallelSearches({\r\n        searchAPI,\r\n        query,\r\n        date,\r\n        country,\r\n        safeSearch,\r\n        images,\r\n        videos,\r\n        news,\r\n        logger,\r\n      });\r\n\r\n      onSearchResults?.(searchResult);\r\n\r\n      const processedSources = await sourceProcessor.processSources({\r\n        query,\r\n        news,\r\n        result: searchResult,\r\n        proMode,\r\n        onGetHighlights,\r\n        numElements: maxSources,\r\n      });\r\n\r\n      return expandHighlights(processedSources);\r\n    } catch (error) {\r\n      logger.error('Error in search:', error);\r\n      return {\r\n        organic: [],\r\n        topStories: [],\r\n        images: [],\r\n        videos: [],\r\n        news: [],\r\n        relatedSearches: [],\r\n        error: error instanceof Error ? error.message : String(error),\r\n      };\r\n    }\r\n  };\r\n}\r\n\r\nfunction createOnSearchResults({\r\n  runnableConfig,\r\n  onSearchResults,\r\n}: {\r\n  runnableConfig: RunnableConfig;\r\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\r\n}) {\r\n  return function (results: t.SearchResult): void {\r\n    if (!onSearchResults) {\r\n      return;\r\n    }\r\n    onSearchResults(results, runnableConfig);\r\n  };\r\n}\r\n\r\nfunction createTool({\r\n  schema,\r\n  search,\r\n  onSearchResults: _onSearchResults,\r\n}: {\r\n  schema: t.SearchToolSchema;\r\n  search: ReturnType<typeof createSearchProcessor>;\r\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\r\n}): DynamicStructuredTool<typeof schema> {\r\n  return tool<typeof schema>(\r\n    async (params, runnableConfig) => {\r\n      const { query, date, country: _c, images, videos, news } = params;\r\n      const country = typeof _c === 'string' && _c ? _c : undefined;\r\n      const searchResult = await search({\r\n        query,\r\n        date,\r\n        country,\r\n        images,\r\n        videos,\r\n        news,\r\n        onSearchResults: createOnSearchResults({\r\n          runnableConfig,\r\n          onSearchResults: _onSearchResults,\r\n        }),\r\n      });\r\n      const turn = runnableConfig.toolCall?.turn ?? 0;\r\n      const { output, references } = formatResultsForLLM(turn, searchResult);\r\n      const data: t.SearchResultData = { turn, ...searchResult, references };\r\n      return [output, { [Constants.WEB_SEARCH]: data }];\r\n    },\r\n    {\r\n      name: Constants.WEB_SEARCH,\r\n      description: `Real-time search. Results have required citation anchors.\r\n\r\nNote: Use ONCE per reply unless instructed otherwise.\r\n\r\nAnchors:\r\n- \\\\ue202turnXtypeY\r\n- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx\r\n\r\nSpecial Markers:\r\n- \\\\ue203...\\\\ue204 — highlight start/end of cited text (for Standalone or Group citations)\r\n- \\\\ue200...\\\\ue201 — group block (e.g. \\\\ue200\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\r\n\r\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\r\nUse anchor marker(s) immediately after the statement:\r\n- Standalone: \"Pure functions produce same output. \\\\ue202turn0search0\"\r\n- Standalone (multiple): \"Today's News \\\\ue202turn0search0\\\\ue202turn0news0\"\r\n- Highlight: \"\\\\ue203Highlight text.\\\\ue204\\\\ue202turn0news1\"\r\n- Group: \"Sources. \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\r\n- Group Highlight: \"\\\\ue203Highlight for group.\\\\ue204 \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\r\n- Image: \"See photo \\\\ue202turn0image0.\"\r\n\r\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\r\n`.trim(),\r\n      schema: schema,\r\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Creates a search tool with a schema that dynamically includes the country field\r\n * only when the searchProvider is 'serper'.\r\n *\r\n * Supports multiple scraper providers:\r\n * - Firecrawl (default): Full-featured web scraping with multiple formats\r\n * - Serper: Lightweight scraping using Serper's scrape API\r\n *\r\n * @example\r\n * ```typescript\r\n * // Using Firecrawl scraper (default)\r\n * const searchTool = createSearchTool({\r\n *   searchProvider: 'serper',\r\n *   scraperProvider: 'firecrawl',\r\n *   firecrawlApiKey: 'your-firecrawl-key'\r\n * });\r\n *\r\n * // Using Serper scraper\r\n * const searchTool = createSearchTool({\r\n *   searchProvider: 'serper',\r\n *   scraperProvider: 'serper',\r\n *   serperApiKey: 'your-serper-key'\r\n * });\r\n * ```\r\n *\r\n * @param config - The search tool configuration\r\n * @returns A DynamicStructuredTool with a schema that depends on the searchProvider\r\n */\r\nexport const createSearchTool = (\r\n  config: t.SearchToolConfig = {}\r\n): DynamicStructuredTool<typeof toolSchema> => {\r\n  const {\r\n    searchProvider = 'serper',\r\n    serperApiKey,\r\n    searxngInstanceUrl,\r\n    searxngApiKey,\r\n    rerankerType = 'cohere',\r\n    topResults = 5,\r\n    strategies = ['no_extraction'],\r\n    filterContent = true,\r\n    safeSearch = 1,\r\n    scraperProvider = 'firecrawl',\r\n    firecrawlApiKey,\r\n    firecrawlApiUrl,\r\n    firecrawlVersion,\r\n    firecrawlOptions,\r\n    serperScraperOptions,\r\n    scraperTimeout,\r\n    jinaApiKey,\r\n    jinaApiUrl,\r\n    cohereApiKey,\r\n    onSearchResults: _onSearchResults,\r\n    onGetHighlights,\r\n  } = config;\r\n\r\n  const logger = config.logger || createDefaultLogger();\r\n\r\n  const schemaObject: {\r\n    query: z.ZodString;\r\n    date: z.ZodOptional<z.ZodNativeEnum<typeof DATE_RANGE>>;\r\n    country?: z.ZodOptional<z.ZodString>;\r\n    images: z.ZodOptional<z.ZodBoolean>;\r\n    videos: z.ZodOptional<z.ZodBoolean>;\r\n    news: z.ZodOptional<z.ZodBoolean>;\r\n  } = {\r\n    query: querySchema,\r\n    date: dateSchema,\r\n    images: imagesSchema,\r\n    videos: videosSchema,\r\n    news: newsSchema,\r\n  };\r\n\r\n  if (searchProvider === 'serper') {\r\n    schemaObject.country = countrySchema;\r\n  }\r\n\r\n  const toolSchema = z.object(schemaObject);\r\n\r\n  const searchAPI = createSearchAPI({\r\n    searchProvider,\r\n    serperApiKey,\r\n    searxngInstanceUrl,\r\n    searxngApiKey,\r\n  });\r\n\r\n  /** Create scraper based on scraperProvider */\r\n  let scraperInstance: t.BaseScraper;\r\n\r\n  if (scraperProvider === 'serper') {\r\n    scraperInstance = createSerperScraper({\r\n      ...serperScraperOptions,\r\n      apiKey: serperApiKey,\r\n      timeout: scraperTimeout ?? serperScraperOptions?.timeout,\r\n      logger,\r\n    });\r\n  } else {\r\n    scraperInstance = createFirecrawlScraper({\r\n      ...firecrawlOptions,\r\n      apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\r\n      apiUrl: firecrawlApiUrl,\r\n      version: firecrawlVersion,\r\n      timeout: scraperTimeout ?? firecrawlOptions?.timeout,\r\n      formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],\r\n      logger,\r\n    });\r\n  }\r\n\r\n  const selectedReranker = createReranker({\r\n    rerankerType,\r\n    jinaApiKey,\r\n    jinaApiUrl,\r\n    cohereApiKey,\r\n    logger,\r\n  });\r\n\r\n  if (!selectedReranker) {\r\n    logger.warn('No reranker selected. Using default ranking.');\r\n  }\r\n\r\n  const sourceProcessor = createSourceProcessor(\r\n    {\r\n      reranker: selectedReranker,\r\n      topResults,\r\n      strategies,\r\n      filterContent,\r\n      logger,\r\n    },\r\n    scraperInstance\r\n  );\r\n\r\n  const search = createSearchProcessor({\r\n    searchAPI,\r\n    safeSearch,\r\n    sourceProcessor,\r\n    onGetHighlights,\r\n    logger,\r\n  });\r\n\r\n  return createTool({\r\n    search,\r\n    schema: toolSchema,\r\n    onSearchResults: _onSearchResults,\r\n  });\r\n};\r\n"],"names":[],"mappings":";;;;;;;;;;;;AAsBA;;AAEG;AACH,eAAe,uBAAuB,CAAC,EACrC,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,GAWP,EAAA;;AAEC,IAAA,MAAM,WAAW,GAA8B;;QAE7C,SAAS,CAAC,UAAU,CAAC;YACnB,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;SACX,CAAC;KACH;IAED,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,IAAI,EAAE;QACR,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,MAAM;SACb;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;YAC3C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAuB,oBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACvF;SACF,CAAC,CACL;;;IAIH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;;AAG9C,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;AAC7B,IAAA,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;;;IAItD,MAAM,aAAa,GAAG,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE;;AAG5C,IAAA,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACrE,QAAA,MAAM,wBAAwB,GAAG,aAAa,CAAC;AAC5C,aAAA,MAAM,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,EAAE;AACxE,aAAA,GAAG,CAAC,CAAC,QAAQ,MAAM;AAClB,YAAA,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;AAC3B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;AAC7B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;AACjC,YAAA,SAAS,EAAE,KAAK;AACjB,SAAA,CAAC,CAAC;QACL,aAAa,CAAC,UAAU,GAAG;AACzB,YAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,YAAA,GAAG,wBAAwB;SAC5B;QACD,OAAO,aAAa,CAAC,IAAI;;IAG3B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAI;QAClC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE;AAC/C,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,gBAAA,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM;AAC3D,oBAAA,GAAG,QAAQ;AACX,oBAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AAC1B,iBAAA,CAAC,CAAC;gBACH,aAAa,CAAC,UAAU,GAAG;AACzB,oBAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,oBAAA,GAAG,gBAAgB;iBACpB;;;AAGP,KAAC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;AAC/C;AAEA,SAAS,qBAAqB,CAAC,EAC7B,SAAS,EACT,UAAU,EACV,eAAe,EACf,eAAe,EACf,MAAM,GAOP,EAAA;AACC,IAAA,OAAO,gBAAgB,EACrB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,EACf,MAAM,GAAG,KAAK,EACd,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,KAAK,GAWb,EAAA;AACC,QAAA,IAAI;;AAEF,YAAA,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC;gBACjD,SAAS;gBACT,KAAK;gBACL,IAAI;gBACJ,OAAO;gBACP,UAAU;gBACV,MAAM;gBACN,MAAM;gBACN,IAAI;gBACJ,MAAM;AACP,aAAA,CAAC;AAEF,YAAA,eAAe,GAAG,YAAY,CAAC;AAE/B,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC5D,KAAK;gBACL,IAAI;AACJ,gBAAA,MAAM,EAAE,YAAY;gBACpB,OAAO;gBACP,eAAe;AACf,gBAAA,WAAW,EAAE,UAAU;AACxB,aAAA,CAAC;AAEF,YAAA,OAAO,gBAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACvC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,EAAE;AACR,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;AACH;AAEA,SAAS,qBAAqB,CAAC,EAC7B,cAAc,EACd,eAAe,GAIhB,EAAA;AACC,IAAA,OAAO,UAAU,OAAuB,EAAA;QACtC,IAAI,CAAC,eAAe,EAAE;YACpB;;AAEF,QAAA,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC;AAC1C,KAAC;AACH;AAEA,SAAS,UAAU,CAAC,EAClB,MAAM,EACN,MAAM,EACN,eAAe,EAAE,gBAAgB,GAKlC,EAAA;IACC,OAAO,IAAI,CACT,OAAO,MAAM,EAAE,cAAc,KAAI;AAC/B,QAAA,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;AACjE,QAAA,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,SAAS;AAC7D,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;YACL,IAAI;YACJ,OAAO;YACP,MAAM;YACN,MAAM;YACN,IAAI;YACJ,eAAe,EAAE,qBAAqB,CAAC;gBACrC,cAAc;AACd,gBAAA,eAAe,EAAE,gBAAgB;aAClC,CAAC;AACH,SAAA,CAAC;QACF,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC/C,QAAA,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,YAAY,CAAC;QACtE,MAAM,IAAI,GAAuB,EAAE,IAAI,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;AACtE,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;AACnD,KAAC,EACD;QACE,IAAI,EAAE,SAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;;;;AAsBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,MAAM;QACd,cAAc,EAAE,SAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BG;MACU,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACa;AAC5C,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,UAAU,GAAG,CAAC,EACd,eAAe,GAAG,WAAW,EAC7B,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,UAAU,EACV,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,EACjC,eAAe,GAChB,GAAG,MAAM;IAEV,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAErD,IAAA,MAAM,YAAY,GAOd;AACF,QAAA,KAAK,EAAE,WAAW;AAClB,QAAA,IAAI,EAAE,UAAU;AAChB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,IAAI,EAAE,UAAU;KACjB;AAED,IAAA,IAAI,cAAc,KAAK,QAAQ,EAAE;AAC/B,QAAA,YAAY,CAAC,OAAO,GAAG,aAAa;;IAGtC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAEzC,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;;AAGF,IAAA,IAAI,eAA8B;AAElC,IAAA,IAAI,eAAe,KAAK,QAAQ,EAAE;QAChC,eAAe,GAAG,mBAAmB,CAAC;AACpC,YAAA,GAAG,oBAAoB;AACvB,YAAA,MAAM,EAAE,YAAY;AACpB,YAAA,OAAO,EAAE,cAAc,IAAI,oBAAoB,EAAE,OAAO;YACxD,MAAM;AACP,SAAA,CAAC;;SACG;QACL,eAAe,GAAG,sBAAsB,CAAC;AACvC,YAAA,GAAG,gBAAgB;AACnB,YAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,YAAA,MAAM,EAAE,eAAe;AACvB,YAAA,OAAO,EAAE,gBAAgB;AACzB,YAAA,OAAO,EAAE,cAAc,IAAI,gBAAgB,EAAE,OAAO;YACpD,OAAO,EAAE,gBAAgB,EAAE,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;YAC7D,MAAM;AACP,SAAA,CAAC;;IAGJ,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,UAAU;QACV,YAAY;QACZ,MAAM;AACP,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG7D,MAAM,eAAe,GAAG,qBAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,UAAU;QAGV,MAAM;KACP,EACD,eAAe,CAChB;IAED,MAAM,MAAM,GAAG,qBAAqB,CAAC;QACnC,SAAS;QACT,UAAU;QACV,eAAe;QACf,eAAe;QACf,MAAM;AACP,KAAA,CAAC;AAEF,IAAA,OAAO,UAAU,CAAC;QAChB,MAAM;AACN,QAAA,MAAM,EAAE,UAAU;AAClB,QAAA,eAAe,EAAE,gBAAgB;AAClC,KAAA,CAAC;AACJ;;;;"}

package/dist/esm/tools/search/utils.mjs

@@ -0,0 +1,61 @@
+/* eslint-disable no-console */
+/**
+ * Singleton instance of the default logger
+ */
+let defaultLoggerInstance = null;
+/**
+ * Creates a default logger that maps to console methods
+ * Uses a singleton pattern to avoid creating multiple instances
+ * @returns A default logger that implements the Logger interface
+ */
+const createDefaultLogger = () => {
+    if (!defaultLoggerInstance) {
+        defaultLoggerInstance = {
+            error: console.error,
+            warn: console.warn,
+            info: console.info,
+            debug: console.debug,
+        };
+    }
+    return defaultLoggerInstance;
+};
+const fileExtRegex = /\.(pdf|jpe?g|png|gif|svg|webp|bmp|ico|tiff?|avif|heic|doc[xm]?|xls[xm]?|ppt[xm]?|zip|rar|mp[34]|mov|avi|wav)(?:\?.*)?$/i;
+const getDomainName = (link, metadata, logger) => {
+    try {
+        const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');
+        const domain = new URL(url).hostname.replace(/^www\./, '');
+        if (domain) {
+            return domain;
+        }
+    }
+    catch (e) {
+        // URL parsing failed
+        if (logger) {
+            logger.error('Error parsing URL:', e);
+        }
+        else {
+            console.error('Error parsing URL:', e);
+        }
+    }
+    return;
+};
+function getAttribution(link, metadata, logger) {
+    if (!metadata)
+        return getDomainName(link, metadata, logger);
+    const twitterSite = metadata['twitter:site'];
+    const twitterSiteFormatted = typeof twitterSite === 'string' ? twitterSite.replace(/^@/, '') : undefined;
+    const possibleAttributions = [
+        metadata.ogSiteName,
+        metadata['og:site_name'],
+        metadata.title?.split('|').pop()?.trim(),
+        twitterSiteFormatted,
+    ];
+    const attribution = possibleAttributions.find((attr) => attr != null && typeof attr === 'string' && attr.trim() !== '');
+    if (attribution != null) {
+        return attribution;
+    }
+    return getDomainName(link, metadata, logger);
+}
+
+export { createDefaultLogger, fileExtRegex, getAttribution, getDomainName };
+//# sourceMappingURL=utils.mjs.map

package/dist/esm/tools/search/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.mjs","sources":["../../../../src/tools/search/utils.ts"],"sourcesContent":["/* eslint-disable no-console */\r\n\r\nimport type * as t from './types';\r\n\r\n/**\r\n * Singleton instance of the default logger\r\n */\r\nlet defaultLoggerInstance: t.Logger | null = null;\r\n\r\n/**\r\n * Creates a default logger that maps to console methods\r\n * Uses a singleton pattern to avoid creating multiple instances\r\n * @returns A default logger that implements the Logger interface\r\n */\r\nexport const createDefaultLogger = (): t.Logger => {\r\n  if (!defaultLoggerInstance) {\r\n    defaultLoggerInstance = {\r\n      error: console.error,\r\n      warn: console.warn,\r\n      info: console.info,\r\n      debug: console.debug,\r\n    } as t.Logger;\r\n  }\r\n  return defaultLoggerInstance;\r\n};\r\n\r\nexport const fileExtRegex =\r\n  /\\.(pdf|jpe?g|png|gif|svg|webp|bmp|ico|tiff?|avif|heic|doc[xm]?|xls[xm]?|ppt[xm]?|zip|rar|mp[34]|mov|avi|wav)(?:\\?.*)?$/i;\r\n\r\nexport const getDomainName = (\r\n  link: string,\r\n  metadata?: t.ScrapeMetadata,\r\n  logger?: t.Logger\r\n): string | undefined => {\r\n  try {\r\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\r\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\r\n    if (domain) {\r\n      return domain;\r\n    }\r\n  } catch (e) {\r\n    // URL parsing failed\r\n    if (logger) {\r\n      logger.error('Error parsing URL:', e);\r\n    } else {\r\n      console.error('Error parsing URL:', e);\r\n    }\r\n  }\r\n\r\n  return;\r\n};\r\n\r\nexport function getAttribution(\r\n  link: string,\r\n  metadata?: t.ScrapeMetadata,\r\n  logger?: t.Logger\r\n): string | undefined {\r\n  if (!metadata) return getDomainName(link, metadata, logger);\r\n\r\n  const twitterSite = metadata['twitter:site'];\r\n  const twitterSiteFormatted =\r\n    typeof twitterSite === 'string' ? twitterSite.replace(/^@/, '') : undefined;\r\n\r\n  const possibleAttributions = [\r\n    metadata.ogSiteName,\r\n    metadata['og:site_name'],\r\n    metadata.title?.split('|').pop()?.trim(),\r\n    twitterSiteFormatted,\r\n  ];\r\n\r\n  const attribution = possibleAttributions.find(\r\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\r\n  );\r\n  if (attribution != null) {\r\n    return attribution;\r\n  }\r\n\r\n  return getDomainName(link, metadata, logger);\r\n}\r\n"],"names":[],"mappings":"AAAA;AAIA;;AAEG;AACH,IAAI,qBAAqB,GAAoB,IAAI;AAEjD;;;;AAIG;AACI,MAAM,mBAAmB,GAAG,MAAe;IAChD,IAAI,CAAC,qBAAqB,EAAE;AAC1B,QAAA,qBAAqB,GAAG;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK;SACT;;AAEf,IAAA,OAAO,qBAAqB;AAC9B;AAEO,MAAM,YAAY,GACvB;AAEW,MAAA,aAAa,GAAG,CAC3B,IAAY,EACZ,QAA2B,EAC3B,MAAiB,KACK;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;QAEV,IAAI,MAAM,EAAE;AACV,YAAA,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;aAChC;AACL,YAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;;IAI1C;AACF;SAEgB,cAAc,CAC5B,IAAY,EACZ,QAA2B,EAC3B,MAAiB,EAAA;AAEjB,IAAA,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;AAE3D,IAAA,MAAM,WAAW,GAAG,QAAQ,CAAC,cAAc,CAAC;IAC5C,MAAM,oBAAoB,GACxB,OAAO,WAAW,KAAK,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,SAAS;AAE7E,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,oBAAoB;KACrB;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;IAGpB,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;AAC9C;;;;"}

package/dist/esm/utils/graph.mjs

@@ -0,0 +1,13 @@
+const resetIfNotEmpty = (value, resetValue) => {
+    if (Array.isArray(value)) {
+        return value.length > 0 ? resetValue : value;
+    }
+    if (value instanceof Set || value instanceof Map) {
+        return value.size > 0 ? resetValue : value;
+    }
+    return value !== undefined ? resetValue : value;
+};
+const joinKeys = (args) => args.join('_');
+
+export { joinKeys, resetIfNotEmpty };
+//# sourceMappingURL=graph.mjs.map

package/dist/esm/utils/graph.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph.mjs","sources":["../../../src/utils/graph.ts"],"sourcesContent":["export const resetIfNotEmpty = <T>(value: T, resetValue: T): T => {\r\n  if (Array.isArray(value)) {\r\n    return value.length > 0 ? resetValue : value;\r\n  }\r\n  if (value instanceof Set || value instanceof Map) {\r\n    return value.size > 0 ? resetValue : value;\r\n  }\r\n  return value !== undefined ? resetValue : value;\r\n};\r\n\r\nexport const joinKeys = (args: (string | number | undefined)[]): string => args.join('_');"],"names":[],"mappings":"MAAa,eAAe,GAAG,CAAI,KAAQ,EAAE,UAAa,KAAO;AAC/D,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,QAAA,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,UAAU,GAAG,KAAK;;IAE9C,IAAI,KAAK,YAAY,GAAG,IAAI,KAAK,YAAY,GAAG,EAAE;AAChD,QAAA,OAAO,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,UAAU,GAAG,KAAK;;IAE5C,OAAO,KAAK,KAAK,SAAS,GAAG,UAAU,GAAG,KAAK;AACjD;AAEO,MAAM,QAAQ,GAAG,CAAC,IAAqC,KAAa,IAAI,CAAC,IAAI,CAAC,GAAG;;;;"}

package/dist/esm/utils/llm.mjs

@@ -0,0 +1,25 @@
+import { Providers } from '../common/enum.mjs';
+
+// src/utils/llm.ts
+function isOpenAILike(provider) {
+    if (provider == null) {
+        return false;
+    }
+    return [
+        Providers.OPENAI,
+        Providers.AZURE,
+        Providers.OPENROUTER,
+        Providers.XAI,
+        Providers.DEEPSEEK,
+        Providers.OLLAMA,
+    ].includes(provider);
+}
+function isGoogleLike(provider) {
+    if (provider == null) {
+        return false;
+    }
+    return [Providers.GOOGLE, Providers.VERTEXAI].includes(provider);
+}
+
+export { isGoogleLike, isOpenAILike };
+//# sourceMappingURL=llm.mjs.map

package/dist/esm/utils/llm.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.mjs","sources":["../../../src/utils/llm.ts"],"sourcesContent":["// src/utils/llm.ts\r\nimport { Providers } from '@/common';\r\n\r\nexport function isOpenAILike(provider?: string | Providers): boolean {\r\n  if (provider == null) {\r\n    return false;\r\n  }\r\n  return (\r\n    [\r\n      Providers.OPENAI,\r\n      Providers.AZURE,\r\n      Providers.OPENROUTER,\r\n      Providers.XAI,\r\n      Providers.DEEPSEEK,\r\n      Providers.OLLAMA,\r\n    ] as string[]\r\n  ).includes(provider);\r\n}\r\n\r\nexport function isGoogleLike(provider?: string | Providers): boolean {\r\n  if (provider == null) {\r\n    return false;\r\n  }\r\n  return ([Providers.GOOGLE, Providers.VERTEXAI] as string[]).includes(\r\n    provider\r\n  );\r\n}\r\n"],"names":[],"mappings":";;AAAA;AAGM,SAAU,YAAY,CAAC,QAA6B,EAAA;AACxD,IAAA,IAAI,QAAQ,IAAI,IAAI,EAAE;AACpB,QAAA,OAAO,KAAK;;IAEd,OACE;AACE,QAAA,SAAS,CAAC,MAAM;AAChB,QAAA,SAAS,CAAC,KAAK;AACf,QAAA,SAAS,CAAC,UAAU;AACpB,QAAA,SAAS,CAAC,GAAG;AACb,QAAA,SAAS,CAAC,QAAQ;AAClB,QAAA,SAAS,CAAC,MAAM;AAEnB,KAAA,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB;AAEM,SAAU,YAAY,CAAC,QAA6B,EAAA;AACxD,IAAA,IAAI,QAAQ,IAAI,IAAI,EAAE;AACpB,QAAA,OAAO,KAAK;;AAEd,IAAA,OAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAc,CAAC,QAAQ,CAClE,QAAQ,CACT;AACH;;;;"}