@librechat/agents 3.2.63 → 3.2.65

package/dist/esm/tools/search/search.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"search.mjs","names":[],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createKeenableAPI } from './keenable-search';\nimport { createTavilyAPI } from './tavily-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\nconst DEFAULT_CHUNK_SIZE = 150;\nconst DEFAULT_CHUNK_OVERLAP = 50;\n\n/** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /\n * `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The\n * overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`\n * throws when overlap >= size. */\nfunction resolveChunkOptions(\n  chunkSize?: number,\n  chunkOverlap?: number\n): { chunkSize: number; chunkOverlap: number } {\n  const resolve = (\n    configValue: number | undefined,\n    envVar: string,\n    fallback: number\n  ): number => {\n    if (configValue != null && configValue > 0) {\n      return configValue;\n    }\n    const envValue = Number(process.env[envVar]);\n    if (Number.isFinite(envValue) && envValue > 0) {\n      return envValue;\n    }\n    return fallback;\n  };\n\n  const size = resolve(chunkSize, 'SEARCH_CHUNK_SIZE', DEFAULT_CHUNK_SIZE);\n  let overlap = resolve(\n    chunkOverlap,\n    'SEARCH_CHUNK_OVERLAP',\n    DEFAULT_CHUNK_OVERLAP\n  );\n  if (overlap >= size) {\n    overlap = Math.floor(size / 3);\n  }\n  return { chunkSize: size, chunkOverlap: overlap };\n}\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n  if (maxContentLength != null && maxContentLength > 0) {\n    return maxContentLength;\n  }\n  const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n  if (Number.isFinite(envValue) && envValue > 0) {\n    return envValue;\n  }\n  return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n  return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n  chunkOptions,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  maxContentLength?: number;\n  chunkOptions?: { chunkSize: number; chunkOverlap: number };\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(\n      truncateContent(content, maxContentLength),\n      chunkOptions\n    );\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n  } = config;\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n  } else if (searchProvider.toLowerCase() === 'tavily') {\n    return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n  } else if (searchProvider.toLowerCase() === 'keenable') {\n    return createKeenableAPI(\n      keenableApiKey,\n      keenableApiUrl,\n      keenableSearchOptions\n    );\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', or 'keenable'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n  const chunkOptions = resolveChunkOptions(\n    config.chunkSize,\n    config.chunkOverlap\n  );\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const processResponse = (\n    url: string,\n    response: t.AnyScraperResponse\n  ): t.ScrapeResult => {\n    const rawMetadata = scraper.extractMetadata(response);\n    const metadata =\n      Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n    const attribution = getAttribution(url, metadata, logger_);\n\n    if (response.success && response.data) {\n      const [content, references] = scraper.extractContent(response);\n      return {\n        url,\n        references,\n        attribution,\n        content: truncateContent(chunker.cleanText(content), maxContentLength),\n      };\n    }\n\n    logger_.error(\n      `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n    );\n    return { url, attribution, error: true, content: '' };\n  };\n\n  const addHighlights = async (\n    result: t.ScrapeResult,\n    query: string,\n    onGetHighlights: t.SearchToolConfig['onGetHighlights']\n  ): Promise<t.ScrapeResult> => {\n    if (result.error != null) {\n      return result;\n    }\n    try {\n      const highlights = await getHighlights({\n        query,\n        reranker,\n        topResults,\n        content: result.content,\n        maxContentLength,\n        chunkOptions,\n        logger: logger_,\n      });\n      if (onGetHighlights) {\n        onGetHighlights(result.url);\n      }\n      return { ...result, highlights };\n    } catch (error) {\n      logger_.error('Error processing scraped content:', error);\n      return result;\n    }\n  };\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      try {\n        let responses: Array<[string, t.AnyScraperResponse]>;\n\n        if (scraper.scrapeUrls) {\n          responses = await scraper.scrapeUrls(links);\n        } else {\n          responses = await Promise.all(\n            links.map((link) =>\n              scraper\n                .scrapeUrl(link, {})\n                .catch((error): [string, t.AnyScraperResponse] => {\n                  logger_.error(`Error scraping ${link}:`, error);\n                  return [link, { success: false, error: String(error) }];\n                })\n            )\n          );\n        }\n\n        const withHighlights = await Promise.all(\n          responses.map(([url, response]) =>\n            addHighlights(\n              processResponse(url, response),\n              query,\n              onGetHighlights\n            )\n          )\n        );\n        return withHighlights;\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      }\n\n      if (\n        result.data.topStories != null &&\n        result.data.topStories.length > numElements\n      ) {\n        /** Merged news results can far exceed the requested source count;\n         * every entry is formatted into the LLM output, so cap them up\n         * front — before any early return below and before scraping\n         * entries the cap would discard */\n        result.data.topStories = result.data.topStories.slice(0, numElements);\n      }\n\n      if (!result.data.organic) {\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"mappings":";;;;;;AAQA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQ,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAU,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;AACnC,MAAM,qBAAqB;AAC3B,MAAM,wBAAwB;;;;;AAM9B,SAAS,oBACP,WACA,cAC6C;CAC7C,MAAM,WACJ,aACA,QACA,aACW;EACX,IAAI,eAAe,QAAQ,cAAc,GACvC,OAAO;EAET,MAAM,WAAW,OAAO,QAAQ,IAAI,OAAO;EAC3C,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;EAET,OAAO;CACT;CAEA,MAAM,OAAO,QAAQ,WAAW,qBAAqB,kBAAkB;CACvE,IAAI,UAAU,QACZ,cACA,wBACA,qBACF;CACA,IAAI,WAAW,MACb,UAAU,KAAK,MAAM,OAAO,CAAC;CAE/B,OAAO;EAAE,WAAW;EAAM,cAAc;CAAQ;AAClD;;;AAIA,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,cACA,aASwC;CACxC,MAAM,UAAU,UAAU,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,GACzC,YACF;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAehB,MAAM,QAAO,MAZU,MAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;GAClB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAShC,MAAM,QAAO,MANU,MAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;GAClB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,qBACA,gBACA,gBACA,0BACE;CAEJ,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,YAAY;MAC9B,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,aAAa;MACpD,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAO,gBAAgB,cAAc,iBAAiB,mBAAmB;MACpE,IAAI,eAAe,YAAY,MAAM,YAC1C,OAAO,kBACL,gBACA,gBACA,qBACF;MAEA,MAAM,IAAI,MACR,4BAA4B,eAAe,uDAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,eAAe,oBACnB,OAAO,WACP,OAAO,YACT;CACA,MAAM,UAAU,UAAU,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAc,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA;IACA,SAAS,OAAO;IAChB;IACA;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}
+{"version":3,"file":"search.mjs","names":[],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createKeenableAPI } from './keenable-search';\nimport { createTavilyAPI } from './tavily-search';\nimport { createCrwAPI } from './crw-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\nconst DEFAULT_CHUNK_SIZE = 150;\nconst DEFAULT_CHUNK_OVERLAP = 50;\n\n/** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /\n * `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The\n * overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`\n * throws when overlap >= size. */\nfunction resolveChunkOptions(\n  chunkSize?: number,\n  chunkOverlap?: number\n): { chunkSize: number; chunkOverlap: number } {\n  const resolve = (\n    configValue: number | undefined,\n    envVar: string,\n    fallback: number\n  ): number => {\n    if (configValue != null && configValue > 0) {\n      return configValue;\n    }\n    const envValue = Number(process.env[envVar]);\n    if (Number.isFinite(envValue) && envValue > 0) {\n      return envValue;\n    }\n    return fallback;\n  };\n\n  const size = resolve(chunkSize, 'SEARCH_CHUNK_SIZE', DEFAULT_CHUNK_SIZE);\n  let overlap = resolve(\n    chunkOverlap,\n    'SEARCH_CHUNK_OVERLAP',\n    DEFAULT_CHUNK_OVERLAP\n  );\n  if (overlap >= size) {\n    overlap = Math.floor(size / 3);\n  }\n  return { chunkSize: size, chunkOverlap: overlap };\n}\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n  if (maxContentLength != null && maxContentLength > 0) {\n    return maxContentLength;\n  }\n  const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n  if (Number.isFinite(envValue) && envValue > 0) {\n    return envValue;\n  }\n  return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n  return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n  chunkOptions,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  maxContentLength?: number;\n  chunkOptions?: { chunkSize: number; chunkOverlap: number };\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(\n      truncateContent(content, maxContentLength),\n      chunkOptions\n    );\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n    crwApiKey,\n    crwApiUrl,\n    crwSearchOptions,\n  } = config;\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n  } else if (searchProvider.toLowerCase() === 'tavily') {\n    return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n  } else if (searchProvider.toLowerCase() === 'keenable') {\n    return createKeenableAPI(\n      keenableApiKey,\n      keenableApiUrl,\n      keenableSearchOptions\n    );\n  } else if (searchProvider.toLowerCase() === 'crw') {\n    return createCrwAPI(crwApiKey, crwApiUrl, crwSearchOptions);\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', 'keenable', or 'crw'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n  const chunkOptions = resolveChunkOptions(\n    config.chunkSize,\n    config.chunkOverlap\n  );\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const processResponse = (\n    url: string,\n    response: t.AnyScraperResponse\n  ): t.ScrapeResult => {\n    const rawMetadata = scraper.extractMetadata(response);\n    const metadata =\n      Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n    const attribution = getAttribution(url, metadata, logger_);\n\n    if (response.success && response.data) {\n      const [content, references] = scraper.extractContent(response);\n      return {\n        url,\n        references,\n        attribution,\n        content: truncateContent(chunker.cleanText(content), maxContentLength),\n      };\n    }\n\n    logger_.error(\n      `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n    );\n    return { url, attribution, error: true, content: '' };\n  };\n\n  const addHighlights = async (\n    result: t.ScrapeResult,\n    query: string,\n    onGetHighlights: t.SearchToolConfig['onGetHighlights']\n  ): Promise<t.ScrapeResult> => {\n    if (result.error != null) {\n      return result;\n    }\n    try {\n      const highlights = await getHighlights({\n        query,\n        reranker,\n        topResults,\n        content: result.content,\n        maxContentLength,\n        chunkOptions,\n        logger: logger_,\n      });\n      if (onGetHighlights) {\n        onGetHighlights(result.url);\n      }\n      return { ...result, highlights };\n    } catch (error) {\n      logger_.error('Error processing scraped content:', error);\n      return result;\n    }\n  };\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      try {\n        let responses: Array<[string, t.AnyScraperResponse]>;\n\n        if (scraper.scrapeUrls) {\n          responses = await scraper.scrapeUrls(links);\n        } else {\n          responses = await Promise.all(\n            links.map((link) =>\n              scraper\n                .scrapeUrl(link, {})\n                .catch((error): [string, t.AnyScraperResponse] => {\n                  logger_.error(`Error scraping ${link}:`, error);\n                  return [link, { success: false, error: String(error) }];\n                })\n            )\n          );\n        }\n\n        const withHighlights = await Promise.all(\n          responses.map(([url, response]) =>\n            addHighlights(\n              processResponse(url, response),\n              query,\n              onGetHighlights\n            )\n          )\n        );\n        return withHighlights;\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      }\n\n      if (\n        result.data.topStories != null &&\n        result.data.topStories.length > numElements\n      ) {\n        /** Merged news results can far exceed the requested source count;\n         * every entry is formatted into the LLM output, so cap them up\n         * front — before any early return below and before scraping\n         * entries the cap would discard */\n        result.data.topStories = result.data.topStories.slice(0, numElements);\n      }\n\n      if (!result.data.organic) {\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"mappings":";;;;;;;AASA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQ,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAU,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;AACnC,MAAM,qBAAqB;AAC3B,MAAM,wBAAwB;;;;;AAM9B,SAAS,oBACP,WACA,cAC6C;CAC7C,MAAM,WACJ,aACA,QACA,aACW;EACX,IAAI,eAAe,QAAQ,cAAc,GACvC,OAAO;EAET,MAAM,WAAW,OAAO,QAAQ,IAAI,OAAO;EAC3C,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;EAET,OAAO;CACT;CAEA,MAAM,OAAO,QAAQ,WAAW,qBAAqB,kBAAkB;CACvE,IAAI,UAAU,QACZ,cACA,wBACA,qBACF;CACA,IAAI,WAAW,MACb,UAAU,KAAK,MAAM,OAAO,CAAC;CAE/B,OAAO;EAAE,WAAW;EAAM,cAAc;CAAQ;AAClD;;;AAIA,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,cACA,aASwC;CACxC,MAAM,UAAU,UAAU,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,GACzC,YACF;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAehB,MAAM,QAAO,MAZU,MAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;GAClB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAShC,MAAM,QAAO,MANU,MAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;GAClB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,qBACA,gBACA,gBACA,uBACA,WACA,WACA,qBACE;CAEJ,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,YAAY;MAC9B,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,aAAa;MACpD,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAO,gBAAgB,cAAc,iBAAiB,mBAAmB;MACpE,IAAI,eAAe,YAAY,MAAM,YAC1C,OAAO,kBACL,gBACA,gBACA,qBACF;MACK,IAAI,eAAe,YAAY,MAAM,OAC1C,OAAO,aAAa,WAAW,WAAW,gBAAgB;MAE1D,MAAM,IAAI,MACR,4BAA4B,eAAe,8DAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,eAAe,oBACnB,OAAO,WACP,OAAO,YACT;CACA,MAAM,UAAU,UAAU,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAc,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA;IACA,SAAS,OAAO;IAChB;IACA;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}

package/dist/esm/tools/search/tool.mjs

@@ -7,6 +7,7 @@ import { createSearchAPI, createSourceProcessor } from "./search.mjs";
 import { createSerperScraper } from "./serper-scraper.mjs";
 import { createTavilyScraper } from "./tavily-scraper.mjs";
 import { createFirecrawlScraper } from "./firecrawl.mjs";
+import { createCrwScraper } from "./crw-scraper.mjs";
 import { expandHighlights } from "./highlights.mjs";
 import { createReranker } from "./rerankers.mjs";
 import { tool } from "@langchain/core/tools";
@@ -180,7 +181,7 @@ function createTool({ schema, search, maxOutputChars, onSearchResults: _onSearch
 	});
 }
 const createSearchTool = (config = {}) => {
-	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilyExtractUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions, rerankerType = "cohere", rerankerTimeout, topResults = 5, maxContentLength, chunkSize, chunkOverlap, maxOutputChars, strategies = ["no_extraction"], filterContent = true, safeSearch = 1, scraperProvider = "firecrawl", firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, tavilyScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights } = config;
+	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilyExtractUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions, rerankerType = "cohere", rerankerTimeout, topResults = 5, maxContentLength, chunkSize, chunkOverlap, maxOutputChars, strategies = ["no_extraction"], filterContent = true, safeSearch = 1, scraperProvider = "firecrawl", firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, tavilyScraperOptions, crwApiKey, crwApiUrl, crwSearchOptions, crwScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights } = config;
 	const logger = config.logger || createDefaultLogger();
 	const effectiveTavilySearchOptions = searchProvider === "tavily" && config.safeSearch != null ? {
 		...tavilySearchOptions,
@@ -209,7 +210,10 @@ const createSearchTool = (config = {}) => {
 		tavilySearchOptions: effectiveTavilySearchOptions,
 		keenableApiKey,
 		keenableApiUrl,
-		keenableSearchOptions
+		keenableSearchOptions,
+		crwApiKey,
+		crwApiUrl,
+		crwSearchOptions
 	});
 	/** Create scraper based on scraperProvider */
 	let scraperInstance;
@@ -226,6 +230,14 @@ const createSearchTool = (config = {}) => {
 		timeout: scraperTimeout ?? tavilyScraperOptions?.timeout,
 		logger
 	});
+	else if (scraperProvider === "crw") scraperInstance = createCrwScraper({
+		...crwScraperOptions,
+		apiKey: crwScraperOptions?.apiKey ?? crwApiKey ?? process.env.CRW_API_KEY,
+		apiUrl: crwScraperOptions?.apiUrl ?? crwApiUrl,
+		timeout: scraperTimeout ?? crwScraperOptions?.timeout,
+		formats: crwScraperOptions?.formats ?? ["markdown", "rawHtml"],
+		logger
+	});
 	else scraperInstance = createFirecrawlScraper({
 		...firecrawlOptions,
 		apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
@@ -259,7 +271,7 @@ const createSearchTool = (config = {}) => {
 			searchAPI,
 			safeSearch,
 			supportsImages: searchProvider !== "keenable",
-			supportsVideos: searchProvider !== "tavily" && searchProvider !== "keenable",
+			supportsVideos: searchProvider !== "tavily" && searchProvider !== "keenable" && searchProvider !== "crw",
 			supportsNews: searchProvider !== "keenable",
 			sourceProcessor,
 			onGetHighlights,

package/dist/esm/tools/search/tool.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tool.mjs","names":["params"],"sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type { RunnableConfig } from '@langchain/core/runnables';\nimport type * as t from './types';\nimport {\n  WebSearchToolDescription,\n  WebSearchToolName,\n  countrySchema,\n  imagesSchema,\n  videosSchema,\n  querySchema,\n  dateSchema,\n  newsSchema,\n  DATE_RANGE,\n} from './schema';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createSerperScraper } from './serper-scraper';\nimport { createTavilyScraper } from './tavily-scraper';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createDefaultLogger } from './utils';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\n/**\n * Executes parallel searches and merges the results,\n * deduplicating top stories by link\n */\nexport async function executeParallelSearches({\n  searchAPI,\n  query,\n  date,\n  country,\n  safeSearch,\n  images,\n  videos,\n  news,\n  logger,\n}: {\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  images: boolean;\n  videos: boolean;\n  news: boolean;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  // Prepare all search tasks to run in parallel\n  const searchTasks: Promise<t.SearchResult>[] = [\n    // Main search\n    searchAPI.getSources({\n      query,\n      date,\n      country,\n      safeSearch,\n    }),\n  ];\n\n  if (images) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'images',\n        })\n        .catch((error) => {\n          logger.error('Error fetching images:', error);\n          return {\n            success: false,\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (videos) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'videos',\n        })\n        .catch((error) => {\n          logger.error('Error fetching videos:', error);\n          return {\n            success: false,\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (news) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'news',\n        })\n        .catch((error) => {\n          logger.error('Error fetching news:', error);\n          return {\n            success: false,\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n\n  // Run all searches in parallel\n  const results = await Promise.all(searchTasks);\n\n  // Get the main search result (first result)\n  const mainResult = results[0];\n  if (!mainResult.success) {\n    throw new Error(mainResult.error ?? 'Search failed');\n  }\n\n  // Merge additional results with the main results\n  const mergedResults = { ...mainResult.data };\n\n  // Convert existing news to topStories if present\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\n    const existingNewsAsTopStories = mergedResults.news\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\n      .map((newsItem) => ({\n        title: newsItem.title ?? '',\n        link: newsItem.link ?? '',\n        source: newsItem.source ?? '',\n        date: newsItem.date ?? '',\n        imageUrl: newsItem.imageUrl ?? '',\n        processed: false,\n      }));\n    mergedResults.topStories = [\n      ...(mergedResults.topStories ?? []),\n      ...existingNewsAsTopStories,\n    ];\n    delete mergedResults.news;\n  }\n\n  results.slice(1).forEach((result) => {\n    if (result.success && result.data !== undefined) {\n      if (result.data.images !== undefined && result.data.images.length > 0) {\n        mergedResults.images = [\n          ...(mergedResults.images ?? []),\n          ...result.data.images,\n        ];\n      }\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\n        mergedResults.videos = [\n          ...(mergedResults.videos ?? []),\n          ...result.data.videos,\n        ];\n      }\n      if (result.data.news !== undefined && result.data.news.length > 0) {\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\n          ...newsItem,\n          link: newsItem.link ?? '',\n        }));\n        mergedResults.topStories = [\n          ...(mergedResults.topStories ?? []),\n          ...newsAsTopStories,\n        ];\n      }\n    }\n  });\n\n  if (\n    mergedResults.topStories !== undefined &&\n    mergedResults.topStories.length > 1\n  ) {\n    /** The main search's own news results and the parallel news sub-search\n     * frequently return the same stories — keep the first occurrence of each\n     * link so duplicates aren't scraped, reranked, and formatted repeatedly */\n    const seenLinks = new Set<string>();\n    mergedResults.topStories = mergedResults.topStories.filter((story) => {\n      if (!story.link || seenLinks.has(story.link)) {\n        return false;\n      }\n      seenLinks.add(story.link);\n      return true;\n    });\n  }\n\n  return { success: true, data: mergedResults };\n}\n\nfunction createSearchProcessor({\n  searchAPI,\n  safeSearch,\n  supportsImages,\n  supportsVideos,\n  supportsNews,\n  sourceProcessor,\n  onGetHighlights,\n  logger,\n}: {\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  supportsImages: boolean;\n  supportsVideos: boolean;\n  supportsNews: boolean;\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n  logger: t.Logger;\n}) {\n  return async function ({\n    query,\n    date,\n    country,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n    images = false,\n    videos = false,\n    news = false,\n  }: {\n    query: string;\n    country?: string;\n    date?: DATE_RANGE;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\n    images?: boolean;\n    videos?: boolean;\n    news?: boolean;\n  }): Promise<t.SearchResultData> {\n    try {\n      // Execute parallel searches and merge results\n      const searchResult = await executeParallelSearches({\n        searchAPI,\n        query,\n        date,\n        country,\n        safeSearch,\n        images: supportsImages && images,\n        videos: supportsVideos && videos,\n        news: supportsNews && news,\n        logger,\n      });\n\n      onSearchResults?.(searchResult);\n\n      const processedSources = await sourceProcessor.processSources({\n        query,\n        news,\n        result: searchResult,\n        proMode,\n        onGetHighlights,\n        numElements: maxSources,\n      });\n\n      return expandHighlights(processedSources);\n    } catch (error) {\n      logger.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        videos: [],\n        news: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n}\n\nfunction createOnSearchResults({\n  runnableConfig,\n  onSearchResults,\n}: {\n  runnableConfig: RunnableConfig;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}) {\n  return function (results: t.SearchResult): void {\n    if (!onSearchResults) {\n      return;\n    }\n    onSearchResults(results, runnableConfig);\n  };\n}\n\nfunction createTool({\n  schema,\n  search,\n  maxOutputChars,\n  onSearchResults: _onSearchResults,\n}: {\n  schema: Record<string, unknown>;\n  search: ReturnType<typeof createSearchProcessor>;\n  maxOutputChars?: number;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}): DynamicStructuredTool {\n  return tool(\n    async (rawParams, runnableConfig) => {\n      const params = rawParams as SearchToolParams;\n      const { query, date, country: _c, images, videos, news } = params;\n      const country = typeof _c === 'string' && _c ? _c : undefined;\n      const searchResult = await search({\n        query,\n        date,\n        country,\n        images,\n        videos,\n        news,\n        onSearchResults: createOnSearchResults({\n          runnableConfig,\n          onSearchResults: _onSearchResults,\n        }),\n      });\n      const turn = runnableConfig.toolCall?.turn ?? 0;\n      const { output, references } = formatResultsForLLM(\n        turn,\n        searchResult,\n        maxOutputChars\n      );\n      const data: t.SearchResultData = { turn, ...searchResult, references };\n      return [output, { [Constants.WEB_SEARCH]: data }];\n    },\n    {\n      name: WebSearchToolName,\n      description: WebSearchToolDescription,\n      schema: schema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n}\n\n/**\n * Creates a search tool with configurable search and scraper providers.\n *\n * Search providers: Serper (Google results), SearXNG (self-hosted meta-search), Tavily (AI-optimized).\n * Scraper providers: Firecrawl (default, full-featured), Serper (lightweight), Tavily (batch extraction).\n *\n * The country schema field is exposed to the LLM for providers that support localized results.\n */\n/** Input params type for search tool */\ninterface SearchToolParams {\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  images?: boolean;\n  videos?: boolean;\n  news?: boolean;\n}\n\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilyExtractUrl,\n    tavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n    rerankerType = 'cohere',\n    rerankerTimeout,\n    topResults = 5,\n    maxContentLength,\n    chunkSize,\n    chunkOverlap,\n    maxOutputChars,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    safeSearch = 1,\n    scraperProvider = 'firecrawl',\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlVersion,\n    firecrawlOptions,\n    serperScraperOptions,\n    tavilyScraperOptions,\n    scraperTimeout,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n    onGetHighlights,\n  } = config;\n\n  const logger = config.logger || createDefaultLogger();\n  const effectiveTavilySearchOptions =\n    searchProvider === 'tavily' && config.safeSearch != null\n      ? {\n        ...tavilySearchOptions,\n        safeSearch: config.safeSearch !== 0,\n      }\n      : tavilySearchOptions;\n\n  const schemaProperties: Record<string, unknown> = {\n    query: querySchema,\n    date: dateSchema,\n    images: imagesSchema,\n    videos: videosSchema,\n    news: newsSchema,\n  };\n\n  if (searchProvider === 'serper' || searchProvider === 'tavily') {\n    schemaProperties.country = countrySchema;\n  }\n\n  const toolSchema = {\n    type: 'object',\n    properties: schemaProperties,\n    required: ['query'],\n  };\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions: effectiveTavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n  });\n\n  /** Create scraper based on scraperProvider */\n  let scraperInstance: t.BaseScraper;\n\n  if (scraperProvider === 'serper') {\n    scraperInstance = createSerperScraper({\n      ...serperScraperOptions,\n      apiKey: serperApiKey,\n      timeout: scraperTimeout ?? serperScraperOptions?.timeout,\n      logger,\n    });\n  } else if (scraperProvider === 'tavily') {\n    scraperInstance = createTavilyScraper({\n      ...tavilyScraperOptions,\n      apiKey:\n        tavilyScraperOptions?.apiKey ??\n        tavilyApiKey ??\n        process.env.TAVILY_API_KEY,\n      apiUrl: tavilyScraperOptions?.apiUrl ?? tavilyExtractUrl,\n      timeout: scraperTimeout ?? tavilyScraperOptions?.timeout,\n      logger,\n    });\n  } else {\n    scraperInstance = createFirecrawlScraper({\n      ...firecrawlOptions,\n      apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n      apiUrl: firecrawlApiUrl,\n      version: firecrawlVersion,\n      timeout: scraperTimeout ?? firecrawlOptions?.timeout,\n      formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],\n      logger,\n    });\n  }\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    rerankerTimeout,\n    logger,\n  });\n\n  if (!selectedReranker) {\n    logger.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      maxContentLength,\n      chunkSize,\n      chunkOverlap,\n      strategies,\n      filterContent,\n      logger,\n    },\n    scraperInstance\n  );\n\n  const search = createSearchProcessor({\n    searchAPI,\n    safeSearch,\n    // Keenable is organic-only: its API ignores `type`, so image/news\n    // sub-searches would spend rate limit and merge nothing.\n    supportsImages: searchProvider !== 'keenable',\n    supportsVideos:\n      searchProvider !== 'tavily' && searchProvider !== 'keenable',\n    supportsNews: searchProvider !== 'keenable',\n    sourceProcessor,\n    onGetHighlights,\n    logger,\n  });\n\n  return createTool({\n    search,\n    schema: toolSchema,\n    maxOutputChars,\n    onSearchResults: _onSearchResults,\n  });\n};\n"],"mappings":";;;;;;;;;;;;;;;;;AA4BA,eAAsB,wBAAwB,EAC5C,WACA,OACA,MACA,SACA,YACA,QACA,QACA,MACA,UAW0B;CAE1B,MAAM,cAAyC,CAE7C,UAAU,WAAW;EACnB;EACA;EACA;EACA;CACF,CAAC,CACH;CAEA,IAAI,QACF,YAAY,KACV,UACG,WAAW;EACV;EACA;EACA;EACA;EACA,MAAM;CACR,CAAC,CAAC,CACD,OAAO,UAAU;EAChB,OAAO,MAAM,0BAA0B,KAAK;EAC5C,OAAO;GACL,SAAS;GACT,OAAO,yBAAyB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACvF;CACF,CAAC,CACL;CAEF,IAAI,QACF,YAAY,KACV,UACG,WAAW;EACV;EACA;EACA;EACA;EACA,MAAM;CACR,CAAC,CAAC,CACD,OAAO,UAAU;EAChB,OAAO,MAAM,0BAA0B,KAAK;EAC5C,OAAO;GACL,SAAS;GACT,OAAO,yBAAyB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACvF;CACF,CAAC,CACL;CAEF,IAAI,MACF,YAAY,KACV,UACG,WAAW;EACV;EACA;EACA;EACA;EACA,MAAM;CACR,CAAC,CAAC,CACD,OAAO,UAAU;EAChB,OAAO,MAAM,wBAAwB,KAAK;EAC1C,OAAO;GACL,SAAS;GACT,OAAO,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrF;CACF,CAAC,CACL;CAIF,MAAM,UAAU,MAAM,QAAQ,IAAI,WAAW;CAG7C,MAAM,aAAa,QAAQ;CAC3B,IAAI,CAAC,WAAW,SACd,MAAM,IAAI,MAAM,WAAW,SAAS,eAAe;CAIrD,MAAM,gBAAgB,EAAE,GAAG,WAAW,KAAK;CAG3C,IAAI,cAAc,SAAS,KAAA,KAAa,cAAc,KAAK,SAAS,GAAG;EACrE,MAAM,2BAA2B,cAAc,KAC5C,QAAQ,aAAa,SAAS,SAAS,KAAA,KAAa,SAAS,SAAS,EAAE,CAAC,CACzE,KAAK,cAAc;GAClB,OAAO,SAAS,SAAS;GACzB,MAAM,SAAS,QAAQ;GACvB,QAAQ,SAAS,UAAU;GAC3B,MAAM,SAAS,QAAQ;GACvB,UAAU,SAAS,YAAY;GAC/B,WAAW;EACb,EAAE;EACJ,cAAc,aAAa,CACzB,GAAI,cAAc,cAAc,CAAC,GACjC,GAAG,wBACL;EACA,OAAO,cAAc;CACvB;CAEA,QAAQ,MAAM,CAAC,CAAC,CAAC,SAAS,WAAW;EACnC,IAAI,OAAO,WAAW,OAAO,SAAS,KAAA,GAAW;GAC/C,IAAI,OAAO,KAAK,WAAW,KAAA,KAAa,OAAO,KAAK,OAAO,SAAS,GAClE,cAAc,SAAS,CACrB,GAAI,cAAc,UAAU,CAAC,GAC7B,GAAG,OAAO,KAAK,MACjB;GAEF,IAAI,OAAO,KAAK,WAAW,KAAA,KAAa,OAAO,KAAK,OAAO,SAAS,GAClE,cAAc,SAAS,CACrB,GAAI,cAAc,UAAU,CAAC,GAC7B,GAAG,OAAO,KAAK,MACjB;GAEF,IAAI,OAAO,KAAK,SAAS,KAAA,KAAa,OAAO,KAAK,KAAK,SAAS,GAAG;IACjE,MAAM,mBAAmB,OAAO,KAAK,KAAK,KAAK,cAAc;KAC3D,GAAG;KACH,MAAM,SAAS,QAAQ;IACzB,EAAE;IACF,cAAc,aAAa,CACzB,GAAI,cAAc,cAAc,CAAC,GACjC,GAAG,gBACL;GACF;EACF;CACF,CAAC;CAED,IACE,cAAc,eAAe,KAAA,KAC7B,cAAc,WAAW,SAAS,GAClC;;;;EAIA,MAAM,4BAAY,IAAI,IAAY;EAClC,cAAc,aAAa,cAAc,WAAW,QAAQ,UAAU;GACpE,IAAI,CAAC,MAAM,QAAQ,UAAU,IAAI,MAAM,IAAI,GACzC,OAAO;GAET,UAAU,IAAI,MAAM,IAAI;GACxB,OAAO;EACT,CAAC;CACH;CAEA,OAAO;EAAE,SAAS;EAAM,MAAM;CAAc;AAC9C;AAEA,SAAS,sBAAsB,EAC7B,WACA,YACA,gBACA,gBACA,cACA,iBACA,iBACA,UAUC;CACD,OAAO,eAAgB,EACrB,OACA,MACA,SACA,UAAU,MACV,aAAa,GACb,iBACA,SAAS,OACT,SAAS,OACT,OAAO,SAWuB;EAC9B,IAAI;GAEF,MAAM,eAAe,MAAM,wBAAwB;IACjD;IACA;IACA;IACA;IACA;IACA,QAAQ,kBAAkB;IAC1B,QAAQ,kBAAkB;IAC1B,MAAM,gBAAgB;IACtB;GACF,CAAC;GAED,kBAAkB,YAAY;GAW9B,OAAO,iBAAiB,MATO,gBAAgB,eAAe;IAC5D;IACA;IACA,QAAQ;IACR;IACA;IACA,aAAa;GACf,CAAC,CAEuC;EAC1C,SAAS,OAAO;GACd,OAAO,MAAM,oBAAoB,KAAK;GACtC,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,QAAQ,CAAC;IACT,MAAM,CAAC;IACP,iBAAiB,CAAC;IAClB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;AACF;AAEA,SAAS,sBAAsB,EAC7B,gBACA,mBAIC;CACD,OAAO,SAAU,SAA+B;EAC9C,IAAI,CAAC,iBACH;EAEF,gBAAgB,SAAS,cAAc;CACzC;AACF;AAEA,SAAS,WAAW,EAClB,QACA,QACA,gBACA,iBAAiB,oBAMO;CACxB,OAAO,KACL,OAAO,WAAW,mBAAmB;EAEnC,MAAM,EAAE,OAAO,MAAM,SAAS,IAAI,QAAQ,QAAQ,SAASA;EAE3D,MAAM,eAAe,MAAM,OAAO;GAChC;GACA;GACA,SAJc,OAAO,OAAO,YAAY,KAAK,KAAK,KAAA;GAKlD;GACA;GACA;GACA,iBAAiB,sBAAsB;IACrC;IACA,iBAAiB;GACnB,CAAC;EACH,CAAC;EACD,MAAM,OAAO,eAAe,UAAU,QAAQ;EAC9C,MAAM,EAAE,QAAQ,eAAe,oBAC7B,MACA,cACA,cACF;EACA,MAAM,OAA2B;GAAE;GAAM,GAAG;GAAc;EAAW;EACrE,OAAO,CAAC,QAAQ,GAAA,eAA0B,KAAK,CAAC;CAClD,GACA;EACE,MAAM;EACN,aAAa;EACL;EACR,gBAAA;CACF,CACF;AACF;AAoBA,MAAa,oBACX,SAA6B,CAAC,MACJ;CAC1B,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,kBACA,qBACA,gBACA,gBACA,uBACA,eAAe,UACf,iBACA,aAAa,GACb,kBACA,WACA,cACA,gBACA,aAAa,CAAC,eAAe,GAC7B,gBAAgB,MAChB,aAAa,GACb,kBAAkB,aAClB,iBACA,iBACA,kBACA,kBACA,sBACA,sBACA,gBACA,YACA,YACA,cACA,iBAAiB,kBACjB,oBACE;CAEJ,MAAM,SAAS,OAAO,UAAU,oBAAoB;CACpD,MAAM,+BACJ,mBAAmB,YAAY,OAAO,cAAc,OAChD;EACA,GAAG;EACH,YAAY,OAAO,eAAe;CACpC,IACE;CAEN,MAAM,mBAA4C;EAChD,OAAO;EACP,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,MAAM;CACR;CAEA,IAAI,mBAAmB,YAAY,mBAAmB,UACpD,iBAAiB,UAAU;CAG7B,MAAM,aAAa;EACjB,MAAM;EACN,YAAY;EACZ,UAAU,CAAC,OAAO;CACpB;CAEA,MAAM,YAAY,gBAAgB;EAChC;EACA;EACA;EACA;EACA;EACA;EACA,qBAAqB;EACrB;EACA;EACA;CACF,CAAC;;CAGD,IAAI;CAEJ,IAAI,oBAAoB,UACtB,kBAAkB,oBAAoB;EACpC,GAAG;EACH,QAAQ;EACR,SAAS,kBAAkB,sBAAsB;EACjD;CACF,CAAC;MACI,IAAI,oBAAoB,UAC7B,kBAAkB,oBAAoB;EACpC,GAAG;EACH,QACE,sBAAsB,UACtB,gBACA,QAAQ,IAAI;EACd,QAAQ,sBAAsB,UAAU;EACxC,SAAS,kBAAkB,sBAAsB;EACjD;CACF,CAAC;MAED,kBAAkB,uBAAuB;EACvC,GAAG;EACH,QAAQ,mBAAmB,QAAQ,IAAI;EACvC,QAAQ;EACR,SAAS;EACT,SAAS,kBAAkB,kBAAkB;EAC7C,SAAS,kBAAkB,WAAW,CAAC,YAAY,SAAS;EAC5D;CACF,CAAC;CAGH,MAAM,mBAAmB,eAAe;EACtC;EACA;EACA;EACA;EACA;EACA;CACF,CAAC;CAED,IAAI,CAAC,kBACH,OAAO,KAAK,8CAA8C;CAG5D,MAAM,kBAAkB,sBACtB;EACE,UAAU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;CACF,GACA,eACF;CAgBA,OAAO,WAAW;EAChB,QAfa,sBAAsB;GACnC;GACA;GAGA,gBAAgB,mBAAmB;GACnC,gBACE,mBAAmB,YAAY,mBAAmB;GACpD,cAAc,mBAAmB;GACjC;GACA;GACA;EACF,CAGO;EACL,QAAQ;EACR;EACA,iBAAiB;CACnB,CAAC;AACH"}
+{"version":3,"file":"tool.mjs","names":["params"],"sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type { RunnableConfig } from '@langchain/core/runnables';\nimport type * as t from './types';\nimport {\n  WebSearchToolDescription,\n  WebSearchToolName,\n  countrySchema,\n  imagesSchema,\n  videosSchema,\n  querySchema,\n  dateSchema,\n  newsSchema,\n  DATE_RANGE,\n} from './schema';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createSerperScraper } from './serper-scraper';\nimport { createTavilyScraper } from './tavily-scraper';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { createCrwScraper } from './crw-scraper';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createDefaultLogger } from './utils';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\n/**\n * Executes parallel searches and merges the results,\n * deduplicating top stories by link\n */\nexport async function executeParallelSearches({\n  searchAPI,\n  query,\n  date,\n  country,\n  safeSearch,\n  images,\n  videos,\n  news,\n  logger,\n}: {\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  images: boolean;\n  videos: boolean;\n  news: boolean;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  // Prepare all search tasks to run in parallel\n  const searchTasks: Promise<t.SearchResult>[] = [\n    // Main search\n    searchAPI.getSources({\n      query,\n      date,\n      country,\n      safeSearch,\n    }),\n  ];\n\n  if (images) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'images',\n        })\n        .catch((error) => {\n          logger.error('Error fetching images:', error);\n          return {\n            success: false,\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (videos) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'videos',\n        })\n        .catch((error) => {\n          logger.error('Error fetching videos:', error);\n          return {\n            success: false,\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (news) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'news',\n        })\n        .catch((error) => {\n          logger.error('Error fetching news:', error);\n          return {\n            success: false,\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n\n  // Run all searches in parallel\n  const results = await Promise.all(searchTasks);\n\n  // Get the main search result (first result)\n  const mainResult = results[0];\n  if (!mainResult.success) {\n    throw new Error(mainResult.error ?? 'Search failed');\n  }\n\n  // Merge additional results with the main results\n  const mergedResults = { ...mainResult.data };\n\n  // Convert existing news to topStories if present\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\n    const existingNewsAsTopStories = mergedResults.news\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\n      .map((newsItem) => ({\n        title: newsItem.title ?? '',\n        link: newsItem.link ?? '',\n        source: newsItem.source ?? '',\n        date: newsItem.date ?? '',\n        imageUrl: newsItem.imageUrl ?? '',\n        processed: false,\n      }));\n    mergedResults.topStories = [\n      ...(mergedResults.topStories ?? []),\n      ...existingNewsAsTopStories,\n    ];\n    delete mergedResults.news;\n  }\n\n  results.slice(1).forEach((result) => {\n    if (result.success && result.data !== undefined) {\n      if (result.data.images !== undefined && result.data.images.length > 0) {\n        mergedResults.images = [\n          ...(mergedResults.images ?? []),\n          ...result.data.images,\n        ];\n      }\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\n        mergedResults.videos = [\n          ...(mergedResults.videos ?? []),\n          ...result.data.videos,\n        ];\n      }\n      if (result.data.news !== undefined && result.data.news.length > 0) {\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\n          ...newsItem,\n          link: newsItem.link ?? '',\n        }));\n        mergedResults.topStories = [\n          ...(mergedResults.topStories ?? []),\n          ...newsAsTopStories,\n        ];\n      }\n    }\n  });\n\n  if (\n    mergedResults.topStories !== undefined &&\n    mergedResults.topStories.length > 1\n  ) {\n    /** The main search's own news results and the parallel news sub-search\n     * frequently return the same stories — keep the first occurrence of each\n     * link so duplicates aren't scraped, reranked, and formatted repeatedly */\n    const seenLinks = new Set<string>();\n    mergedResults.topStories = mergedResults.topStories.filter((story) => {\n      if (!story.link || seenLinks.has(story.link)) {\n        return false;\n      }\n      seenLinks.add(story.link);\n      return true;\n    });\n  }\n\n  return { success: true, data: mergedResults };\n}\n\nfunction createSearchProcessor({\n  searchAPI,\n  safeSearch,\n  supportsImages,\n  supportsVideos,\n  supportsNews,\n  sourceProcessor,\n  onGetHighlights,\n  logger,\n}: {\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  supportsImages: boolean;\n  supportsVideos: boolean;\n  supportsNews: boolean;\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n  logger: t.Logger;\n}) {\n  return async function ({\n    query,\n    date,\n    country,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n    images = false,\n    videos = false,\n    news = false,\n  }: {\n    query: string;\n    country?: string;\n    date?: DATE_RANGE;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\n    images?: boolean;\n    videos?: boolean;\n    news?: boolean;\n  }): Promise<t.SearchResultData> {\n    try {\n      // Execute parallel searches and merge results\n      const searchResult = await executeParallelSearches({\n        searchAPI,\n        query,\n        date,\n        country,\n        safeSearch,\n        images: supportsImages && images,\n        videos: supportsVideos && videos,\n        news: supportsNews && news,\n        logger,\n      });\n\n      onSearchResults?.(searchResult);\n\n      const processedSources = await sourceProcessor.processSources({\n        query,\n        news,\n        result: searchResult,\n        proMode,\n        onGetHighlights,\n        numElements: maxSources,\n      });\n\n      return expandHighlights(processedSources);\n    } catch (error) {\n      logger.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        videos: [],\n        news: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n}\n\nfunction createOnSearchResults({\n  runnableConfig,\n  onSearchResults,\n}: {\n  runnableConfig: RunnableConfig;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}) {\n  return function (results: t.SearchResult): void {\n    if (!onSearchResults) {\n      return;\n    }\n    onSearchResults(results, runnableConfig);\n  };\n}\n\nfunction createTool({\n  schema,\n  search,\n  maxOutputChars,\n  onSearchResults: _onSearchResults,\n}: {\n  schema: Record<string, unknown>;\n  search: ReturnType<typeof createSearchProcessor>;\n  maxOutputChars?: number;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}): DynamicStructuredTool {\n  return tool(\n    async (rawParams, runnableConfig) => {\n      const params = rawParams as SearchToolParams;\n      const { query, date, country: _c, images, videos, news } = params;\n      const country = typeof _c === 'string' && _c ? _c : undefined;\n      const searchResult = await search({\n        query,\n        date,\n        country,\n        images,\n        videos,\n        news,\n        onSearchResults: createOnSearchResults({\n          runnableConfig,\n          onSearchResults: _onSearchResults,\n        }),\n      });\n      const turn = runnableConfig.toolCall?.turn ?? 0;\n      const { output, references } = formatResultsForLLM(\n        turn,\n        searchResult,\n        maxOutputChars\n      );\n      const data: t.SearchResultData = { turn, ...searchResult, references };\n      return [output, { [Constants.WEB_SEARCH]: data }];\n    },\n    {\n      name: WebSearchToolName,\n      description: WebSearchToolDescription,\n      schema: schema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n}\n\n/**\n * Creates a search tool with configurable search and scraper providers.\n *\n * Search providers: Serper (Google results), SearXNG (self-hosted meta-search), Tavily (AI-optimized), fastCRW (Firecrawl-compatible, self-host or cloud).\n * Scraper providers: Firecrawl (default, full-featured), Serper (lightweight), Tavily (batch extraction), fastCRW (Firecrawl-compatible, self-host or cloud).\n *\n * The country schema field is exposed to the LLM for providers that support localized results.\n */\n/** Input params type for search tool */\ninterface SearchToolParams {\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  images?: boolean;\n  videos?: boolean;\n  news?: boolean;\n}\n\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilyExtractUrl,\n    tavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n    rerankerType = 'cohere',\n    rerankerTimeout,\n    topResults = 5,\n    maxContentLength,\n    chunkSize,\n    chunkOverlap,\n    maxOutputChars,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    safeSearch = 1,\n    scraperProvider = 'firecrawl',\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlVersion,\n    firecrawlOptions,\n    serperScraperOptions,\n    tavilyScraperOptions,\n    crwApiKey,\n    crwApiUrl,\n    crwSearchOptions,\n    crwScraperOptions,\n    scraperTimeout,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n    onGetHighlights,\n  } = config;\n\n  const logger = config.logger || createDefaultLogger();\n  const effectiveTavilySearchOptions =\n    searchProvider === 'tavily' && config.safeSearch != null\n      ? {\n        ...tavilySearchOptions,\n        safeSearch: config.safeSearch !== 0,\n      }\n      : tavilySearchOptions;\n\n  const schemaProperties: Record<string, unknown> = {\n    query: querySchema,\n    date: dateSchema,\n    images: imagesSchema,\n    videos: videosSchema,\n    news: newsSchema,\n  };\n\n  if (searchProvider === 'serper' || searchProvider === 'tavily') {\n    schemaProperties.country = countrySchema;\n  }\n\n  const toolSchema = {\n    type: 'object',\n    properties: schemaProperties,\n    required: ['query'],\n  };\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions: effectiveTavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n    crwApiKey,\n    crwApiUrl,\n    crwSearchOptions,\n  });\n\n  /** Create scraper based on scraperProvider */\n  let scraperInstance: t.BaseScraper;\n\n  if (scraperProvider === 'serper') {\n    scraperInstance = createSerperScraper({\n      ...serperScraperOptions,\n      apiKey: serperApiKey,\n      timeout: scraperTimeout ?? serperScraperOptions?.timeout,\n      logger,\n    });\n  } else if (scraperProvider === 'tavily') {\n    scraperInstance = createTavilyScraper({\n      ...tavilyScraperOptions,\n      apiKey:\n        tavilyScraperOptions?.apiKey ??\n        tavilyApiKey ??\n        process.env.TAVILY_API_KEY,\n      apiUrl: tavilyScraperOptions?.apiUrl ?? tavilyExtractUrl,\n      timeout: scraperTimeout ?? tavilyScraperOptions?.timeout,\n      logger,\n    });\n  } else if (scraperProvider === 'crw') {\n    scraperInstance = createCrwScraper({\n      ...crwScraperOptions,\n      apiKey:\n        crwScraperOptions?.apiKey ?? crwApiKey ?? process.env.CRW_API_KEY,\n      apiUrl: crwScraperOptions?.apiUrl ?? crwApiUrl,\n      timeout: scraperTimeout ?? crwScraperOptions?.timeout,\n      formats: crwScraperOptions?.formats ?? ['markdown', 'rawHtml'],\n      logger,\n    });\n  } else {\n    scraperInstance = createFirecrawlScraper({\n      ...firecrawlOptions,\n      apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n      apiUrl: firecrawlApiUrl,\n      version: firecrawlVersion,\n      timeout: scraperTimeout ?? firecrawlOptions?.timeout,\n      formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],\n      logger,\n    });\n  }\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    rerankerTimeout,\n    logger,\n  });\n\n  if (!selectedReranker) {\n    logger.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      maxContentLength,\n      chunkSize,\n      chunkOverlap,\n      strategies,\n      filterContent,\n      logger,\n    },\n    scraperInstance\n  );\n\n  const search = createSearchProcessor({\n    searchAPI,\n    safeSearch,\n    // Keenable is organic-only: its API ignores `type`, so image/news\n    // sub-searches would spend rate limit and merge nothing.\n    supportsImages: searchProvider !== 'keenable',\n    supportsVideos:\n      searchProvider !== 'tavily' &&\n      searchProvider !== 'keenable' &&\n      searchProvider !== 'crw',\n    supportsNews: searchProvider !== 'keenable',\n    sourceProcessor,\n    onGetHighlights,\n    logger,\n  });\n\n  return createTool({\n    search,\n    schema: toolSchema,\n    maxOutputChars,\n    onSearchResults: _onSearchResults,\n  });\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AA6BA,eAAsB,wBAAwB,EAC5C,WACA,OACA,MACA,SACA,YACA,QACA,QACA,MACA,UAW0B;CAE1B,MAAM,cAAyC,CAE7C,UAAU,WAAW;EACnB;EACA;EACA;EACA;CACF,CAAC,CACH;CAEA,IAAI,QACF,YAAY,KACV,UACG,WAAW;EACV;EACA;EACA;EACA;EACA,MAAM;CACR,CAAC,CAAC,CACD,OAAO,UAAU;EAChB,OAAO,MAAM,0BAA0B,KAAK;EAC5C,OAAO;GACL,SAAS;GACT,OAAO,yBAAyB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACvF;CACF,CAAC,CACL;CAEF,IAAI,QACF,YAAY,KACV,UACG,WAAW;EACV;EACA;EACA;EACA;EACA,MAAM;CACR,CAAC,CAAC,CACD,OAAO,UAAU;EAChB,OAAO,MAAM,0BAA0B,KAAK;EAC5C,OAAO;GACL,SAAS;GACT,OAAO,yBAAyB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACvF;CACF,CAAC,CACL;CAEF,IAAI,MACF,YAAY,KACV,UACG,WAAW;EACV;EACA;EACA;EACA;EACA,MAAM;CACR,CAAC,CAAC,CACD,OAAO,UAAU;EAChB,OAAO,MAAM,wBAAwB,KAAK;EAC1C,OAAO;GACL,SAAS;GACT,OAAO,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrF;CACF,CAAC,CACL;CAIF,MAAM,UAAU,MAAM,QAAQ,IAAI,WAAW;CAG7C,MAAM,aAAa,QAAQ;CAC3B,IAAI,CAAC,WAAW,SACd,MAAM,IAAI,MAAM,WAAW,SAAS,eAAe;CAIrD,MAAM,gBAAgB,EAAE,GAAG,WAAW,KAAK;CAG3C,IAAI,cAAc,SAAS,KAAA,KAAa,cAAc,KAAK,SAAS,GAAG;EACrE,MAAM,2BAA2B,cAAc,KAC5C,QAAQ,aAAa,SAAS,SAAS,KAAA,KAAa,SAAS,SAAS,EAAE,CAAC,CACzE,KAAK,cAAc;GAClB,OAAO,SAAS,SAAS;GACzB,MAAM,SAAS,QAAQ;GACvB,QAAQ,SAAS,UAAU;GAC3B,MAAM,SAAS,QAAQ;GACvB,UAAU,SAAS,YAAY;GAC/B,WAAW;EACb,EAAE;EACJ,cAAc,aAAa,CACzB,GAAI,cAAc,cAAc,CAAC,GACjC,GAAG,wBACL;EACA,OAAO,cAAc;CACvB;CAEA,QAAQ,MAAM,CAAC,CAAC,CAAC,SAAS,WAAW;EACnC,IAAI,OAAO,WAAW,OAAO,SAAS,KAAA,GAAW;GAC/C,IAAI,OAAO,KAAK,WAAW,KAAA,KAAa,OAAO,KAAK,OAAO,SAAS,GAClE,cAAc,SAAS,CACrB,GAAI,cAAc,UAAU,CAAC,GAC7B,GAAG,OAAO,KAAK,MACjB;GAEF,IAAI,OAAO,KAAK,WAAW,KAAA,KAAa,OAAO,KAAK,OAAO,SAAS,GAClE,cAAc,SAAS,CACrB,GAAI,cAAc,UAAU,CAAC,GAC7B,GAAG,OAAO,KAAK,MACjB;GAEF,IAAI,OAAO,KAAK,SAAS,KAAA,KAAa,OAAO,KAAK,KAAK,SAAS,GAAG;IACjE,MAAM,mBAAmB,OAAO,KAAK,KAAK,KAAK,cAAc;KAC3D,GAAG;KACH,MAAM,SAAS,QAAQ;IACzB,EAAE;IACF,cAAc,aAAa,CACzB,GAAI,cAAc,cAAc,CAAC,GACjC,GAAG,gBACL;GACF;EACF;CACF,CAAC;CAED,IACE,cAAc,eAAe,KAAA,KAC7B,cAAc,WAAW,SAAS,GAClC;;;;EAIA,MAAM,4BAAY,IAAI,IAAY;EAClC,cAAc,aAAa,cAAc,WAAW,QAAQ,UAAU;GACpE,IAAI,CAAC,MAAM,QAAQ,UAAU,IAAI,MAAM,IAAI,GACzC,OAAO;GAET,UAAU,IAAI,MAAM,IAAI;GACxB,OAAO;EACT,CAAC;CACH;CAEA,OAAO;EAAE,SAAS;EAAM,MAAM;CAAc;AAC9C;AAEA,SAAS,sBAAsB,EAC7B,WACA,YACA,gBACA,gBACA,cACA,iBACA,iBACA,UAUC;CACD,OAAO,eAAgB,EACrB,OACA,MACA,SACA,UAAU,MACV,aAAa,GACb,iBACA,SAAS,OACT,SAAS,OACT,OAAO,SAWuB;EAC9B,IAAI;GAEF,MAAM,eAAe,MAAM,wBAAwB;IACjD;IACA;IACA;IACA;IACA;IACA,QAAQ,kBAAkB;IAC1B,QAAQ,kBAAkB;IAC1B,MAAM,gBAAgB;IACtB;GACF,CAAC;GAED,kBAAkB,YAAY;GAW9B,OAAO,iBAAiB,MATO,gBAAgB,eAAe;IAC5D;IACA;IACA,QAAQ;IACR;IACA;IACA,aAAa;GACf,CAAC,CAEuC;EAC1C,SAAS,OAAO;GACd,OAAO,MAAM,oBAAoB,KAAK;GACtC,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,QAAQ,CAAC;IACT,MAAM,CAAC;IACP,iBAAiB,CAAC;IAClB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;AACF;AAEA,SAAS,sBAAsB,EAC7B,gBACA,mBAIC;CACD,OAAO,SAAU,SAA+B;EAC9C,IAAI,CAAC,iBACH;EAEF,gBAAgB,SAAS,cAAc;CACzC;AACF;AAEA,SAAS,WAAW,EAClB,QACA,QACA,gBACA,iBAAiB,oBAMO;CACxB,OAAO,KACL,OAAO,WAAW,mBAAmB;EAEnC,MAAM,EAAE,OAAO,MAAM,SAAS,IAAI,QAAQ,QAAQ,SAASA;EAE3D,MAAM,eAAe,MAAM,OAAO;GAChC;GACA;GACA,SAJc,OAAO,OAAO,YAAY,KAAK,KAAK,KAAA;GAKlD;GACA;GACA;GACA,iBAAiB,sBAAsB;IACrC;IACA,iBAAiB;GACnB,CAAC;EACH,CAAC;EACD,MAAM,OAAO,eAAe,UAAU,QAAQ;EAC9C,MAAM,EAAE,QAAQ,eAAe,oBAC7B,MACA,cACA,cACF;EACA,MAAM,OAA2B;GAAE;GAAM,GAAG;GAAc;EAAW;EACrE,OAAO,CAAC,QAAQ,GAAA,eAA0B,KAAK,CAAC;CAClD,GACA;EACE,MAAM;EACN,aAAa;EACL;EACR,gBAAA;CACF,CACF;AACF;AAoBA,MAAa,oBACX,SAA6B,CAAC,MACJ;CAC1B,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,kBACA,qBACA,gBACA,gBACA,uBACA,eAAe,UACf,iBACA,aAAa,GACb,kBACA,WACA,cACA,gBACA,aAAa,CAAC,eAAe,GAC7B,gBAAgB,MAChB,aAAa,GACb,kBAAkB,aAClB,iBACA,iBACA,kBACA,kBACA,sBACA,sBACA,WACA,WACA,kBACA,mBACA,gBACA,YACA,YACA,cACA,iBAAiB,kBACjB,oBACE;CAEJ,MAAM,SAAS,OAAO,UAAU,oBAAoB;CACpD,MAAM,+BACJ,mBAAmB,YAAY,OAAO,cAAc,OAChD;EACA,GAAG;EACH,YAAY,OAAO,eAAe;CACpC,IACE;CAEN,MAAM,mBAA4C;EAChD,OAAO;EACP,MAAM;EACN,QAAQ;EACR,QAAQ;EACR,MAAM;CACR;CAEA,IAAI,mBAAmB,YAAY,mBAAmB,UACpD,iBAAiB,UAAU;CAG7B,MAAM,aAAa;EACjB,MAAM;EACN,YAAY;EACZ,UAAU,CAAC,OAAO;CACpB;CAEA,MAAM,YAAY,gBAAgB;EAChC;EACA;EACA;EACA;EACA;EACA;EACA,qBAAqB;EACrB;EACA;EACA;EACA;EACA;EACA;CACF,CAAC;;CAGD,IAAI;CAEJ,IAAI,oBAAoB,UACtB,kBAAkB,oBAAoB;EACpC,GAAG;EACH,QAAQ;EACR,SAAS,kBAAkB,sBAAsB;EACjD;CACF,CAAC;MACI,IAAI,oBAAoB,UAC7B,kBAAkB,oBAAoB;EACpC,GAAG;EACH,QACE,sBAAsB,UACtB,gBACA,QAAQ,IAAI;EACd,QAAQ,sBAAsB,UAAU;EACxC,SAAS,kBAAkB,sBAAsB;EACjD;CACF,CAAC;MACI,IAAI,oBAAoB,OAC7B,kBAAkB,iBAAiB;EACjC,GAAG;EACH,QACE,mBAAmB,UAAU,aAAa,QAAQ,IAAI;EACxD,QAAQ,mBAAmB,UAAU;EACrC,SAAS,kBAAkB,mBAAmB;EAC9C,SAAS,mBAAmB,WAAW,CAAC,YAAY,SAAS;EAC7D;CACF,CAAC;MAED,kBAAkB,uBAAuB;EACvC,GAAG;EACH,QAAQ,mBAAmB,QAAQ,IAAI;EACvC,QAAQ;EACR,SAAS;EACT,SAAS,kBAAkB,kBAAkB;EAC7C,SAAS,kBAAkB,WAAW,CAAC,YAAY,SAAS;EAC5D;CACF,CAAC;CAGH,MAAM,mBAAmB,eAAe;EACtC;EACA;EACA;EACA;EACA;EACA;CACF,CAAC;CAED,IAAI,CAAC,kBACH,OAAO,KAAK,8CAA8C;CAG5D,MAAM,kBAAkB,sBACtB;EACE,UAAU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;CACF,GACA,eACF;CAkBA,OAAO,WAAW;EAChB,QAjBa,sBAAsB;GACnC;GACA;GAGA,gBAAgB,mBAAmB;GACnC,gBACE,mBAAmB,YACnB,mBAAmB,cACnB,mBAAmB;GACrB,cAAc,mBAAmB;GACjC;GACA;GACA;EACF,CAGO;EACL,QAAQ;EACR;EACA,iBAAiB;CACnB,CAAC;AACH"}

package/dist/types/llm/anthropic/index.d.ts

@@ -2,6 +2,7 @@ import { ChatAnthropicMessages } from '@langchain/anthropic';
 import { ChatGenerationChunk } from '@langchain/core/outputs';
 import type { BaseChatModelParams } from '@langchain/core/language_models/chat_models';
 import type { BaseMessage } from '@langchain/core/messages';
+import type { ChatModelStreamEvent } from '@langchain/core/language_models/event';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { AnthropicInput } from '@langchain/anthropic';
 import type { Anthropic } from '@anthropic-ai/sdk';
@@ -47,6 +48,7 @@ export declare class CustomAnthropic extends ChatAnthropicMessages {
     private createGenerationChunk;
     protected createStreamWithRetry(request: AnthropicStreamingMessageCreateParams, options?: AnthropicRequestOptions): ReturnType<ChatAnthropicMessages['createStreamWithRetry']>;
     protected completionWithRetry(request: AnthropicMessageCreateParams, options: AnthropicRequestOptions): ReturnType<ChatAnthropicMessages['completionWithRetry']>;
+    _streamChatModelEvents(messages: BaseMessage[], options: this['ParsedCallOptions'], _runManager?: CallbackManagerForLLMRun): AsyncGenerator<ChatModelStreamEvent>;
     _streamResponseChunks(messages: BaseMessage[], options: this['ParsedCallOptions'], runManager?: CallbackManagerForLLMRun): AsyncGenerator<ChatGenerationChunk>;
 }
 export {};

package/dist/types/llm/anthropic/types.d.ts

@@ -37,6 +37,8 @@ export type AnthropicWebSearchToolResultBlockParam = Anthropic.Messages.WebSearc
 export type AnthropicWebSearchResultBlockParam = Anthropic.Messages.WebSearchResultBlockParam;
 export type AnthropicSearchResultBlockParam = Anthropic.Beta.BetaSearchResultBlockParam;
 export type AnthropicCompactionBlockParam = Anthropic.Beta.BetaCompactionBlockParam;
+export type AnthropicCompactionBlock = Anthropic.Beta.BetaCompactionBlock;
+export type AnthropicCompactionContentBlockDelta = Anthropic.Beta.BetaCompactionContentBlockDelta;
 export type AnthropicOutputConfig = Anthropic.Messages.OutputConfig;
 export type ChatAnthropicOutputFormat = Anthropic.Messages.JSONOutputFormat;
 export type AnthropicContentBlock = AnthropicTextBlockParam | AnthropicImageBlockParam | AnthropicToolUseBlockParam | AnthropicToolResultBlockParam | AnthropicDocumentBlockParam | AnthropicThinkingBlockParam | AnthropicRedactedThinkingBlockParam | AnthropicServerToolUseBlockParam | AnthropicWebSearchToolResultBlockParam | AnthropicWebSearchResultBlockParam | AnthropicCompactionBlockParam;

package/dist/types/llm/anthropic/utils/message_outputs.d.ts

@@ -4,7 +4,7 @@ import type { UsageMetadata } from '@langchain/core/messages';
 import type { ChatGeneration } from '@langchain/core/outputs';
 import type Anthropic from '@anthropic-ai/sdk';
 import type { AnthropicMessageResponse } from '../types';
-interface AnthropicUsageData {
+export interface AnthropicUsageData {
     input_tokens?: number | null;
     output_tokens?: number | null;
     cache_creation_input_tokens?: number | null;
@@ -18,4 +18,3 @@ export declare function _makeMessageChunkFromAnthropicEvent(data: Anthropic.Beta
     chunk: AIMessageChunk;
 } | null;
 export declare function anthropicResponseToChatMessages(messages: AnthropicMessageResponse[], additionalKwargs: Record<string, unknown>): ChatGeneration[];
-export {};

package/dist/types/llm/anthropic/utils/stream_events.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Converts a raw Anthropic SSE event stream into LangChain ChatModelStreamEvents.
+ *
+ * @module
+ */
+import type { ChatModelStreamEvent } from '@langchain/core/language_models/event';
+import type { AnthropicMessageStreamEvent } from '../types';
+interface AnthropicStreamErrorEvent {
+    type: 'error';
+    error: {
+        type: string;
+        message: string;
+    };
+    request_id?: string | null;
+}
+type AnthropicStreamInputEvent = AnthropicMessageStreamEvent | AnthropicStreamErrorEvent;
+export interface ConvertAnthropicStreamOptions {
+    streamUsage?: boolean;
+}
+/**
+ * Convert an async iterable of raw Anthropic stream events into
+ * LangChain `ChatModelStreamEvent`s with typed deltas.
+ */
+export declare function convertAnthropicStream(source: AsyncIterable<AnthropicStreamInputEvent>, options?: ConvertAnthropicStreamOptions): AsyncGenerator<ChatModelStreamEvent>;
+export {};

package/dist/types/tools/search/crw-scraper.d.ts

@@ -0,0 +1,41 @@
+import type * as t from './types';
+/**
+ * fastCRW scraper. Firecrawl-compatible web scraper; single binary;
+ * self-host or cloud. Posts to {base}/v1/scrape.
+ */
+export declare class CrwScraper implements t.BaseScraper {
+    private apiKey;
+    private apiUrl;
+    private defaultFormats;
+    private timeout;
+    private logger;
+    private onlyMainContent?;
+    private includeTags?;
+    private excludeTags?;
+    private waitFor?;
+    private headers?;
+    private renderJs?;
+    private cssSelector?;
+    private xpath?;
+    private proxy?;
+    private stealth?;
+    constructor(config?: t.CrwScraperConfig);
+    scrapeUrl(url: string, options?: t.CrwScrapeOptions): Promise<[string, t.CrwScrapeResponse]>;
+    /**
+     * Extract content from scrape response. Mirrors FirecrawlScraper — reads
+     * response.data.*, which normalizeCrwResponse guarantees, preserving the
+     * processContent ref-markers used by the reranker. Parameter is typed as the
+     * NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:
+     * TS class-method bivariance accepts this against BaseScraper's
+     * AnyScraperResponse, AND it lets us read response.data.plainText (a
+     * CrwScrapeResponse-only field) with no cast.
+     */
+    extractContent(response: t.CrwScrapeResponse): [string, undefined | t.References];
+    extractMetadata(response: t.CrwScrapeResponse): t.ScrapeMetadata;
+}
+/**
+ * Create a fastCRW scraper instance
+ * @param config Scraper configuration
+ * @returns fastCRW scraper instance
+ */
+export declare const createCrwScraper: (config?: t.CrwScraperConfig) => CrwScraper;

package/dist/types/tools/search/crw-search.d.ts

@@ -0,0 +1,4 @@
+import type * as t from './types';
+export declare const createCrwAPI: (apiKey?: string, apiUrl?: string, options?: t.CrwSearchOptions) => {
+    getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;
+};

package/dist/types/tools/search/types.d.ts

@@ -2,8 +2,8 @@ import type { RunnableConfig } from '@langchain/core/runnables';
 import type { Logger as WinstonLogger } from 'winston';
 import type { BaseReranker } from './rerankers';
 import { DATE_RANGE } from './schema';
-export type SearchProvider = 'serper' | 'searxng' | 'tavily' | 'keenable';
-export type ScraperProvider = 'firecrawl' | 'serper' | 'tavily';
+export type SearchProvider = 'serper' | 'searxng' | 'tavily' | 'keenable' | 'crw';
+export type ScraperProvider = 'firecrawl' | 'serper' | 'tavily' | 'crw';
 export type RerankerType = 'infinity' | 'jina' | 'cohere' | 'none';
 export interface Highlight {
     score: number;
@@ -91,6 +91,48 @@ export interface TavilySearchPayload {
     include_favicon?: boolean;
     chunks_per_source?: number;
 }
+export interface CrwSearchOptions {
+    /** Max results to request (maps to `limit`; clamped 1..20). */
+    maxResults?: number;
+    /** Add 'images' to the sources array. */
+    includeImages?: boolean;
+    timeout?: number;
+}
+export type CrwSearchSource = 'web' | 'images' | 'news';
+export interface CrwSearchPayload {
+    query: string;
+    limit: number;
+    sources?: CrwSearchSource[];
+    tbs?: string;
+}
+export interface CrwSearchResult {
+    title?: string;
+    url?: string;
+    description?: string;
+    snippet?: string;
+    position?: number;
+    category?: string;
+}
+export interface CrwImageSearchResult extends CrwSearchResult {
+    imageUrl?: string;
+    imageFormat?: string;
+}
+export interface CrwNewsSearchResult extends CrwSearchResult {
+    publishedDate?: string;
+}
+export interface CrwSearchGroups {
+    web?: CrwSearchResult[];
+    images?: CrwImageSearchResult[];
+    news?: CrwNewsSearchResult[];
+}
+export interface CrwSearchResponse {
+    success: boolean;
+    data?: (CrwSearchGroups & {
+        results?: CrwSearchGroups;
+    }) | CrwSearchResult[];
+    error?: string;
+    error_code?: string;
+}
 export interface SearchConfig {
     searchProvider?: SearchProvider;
     serperApiKey?: string;
@@ -103,6 +145,9 @@ export interface SearchConfig {
     keenableApiKey?: string;
     keenableApiUrl?: string;
     keenableSearchOptions?: KeenableSearchOptions;
+    crwApiKey?: string;
+    crwApiUrl?: string;
+    crwSearchOptions?: CrwSearchOptions;
 }
 export interface KeenableSearchOptions {
     maxResults?: number;
@@ -224,6 +269,7 @@ export type SafeSearchLevel = 0 | 1 | 2;
 export type Logger = WinstonLogger;
 export interface SearchToolConfig extends SearchConfig, ProcessSourcesConfig, FirecrawlConfig {
     tavilyScraperOptions?: TavilyScraperConfig;
+    crwScraperOptions?: CrwScraperConfig;
     /** Max chars of highlight content this tool feeds the MODEL per search (the
      * dominant, otherwise-unbounded part of the output). Distinct from
      * `maxContentLength`, which caps scraped/reranked content per source — full
@@ -255,7 +301,7 @@ export type UsedReferences = {
     originalIndex: number;
     reference: MediaReference;
 }[];
-export type AnyScraperResponse = FirecrawlScrapeResponse | SerperScrapeResponse | TavilyScrapeResponse;
+export type AnyScraperResponse = FirecrawlScrapeResponse | SerperScrapeResponse | TavilyScrapeResponse | CrwScrapeResponse;
 /** Base Scraper Interface */
 export interface BaseScraper {
     scrapeUrl(url: string, options?: unknown): Promise<[string, AnyScraperResponse]>;
@@ -265,6 +311,24 @@ export interface BaseScraper {
 }
 /** Firecrawl */
 export type FirecrawlScrapeOptions = Omit<FirecrawlScraperConfig, 'apiKey' | 'apiUrl' | 'version' | 'logger'>;
+export interface CrwScraperConfig {
+    apiKey?: string;
+    apiUrl?: string;
+    formats?: string[];
+    timeout?: number;
+    logger?: Logger;
+    onlyMainContent?: boolean;
+    includeTags?: string[];
+    excludeTags?: string[];
+    waitFor?: number;
+    headers?: Record<string, string>;
+    renderJs?: boolean | null;
+    cssSelector?: string;
+    xpath?: string;
+    proxy?: string;
+    stealth?: boolean;
+}
+export type CrwScrapeOptions = Omit<CrwScraperConfig, 'apiKey' | 'apiUrl' | 'logger'>;
 export type SerperScrapeOptions = Omit<SerperScraperConfig, 'apiKey' | 'apiUrl' | 'logger'>;
 export type TavilyScrapeOptions = Omit<TavilyScraperConfig, 'apiKey' | 'apiUrl' | 'logger'>;
 export interface TavilyExtractPayload {
@@ -341,6 +405,27 @@ export interface FirecrawlScrapeResponse {
     };
     error?: string;
 }
+export interface CrwScrapeData {
+    markdown?: string;
+    html?: string;
+    rawHtml?: string;
+    plainText?: string;
+    screenshot?: string;
+    links?: string[];
+    metadata?: ScrapeMetadata;
+}
+export interface CrwScrapeResponse {
+    success: boolean;
+    data?: CrwScrapeData;
+    error?: string;
+    error_code?: string;
+}
+export interface CrwRawScrapeResponse extends CrwScrapeData {
+    success?: boolean;
+    data?: CrwScrapeData;
+    error?: string;
+    error_code?: string;
+}
 export interface SerperScrapeResponse {
     success: boolean;
     data?: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "3.2.63",
+  "version": "3.2.65",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/graphs/Graph.ts

@@ -1130,6 +1130,9 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
   /* Misc.*/
 
   getRunMessages(): BaseMessage[] | undefined {
+    if (this.messages == null) {
+      return this.cachedRunMessages;
+    }
     if (this.messages.length === 0 && this.cachedRunMessages != null) {
       return this.cachedRunMessages;
     }
@@ -1137,6 +1140,12 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
   }
 
   getContentParts(): t.MessageContentComplex[] | undefined {
+    // `messages` can be null/undefined on a graph that has been disposed
+    // (clearHeavyState) but is still reachable via a cache (e.g. RedisJobStore's
+    // WeakRef) during a HITL resume/reconnect. Guard instead of dereferencing null.
+    if (this.messages == null) {
+      return undefined;
+    }
     return convertMessagesToContent(this.messages.slice(this.startIndex));
   }
 
@@ -1169,6 +1178,12 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
    * Get all run steps, optionally filtered by agent ID
    */
   getRunSteps(agentId?: string): t.RunStep[] {
+    // `contentData` can be null/undefined on a disposed-but-cached graph during a
+    // HITL resume/reconnect; without this guard `[...this.contentData]` throws
+    // "this.contentData is not iterable".
+    if (this.contentData == null) {
+      return [];
+    }
     if (agentId == null || agentId === '') {
       return [...this.contentData];
     }