illuma-agents 1.0.7 → 1.0.9

package/dist/esm/tools/search/search.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"search.mjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\r\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\r\nimport type * as t from './types';\r\nimport { getAttribution, createDefaultLogger } from './utils';\r\nimport { BaseReranker } from './rerankers';\r\n\r\nconst chunker = {\r\n  cleanText: (text: string): string => {\r\n    if (!text) return '';\r\n\r\n    /** Normalized all line endings to '\\n' */\r\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\r\n\r\n    /** Handle multiple backslashes followed by newlines\r\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\r\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\r\n\r\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\r\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\r\n\r\n    /** Cleaned up excessive spaces and tabs */\r\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\r\n\r\n    return cleanedSpaces.trim();\r\n  },\r\n  splitText: async (\r\n    text: string,\r\n    options?: {\r\n      chunkSize?: number;\r\n      chunkOverlap?: number;\r\n      separators?: string[];\r\n    }\r\n  ): Promise<string[]> => {\r\n    const chunkSize = options?.chunkSize ?? 150;\r\n    const chunkOverlap = options?.chunkOverlap ?? 50;\r\n    const separators = options?.separators || ['\\n\\n', '\\n'];\r\n\r\n    const splitter = new RecursiveCharacterTextSplitter({\r\n      separators,\r\n      chunkSize,\r\n      chunkOverlap,\r\n    });\r\n\r\n    return await splitter.splitText(text);\r\n  },\r\n\r\n  splitTexts: async (\r\n    texts: string[],\r\n    options?: {\r\n      chunkSize?: number;\r\n      chunkOverlap?: number;\r\n      separators?: string[];\r\n    },\r\n    logger?: t.Logger\r\n  ): Promise<string[][]> => {\r\n    // Split multiple texts\r\n    const logger_ = logger || createDefaultLogger();\r\n    const promises = texts.map((text) =>\r\n      chunker.splitText(text, options).catch((error) => {\r\n        logger_.error('Error splitting text:', error);\r\n        return [text];\r\n      })\r\n    );\r\n    return Promise.all(promises);\r\n  },\r\n};\r\n\r\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\r\n  return (link: string, update?: Partial<t.ValidSource>): void => {\r\n    const source = sourceMap.get(link);\r\n    if (source) {\r\n      sourceMap.set(link, {\r\n        ...source,\r\n        ...update,\r\n      });\r\n    }\r\n  };\r\n}\r\n\r\nconst getHighlights = async ({\r\n  query,\r\n  content,\r\n  reranker,\r\n  topResults = 5,\r\n  logger,\r\n}: {\r\n  content: string;\r\n  query: string;\r\n  reranker?: BaseReranker;\r\n  topResults?: number;\r\n  logger?: t.Logger;\r\n}): Promise<t.Highlight[] | undefined> => {\r\n  const logger_ = logger || createDefaultLogger();\r\n\r\n  if (!content) {\r\n    logger_.warn('No content provided for highlights');\r\n    return;\r\n  }\r\n  if (!reranker) {\r\n    logger_.warn('No reranker provided for highlights');\r\n    return;\r\n  }\r\n\r\n  try {\r\n    const documents = await chunker.splitText(content);\r\n    if (Array.isArray(documents)) {\r\n      return await reranker.rerank(query, documents, topResults);\r\n    } else {\r\n      logger_.error(\r\n        'Expected documents to be an array, got:',\r\n        typeof documents\r\n      );\r\n      return;\r\n    }\r\n  } catch (error) {\r\n    logger_.error('Error in content processing:', error);\r\n    return;\r\n  }\r\n};\r\n\r\nconst createSerperAPI = (\r\n  apiKey?: string\r\n): {\r\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\r\n} => {\r\n  const config = {\r\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\r\n    apiUrl: 'https://google.serper.dev/search',\r\n    timeout: 10000,\r\n  };\r\n\r\n  if (config.apiKey == null || config.apiKey === '') {\r\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\r\n  }\r\n\r\n  const getSources = async ({\r\n    query,\r\n    date,\r\n    country,\r\n    safeSearch,\r\n    numResults = 8,\r\n    type,\r\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\r\n    if (!query.trim()) {\r\n      return { success: false, error: 'Query cannot be empty' };\r\n    }\r\n\r\n    try {\r\n      const safe = ['off', 'moderate', 'active'] as const;\r\n      const payload: t.SerperSearchPayload = {\r\n        q: query,\r\n        safe: safe[safeSearch ?? 1],\r\n        num: Math.min(Math.max(1, numResults), 10),\r\n      };\r\n\r\n      // Set the search type if provided\r\n      if (type) {\r\n        payload.type = type;\r\n      }\r\n\r\n      if (date != null) {\r\n        payload.tbs = `qdr:${date}`;\r\n      }\r\n\r\n      if (country != null && country !== '') {\r\n        payload['gl'] = country.toLowerCase();\r\n      }\r\n\r\n      // Determine the API endpoint based on the search type\r\n      let apiEndpoint = config.apiUrl;\r\n      if (type === 'images') {\r\n        apiEndpoint = 'https://google.serper.dev/images';\r\n      } else if (type === 'videos') {\r\n        apiEndpoint = 'https://google.serper.dev/videos';\r\n      } else if (type === 'news') {\r\n        apiEndpoint = 'https://google.serper.dev/news';\r\n      }\r\n\r\n      const response = await axios.post<t.SerperResultData>(\r\n        apiEndpoint,\r\n        payload,\r\n        {\r\n          headers: {\r\n            'X-API-KEY': config.apiKey,\r\n            'Content-Type': 'application/json',\r\n          },\r\n          timeout: config.timeout,\r\n        }\r\n      );\r\n\r\n      const data = response.data;\r\n      const results: t.SearchResultData = {\r\n        organic: data.organic,\r\n        images: data.images ?? [],\r\n        answerBox: data.answerBox,\r\n        topStories: data.topStories ?? [],\r\n        peopleAlsoAsk: data.peopleAlsoAsk,\r\n        knowledgeGraph: data.knowledgeGraph,\r\n        relatedSearches: data.relatedSearches,\r\n        videos: data.videos ?? [],\r\n        news: data.news ?? [],\r\n      };\r\n\r\n      return { success: true, data: results };\r\n    } catch (error) {\r\n      const errorMessage =\r\n        error instanceof Error ? error.message : String(error);\r\n      return { success: false, error: `API request failed: ${errorMessage}` };\r\n    }\r\n  };\r\n\r\n  return { getSources };\r\n};\r\n\r\nconst createSearXNGAPI = (\r\n  instanceUrl?: string,\r\n  apiKey?: string\r\n): {\r\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\r\n} => {\r\n  const config = {\r\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\r\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\r\n    defaultLocation: 'all',\r\n    timeout: 10000,\r\n  };\r\n\r\n  if (config.instanceUrl == null || config.instanceUrl === '') {\r\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\r\n  }\r\n\r\n  const getSources = async ({\r\n    query,\r\n    numResults = 8,\r\n    safeSearch,\r\n    type,\r\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\r\n    if (!query.trim()) {\r\n      return { success: false, error: 'Query cannot be empty' };\r\n    }\r\n\r\n    try {\r\n      // Ensure the instance URL ends with /search\r\n      if (config.instanceUrl == null || config.instanceUrl === '') {\r\n        return { success: false, error: 'Instance URL is not defined' };\r\n      }\r\n\r\n      let searchUrl = config.instanceUrl;\r\n      if (!searchUrl.endsWith('/search')) {\r\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\r\n      }\r\n\r\n      // Determine the search category based on the type\r\n      let category = 'general';\r\n      if (type === 'images') {\r\n        category = 'images';\r\n      } else if (type === 'videos') {\r\n        category = 'videos';\r\n      } else if (type === 'news') {\r\n        category = 'news';\r\n      }\r\n\r\n      // Prepare parameters for SearXNG\r\n      const params: t.SearxNGSearchPayload = {\r\n        q: query,\r\n        format: 'json',\r\n        pageno: 1,\r\n        categories: category,\r\n        language: 'all',\r\n        safesearch: safeSearch,\r\n        engines: 'google,bing,duckduckgo',\r\n      };\r\n\r\n      const headers: Record<string, string> = {\r\n        'Content-Type': 'application/json',\r\n      };\r\n\r\n      if (config.apiKey != null && config.apiKey !== '') {\r\n        headers['X-API-Key'] = config.apiKey;\r\n      }\r\n\r\n      const response = await axios.get(searchUrl, {\r\n        headers,\r\n        params,\r\n        timeout: config.timeout,\r\n      });\r\n\r\n      const data = response.data;\r\n\r\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\r\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\r\n        const url = result.url?.toLowerCase() ?? '';\r\n        const title = result.title?.toLowerCase() ?? '';\r\n\r\n        // News-related keywords in title/content\r\n        const newsKeywords = [\r\n          'breaking news',\r\n          'latest news',\r\n          'top stories',\r\n          'news today',\r\n          'developing story',\r\n          'trending news',\r\n          'news',\r\n        ];\r\n\r\n        // Check if title/content contains news keywords\r\n        const hasNewsKeywords = newsKeywords.some(\r\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\r\n        );\r\n\r\n        // Check if URL contains news-related paths\r\n        const hasNewsPath =\r\n          url.includes('/news/') ||\r\n          url.includes('/world/') ||\r\n          url.includes('/politics/') ||\r\n          url.includes('/breaking/');\r\n\r\n        return hasNewsKeywords || hasNewsPath;\r\n      };\r\n\r\n      // Transform SearXNG results to match SerperAPI format\r\n      const organicResults = (data.results ?? [])\r\n        .slice(0, numResults)\r\n        .map((result: t.SearXNGResult, index: number) => {\r\n          let attribution = '';\r\n          try {\r\n            attribution = new URL(result.url ?? '').hostname;\r\n          } catch {\r\n            attribution = '';\r\n          }\r\n\r\n          return {\r\n            position: index + 1,\r\n            title: result.title ?? '',\r\n            link: result.url ?? '',\r\n            snippet: result.content ?? '',\r\n            date: result.publishedDate ?? '',\r\n            attribution,\r\n          };\r\n        });\r\n\r\n      const imageResults = (data.results ?? [])\r\n        .filter((result: t.SearXNGResult) => result.img_src)\r\n        .slice(0, 6)\r\n        .map((result: t.SearXNGResult, index: number) => ({\r\n          title: result.title ?? '',\r\n          imageUrl: result.img_src ?? '',\r\n          position: index + 1,\r\n          source: new URL(result.url ?? '').hostname,\r\n          domain: new URL(result.url ?? '').hostname,\r\n          link: result.url ?? '',\r\n        }));\r\n\r\n      // Extract news results from organic results\r\n      const newsResults = (data.results ?? [])\r\n        .filter(isNewsResult)\r\n        .map((result: t.SearXNGResult, index: number) => {\r\n          let attribution = '';\r\n          try {\r\n            attribution = new URL(result.url ?? '').hostname;\r\n          } catch {\r\n            attribution = '';\r\n          }\r\n\r\n          return {\r\n            title: result.title ?? '',\r\n            link: result.url ?? '',\r\n            snippet: result.content ?? '',\r\n            date: result.publishedDate ?? '',\r\n            source: attribution,\r\n            imageUrl: result.img_src ?? '',\r\n            position: index + 1,\r\n          };\r\n        });\r\n\r\n      const topStories = newsResults.slice(0, 5);\r\n\r\n      const relatedSearches = Array.isArray(data.suggestions)\r\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\r\n        : [];\r\n\r\n      const results: t.SearchResultData = {\r\n        organic: organicResults,\r\n        images: imageResults,\r\n        topStories: topStories, // Use first 5 extracted news as top stories\r\n        relatedSearches,\r\n        videos: [],\r\n        news: newsResults,\r\n        // Add empty arrays for other Serper fields to maintain parity\r\n        places: [],\r\n        shopping: [],\r\n        peopleAlsoAsk: [],\r\n        knowledgeGraph: undefined,\r\n        answerBox: undefined,\r\n      };\r\n\r\n      return { success: true, data: results };\r\n    } catch (error) {\r\n      const errorMessage =\r\n        error instanceof Error ? error.message : String(error);\r\n      return {\r\n        success: false,\r\n        error: `SearXNG API request failed: ${errorMessage}`,\r\n      };\r\n    }\r\n  };\r\n\r\n  return { getSources };\r\n};\r\n\r\nexport const createSearchAPI = (\r\n  config: t.SearchConfig\r\n): {\r\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\r\n} => {\r\n  const {\r\n    searchProvider = 'serper',\r\n    serperApiKey,\r\n    searxngInstanceUrl,\r\n    searxngApiKey,\r\n  } = config;\r\n\r\n  if (searchProvider.toLowerCase() === 'serper') {\r\n    return createSerperAPI(serperApiKey);\r\n  } else if (searchProvider.toLowerCase() === 'searxng') {\r\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\r\n  } else {\r\n    throw new Error(\r\n      `Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`\r\n    );\r\n  }\r\n};\r\n\r\nexport const createSourceProcessor = (\r\n  config: t.ProcessSourcesConfig = {},\r\n  scraperInstance?: t.BaseScraper\r\n): {\r\n  processSources: (\r\n    fields: t.ProcessSourcesFields\r\n  ) => Promise<t.SearchResultData>;\r\n  topResults: number;\r\n} => {\r\n  if (!scraperInstance) {\r\n    throw new Error('Scraper instance is required');\r\n  }\r\n  const {\r\n    topResults = 5,\r\n    // strategies = ['no_extraction'],\r\n    // filterContent = true,\r\n    reranker,\r\n    logger,\r\n  } = config;\r\n\r\n  const logger_ = logger || createDefaultLogger();\r\n  const scraper = scraperInstance;\r\n\r\n  const webScraper = {\r\n    scrapeMany: async ({\r\n      query,\r\n      links,\r\n      onGetHighlights,\r\n    }: {\r\n      query: string;\r\n      links: string[];\r\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\r\n    }): Promise<Array<t.ScrapeResult>> => {\r\n      logger_.debug(`Scraping ${links.length} links`);\r\n      const promises: Array<Promise<t.ScrapeResult>> = [];\r\n      try {\r\n        for (let i = 0; i < links.length; i++) {\r\n          const currentLink = links[i];\r\n          const promise: Promise<t.ScrapeResult> = scraper\r\n            .scrapeUrl(currentLink, {})\r\n            .then(([url, response]) => {\r\n              const attribution = getAttribution(\r\n                url,\r\n                response.data?.metadata,\r\n                logger_\r\n              );\r\n              if (response.success && response.data) {\r\n                const [content, references] = scraper.extractContent(response);\r\n                return {\r\n                  url,\r\n                  references,\r\n                  attribution,\r\n                  content: chunker.cleanText(content),\r\n                } as t.ScrapeResult;\r\n              } else {\r\n                logger_.error(\r\n                  `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\r\n                );\r\n              }\r\n\r\n              return {\r\n                url,\r\n                attribution,\r\n                error: true,\r\n                content: '',\r\n              } as t.ScrapeResult;\r\n            })\r\n            .then(async (result) => {\r\n              try {\r\n                if (result.error != null) {\r\n                  logger_.error(\r\n                    `Error scraping ${result.url}: ${result.content}`\r\n                  );\r\n                  return {\r\n                    ...result,\r\n                  };\r\n                }\r\n                const highlights = await getHighlights({\r\n                  query,\r\n                  reranker,\r\n                  content: result.content,\r\n                  logger: logger_,\r\n                });\r\n                if (onGetHighlights) {\r\n                  onGetHighlights(result.url);\r\n                }\r\n                return {\r\n                  ...result,\r\n                  highlights,\r\n                };\r\n              } catch (error) {\r\n                logger_.error('Error processing scraped content:', error);\r\n                return {\r\n                  ...result,\r\n                };\r\n              }\r\n            })\r\n            .catch((error) => {\r\n              logger_.error(`Error scraping ${currentLink}:`, error);\r\n              return {\r\n                url: currentLink,\r\n                error: true,\r\n                content: '',\r\n              };\r\n            });\r\n          promises.push(promise);\r\n        }\r\n        return await Promise.all(promises);\r\n      } catch (error) {\r\n        logger_.error('Error in scrapeMany:', error);\r\n        return [];\r\n      }\r\n    },\r\n  };\r\n\r\n  const fetchContents = async ({\r\n    links,\r\n    query,\r\n    target,\r\n    onGetHighlights,\r\n    onContentScraped,\r\n  }: {\r\n    links: string[];\r\n    query: string;\r\n    target: number;\r\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\r\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\r\n  }): Promise<void> => {\r\n    const initialLinks = links.slice(0, target);\r\n    // const remainingLinks = links.slice(target).reverse();\r\n    const results = await webScraper.scrapeMany({\r\n      query,\r\n      links: initialLinks,\r\n      onGetHighlights,\r\n    });\r\n    for (const result of results) {\r\n      if (result.error === true) {\r\n        continue;\r\n      }\r\n      const { url, content, attribution, references, highlights } = result;\r\n      onContentScraped?.(url, {\r\n        content,\r\n        attribution,\r\n        references,\r\n        highlights,\r\n      });\r\n    }\r\n  };\r\n\r\n  const processSources = async ({\r\n    result,\r\n    numElements,\r\n    query,\r\n    news,\r\n    proMode = true,\r\n    onGetHighlights,\r\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\r\n    try {\r\n      if (!result.data) {\r\n        return {\r\n          organic: [],\r\n          topStories: [],\r\n          images: [],\r\n          relatedSearches: [],\r\n        };\r\n      } else if (!result.data.organic) {\r\n        return result.data;\r\n      }\r\n\r\n      if (!proMode) {\r\n        const wikiSources = result.data.organic.filter((source) =>\r\n          source.link.includes('wikipedia.org')\r\n        );\r\n\r\n        if (!wikiSources.length) {\r\n          return result.data;\r\n        }\r\n\r\n        const wikiSourceMap = new Map<string, t.ValidSource>();\r\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\r\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\r\n        await fetchContents({\r\n          query,\r\n          target: 1,\r\n          onGetHighlights,\r\n          onContentScraped,\r\n          links: [wikiSources[0].link],\r\n        });\r\n\r\n        for (let i = 0; i < result.data.organic.length; i++) {\r\n          const source = result.data.organic[i];\r\n          const updatedSource = wikiSourceMap.get(source.link);\r\n          if (updatedSource) {\r\n            result.data.organic[i] = {\r\n              ...source,\r\n              ...updatedSource,\r\n            };\r\n          }\r\n        }\r\n\r\n        return result.data;\r\n      }\r\n\r\n      const sourceMap = new Map<string, t.ValidSource>();\r\n      const organicLinksSet = new Set<string>();\r\n\r\n      // Collect organic links\r\n      const organicLinks = collectLinks(\r\n        result.data.organic,\r\n        sourceMap,\r\n        organicLinksSet\r\n      );\r\n\r\n      // Collect top story links, excluding any that are already in organic links\r\n      const topStories = result.data.topStories ?? [];\r\n      const topStoryLinks = collectLinks(\r\n        topStories,\r\n        sourceMap,\r\n        organicLinksSet\r\n      );\r\n\r\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\r\n        return result.data;\r\n      }\r\n\r\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\r\n      const promises: Promise<void>[] = [];\r\n\r\n      // Process organic links\r\n      if (organicLinks.length > 0) {\r\n        promises.push(\r\n          fetchContents({\r\n            query,\r\n            onGetHighlights,\r\n            onContentScraped,\r\n            links: organicLinks,\r\n            target: numElements,\r\n          })\r\n        );\r\n      }\r\n\r\n      // Process top story links\r\n      if (news && topStoryLinks.length > 0) {\r\n        promises.push(\r\n          fetchContents({\r\n            query,\r\n            onGetHighlights,\r\n            onContentScraped,\r\n            links: topStoryLinks,\r\n            target: numElements,\r\n          })\r\n        );\r\n      }\r\n\r\n      await Promise.all(promises);\r\n\r\n      if (result.data.organic.length > 0) {\r\n        updateSourcesWithContent(result.data.organic, sourceMap);\r\n      }\r\n\r\n      if (news && topStories.length > 0) {\r\n        updateSourcesWithContent(topStories, sourceMap);\r\n      }\r\n\r\n      return result.data;\r\n    } catch (error) {\r\n      logger_.error('Error in processSources:', error);\r\n      return {\r\n        organic: [],\r\n        topStories: [],\r\n        images: [],\r\n        relatedSearches: [],\r\n        ...result.data,\r\n        error: error instanceof Error ? error.message : String(error),\r\n      };\r\n    }\r\n  };\r\n\r\n  return {\r\n    processSources,\r\n    topResults,\r\n  };\r\n};\r\n\r\n/** Helper function to collect links and update sourceMap */\r\nfunction collectLinks(\r\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\r\n  sourceMap: Map<string, t.ValidSource>,\r\n  existingLinksSet?: Set<string>\r\n): string[] {\r\n  const links: string[] = [];\r\n\r\n  for (const source of sources) {\r\n    if (source.link) {\r\n      // For topStories, only add if not already in organic links\r\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\r\n        continue;\r\n      }\r\n\r\n      links.push(source.link);\r\n      if (existingLinksSet) {\r\n        existingLinksSet.add(source.link);\r\n      }\r\n      sourceMap.set(source.link, source as t.ValidSource);\r\n    }\r\n  }\r\n\r\n  return links;\r\n}\r\n\r\n/** Helper function to update sources with scraped content */\r\nfunction updateSourcesWithContent<T extends t.ValidSource>(\r\n  sources: T[],\r\n  sourceMap: Map<string, t.ValidSource>\r\n): void {\r\n  for (let i = 0; i < sources.length; i++) {\r\n    const source = sources[i];\r\n    const updatedSource = sourceMap.get(source.link);\r\n    if (updatedSource) {\r\n      sources[i] = {\r\n        ...source,\r\n        ...updatedSource,\r\n      } as T;\r\n    }\r\n  }\r\n}\r\n"],"names":[],"mappings":";;;;AAMA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;KAC5B;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAI,8BAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;KACtC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAI,mBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;SACd,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;KAC7B;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;;AAEN,KAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAI,mBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;;IAEF,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;;AAGF,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;;aACrD;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;;;IAEF,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;;AAEJ,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;;IAG7D,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;AACnD,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;;AAGrB,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAO,IAAA,EAAA,IAAI,EAAE;;YAG7B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;;;AAIvC,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;;YAGhD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAuB,oBAAA,EAAA,YAAY,CAAE,CAAA,EAAE;;AAE3E,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QACA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;;AAGrE,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;;AAGjE,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;;;YAItD,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;;;AAInB,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;;YAGtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,aAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,aAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,aAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAA+B,4BAAA,EAAA,YAAY,CAAE,CAAA;aACrD;;AAEL,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEY,MAAA,eAAe,GAAG,CAC7B,MAAsB,KAGpB;AACF,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,GACd,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;AAC7C,QAAA,OAAO,eAAe,CAAC,YAAY,CAAC;;AAC/B,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;;SACrD;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,+BAAA,CAAiC,CAC5E;;AAEL;AAEa,MAAA,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;;IAEjD,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAI,mBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAQ,MAAA,CAAA,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB,wBAAA,MAAM,WAAW,GAAG,cAAc,CAChC,GAAG,EACH,QAAQ,CAAC,IAAI,EAAE,QAAQ,EACvB,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;;6BACd;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;;wBAGH,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,qBAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;;AAEH,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;;4BAE7B,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;;wBACD,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;;AAEL,qBAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAG,CAAA,CAAA,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,qBAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;;AAExB,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;;YAClC,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;;SAEZ;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;;AAEF,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;;AAEN,KAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,GACQ,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;;AACI,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;;YAGpB,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;;AAGpB,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;;;gBAIL,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;;YAIH,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;AAGH,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;YAG1D,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;;YAGjD,OAAO,MAAM,CAAC,IAAI;;QAClB,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;;AAGF,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;;YAEnC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;;;AAIvD,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;;;AAGZ;;;;"}
+{"version":3,"file":"search.mjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(content);\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  } = config;\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      const promises: Array<Promise<t.ScrapeResult>> = [];\n      try {\n        for (let i = 0; i < links.length; i++) {\n          const currentLink = links[i];\n          const promise: Promise<t.ScrapeResult> = scraper\n            .scrapeUrl(currentLink, {})\n            .then(([url, response]) => {\n              const attribution = getAttribution(\n                url,\n                response.data?.metadata,\n                logger_\n              );\n              if (response.success && response.data) {\n                const [content, references] = scraper.extractContent(response);\n                return {\n                  url,\n                  references,\n                  attribution,\n                  content: chunker.cleanText(content),\n                } as t.ScrapeResult;\n              } else {\n                logger_.error(\n                  `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n                );\n              }\n\n              return {\n                url,\n                attribution,\n                error: true,\n                content: '',\n              } as t.ScrapeResult;\n            })\n            .then(async (result) => {\n              try {\n                if (result.error != null) {\n                  logger_.error(\n                    `Error scraping ${result.url}: ${result.content}`\n                  );\n                  return {\n                    ...result,\n                  };\n                }\n                const highlights = await getHighlights({\n                  query,\n                  reranker,\n                  content: result.content,\n                  logger: logger_,\n                });\n                if (onGetHighlights) {\n                  onGetHighlights(result.url);\n                }\n                return {\n                  ...result,\n                  highlights,\n                };\n              } catch (error) {\n                logger_.error('Error processing scraped content:', error);\n                return {\n                  ...result,\n                };\n              }\n            })\n            .catch((error) => {\n              logger_.error(`Error scraping ${currentLink}:`, error);\n              return {\n                url: currentLink,\n                error: true,\n                content: '',\n              };\n            });\n          promises.push(promise);\n        }\n        return await Promise.all(promises);\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n    skipScraping = false,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      } else if (!result.data.organic) {\n        return result.data;\n      }\n\n      // If content was already extracted directly (e.g., direct URL extraction), skip scraping\n      if (skipScraping) {\n        logger_.debug('Skipping additional scraping - content already extracted');\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"names":[],"mappings":";;;;AAMA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;KAC5B;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAI,8BAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;KACtC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAI,mBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;SACd,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;KAC7B;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;;AAEN,KAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAI,mBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;;IAEF,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;;AAGF,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;;aACrD;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;;;IAEF,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;;AAEJ,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;;IAG7D,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;AACnD,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;;AAGrB,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAO,IAAA,EAAA,IAAI,EAAE;;YAG7B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;;;AAIvC,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;;YAGhD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAuB,oBAAA,EAAA,YAAY,CAAE,CAAA,EAAE;;AAE3E,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QACA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;;AAGrE,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;;AAGjE,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;;;YAItD,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;;;AAInB,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;;YAGtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,aAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,aAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,aAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAA+B,4BAAA,EAAA,YAAY,CAAE,CAAA;aACrD;;AAEL,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEY,MAAA,eAAe,GAAG,CAC7B,MAAsB,KAGpB;AACF,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,GACd,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;AAC7C,QAAA,OAAO,eAAe,CAAC,YAAY,CAAC;;AAC/B,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;;SACrD;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,+BAAA,CAAiC,CAC5E;;AAEL;AAEa,MAAA,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;;IAEjD,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAI,mBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAQ,MAAA,CAAA,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB,wBAAA,MAAM,WAAW,GAAG,cAAc,CAChC,GAAG,EACH,QAAQ,CAAC,IAAI,EAAE,QAAQ,EACvB,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;;6BACd;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;;wBAGH,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,qBAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;;AAEH,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;;4BAE7B,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;;wBACD,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;;AAEL,qBAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAG,CAAA,CAAA,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,qBAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;;AAExB,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;;YAClC,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;;SAEZ;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;;AAEF,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;;AAEN,KAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,EACf,YAAY,GAAG,KAAK,GACG,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;;AACI,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;;;YAIpB,IAAI,YAAY,EAAE;AAChB,gBAAA,OAAO,CAAC,KAAK,CAAC,0DAA0D,CAAC;gBACzE,OAAO,MAAM,CAAC,IAAI;;YAGpB,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;;AAGpB,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;;;gBAIL,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;;YAIH,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;AAGH,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;YAG1D,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;;YAGjD,OAAO,MAAM,CAAC,IAAI;;QAClB,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;;AAGF,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;;YAEnC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;;;AAIvD,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;;;AAGZ;;;;"}

package/dist/esm/tools/search/serper-scraper.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"serper-scraper.mjs","sources":["../../../../src/tools/search/serper-scraper.ts"],"sourcesContent":["import axios from 'axios';\r\nimport type * as t from './types';\r\nimport { createDefaultLogger } from './utils';\r\n\r\n/**\r\n * Serper scraper implementation\r\n * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages\r\n *\r\n * Features:\r\n * - Simple API with single endpoint\r\n * - Returns both text and markdown content\r\n * - Includes metadata from scraped pages\r\n * - Credits-based pricing model\r\n *\r\n * @example\r\n * ```typescript\r\n * const scraper = createSerperScraper({\r\n *   apiKey: 'your-serper-api-key',\r\n *   includeMarkdown: true,\r\n *   timeout: 10000\r\n * });\r\n *\r\n * const [url, response] = await scraper.scrapeUrl('https://example.com');\r\n * if (response.success) {\r\n *   const [content] = scraper.extractContent(response);\r\n *   console.log(content);\r\n * }\r\n * ```\r\n */\r\nexport class SerperScraper implements t.BaseScraper {\r\n  private apiKey: string;\r\n  private apiUrl: string;\r\n  private timeout: number;\r\n  private logger: t.Logger;\r\n  private includeMarkdown: boolean;\r\n\r\n  constructor(config: t.SerperScraperConfig = {}) {\r\n    this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';\r\n\r\n    this.apiUrl =\r\n      config.apiUrl ??\r\n      process.env.SERPER_SCRAPE_URL ??\r\n      'https://scrape.serper.dev';\r\n\r\n    this.timeout = config.timeout ?? 7500;\r\n    this.includeMarkdown = config.includeMarkdown ?? true;\r\n\r\n    this.logger = config.logger || createDefaultLogger();\r\n\r\n    if (!this.apiKey) {\r\n      this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');\r\n    }\r\n\r\n    this.logger.debug(\r\n      `Serper scraper initialized with API URL: ${this.apiUrl}`\r\n    );\r\n  }\r\n\r\n  /**\r\n   * Scrape a single URL\r\n   * @param url URL to scrape\r\n   * @param options Scrape options\r\n   * @returns Scrape response\r\n   */\r\n  async scrapeUrl(\r\n    url: string,\r\n    options: t.SerperScrapeOptions = {}\r\n  ): Promise<[string, t.SerperScrapeResponse]> {\r\n    if (!this.apiKey) {\r\n      return [\r\n        url,\r\n        {\r\n          success: false,\r\n          error: 'SERPER_API_KEY is not set',\r\n        },\r\n      ];\r\n    }\r\n\r\n    try {\r\n      const payload = {\r\n        url,\r\n        includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,\r\n      };\r\n\r\n      const response = await axios.post(this.apiUrl, payload, {\r\n        headers: {\r\n          'X-API-KEY': this.apiKey,\r\n          'Content-Type': 'application/json',\r\n        },\r\n        timeout: options.timeout ?? this.timeout,\r\n      });\r\n\r\n      return [url, { success: true, data: response.data }];\r\n    } catch (error) {\r\n      const errorMessage =\r\n        error instanceof Error ? error.message : String(error);\r\n      return [\r\n        url,\r\n        {\r\n          success: false,\r\n          error: `Serper Scrape API request failed: ${errorMessage}`,\r\n        },\r\n      ];\r\n    }\r\n  }\r\n\r\n  /**\r\n   * Extract content from scrape response\r\n   * @param response Scrape response\r\n   * @returns Extracted content or empty string if not available\r\n   */\r\n  extractContent(\r\n    response: t.SerperScrapeResponse\r\n  ): [string, undefined | t.References] {\r\n    if (!response.success || !response.data) {\r\n      return ['', undefined];\r\n    }\r\n\r\n    if (response.data.markdown != null) {\r\n      return [response.data.markdown, undefined];\r\n    }\r\n\r\n    if (response.data.text != null) {\r\n      return [response.data.text, undefined];\r\n    }\r\n\r\n    return ['', undefined];\r\n  }\r\n\r\n  /**\r\n   * Extract metadata from scrape response\r\n   * @param response Scrape response\r\n   * @returns Metadata object\r\n   */\r\n  extractMetadata(\r\n    response: t.SerperScrapeResponse\r\n  ): Record<string, string | number | boolean | null | undefined> {\r\n    if (!response.success || !response.data || !response.data.metadata) {\r\n      return {};\r\n    }\r\n\r\n    return response.data.metadata;\r\n  }\r\n}\r\n\r\n/**\r\n * Create a Serper scraper instance\r\n * @param config Scraper configuration\r\n * @returns Serper scraper instance\r\n */\r\nexport const createSerperScraper = (\r\n  config: t.SerperScraperConfig = {}\r\n): SerperScraper => {\r\n  return new SerperScraper(config);\r\n};\r\n"],"names":[],"mappings":";;;AAIA;;;;;;;;;;;;;;;;;;;;;;;;AAwBG;MACU,aAAa,CAAA;AAChB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,eAAe;AAEvB,IAAA,WAAA,CAAY,SAAgC,EAAE,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;AAE/D,QAAA,IAAI,CAAC,MAAM;AACT,YAAA,MAAM,CAAC,MAAM;gBACb,OAAO,CAAC,GAAG,CAAC,iBAAiB;AAC7B,gBAAA,2BAA2B;QAE7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC;;QAGxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA4C,yCAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC1D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,2BAA2B;AACnC,iBAAA;aACF;;AAGH,QAAA,IAAI;AACF,YAAA,MAAM,OAAO,GAAG;gBACd,GAAG;AACH,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;aACjE;AAED,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,IAAI,CAAC,MAAM;AACxB,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;AACD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACzC,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;;QACpD,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAqC,kCAAA,EAAA,YAAY,CAAE,CAAA;AAC3D,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAgC,EAAA;QAEhC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;QAGxB,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;QAG5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;AAGxC,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CACb,QAAgC,EAAA;AAEhC,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,mBAAmB,GAAG,CACjC,MAAgC,GAAA,EAAE,KACjB;AACjB,IAAA,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC;AAClC;;;;"}
+{"version":3,"file":"serper-scraper.mjs","sources":["../../../../src/tools/search/serper-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Serper scraper implementation\n * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages\n *\n * Features:\n * - Simple API with single endpoint\n * - Returns both text and markdown content\n * - Includes metadata from scraped pages\n * - Credits-based pricing model\n *\n * @example\n * ```typescript\n * const scraper = createSerperScraper({\n *   apiKey: 'your-serper-api-key',\n *   includeMarkdown: true,\n *   timeout: 10000\n * });\n *\n * const [url, response] = await scraper.scrapeUrl('https://example.com');\n * if (response.success) {\n *   const [content] = scraper.extractContent(response);\n *   console.log(content);\n * }\n * ```\n */\nexport class SerperScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private logger: t.Logger;\n  private includeMarkdown: boolean;\n\n  constructor(config: t.SerperScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';\n\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.SERPER_SCRAPE_URL ??\n      'https://scrape.serper.dev';\n\n    this.timeout = config.timeout ?? 7500;\n    this.includeMarkdown = config.includeMarkdown ?? true;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Serper scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.SerperScrapeOptions = {}\n  ): Promise<[string, t.SerperScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'SERPER_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = {\n        url,\n        includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,\n      };\n\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'X-API-KEY': this.apiKey,\n          'Content-Type': 'application/json',\n        },\n        timeout: options.timeout ?? this.timeout,\n      });\n\n      return [url, { success: true, data: response.data }];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Serper Scrape API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.SerperScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    if (response.data.text != null) {\n      return [response.data.text, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(\n    response: t.SerperScrapeResponse\n  ): Record<string, string | number | boolean | null | undefined> {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Serper scraper instance\n * @param config Scraper configuration\n * @returns Serper scraper instance\n */\nexport const createSerperScraper = (\n  config: t.SerperScraperConfig = {}\n): SerperScraper => {\n  return new SerperScraper(config);\n};\n"],"names":[],"mappings":";;;AAIA;;;;;;;;;;;;;;;;;;;;;;;;AAwBG;MACU,aAAa,CAAA;AAChB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,eAAe;AAEvB,IAAA,WAAA,CAAY,SAAgC,EAAE,EAAA;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;AAE/D,QAAA,IAAI,CAAC,MAAM;AACT,YAAA,MAAM,CAAC,MAAM;gBACb,OAAO,CAAC,GAAG,CAAC,iBAAiB;AAC7B,gBAAA,2BAA2B;QAE7B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAAC;;QAGxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA4C,yCAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC1D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAiC,EAAE,EAAA;AAEnC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,2BAA2B;AACnC,iBAAA;aACF;;AAGH,QAAA,IAAI;AACF,YAAA,MAAM,OAAO,GAAG;gBACd,GAAG;AACH,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;aACjE;AAED,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,IAAI,CAAC,MAAM;AACxB,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;AACD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACzC,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;;QACpD,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAqC,kCAAA,EAAA,YAAY,CAAE,CAAA;AAC3D,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAgC,EAAA;QAEhC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;QAGxB,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;QAG5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;AAGxC,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CACb,QAAgC,EAAA;AAEhC,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,mBAAmB,GAAG,CACjC,MAAgC,GAAA,EAAE,KACjB;AACjB,IAAA,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC;AAClC;;;;"}

package/dist/esm/tools/search/tool.mjs

@@ -10,6 +10,115 @@ import { createDefaultLogger } from './utils.mjs';
 import { createReranker } from './rerankers.mjs';
 import { Constants } from '../../common/enum.mjs';
 
+/**
+ * URL regex pattern to detect direct URLs in query
+ */
+const URL_PATTERN = /https?:\/\/[^\s<>"{}|\\^`[\]]+/gi;
+/**
+ * Extracts URLs from a query string
+ * @param query - The search query
+ * @returns Array of URLs found in the query
+ */
+function extractUrlsFromQuery(query) {
+    const matches = query.match(URL_PATTERN);
+    return matches ?? [];
+}
+/**
+ * Checks if the query is primarily a URL request (contains URL and minimal other text)
+ * @param query - The search query
+ * @returns True if the query appears to be a direct URL request
+ */
+function isDirectUrlRequest(query) {
+    const urls = extractUrlsFromQuery(query);
+    if (urls.length === 0) {
+        return false;
+    }
+    // Remove URLs from query and check remaining text
+    let remainingText = query;
+    for (const url of urls) {
+        remainingText = remainingText.replace(url, '');
+    }
+    // Clean up and check if remaining text is minimal (just filler words or questions about the URL)
+    remainingText = remainingText.trim().toLowerCase();
+    // If very little text remains, it's likely a direct URL request
+    if (remainingText.length < 50) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Directly extracts content from URLs using the scraper
+ * @param urls - URLs to extract content from
+ * @param scraper - The scraper instance to use
+ * @param logger - Logger instance
+ * @returns Search result with extracted content
+ */
+async function extractDirectUrlContent({ urls, scraper, logger, }) {
+    try {
+        const results = [];
+        for (const url of urls) {
+            try {
+                logger.debug(`Direct URL extraction: ${url}`);
+                const [, response] = await scraper.scrapeUrl(url);
+                if (response.success && response.data) {
+                    const [content, references] = scraper.extractContent(response);
+                    const metadata = scraper.extractMetadata(response);
+                    // Helper to safely extract string from metadata
+                    const getString = (value) => {
+                        return typeof value === 'string' ? value : undefined;
+                    };
+                    results.push({
+                        position: results.length + 1,
+                        title: getString(metadata.title) ?? getString(metadata.ogTitle) ?? url,
+                        link: url,
+                        snippet: getString(metadata.description) ?? getString(metadata.ogDescription) ?? '',
+                        content: content,
+                        references: references,
+                        processed: true,
+                    });
+                }
+                else {
+                    logger.warn(`Failed to extract content from ${url}: ${response.error}`);
+                    // Still add the URL as a result, but without content
+                    results.push({
+                        position: results.length + 1,
+                        title: url,
+                        link: url,
+                        snippet: response.error ?? 'Failed to extract content',
+                        processed: false,
+                    });
+                }
+            }
+            catch (error) {
+                logger.error(`Error extracting URL ${url}:`, error);
+                results.push({
+                    position: results.length + 1,
+                    title: url,
+                    link: url,
+                    snippet: error instanceof Error ? error.message : String(error),
+                    processed: false,
+                });
+            }
+        }
+        return {
+            success: true,
+            data: {
+                organic: results,
+                topStories: [],
+                images: [],
+                videos: [],
+                relatedSearches: [],
+            },
+        };
+    }
+    catch (error) {
+        logger.error('Error in direct URL extraction:', error);
+        return {
+            success: false,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+}
 /**
  * Executes parallel searches and merges the results
  */
@@ -130,21 +239,36 @@ async function executeParallelSearches({ searchAPI, query, date, country, safeSe
     });
     return { success: true, data: mergedResults };
 }
-function createSearchProcessor({ searchAPI, safeSearch, sourceProcessor, onGetHighlights, logger, }) {
+function createSearchProcessor({ searchAPI, safeSearch, sourceProcessor, scraper, onGetHighlights, logger, }) {
     return async function ({ query, date, country, proMode = true, maxSources = 5, onSearchResults, images = false, videos = false, news = false, }) {
         try {
-            // Execute parallel searches and merge results
-            const searchResult = await executeParallelSearches({
-                searchAPI,
-                query,
-                date,
-                country,
-                safeSearch,
-                images,
-                videos,
-                news,
-                logger,
-            });
+            // Check if query contains direct URLs for extraction
+            const urls = extractUrlsFromQuery(query);
+            const isDirectUrl = isDirectUrlRequest(query);
+            let searchResult;
+            if (isDirectUrl && urls.length > 0) {
+                // Direct URL extraction mode - skip search API and extract directly
+                logger.debug(`Direct URL extraction mode for: ${urls.join(', ')}`);
+                searchResult = await extractDirectUrlContent({
+                    urls,
+                    scraper,
+                    logger,
+                });
+            }
+            else {
+                // Normal search mode - execute parallel searches and merge results
+                searchResult = await executeParallelSearches({
+                    searchAPI,
+                    query,
+                    date,
+                    country,
+                    safeSearch,
+                    images,
+                    videos,
+                    news,
+                    logger,
+                });
+            }
             onSearchResults?.(searchResult);
             const processedSources = await sourceProcessor.processSources({
                 query,
@@ -153,6 +277,8 @@ function createSearchProcessor({ searchAPI, safeSearch, sourceProcessor, onGetHi
                 proMode,
                 onGetHighlights,
                 numElements: maxSources,
+                // Skip additional scraping if we already extracted content directly
+                skipScraping: isDirectUrl,
             });
             return expandHighlights(processedSources);
         }
@@ -200,28 +326,28 @@ function createTool({ schema, search, onSearchResults: _onSearchResults, }) {
         return [output, { [Constants.WEB_SEARCH]: data }];
     }, {
         name: Constants.WEB_SEARCH,
-        description: `Real-time search. Results have required citation anchors.
-
-Note: Use ONCE per reply unless instructed otherwise.
-
-Anchors:
-- \\ue202turnXtypeY
-- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx
-
-Special Markers:
-- \\ue203...\\ue204 — highlight start/end of cited text (for Standalone or Group citations)
-- \\ue200...\\ue201 — group block (e.g. \\ue200\\ue202turn0search1\\ue202turn0news2\\ue201)
-
-**CITE EVERY NON-OBVIOUS FACT/QUOTE:**
-Use anchor marker(s) immediately after the statement:
-- Standalone: "Pure functions produce same output. \\ue202turn0search0"
-- Standalone (multiple): "Today's News \\ue202turn0search0\\ue202turn0news0"
-- Highlight: "\\ue203Highlight text.\\ue204\\ue202turn0news1"
-- Group: "Sources. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
-- Group Highlight: "\\ue203Highlight for group.\\ue204 \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
-- Image: "See photo \\ue202turn0image0."
-
-**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**
+        description: `Real-time search. Results have required citation anchors.
+
+Note: Use ONCE per reply unless instructed otherwise.
+
+Anchors:
+- \\ue202turnXtypeY
+- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx
+
+Special Markers:
+- \\ue203...\\ue204 — highlight start/end of cited text (for Standalone or Group citations)
+- \\ue200...\\ue201 — group block (e.g. \\ue200\\ue202turn0search1\\ue202turn0news2\\ue201)
+
+**CITE EVERY NON-OBVIOUS FACT/QUOTE:**
+Use anchor marker(s) immediately after the statement:
+- Standalone: "Pure functions produce same output. \\ue202turn0search0"
+- Standalone (multiple): "Today's News \\ue202turn0search0\\ue202turn0news0"
+- Highlight: "\\ue203Highlight text.\\ue204\\ue202turn0news1"
+- Group: "Sources. \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
+- Group Highlight: "\\ue203Highlight for group.\\ue204 \\ue200\\ue202turn0search0\\ue202turn0news1\\ue201"
+- Image: "See photo \\ue202turn0image0."
+
+**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**
 `.trim(),
         schema: schema,
         responseFormat: Constants.CONTENT_AND_ARTIFACT,
@@ -315,6 +441,7 @@ const createSearchTool = (config = {}) => {
         searchAPI,
         safeSearch,
         sourceProcessor,
+        scraper: scraperInstance,
         onGetHighlights,
         logger,
     });

package/dist/esm/tools/search/tool.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tool.mjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\nimport type { RunnableConfig } from '@langchain/core/runnables';\r\nimport type * as t from './types';\r\nimport {\r\n  DATE_RANGE,\r\n  querySchema,\r\n  dateSchema,\r\n  countrySchema,\r\n  imagesSchema,\r\n  videosSchema,\r\n  newsSchema,\r\n} from './schema';\r\nimport { createSearchAPI, createSourceProcessor } from './search';\r\nimport { createSerperScraper } from './serper-scraper';\r\nimport { createFirecrawlScraper } from './firecrawl';\r\nimport { expandHighlights } from './highlights';\r\nimport { formatResultsForLLM } from './format';\r\nimport { createDefaultLogger } from './utils';\r\nimport { createReranker } from './rerankers';\r\nimport { Constants } from '@/common';\r\n\r\n/**\r\n * Executes parallel searches and merges the results\r\n */\r\nasync function executeParallelSearches({\r\n  searchAPI,\r\n  query,\r\n  date,\r\n  country,\r\n  safeSearch,\r\n  images,\r\n  videos,\r\n  news,\r\n  logger,\r\n}: {\r\n  searchAPI: ReturnType<typeof createSearchAPI>;\r\n  query: string;\r\n  date?: DATE_RANGE;\r\n  country?: string;\r\n  safeSearch: t.SearchToolConfig['safeSearch'];\r\n  images: boolean;\r\n  videos: boolean;\r\n  news: boolean;\r\n  logger: t.Logger;\r\n}): Promise<t.SearchResult> {\r\n  // Prepare all search tasks to run in parallel\r\n  const searchTasks: Promise<t.SearchResult>[] = [\r\n    // Main search\r\n    searchAPI.getSources({\r\n      query,\r\n      date,\r\n      country,\r\n      safeSearch,\r\n    }),\r\n  ];\r\n\r\n  if (images) {\r\n    searchTasks.push(\r\n      searchAPI\r\n        .getSources({\r\n          query,\r\n          date,\r\n          country,\r\n          safeSearch,\r\n          type: 'images',\r\n        })\r\n        .catch((error) => {\r\n          logger.error('Error fetching images:', error);\r\n          return {\r\n            success: false,\r\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\r\n          };\r\n        })\r\n    );\r\n  }\r\n  if (videos) {\r\n    searchTasks.push(\r\n      searchAPI\r\n        .getSources({\r\n          query,\r\n          date,\r\n          country,\r\n          safeSearch,\r\n          type: 'videos',\r\n        })\r\n        .catch((error) => {\r\n          logger.error('Error fetching videos:', error);\r\n          return {\r\n            success: false,\r\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\r\n          };\r\n        })\r\n    );\r\n  }\r\n  if (news) {\r\n    searchTasks.push(\r\n      searchAPI\r\n        .getSources({\r\n          query,\r\n          date,\r\n          country,\r\n          safeSearch,\r\n          type: 'news',\r\n        })\r\n        .catch((error) => {\r\n          logger.error('Error fetching news:', error);\r\n          return {\r\n            success: false,\r\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\r\n          };\r\n        })\r\n    );\r\n  }\r\n\r\n  // Run all searches in parallel\r\n  const results = await Promise.all(searchTasks);\r\n\r\n  // Get the main search result (first result)\r\n  const mainResult = results[0];\r\n  if (!mainResult.success) {\r\n    throw new Error(mainResult.error ?? 'Search failed');\r\n  }\r\n\r\n  // Merge additional results with the main results\r\n  const mergedResults = { ...mainResult.data };\r\n\r\n  // Convert existing news to topStories if present\r\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\r\n    const existingNewsAsTopStories = mergedResults.news\r\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\r\n      .map((newsItem) => ({\r\n        title: newsItem.title ?? '',\r\n        link: newsItem.link ?? '',\r\n        source: newsItem.source ?? '',\r\n        date: newsItem.date ?? '',\r\n        imageUrl: newsItem.imageUrl ?? '',\r\n        processed: false,\r\n      }));\r\n    mergedResults.topStories = [\r\n      ...(mergedResults.topStories ?? []),\r\n      ...existingNewsAsTopStories,\r\n    ];\r\n    delete mergedResults.news;\r\n  }\r\n\r\n  results.slice(1).forEach((result) => {\r\n    if (result.success && result.data !== undefined) {\r\n      if (result.data.images !== undefined && result.data.images.length > 0) {\r\n        mergedResults.images = [\r\n          ...(mergedResults.images ?? []),\r\n          ...result.data.images,\r\n        ];\r\n      }\r\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\r\n        mergedResults.videos = [\r\n          ...(mergedResults.videos ?? []),\r\n          ...result.data.videos,\r\n        ];\r\n      }\r\n      if (result.data.news !== undefined && result.data.news.length > 0) {\r\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\r\n          ...newsItem,\r\n          link: newsItem.link ?? '',\r\n        }));\r\n        mergedResults.topStories = [\r\n          ...(mergedResults.topStories ?? []),\r\n          ...newsAsTopStories,\r\n        ];\r\n      }\r\n    }\r\n  });\r\n\r\n  return { success: true, data: mergedResults };\r\n}\r\n\r\nfunction createSearchProcessor({\r\n  searchAPI,\r\n  safeSearch,\r\n  sourceProcessor,\r\n  onGetHighlights,\r\n  logger,\r\n}: {\r\n  safeSearch: t.SearchToolConfig['safeSearch'];\r\n  searchAPI: ReturnType<typeof createSearchAPI>;\r\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\r\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\r\n  logger: t.Logger;\r\n}) {\r\n  return async function ({\r\n    query,\r\n    date,\r\n    country,\r\n    proMode = true,\r\n    maxSources = 5,\r\n    onSearchResults,\r\n    images = false,\r\n    videos = false,\r\n    news = false,\r\n  }: {\r\n    query: string;\r\n    country?: string;\r\n    date?: DATE_RANGE;\r\n    proMode?: boolean;\r\n    maxSources?: number;\r\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\r\n    images?: boolean;\r\n    videos?: boolean;\r\n    news?: boolean;\r\n  }): Promise<t.SearchResultData> {\r\n    try {\r\n      // Execute parallel searches and merge results\r\n      const searchResult = await executeParallelSearches({\r\n        searchAPI,\r\n        query,\r\n        date,\r\n        country,\r\n        safeSearch,\r\n        images,\r\n        videos,\r\n        news,\r\n        logger,\r\n      });\r\n\r\n      onSearchResults?.(searchResult);\r\n\r\n      const processedSources = await sourceProcessor.processSources({\r\n        query,\r\n        news,\r\n        result: searchResult,\r\n        proMode,\r\n        onGetHighlights,\r\n        numElements: maxSources,\r\n      });\r\n\r\n      return expandHighlights(processedSources);\r\n    } catch (error) {\r\n      logger.error('Error in search:', error);\r\n      return {\r\n        organic: [],\r\n        topStories: [],\r\n        images: [],\r\n        videos: [],\r\n        news: [],\r\n        relatedSearches: [],\r\n        error: error instanceof Error ? error.message : String(error),\r\n      };\r\n    }\r\n  };\r\n}\r\n\r\nfunction createOnSearchResults({\r\n  runnableConfig,\r\n  onSearchResults,\r\n}: {\r\n  runnableConfig: RunnableConfig;\r\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\r\n}) {\r\n  return function (results: t.SearchResult): void {\r\n    if (!onSearchResults) {\r\n      return;\r\n    }\r\n    onSearchResults(results, runnableConfig);\r\n  };\r\n}\r\n\r\nfunction createTool({\r\n  schema,\r\n  search,\r\n  onSearchResults: _onSearchResults,\r\n}: {\r\n  schema: t.SearchToolSchema;\r\n  search: ReturnType<typeof createSearchProcessor>;\r\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\r\n}): DynamicStructuredTool<typeof schema> {\r\n  return tool<typeof schema>(\r\n    async (params, runnableConfig) => {\r\n      const { query, date, country: _c, images, videos, news } = params;\r\n      const country = typeof _c === 'string' && _c ? _c : undefined;\r\n      const searchResult = await search({\r\n        query,\r\n        date,\r\n        country,\r\n        images,\r\n        videos,\r\n        news,\r\n        onSearchResults: createOnSearchResults({\r\n          runnableConfig,\r\n          onSearchResults: _onSearchResults,\r\n        }),\r\n      });\r\n      const turn = runnableConfig.toolCall?.turn ?? 0;\r\n      const { output, references } = formatResultsForLLM(turn, searchResult);\r\n      const data: t.SearchResultData = { turn, ...searchResult, references };\r\n      return [output, { [Constants.WEB_SEARCH]: data }];\r\n    },\r\n    {\r\n      name: Constants.WEB_SEARCH,\r\n      description: `Real-time search. Results have required citation anchors.\r\n\r\nNote: Use ONCE per reply unless instructed otherwise.\r\n\r\nAnchors:\r\n- \\\\ue202turnXtypeY\r\n- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx\r\n\r\nSpecial Markers:\r\n- \\\\ue203...\\\\ue204 — highlight start/end of cited text (for Standalone or Group citations)\r\n- \\\\ue200...\\\\ue201 — group block (e.g. \\\\ue200\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\r\n\r\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\r\nUse anchor marker(s) immediately after the statement:\r\n- Standalone: \"Pure functions produce same output. \\\\ue202turn0search0\"\r\n- Standalone (multiple): \"Today's News \\\\ue202turn0search0\\\\ue202turn0news0\"\r\n- Highlight: \"\\\\ue203Highlight text.\\\\ue204\\\\ue202turn0news1\"\r\n- Group: \"Sources. \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\r\n- Group Highlight: \"\\\\ue203Highlight for group.\\\\ue204 \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\r\n- Image: \"See photo \\\\ue202turn0image0.\"\r\n\r\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\r\n`.trim(),\r\n      schema: schema,\r\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\r\n    }\r\n  );\r\n}\r\n\r\n/**\r\n * Creates a search tool with a schema that dynamically includes the country field\r\n * only when the searchProvider is 'serper'.\r\n *\r\n * Supports multiple scraper providers:\r\n * - Firecrawl (default): Full-featured web scraping with multiple formats\r\n * - Serper: Lightweight scraping using Serper's scrape API\r\n *\r\n * @example\r\n * ```typescript\r\n * // Using Firecrawl scraper (default)\r\n * const searchTool = createSearchTool({\r\n *   searchProvider: 'serper',\r\n *   scraperProvider: 'firecrawl',\r\n *   firecrawlApiKey: 'your-firecrawl-key'\r\n * });\r\n *\r\n * // Using Serper scraper\r\n * const searchTool = createSearchTool({\r\n *   searchProvider: 'serper',\r\n *   scraperProvider: 'serper',\r\n *   serperApiKey: 'your-serper-key'\r\n * });\r\n * ```\r\n *\r\n * @param config - The search tool configuration\r\n * @returns A DynamicStructuredTool with a schema that depends on the searchProvider\r\n */\r\nexport const createSearchTool = (\r\n  config: t.SearchToolConfig = {}\r\n): DynamicStructuredTool<typeof toolSchema> => {\r\n  const {\r\n    searchProvider = 'serper',\r\n    serperApiKey,\r\n    searxngInstanceUrl,\r\n    searxngApiKey,\r\n    rerankerType = 'cohere',\r\n    topResults = 5,\r\n    strategies = ['no_extraction'],\r\n    filterContent = true,\r\n    safeSearch = 1,\r\n    scraperProvider = 'firecrawl',\r\n    firecrawlApiKey,\r\n    firecrawlApiUrl,\r\n    firecrawlVersion,\r\n    firecrawlOptions,\r\n    serperScraperOptions,\r\n    scraperTimeout,\r\n    jinaApiKey,\r\n    jinaApiUrl,\r\n    cohereApiKey,\r\n    onSearchResults: _onSearchResults,\r\n    onGetHighlights,\r\n  } = config;\r\n\r\n  const logger = config.logger || createDefaultLogger();\r\n\r\n  const schemaObject: {\r\n    query: z.ZodString;\r\n    date: z.ZodOptional<z.ZodNativeEnum<typeof DATE_RANGE>>;\r\n    country?: z.ZodOptional<z.ZodString>;\r\n    images: z.ZodOptional<z.ZodBoolean>;\r\n    videos: z.ZodOptional<z.ZodBoolean>;\r\n    news: z.ZodOptional<z.ZodBoolean>;\r\n  } = {\r\n    query: querySchema,\r\n    date: dateSchema,\r\n    images: imagesSchema,\r\n    videos: videosSchema,\r\n    news: newsSchema,\r\n  };\r\n\r\n  if (searchProvider === 'serper') {\r\n    schemaObject.country = countrySchema;\r\n  }\r\n\r\n  const toolSchema = z.object(schemaObject);\r\n\r\n  const searchAPI = createSearchAPI({\r\n    searchProvider,\r\n    serperApiKey,\r\n    searxngInstanceUrl,\r\n    searxngApiKey,\r\n  });\r\n\r\n  /** Create scraper based on scraperProvider */\r\n  let scraperInstance: t.BaseScraper;\r\n\r\n  if (scraperProvider === 'serper') {\r\n    scraperInstance = createSerperScraper({\r\n      ...serperScraperOptions,\r\n      apiKey: serperApiKey,\r\n      timeout: scraperTimeout ?? serperScraperOptions?.timeout,\r\n      logger,\r\n    });\r\n  } else {\r\n    scraperInstance = createFirecrawlScraper({\r\n      ...firecrawlOptions,\r\n      apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\r\n      apiUrl: firecrawlApiUrl,\r\n      version: firecrawlVersion,\r\n      timeout: scraperTimeout ?? firecrawlOptions?.timeout,\r\n      formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],\r\n      logger,\r\n    });\r\n  }\r\n\r\n  const selectedReranker = createReranker({\r\n    rerankerType,\r\n    jinaApiKey,\r\n    jinaApiUrl,\r\n    cohereApiKey,\r\n    logger,\r\n  });\r\n\r\n  if (!selectedReranker) {\r\n    logger.warn('No reranker selected. Using default ranking.');\r\n  }\r\n\r\n  const sourceProcessor = createSourceProcessor(\r\n    {\r\n      reranker: selectedReranker,\r\n      topResults,\r\n      strategies,\r\n      filterContent,\r\n      logger,\r\n    },\r\n    scraperInstance\r\n  );\r\n\r\n  const search = createSearchProcessor({\r\n    searchAPI,\r\n    safeSearch,\r\n    sourceProcessor,\r\n    onGetHighlights,\r\n    logger,\r\n  });\r\n\r\n  return createTool({\r\n    search,\r\n    schema: toolSchema,\r\n    onSearchResults: _onSearchResults,\r\n  });\r\n};\r\n"],"names":[],"mappings":";;;;;;;;;;;;AAsBA;;AAEG;AACH,eAAe,uBAAuB,CAAC,EACrC,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,GAWP,EAAA;;AAEC,IAAA,MAAM,WAAW,GAA8B;;QAE7C,SAAS,CAAC,UAAU,CAAC;YACnB,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;SACX,CAAC;KACH;IAED,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,IAAI,EAAE;QACR,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,MAAM;SACb;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;YAC3C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAuB,oBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACvF;SACF,CAAC,CACL;;;IAIH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;;AAG9C,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;AAC7B,IAAA,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;;;IAItD,MAAM,aAAa,GAAG,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE;;AAG5C,IAAA,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACrE,QAAA,MAAM,wBAAwB,GAAG,aAAa,CAAC;AAC5C,aAAA,MAAM,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,EAAE;AACxE,aAAA,GAAG,CAAC,CAAC,QAAQ,MAAM;AAClB,YAAA,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;AAC3B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;AAC7B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;AACjC,YAAA,SAAS,EAAE,KAAK;AACjB,SAAA,CAAC,CAAC;QACL,aAAa,CAAC,UAAU,GAAG;AACzB,YAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,YAAA,GAAG,wBAAwB;SAC5B;QACD,OAAO,aAAa,CAAC,IAAI;;IAG3B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAI;QAClC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE;AAC/C,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,gBAAA,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM;AAC3D,oBAAA,GAAG,QAAQ;AACX,oBAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AAC1B,iBAAA,CAAC,CAAC;gBACH,aAAa,CAAC,UAAU,GAAG;AACzB,oBAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,oBAAA,GAAG,gBAAgB;iBACpB;;;AAGP,KAAC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;AAC/C;AAEA,SAAS,qBAAqB,CAAC,EAC7B,SAAS,EACT,UAAU,EACV,eAAe,EACf,eAAe,EACf,MAAM,GAOP,EAAA;AACC,IAAA,OAAO,gBAAgB,EACrB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,EACf,MAAM,GAAG,KAAK,EACd,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,KAAK,GAWb,EAAA;AACC,QAAA,IAAI;;AAEF,YAAA,MAAM,YAAY,GAAG,MAAM,uBAAuB,CAAC;gBACjD,SAAS;gBACT,KAAK;gBACL,IAAI;gBACJ,OAAO;gBACP,UAAU;gBACV,MAAM;gBACN,MAAM;gBACN,IAAI;gBACJ,MAAM;AACP,aAAA,CAAC;AAEF,YAAA,eAAe,GAAG,YAAY,CAAC;AAE/B,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC5D,KAAK;gBACL,IAAI;AACJ,gBAAA,MAAM,EAAE,YAAY;gBACpB,OAAO;gBACP,eAAe;AACf,gBAAA,WAAW,EAAE,UAAU;AACxB,aAAA,CAAC;AAEF,YAAA,OAAO,gBAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACvC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,EAAE;AACR,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;AACH;AAEA,SAAS,qBAAqB,CAAC,EAC7B,cAAc,EACd,eAAe,GAIhB,EAAA;AACC,IAAA,OAAO,UAAU,OAAuB,EAAA;QACtC,IAAI,CAAC,eAAe,EAAE;YACpB;;AAEF,QAAA,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC;AAC1C,KAAC;AACH;AAEA,SAAS,UAAU,CAAC,EAClB,MAAM,EACN,MAAM,EACN,eAAe,EAAE,gBAAgB,GAKlC,EAAA;IACC,OAAO,IAAI,CACT,OAAO,MAAM,EAAE,cAAc,KAAI;AAC/B,QAAA,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;AACjE,QAAA,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,SAAS;AAC7D,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;YACL,IAAI;YACJ,OAAO;YACP,MAAM;YACN,MAAM;YACN,IAAI;YACJ,eAAe,EAAE,qBAAqB,CAAC;gBACrC,cAAc;AACd,gBAAA,eAAe,EAAE,gBAAgB;aAClC,CAAC;AACH,SAAA,CAAC;QACF,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC/C,QAAA,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,YAAY,CAAC;QACtE,MAAM,IAAI,GAAuB,EAAE,IAAI,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;AACtE,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;AACnD,KAAC,EACD;QACE,IAAI,EAAE,SAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;;;;AAsBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,MAAM;QACd,cAAc,EAAE,SAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BG;MACU,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACa;AAC5C,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,UAAU,GAAG,CAAC,EACd,eAAe,GAAG,WAAW,EAC7B,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,UAAU,EACV,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,EACjC,eAAe,GAChB,GAAG,MAAM;IAEV,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAErD,IAAA,MAAM,YAAY,GAOd;AACF,QAAA,KAAK,EAAE,WAAW;AAClB,QAAA,IAAI,EAAE,UAAU;AAChB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,IAAI,EAAE,UAAU;KACjB;AAED,IAAA,IAAI,cAAc,KAAK,QAAQ,EAAE;AAC/B,QAAA,YAAY,CAAC,OAAO,GAAG,aAAa;;IAGtC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAEzC,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;;AAGF,IAAA,IAAI,eAA8B;AAElC,IAAA,IAAI,eAAe,KAAK,QAAQ,EAAE;QAChC,eAAe,GAAG,mBAAmB,CAAC;AACpC,YAAA,GAAG,oBAAoB;AACvB,YAAA,MAAM,EAAE,YAAY;AACpB,YAAA,OAAO,EAAE,cAAc,IAAI,oBAAoB,EAAE,OAAO;YACxD,MAAM;AACP,SAAA,CAAC;;SACG;QACL,eAAe,GAAG,sBAAsB,CAAC;AACvC,YAAA,GAAG,gBAAgB;AACnB,YAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,YAAA,MAAM,EAAE,eAAe;AACvB,YAAA,OAAO,EAAE,gBAAgB;AACzB,YAAA,OAAO,EAAE,cAAc,IAAI,gBAAgB,EAAE,OAAO;YACpD,OAAO,EAAE,gBAAgB,EAAE,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;YAC7D,MAAM;AACP,SAAA,CAAC;;IAGJ,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,UAAU;QACV,YAAY;QACZ,MAAM;AACP,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG7D,MAAM,eAAe,GAAG,qBAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,UAAU;QAGV,MAAM;KACP,EACD,eAAe,CAChB;IAED,MAAM,MAAM,GAAG,qBAAqB,CAAC;QACnC,SAAS;QACT,UAAU;QACV,eAAe;QACf,eAAe;QACf,MAAM;AACP,KAAA,CAAC;AAEF,IAAA,OAAO,UAAU,CAAC;QAChB,MAAM;AACN,QAAA,MAAM,EAAE,UAAU;AAClB,QAAA,eAAe,EAAE,gBAAgB;AAClC,KAAA,CAAC;AACJ;;;;"}
+{"version":3,"file":"tool.mjs","sources":["../../../../src/tools/search/tool.ts"],"sourcesContent":["import { z } from 'zod';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type { RunnableConfig } from '@langchain/core/runnables';\nimport type * as t from './types';\nimport {\n  DATE_RANGE,\n  querySchema,\n  dateSchema,\n  countrySchema,\n  imagesSchema,\n  videosSchema,\n  newsSchema,\n} from './schema';\nimport { createSearchAPI, createSourceProcessor } from './search';\nimport { createSerperScraper } from './serper-scraper';\nimport { createFirecrawlScraper } from './firecrawl';\nimport { expandHighlights } from './highlights';\nimport { formatResultsForLLM } from './format';\nimport { createDefaultLogger } from './utils';\nimport { createReranker } from './rerankers';\nimport { Constants } from '@/common';\n\n/**\n * URL regex pattern to detect direct URLs in query\n */\nconst URL_PATTERN = /https?:\\/\\/[^\\s<>\"{}|\\\\^`[\\]]+/gi;\n\n/**\n * Extracts URLs from a query string\n * @param query - The search query\n * @returns Array of URLs found in the query\n */\nfunction extractUrlsFromQuery(query: string): string[] {\n  const matches = query.match(URL_PATTERN);\n  return matches ?? [];\n}\n\n/**\n * Checks if the query is primarily a URL request (contains URL and minimal other text)\n * @param query - The search query\n * @returns True if the query appears to be a direct URL request\n */\nfunction isDirectUrlRequest(query: string): boolean {\n  const urls = extractUrlsFromQuery(query);\n  if (urls.length === 0) {\n    return false;\n  }\n\n  // Remove URLs from query and check remaining text\n  let remainingText = query;\n  for (const url of urls) {\n    remainingText = remainingText.replace(url, '');\n  }\n\n  // Clean up and check if remaining text is minimal (just filler words or questions about the URL)\n  remainingText = remainingText.trim().toLowerCase();\n\n  // If very little text remains, it's likely a direct URL request\n  if (remainingText.length < 50) {\n    return true;\n  }\n\n  return false;\n}\n\n/**\n * Directly extracts content from URLs using the scraper\n * @param urls - URLs to extract content from\n * @param scraper - The scraper instance to use\n * @param logger - Logger instance\n * @returns Search result with extracted content\n */\nasync function extractDirectUrlContent({\n  urls,\n  scraper,\n  logger,\n}: {\n  urls: string[];\n  scraper: t.BaseScraper;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  try {\n    const results: t.ProcessedOrganic[] = [];\n\n    for (const url of urls) {\n      try {\n        logger.debug(`Direct URL extraction: ${url}`);\n        const [, response] = await scraper.scrapeUrl(url);\n\n        if (response.success && response.data) {\n          const [content, references] = scraper.extractContent(response);\n          const metadata = scraper.extractMetadata(response);\n\n          // Helper to safely extract string from metadata\n          const getString = (value: unknown): string | undefined => {\n            return typeof value === 'string' ? value : undefined;\n          };\n\n          results.push({\n            position: results.length + 1,\n            title: getString(metadata.title) ?? getString(metadata.ogTitle) ?? url,\n            link: url,\n            snippet: getString(metadata.description) ?? getString(metadata.ogDescription) ?? '',\n            content: content,\n            references: references,\n            processed: true,\n          });\n        } else {\n          logger.warn(`Failed to extract content from ${url}: ${response.error}`);\n          // Still add the URL as a result, but without content\n          results.push({\n            position: results.length + 1,\n            title: url,\n            link: url,\n            snippet: response.error ?? 'Failed to extract content',\n            processed: false,\n          });\n        }\n      } catch (error) {\n        logger.error(`Error extracting URL ${url}:`, error);\n        results.push({\n          position: results.length + 1,\n          title: url,\n          link: url,\n          snippet: error instanceof Error ? error.message : String(error),\n          processed: false,\n        });\n      }\n    }\n\n    return {\n      success: true,\n      data: {\n        organic: results,\n        topStories: [],\n        images: [],\n        videos: [],\n        relatedSearches: [],\n      },\n    };\n  } catch (error) {\n    logger.error('Error in direct URL extraction:', error);\n    return {\n      success: false,\n      error: error instanceof Error ? error.message : String(error),\n    };\n  }\n}\n\n/**\n * Executes parallel searches and merges the results\n */\nasync function executeParallelSearches({\n  searchAPI,\n  query,\n  date,\n  country,\n  safeSearch,\n  images,\n  videos,\n  news,\n  logger,\n}: {\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  query: string;\n  date?: DATE_RANGE;\n  country?: string;\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  images: boolean;\n  videos: boolean;\n  news: boolean;\n  logger: t.Logger;\n}): Promise<t.SearchResult> {\n  // Prepare all search tasks to run in parallel\n  const searchTasks: Promise<t.SearchResult>[] = [\n    // Main search\n    searchAPI.getSources({\n      query,\n      date,\n      country,\n      safeSearch,\n    }),\n  ];\n\n  if (images) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'images',\n        })\n        .catch((error) => {\n          logger.error('Error fetching images:', error);\n          return {\n            success: false,\n            error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (videos) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'videos',\n        })\n        .catch((error) => {\n          logger.error('Error fetching videos:', error);\n          return {\n            success: false,\n            error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n  if (news) {\n    searchTasks.push(\n      searchAPI\n        .getSources({\n          query,\n          date,\n          country,\n          safeSearch,\n          type: 'news',\n        })\n        .catch((error) => {\n          logger.error('Error fetching news:', error);\n          return {\n            success: false,\n            error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,\n          };\n        })\n    );\n  }\n\n  // Run all searches in parallel\n  const results = await Promise.all(searchTasks);\n\n  // Get the main search result (first result)\n  const mainResult = results[0];\n  if (!mainResult.success) {\n    throw new Error(mainResult.error ?? 'Search failed');\n  }\n\n  // Merge additional results with the main results\n  const mergedResults = { ...mainResult.data };\n\n  // Convert existing news to topStories if present\n  if (mergedResults.news !== undefined && mergedResults.news.length > 0) {\n    const existingNewsAsTopStories = mergedResults.news\n      .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')\n      .map((newsItem) => ({\n        title: newsItem.title ?? '',\n        link: newsItem.link ?? '',\n        source: newsItem.source ?? '',\n        date: newsItem.date ?? '',\n        imageUrl: newsItem.imageUrl ?? '',\n        processed: false,\n      }));\n    mergedResults.topStories = [\n      ...(mergedResults.topStories ?? []),\n      ...existingNewsAsTopStories,\n    ];\n    delete mergedResults.news;\n  }\n\n  results.slice(1).forEach((result) => {\n    if (result.success && result.data !== undefined) {\n      if (result.data.images !== undefined && result.data.images.length > 0) {\n        mergedResults.images = [\n          ...(mergedResults.images ?? []),\n          ...result.data.images,\n        ];\n      }\n      if (result.data.videos !== undefined && result.data.videos.length > 0) {\n        mergedResults.videos = [\n          ...(mergedResults.videos ?? []),\n          ...result.data.videos,\n        ];\n      }\n      if (result.data.news !== undefined && result.data.news.length > 0) {\n        const newsAsTopStories = result.data.news.map((newsItem) => ({\n          ...newsItem,\n          link: newsItem.link ?? '',\n        }));\n        mergedResults.topStories = [\n          ...(mergedResults.topStories ?? []),\n          ...newsAsTopStories,\n        ];\n      }\n    }\n  });\n\n  return { success: true, data: mergedResults };\n}\n\nfunction createSearchProcessor({\n  searchAPI,\n  safeSearch,\n  sourceProcessor,\n  scraper,\n  onGetHighlights,\n  logger,\n}: {\n  safeSearch: t.SearchToolConfig['safeSearch'];\n  searchAPI: ReturnType<typeof createSearchAPI>;\n  sourceProcessor: ReturnType<typeof createSourceProcessor>;\n  scraper: t.BaseScraper;\n  onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n  logger: t.Logger;\n}) {\n  return async function ({\n    query,\n    date,\n    country,\n    proMode = true,\n    maxSources = 5,\n    onSearchResults,\n    images = false,\n    videos = false,\n    news = false,\n  }: {\n    query: string;\n    country?: string;\n    date?: DATE_RANGE;\n    proMode?: boolean;\n    maxSources?: number;\n    onSearchResults: t.SearchToolConfig['onSearchResults'];\n    images?: boolean;\n    videos?: boolean;\n    news?: boolean;\n  }): Promise<t.SearchResultData> {\n    try {\n      // Check if query contains direct URLs for extraction\n      const urls = extractUrlsFromQuery(query);\n      const isDirectUrl = isDirectUrlRequest(query);\n\n      let searchResult: t.SearchResult;\n\n      if (isDirectUrl && urls.length > 0) {\n        // Direct URL extraction mode - skip search API and extract directly\n        logger.debug(`Direct URL extraction mode for: ${urls.join(', ')}`);\n        searchResult = await extractDirectUrlContent({\n          urls,\n          scraper,\n          logger,\n        });\n      } else {\n        // Normal search mode - execute parallel searches and merge results\n        searchResult = await executeParallelSearches({\n          searchAPI,\n          query,\n          date,\n          country,\n          safeSearch,\n          images,\n          videos,\n          news,\n          logger,\n        });\n      }\n\n      onSearchResults?.(searchResult);\n\n      const processedSources = await sourceProcessor.processSources({\n        query,\n        news,\n        result: searchResult,\n        proMode,\n        onGetHighlights,\n        numElements: maxSources,\n        // Skip additional scraping if we already extracted content directly\n        skipScraping: isDirectUrl,\n      });\n\n      return expandHighlights(processedSources);\n    } catch (error) {\n      logger.error('Error in search:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        videos: [],\n        news: [],\n        relatedSearches: [],\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n}\n\nfunction createOnSearchResults({\n  runnableConfig,\n  onSearchResults,\n}: {\n  runnableConfig: RunnableConfig;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}) {\n  return function (results: t.SearchResult): void {\n    if (!onSearchResults) {\n      return;\n    }\n    onSearchResults(results, runnableConfig);\n  };\n}\n\nfunction createTool({\n  schema,\n  search,\n  onSearchResults: _onSearchResults,\n}: {\n  schema: t.SearchToolSchema;\n  search: ReturnType<typeof createSearchProcessor>;\n  onSearchResults: t.SearchToolConfig['onSearchResults'];\n}): DynamicStructuredTool<typeof schema> {\n  return tool<typeof schema>(\n    async (params, runnableConfig) => {\n      const { query, date, country: _c, images, videos, news } = params;\n      const country = typeof _c === 'string' && _c ? _c : undefined;\n      const searchResult = await search({\n        query,\n        date,\n        country,\n        images,\n        videos,\n        news,\n        onSearchResults: createOnSearchResults({\n          runnableConfig,\n          onSearchResults: _onSearchResults,\n        }),\n      });\n      const turn = runnableConfig.toolCall?.turn ?? 0;\n      const { output, references } = formatResultsForLLM(turn, searchResult);\n      const data: t.SearchResultData = { turn, ...searchResult, references };\n      return [output, { [Constants.WEB_SEARCH]: data }];\n    },\n    {\n      name: Constants.WEB_SEARCH,\n      description: `Real-time search. Results have required citation anchors.\n\nNote: Use ONCE per reply unless instructed otherwise.\n\nAnchors:\n- \\\\ue202turnXtypeY\n- X = turn idx, type = 'search' | 'news' | 'image' | 'ref', Y = item idx\n\nSpecial Markers:\n- \\\\ue203...\\\\ue204 — highlight start/end of cited text (for Standalone or Group citations)\n- \\\\ue200...\\\\ue201 — group block (e.g. \\\\ue200\\\\ue202turn0search1\\\\ue202turn0news2\\\\ue201)\n\n**CITE EVERY NON-OBVIOUS FACT/QUOTE:**\nUse anchor marker(s) immediately after the statement:\n- Standalone: \"Pure functions produce same output. \\\\ue202turn0search0\"\n- Standalone (multiple): \"Today's News \\\\ue202turn0search0\\\\ue202turn0news0\"\n- Highlight: \"\\\\ue203Highlight text.\\\\ue204\\\\ue202turn0news1\"\n- Group: \"Sources. \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Group Highlight: \"\\\\ue203Highlight for group.\\\\ue204 \\\\ue200\\\\ue202turn0search0\\\\ue202turn0news1\\\\ue201\"\n- Image: \"See photo \\\\ue202turn0image0.\"\n\n**NEVER use markdown links, [1], or footnotes. CITE ONLY with anchors provided.**\n`.trim(),\n      schema: schema,\n      responseFormat: Constants.CONTENT_AND_ARTIFACT,\n    }\n  );\n}\n\n/**\n * Creates a search tool with a schema that dynamically includes the country field\n * only when the searchProvider is 'serper'.\n *\n * Supports multiple scraper providers:\n * - Firecrawl (default): Full-featured web scraping with multiple formats\n * - Serper: Lightweight scraping using Serper's scrape API\n *\n * @example\n * ```typescript\n * // Using Firecrawl scraper (default)\n * const searchTool = createSearchTool({\n *   searchProvider: 'serper',\n *   scraperProvider: 'firecrawl',\n *   firecrawlApiKey: 'your-firecrawl-key'\n * });\n *\n * // Using Serper scraper\n * const searchTool = createSearchTool({\n *   searchProvider: 'serper',\n *   scraperProvider: 'serper',\n *   serperApiKey: 'your-serper-key'\n * });\n * ```\n *\n * @param config - The search tool configuration\n * @returns A DynamicStructuredTool with a schema that depends on the searchProvider\n */\nexport const createSearchTool = (\n  config: t.SearchToolConfig = {}\n): DynamicStructuredTool<typeof toolSchema> => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    rerankerType = 'cohere',\n    topResults = 5,\n    strategies = ['no_extraction'],\n    filterContent = true,\n    safeSearch = 1,\n    scraperProvider = 'firecrawl',\n    firecrawlApiKey,\n    firecrawlApiUrl,\n    firecrawlVersion,\n    firecrawlOptions,\n    serperScraperOptions,\n    scraperTimeout,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    onSearchResults: _onSearchResults,\n    onGetHighlights,\n  } = config;\n\n  const logger = config.logger || createDefaultLogger();\n\n  const schemaObject: {\n    query: z.ZodString;\n    date: z.ZodOptional<z.ZodNativeEnum<typeof DATE_RANGE>>;\n    country?: z.ZodOptional<z.ZodString>;\n    images: z.ZodOptional<z.ZodBoolean>;\n    videos: z.ZodOptional<z.ZodBoolean>;\n    news: z.ZodOptional<z.ZodBoolean>;\n  } = {\n    query: querySchema,\n    date: dateSchema,\n    images: imagesSchema,\n    videos: videosSchema,\n    news: newsSchema,\n  };\n\n  if (searchProvider === 'serper') {\n    schemaObject.country = countrySchema;\n  }\n\n  const toolSchema = z.object(schemaObject);\n\n  const searchAPI = createSearchAPI({\n    searchProvider,\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  });\n\n  /** Create scraper based on scraperProvider */\n  let scraperInstance: t.BaseScraper;\n\n  if (scraperProvider === 'serper') {\n    scraperInstance = createSerperScraper({\n      ...serperScraperOptions,\n      apiKey: serperApiKey,\n      timeout: scraperTimeout ?? serperScraperOptions?.timeout,\n      logger,\n    });\n  } else {\n    scraperInstance = createFirecrawlScraper({\n      ...firecrawlOptions,\n      apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,\n      apiUrl: firecrawlApiUrl,\n      version: firecrawlVersion,\n      timeout: scraperTimeout ?? firecrawlOptions?.timeout,\n      formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],\n      logger,\n    });\n  }\n\n  const selectedReranker = createReranker({\n    rerankerType,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    logger,\n  });\n\n  if (!selectedReranker) {\n    logger.warn('No reranker selected. Using default ranking.');\n  }\n\n  const sourceProcessor = createSourceProcessor(\n    {\n      reranker: selectedReranker,\n      topResults,\n      strategies,\n      filterContent,\n      logger,\n    },\n    scraperInstance\n  );\n\n  const search = createSearchProcessor({\n    searchAPI,\n    safeSearch,\n    sourceProcessor,\n    scraper: scraperInstance,\n    onGetHighlights,\n    logger,\n  });\n\n  return createTool({\n    search,\n    schema: toolSchema,\n    onSearchResults: _onSearchResults,\n  });\n};\n"],"names":[],"mappings":";;;;;;;;;;;;AAsBA;;AAEG;AACH,MAAM,WAAW,GAAG,kCAAkC;AAEtD;;;;AAIG;AACH,SAAS,oBAAoB,CAAC,KAAa,EAAA;IACzC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC;IACxC,OAAO,OAAO,IAAI,EAAE;AACtB;AAEA;;;;AAIG;AACH,SAAS,kBAAkB,CAAC,KAAa,EAAA;AACvC,IAAA,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC;AACxC,IAAA,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;AACrB,QAAA,OAAO,KAAK;;;IAId,IAAI,aAAa,GAAG,KAAK;AACzB,IAAA,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;;;IAIhD,aAAa,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE;;AAGlD,IAAA,IAAI,aAAa,CAAC,MAAM,GAAG,EAAE,EAAE;AAC7B,QAAA,OAAO,IAAI;;AAGb,IAAA,OAAO,KAAK;AACd;AAEA;;;;;;AAMG;AACH,eAAe,uBAAuB,CAAC,EACrC,IAAI,EACJ,OAAO,EACP,MAAM,GAKP,EAAA;AACC,IAAA,IAAI;QACF,MAAM,OAAO,GAAyB,EAAE;AAExC,QAAA,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;AACtB,YAAA,IAAI;AACF,gBAAA,MAAM,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAA,CAAE,CAAC;AAC7C,gBAAA,MAAM,GAAG,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC;gBAEjD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,oBAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;oBAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC;;AAGlD,oBAAA,MAAM,SAAS,GAAG,CAAC,KAAc,KAAwB;AACvD,wBAAA,OAAO,OAAO,KAAK,KAAK,QAAQ,GAAG,KAAK,GAAG,SAAS;AACtD,qBAAC;oBAED,OAAO,CAAC,IAAI,CAAC;AACX,wBAAA,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;AAC5B,wBAAA,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG;AACtE,wBAAA,IAAI,EAAE,GAAG;AACT,wBAAA,OAAO,EAAE,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE;AACnF,wBAAA,OAAO,EAAE,OAAO;AAChB,wBAAA,UAAU,EAAE,UAAU;AACtB,wBAAA,SAAS,EAAE,IAAI;AAChB,qBAAA,CAAC;;qBACG;oBACL,MAAM,CAAC,IAAI,CAAC,CAAkC,+BAAA,EAAA,GAAG,CAAK,EAAA,EAAA,QAAQ,CAAC,KAAK,CAAE,CAAA,CAAC;;oBAEvE,OAAO,CAAC,IAAI,CAAC;AACX,wBAAA,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;AAC5B,wBAAA,KAAK,EAAE,GAAG;AACV,wBAAA,IAAI,EAAE,GAAG;AACT,wBAAA,OAAO,EAAE,QAAQ,CAAC,KAAK,IAAI,2BAA2B;AACtD,wBAAA,SAAS,EAAE,KAAK;AACjB,qBAAA,CAAC;;;YAEJ,OAAO,KAAK,EAAE;gBACd,MAAM,CAAC,KAAK,CAAC,CAAA,qBAAA,EAAwB,GAAG,CAAG,CAAA,CAAA,EAAE,KAAK,CAAC;gBACnD,OAAO,CAAC,IAAI,CAAC;AACX,oBAAA,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;AAC5B,oBAAA,KAAK,EAAE,GAAG;AACV,oBAAA,IAAI,EAAE,GAAG;AACT,oBAAA,OAAO,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AAC/D,oBAAA,SAAS,EAAE,KAAK;AACjB,iBAAA,CAAC;;;QAIN,OAAO;AACL,YAAA,OAAO,EAAE,IAAI;AACb,YAAA,IAAI,EAAE;AACJ,gBAAA,OAAO,EAAE,OAAO;AAChB,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;AACpB,aAAA;SACF;;IACD,OAAO,KAAK,EAAE;AACd,QAAA,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC;QACtD,OAAO;AACL,YAAA,OAAO,EAAE,KAAK;AACd,YAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;SAC9D;;AAEL;AAEA;;AAEG;AACH,eAAe,uBAAuB,CAAC,EACrC,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,GAWP,EAAA;;AAEC,IAAA,MAAM,WAAW,GAA8B;;QAE7C,SAAS,CAAC,UAAU,CAAC;YACnB,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;SACX,CAAC;KACH;IAED,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,MAAM,EAAE;QACV,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,QAAQ;SACf;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC;YAC7C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAyB,sBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACzF;SACF,CAAC,CACL;;IAEH,IAAI,IAAI,EAAE;QACR,WAAW,CAAC,IAAI,CACd;AACG,aAAA,UAAU,CAAC;YACV,KAAK;YACL,IAAI;YACJ,OAAO;YACP,UAAU;AACV,YAAA,IAAI,EAAE,MAAM;SACb;AACA,aAAA,KAAK,CAAC,CAAC,KAAK,KAAI;AACf,YAAA,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;YAC3C,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;AACd,gBAAA,KAAK,EAAE,CAAuB,oBAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA;aACvF;SACF,CAAC,CACL;;;IAIH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;;AAG9C,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;AAC7B,IAAA,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,eAAe,CAAC;;;IAItD,MAAM,aAAa,GAAG,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE;;AAG5C,IAAA,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACrE,QAAA,MAAM,wBAAwB,GAAG,aAAa,CAAC;AAC5C,aAAA,MAAM,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,EAAE;AACxE,aAAA,GAAG,CAAC,CAAC,QAAQ,MAAM;AAClB,YAAA,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;AAC3B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;AAC7B,YAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AACzB,YAAA,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;AACjC,YAAA,SAAS,EAAE,KAAK;AACjB,SAAA,CAAC,CAAC;QACL,aAAa,CAAC,UAAU,GAAG;AACzB,YAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,YAAA,GAAG,wBAAwB;SAC5B;QACD,OAAO,aAAa,CAAC,IAAI;;IAG3B,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,KAAI;QAClC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE;AAC/C,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBACrE,aAAa,CAAC,MAAM,GAAG;AACrB,oBAAA,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE,CAAC;AAC/B,oBAAA,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM;iBACtB;;AAEH,YAAA,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,gBAAA,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,MAAM;AAC3D,oBAAA,GAAG,QAAQ;AACX,oBAAA,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;AAC1B,iBAAA,CAAC,CAAC;gBACH,aAAa,CAAC,UAAU,GAAG;AACzB,oBAAA,IAAI,aAAa,CAAC,UAAU,IAAI,EAAE,CAAC;AACnC,oBAAA,GAAG,gBAAgB;iBACpB;;;AAGP,KAAC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;AAC/C;AAEA,SAAS,qBAAqB,CAAC,EAC7B,SAAS,EACT,UAAU,EACV,eAAe,EACf,OAAO,EACP,eAAe,EACf,MAAM,GAQP,EAAA;AACC,IAAA,OAAO,gBAAgB,EACrB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,eAAe,EACf,MAAM,GAAG,KAAK,EACd,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,KAAK,GAWb,EAAA;AACC,QAAA,IAAI;;AAEF,YAAA,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC;AACxC,YAAA,MAAM,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC;AAE7C,YAAA,IAAI,YAA4B;YAEhC,IAAI,WAAW,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;;AAElC,gBAAA,MAAM,CAAC,KAAK,CAAC,CAAA,gCAAA,EAAmC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAE,CAAA,CAAC;gBAClE,YAAY,GAAG,MAAM,uBAAuB,CAAC;oBAC3C,IAAI;oBACJ,OAAO;oBACP,MAAM;AACP,iBAAA,CAAC;;iBACG;;gBAEL,YAAY,GAAG,MAAM,uBAAuB,CAAC;oBAC3C,SAAS;oBACT,KAAK;oBACL,IAAI;oBACJ,OAAO;oBACP,UAAU;oBACV,MAAM;oBACN,MAAM;oBACN,IAAI;oBACJ,MAAM;AACP,iBAAA,CAAC;;AAGJ,YAAA,eAAe,GAAG,YAAY,CAAC;AAE/B,YAAA,MAAM,gBAAgB,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC;gBAC5D,KAAK;gBACL,IAAI;AACJ,gBAAA,MAAM,EAAE,YAAY;gBACpB,OAAO;gBACP,eAAe;AACf,gBAAA,WAAW,EAAE,UAAU;;AAEvB,gBAAA,YAAY,EAAE,WAAW;AAC1B,aAAA,CAAC;AAEF,YAAA,OAAO,gBAAgB,CAAC,gBAAgB,CAAC;;QACzC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;YACvC,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,EAAE;AACR,gBAAA,eAAe,EAAE,EAAE;AACnB,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;AACH;AAEA,SAAS,qBAAqB,CAAC,EAC7B,cAAc,EACd,eAAe,GAIhB,EAAA;AACC,IAAA,OAAO,UAAU,OAAuB,EAAA;QACtC,IAAI,CAAC,eAAe,EAAE;YACpB;;AAEF,QAAA,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC;AAC1C,KAAC;AACH;AAEA,SAAS,UAAU,CAAC,EAClB,MAAM,EACN,MAAM,EACN,eAAe,EAAE,gBAAgB,GAKlC,EAAA;IACC,OAAO,IAAI,CACT,OAAO,MAAM,EAAE,cAAc,KAAI;AAC/B,QAAA,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM;AACjE,QAAA,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,SAAS;AAC7D,QAAA,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC;YAChC,KAAK;YACL,IAAI;YACJ,OAAO;YACP,MAAM;YACN,MAAM;YACN,IAAI;YACJ,eAAe,EAAE,qBAAqB,CAAC;gBACrC,cAAc;AACd,gBAAA,eAAe,EAAE,gBAAgB;aAClC,CAAC;AACH,SAAA,CAAC;QACF,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC;AAC/C,QAAA,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,YAAY,CAAC;QACtE,MAAM,IAAI,GAAuB,EAAE,IAAI,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;AACtE,QAAA,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,GAAG,IAAI,EAAE,CAAC;AACnD,KAAC,EACD;QACE,IAAI,EAAE,SAAS,CAAC,UAAU;AAC1B,QAAA,WAAW,EAAE,CAAA;;;;;;;;;;;;;;;;;;;;;;AAsBlB,CAAA,CAAC,IAAI,EAAE;AACF,QAAA,MAAM,EAAE,MAAM;QACd,cAAc,EAAE,SAAS,CAAC,oBAAoB;AAC/C,KAAA,CACF;AACH;AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BG;MACU,gBAAgB,GAAG,CAC9B,MAA6B,GAAA,EAAE,KACa;AAC5C,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,YAAY,GAAG,QAAQ,EACvB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,eAAe,CAAC,EAC9B,aAAa,GAAG,IAAI,EACpB,UAAU,GAAG,CAAC,EACd,eAAe,GAAG,WAAW,EAC7B,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,UAAU,EACV,UAAU,EACV,YAAY,EACZ,eAAe,EAAE,gBAAgB,EACjC,eAAe,GAChB,GAAG,MAAM;IAEV,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE;AAErD,IAAA,MAAM,YAAY,GAOd;AACF,QAAA,KAAK,EAAE,WAAW;AAClB,QAAA,IAAI,EAAE,UAAU;AAChB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,MAAM,EAAE,YAAY;AACpB,QAAA,IAAI,EAAE,UAAU;KACjB;AAED,IAAA,IAAI,cAAc,KAAK,QAAQ,EAAE;AAC/B,QAAA,YAAY,CAAC,OAAO,GAAG,aAAa;;IAGtC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;IAEzC,MAAM,SAAS,GAAG,eAAe,CAAC;QAChC,cAAc;QACd,YAAY;QACZ,kBAAkB;QAClB,aAAa;AACd,KAAA,CAAC;;AAGF,IAAA,IAAI,eAA8B;AAElC,IAAA,IAAI,eAAe,KAAK,QAAQ,EAAE;QAChC,eAAe,GAAG,mBAAmB,CAAC;AACpC,YAAA,GAAG,oBAAoB;AACvB,YAAA,MAAM,EAAE,YAAY;AACpB,YAAA,OAAO,EAAE,cAAc,IAAI,oBAAoB,EAAE,OAAO;YACxD,MAAM;AACP,SAAA,CAAC;;SACG;QACL,eAAe,GAAG,sBAAsB,CAAC;AACvC,YAAA,GAAG,gBAAgB;AACnB,YAAA,MAAM,EAAE,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB;AACxD,YAAA,MAAM,EAAE,eAAe;AACvB,YAAA,OAAO,EAAE,gBAAgB;AACzB,YAAA,OAAO,EAAE,cAAc,IAAI,gBAAgB,EAAE,OAAO;YACpD,OAAO,EAAE,gBAAgB,EAAE,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;YAC7D,MAAM;AACP,SAAA,CAAC;;IAGJ,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,YAAY;QACZ,UAAU;QACV,UAAU;QACV,YAAY;QACZ,MAAM;AACP,KAAA,CAAC;IAEF,IAAI,CAAC,gBAAgB,EAAE;AACrB,QAAA,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC;;IAG7D,MAAM,eAAe,GAAG,qBAAqB,CAC3C;AACE,QAAA,QAAQ,EAAE,gBAAgB;QAC1B,UAAU;QAGV,MAAM;KACP,EACD,eAAe,CAChB;IAED,MAAM,MAAM,GAAG,qBAAqB,CAAC;QACnC,SAAS;QACT,UAAU;QACV,eAAe;AACf,QAAA,OAAO,EAAE,eAAe;QACxB,eAAe;QACf,MAAM;AACP,KAAA,CAAC;AAEF,IAAA,OAAO,UAAU,CAAC;QAChB,MAAM;AACN,QAAA,MAAM,EAAE,UAAU;AAClB,QAAA,eAAe,EAAE,gBAAgB;AAClC,KAAA,CAAC;AACJ;;;;"}

package/dist/esm/tools/search/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"utils.mjs","sources":["../../../../src/tools/search/utils.ts"],"sourcesContent":["/* eslint-disable no-console */\r\n\r\nimport type * as t from './types';\r\n\r\n/**\r\n * Singleton instance of the default logger\r\n */\r\nlet defaultLoggerInstance: t.Logger | null = null;\r\n\r\n/**\r\n * Creates a default logger that maps to console methods\r\n * Uses a singleton pattern to avoid creating multiple instances\r\n * @returns A default logger that implements the Logger interface\r\n */\r\nexport const createDefaultLogger = (): t.Logger => {\r\n  if (!defaultLoggerInstance) {\r\n    defaultLoggerInstance = {\r\n      error: console.error,\r\n      warn: console.warn,\r\n      info: console.info,\r\n      debug: console.debug,\r\n    } as t.Logger;\r\n  }\r\n  return defaultLoggerInstance;\r\n};\r\n\r\nexport const fileExtRegex =\r\n  /\\.(pdf|jpe?g|png|gif|svg|webp|bmp|ico|tiff?|avif|heic|doc[xm]?|xls[xm]?|ppt[xm]?|zip|rar|mp[34]|mov|avi|wav)(?:\\?.*)?$/i;\r\n\r\nexport const getDomainName = (\r\n  link: string,\r\n  metadata?: t.ScrapeMetadata,\r\n  logger?: t.Logger\r\n): string | undefined => {\r\n  try {\r\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\r\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\r\n    if (domain) {\r\n      return domain;\r\n    }\r\n  } catch (e) {\r\n    // URL parsing failed\r\n    if (logger) {\r\n      logger.error('Error parsing URL:', e);\r\n    } else {\r\n      console.error('Error parsing URL:', e);\r\n    }\r\n  }\r\n\r\n  return;\r\n};\r\n\r\nexport function getAttribution(\r\n  link: string,\r\n  metadata?: t.ScrapeMetadata,\r\n  logger?: t.Logger\r\n): string | undefined {\r\n  if (!metadata) return getDomainName(link, metadata, logger);\r\n\r\n  const twitterSite = metadata['twitter:site'];\r\n  const twitterSiteFormatted =\r\n    typeof twitterSite === 'string' ? twitterSite.replace(/^@/, '') : undefined;\r\n\r\n  const possibleAttributions = [\r\n    metadata.ogSiteName,\r\n    metadata['og:site_name'],\r\n    metadata.title?.split('|').pop()?.trim(),\r\n    twitterSiteFormatted,\r\n  ];\r\n\r\n  const attribution = possibleAttributions.find(\r\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\r\n  );\r\n  if (attribution != null) {\r\n    return attribution;\r\n  }\r\n\r\n  return getDomainName(link, metadata, logger);\r\n}\r\n"],"names":[],"mappings":"AAAA;AAIA;;AAEG;AACH,IAAI,qBAAqB,GAAoB,IAAI;AAEjD;;;;AAIG;AACI,MAAM,mBAAmB,GAAG,MAAe;IAChD,IAAI,CAAC,qBAAqB,EAAE;AAC1B,QAAA,qBAAqB,GAAG;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK;SACT;;AAEf,IAAA,OAAO,qBAAqB;AAC9B;AAEO,MAAM,YAAY,GACvB;AAEW,MAAA,aAAa,GAAG,CAC3B,IAAY,EACZ,QAA2B,EAC3B,MAAiB,KACK;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;QAEV,IAAI,MAAM,EAAE;AACV,YAAA,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;aAChC;AACL,YAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;;IAI1C;AACF;SAEgB,cAAc,CAC5B,IAAY,EACZ,QAA2B,EAC3B,MAAiB,EAAA;AAEjB,IAAA,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;AAE3D,IAAA,MAAM,WAAW,GAAG,QAAQ,CAAC,cAAc,CAAC;IAC5C,MAAM,oBAAoB,GACxB,OAAO,WAAW,KAAK,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,SAAS;AAE7E,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,oBAAoB;KACrB;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;IAGpB,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;AAC9C;;;;"}
+{"version":3,"file":"utils.mjs","sources":["../../../../src/tools/search/utils.ts"],"sourcesContent":["/* eslint-disable no-console */\n\nimport type * as t from './types';\n\n/**\n * Singleton instance of the default logger\n */\nlet defaultLoggerInstance: t.Logger | null = null;\n\n/**\n * Creates a default logger that maps to console methods\n * Uses a singleton pattern to avoid creating multiple instances\n * @returns A default logger that implements the Logger interface\n */\nexport const createDefaultLogger = (): t.Logger => {\n  if (!defaultLoggerInstance) {\n    defaultLoggerInstance = {\n      error: console.error,\n      warn: console.warn,\n      info: console.info,\n      debug: console.debug,\n    } as t.Logger;\n  }\n  return defaultLoggerInstance;\n};\n\nexport const fileExtRegex =\n  /\\.(pdf|jpe?g|png|gif|svg|webp|bmp|ico|tiff?|avif|heic|doc[xm]?|xls[xm]?|ppt[xm]?|zip|rar|mp[34]|mov|avi|wav)(?:\\?.*)?$/i;\n\nexport const getDomainName = (\n  link: string,\n  metadata?: t.ScrapeMetadata,\n  logger?: t.Logger\n): string | undefined => {\n  try {\n    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');\n    const domain = new URL(url).hostname.replace(/^www\\./, '');\n    if (domain) {\n      return domain;\n    }\n  } catch (e) {\n    // URL parsing failed\n    if (logger) {\n      logger.error('Error parsing URL:', e);\n    } else {\n      console.error('Error parsing URL:', e);\n    }\n  }\n\n  return;\n};\n\nexport function getAttribution(\n  link: string,\n  metadata?: t.ScrapeMetadata,\n  logger?: t.Logger\n): string | undefined {\n  if (!metadata) return getDomainName(link, metadata, logger);\n\n  const twitterSite = metadata['twitter:site'];\n  const twitterSiteFormatted =\n    typeof twitterSite === 'string' ? twitterSite.replace(/^@/, '') : undefined;\n\n  const possibleAttributions = [\n    metadata.ogSiteName,\n    metadata['og:site_name'],\n    metadata.title?.split('|').pop()?.trim(),\n    twitterSiteFormatted,\n  ];\n\n  const attribution = possibleAttributions.find(\n    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''\n  );\n  if (attribution != null) {\n    return attribution;\n  }\n\n  return getDomainName(link, metadata, logger);\n}\n"],"names":[],"mappings":"AAAA;AAIA;;AAEG;AACH,IAAI,qBAAqB,GAAoB,IAAI;AAEjD;;;;AAIG;AACI,MAAM,mBAAmB,GAAG,MAAe;IAChD,IAAI,CAAC,qBAAqB,EAAE;AAC1B,QAAA,qBAAqB,GAAG;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK;SACT;;AAEf,IAAA,OAAO,qBAAqB;AAC9B;AAEO,MAAM,YAAY,GACvB;AAEW,MAAA,aAAa,GAAG,CAC3B,IAAY,EACZ,QAA2B,EAC3B,MAAiB,KACK;AACtB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,QAAQ,EAAE,SAAS,IAAI,QAAQ,EAAE,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AAChE,QAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;QAC1D,IAAI,MAAM,EAAE;AACV,YAAA,OAAO,MAAM;;;IAEf,OAAO,CAAC,EAAE;;QAEV,IAAI,MAAM,EAAE;AACV,YAAA,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;aAChC;AACL,YAAA,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC;;;IAI1C;AACF;SAEgB,cAAc,CAC5B,IAAY,EACZ,QAA2B,EAC3B,MAAiB,EAAA;AAEjB,IAAA,IAAI,CAAC,QAAQ;QAAE,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;AAE3D,IAAA,MAAM,WAAW,GAAG,QAAQ,CAAC,cAAc,CAAC;IAC5C,MAAM,oBAAoB,GACxB,OAAO,WAAW,KAAK,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,SAAS;AAE7E,IAAA,MAAM,oBAAoB,GAAG;AAC3B,QAAA,QAAQ,CAAC,UAAU;QACnB,QAAQ,CAAC,cAAc,CAAC;AACxB,QAAA,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE;QACxC,oBAAoB;KACrB;IAED,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,CAC3C,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CACzE;AACD,IAAA,IAAI,WAAW,IAAI,IAAI,EAAE;AACvB,QAAA,OAAO,WAAW;;IAGpB,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;AAC9C;;;;"}

package/dist/esm/utils/graph.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"graph.mjs","sources":["../../../src/utils/graph.ts"],"sourcesContent":["export const resetIfNotEmpty = <T>(value: T, resetValue: T): T => {\r\n  if (Array.isArray(value)) {\r\n    return value.length > 0 ? resetValue : value;\r\n  }\r\n  if (value instanceof Set || value instanceof Map) {\r\n    return value.size > 0 ? resetValue : value;\r\n  }\r\n  return value !== undefined ? resetValue : value;\r\n};\r\n\r\nexport const joinKeys = (args: (string | number | undefined)[]): string => args.join('_');"],"names":[],"mappings":"MAAa,eAAe,GAAG,CAAI,KAAQ,EAAE,UAAa,KAAO;AAC/D,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,QAAA,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,UAAU,GAAG,KAAK;;IAE9C,IAAI,KAAK,YAAY,GAAG,IAAI,KAAK,YAAY,GAAG,EAAE;AAChD,QAAA,OAAO,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,UAAU,GAAG,KAAK;;IAE5C,OAAO,KAAK,KAAK,SAAS,GAAG,UAAU,GAAG,KAAK;AACjD;AAEO,MAAM,QAAQ,GAAG,CAAC,IAAqC,KAAa,IAAI,CAAC,IAAI,CAAC,GAAG;;;;"}
+{"version":3,"file":"graph.mjs","sources":["../../../src/utils/graph.ts"],"sourcesContent":["export const resetIfNotEmpty = <T>(value: T, resetValue: T): T => {\n  if (Array.isArray(value)) {\n    return value.length > 0 ? resetValue : value;\n  }\n  if (value instanceof Set || value instanceof Map) {\n    return value.size > 0 ? resetValue : value;\n  }\n  return value !== undefined ? resetValue : value;\n};\n\nexport const joinKeys = (args: (string | number | undefined)[]): string => args.join('_');"],"names":[],"mappings":"MAAa,eAAe,GAAG,CAAI,KAAQ,EAAE,UAAa,KAAO;AAC/D,IAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,QAAA,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,UAAU,GAAG,KAAK;;IAE9C,IAAI,KAAK,YAAY,GAAG,IAAI,KAAK,YAAY,GAAG,EAAE;AAChD,QAAA,OAAO,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,UAAU,GAAG,KAAK;;IAE5C,OAAO,KAAK,KAAK,SAAS,GAAG,UAAU,GAAG,KAAK;AACjD;AAEO,MAAM,QAAQ,GAAG,CAAC,IAAqC,KAAa,IAAI,CAAC,IAAI,CAAC,GAAG;;;;"}