illuma-agents 1.0.8 → 1.0.9

package/dist/cjs/tools/search/firecrawl.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\r\nimport { processContent } from './content';\r\nimport type * as t from './types';\r\nimport { createDefaultLogger } from './utils';\r\n\r\n/**\r\n * Firecrawl scraper implementation\r\n * Uses the Firecrawl API to scrape web pages\r\n */\r\nexport class FirecrawlScraper implements t.BaseScraper {\r\n  private apiKey: string;\r\n  private apiUrl: string;\r\n  private version: string;\r\n  private defaultFormats: string[];\r\n  private timeout: number;\r\n  private logger: t.Logger;\r\n  private includeTags?: string[];\r\n  private excludeTags?: string[];\r\n  private waitFor?: number;\r\n  private maxAge?: number;\r\n  private mobile?: boolean;\r\n  private skipTlsVerification?: boolean;\r\n  private blockAds?: boolean;\r\n  private removeBase64Images?: boolean;\r\n  private parsePDF?: boolean;\r\n  private storeInCache?: boolean;\r\n  private zeroDataRetention?: boolean;\r\n  private headers?: Record<string, string>;\r\n  private location?: { country?: string; languages?: string[] };\r\n  private onlyMainContent?: boolean;\r\n  private changeTrackingOptions?: object;\r\n\r\n  constructor(config: t.FirecrawlScraperConfig = {}) {\r\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\r\n\r\n    this.version = config.version ?? 'v2';\r\n\r\n    const baseUrl =\r\n      config.apiUrl ??\r\n      process.env.FIRECRAWL_BASE_URL ??\r\n      'https://api.firecrawl.dev';\r\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\r\n\r\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\r\n    this.timeout = config.timeout ?? 7500;\r\n\r\n    this.logger = config.logger || createDefaultLogger();\r\n\r\n    this.includeTags = config.includeTags;\r\n    this.excludeTags = config.excludeTags;\r\n    this.waitFor = config.waitFor;\r\n    this.maxAge = config.maxAge;\r\n    this.mobile = config.mobile;\r\n    this.skipTlsVerification = config.skipTlsVerification;\r\n    this.blockAds = config.blockAds;\r\n    this.removeBase64Images = config.removeBase64Images;\r\n    this.parsePDF = config.parsePDF;\r\n    this.storeInCache = config.storeInCache;\r\n    this.zeroDataRetention = config.zeroDataRetention;\r\n    this.headers = config.headers;\r\n    this.location = config.location;\r\n    this.onlyMainContent = config.onlyMainContent;\r\n    this.changeTrackingOptions = config.changeTrackingOptions;\r\n\r\n    if (!this.apiKey) {\r\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\r\n    }\r\n\r\n    this.logger.debug(\r\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\r\n    );\r\n  }\r\n\r\n  /**\r\n   * Scrape a single URL\r\n   * @param url URL to scrape\r\n   * @param options Scrape options\r\n   * @returns Scrape response\r\n   */\r\n  async scrapeUrl(\r\n    url: string,\r\n    options: t.FirecrawlScrapeOptions = {}\r\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\r\n    if (!this.apiKey) {\r\n      return [\r\n        url,\r\n        {\r\n          success: false,\r\n          error: 'FIRECRAWL_API_KEY is not set',\r\n        },\r\n      ];\r\n    }\r\n\r\n    try {\r\n      const payload = omitUndefined({\r\n        url,\r\n        formats: options.formats ?? this.defaultFormats,\r\n        includeTags: options.includeTags ?? this.includeTags,\r\n        excludeTags: options.excludeTags ?? this.excludeTags,\r\n        headers: options.headers ?? this.headers,\r\n        waitFor: options.waitFor ?? this.waitFor,\r\n        timeout: options.timeout ?? this.timeout,\r\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\r\n        maxAge: options.maxAge ?? this.maxAge,\r\n        mobile: options.mobile ?? this.mobile,\r\n        skipTlsVerification:\r\n          options.skipTlsVerification ?? this.skipTlsVerification,\r\n        parsePDF: options.parsePDF ?? this.parsePDF,\r\n        location: options.location ?? this.location,\r\n        removeBase64Images:\r\n          options.removeBase64Images ?? this.removeBase64Images,\r\n        blockAds: options.blockAds ?? this.blockAds,\r\n        storeInCache: options.storeInCache ?? this.storeInCache,\r\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\r\n        changeTrackingOptions:\r\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\r\n      });\r\n      const response = await axios.post(this.apiUrl, payload, {\r\n        headers: {\r\n          'Content-Type': 'application/json',\r\n          Authorization: `Bearer ${this.apiKey}`,\r\n        },\r\n        timeout: this.timeout,\r\n      });\r\n\r\n      return [url, response.data];\r\n    } catch (error) {\r\n      const errorMessage =\r\n        error instanceof Error ? error.message : String(error);\r\n      return [\r\n        url,\r\n        {\r\n          success: false,\r\n          error: `Firecrawl API request failed: ${errorMessage}`,\r\n        },\r\n      ];\r\n    }\r\n  }\r\n\r\n  /**\r\n   * Extract content from scrape response\r\n   * @param response Scrape response\r\n   * @returns Extracted content or empty string if not available\r\n   */\r\n  extractContent(\r\n    response: t.FirecrawlScrapeResponse\r\n  ): [string, undefined | t.References] {\r\n    if (!response.success || !response.data) {\r\n      return ['', undefined];\r\n    }\r\n\r\n    if (response.data.markdown != null && response.data.html != null) {\r\n      try {\r\n        const { markdown, ...rest } = processContent(\r\n          response.data.html,\r\n          response.data.markdown\r\n        );\r\n        return [markdown, rest];\r\n      } catch (error) {\r\n        this.logger.error('Error processing content:', error);\r\n        return [response.data.markdown, undefined];\r\n      }\r\n    } else if (response.data.markdown != null) {\r\n      return [response.data.markdown, undefined];\r\n    }\r\n\r\n    // Fall back to HTML content\r\n    if (response.data.html != null) {\r\n      return [response.data.html, undefined];\r\n    }\r\n\r\n    // Fall back to raw HTML content\r\n    if (response.data.rawHtml != null) {\r\n      return [response.data.rawHtml, undefined];\r\n    }\r\n\r\n    return ['', undefined];\r\n  }\r\n\r\n  /**\r\n   * Extract metadata from scrape response\r\n   * @param response Scrape response\r\n   * @returns Metadata object\r\n   */\r\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\r\n    if (!response.success || !response.data || !response.data.metadata) {\r\n      return {};\r\n    }\r\n\r\n    return response.data.metadata;\r\n  }\r\n}\r\n\r\n/**\r\n * Create a Firecrawl scraper instance\r\n * @param config Scraper configuration\r\n * @returns Firecrawl scraper instance\r\n */\r\nexport const createFirecrawlScraper = (\r\n  config: t.FirecrawlScraperConfig = {}\r\n): FirecrawlScraper => {\r\n  return new FirecrawlScraper(config);\r\n};\r\n\r\n// Helper function to clean up payload for firecrawl\r\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\r\n  return Object.fromEntries(\r\n    Object.entries(obj).filter(([, v]) => v !== undefined)\r\n  ) as Partial<T>;\r\n}\r\n"],"names":["createDefaultLogger","processContent"],"mappings":";;;;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,WAAW;AACX,IAAA,WAAW;AACX,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,mBAAmB;AACnB,IAAA,QAAQ;AACR,IAAA,kBAAkB;AAClB,IAAA,QAAQ;AACR,IAAA,YAAY;AACZ,IAAA,iBAAiB;AACjB,IAAA,OAAO;AACP,IAAA,QAAQ;AACR,IAAA,eAAe;AACf,IAAA,qBAAqB;AAE7B,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;QAElE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;AAErC,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAG,EAAA,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA,CAAA,EAAI,IAAI,CAAC,OAAO,SAAS;AAErE,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB;AACrD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB;AACnD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY;AACvC,QAAA,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB;AACjD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe;AAC7C,QAAA,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB;AAEzD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC/C,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;AAChE,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,mBAAmB,EACjB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB;AACzD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB;AACvD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AACvD,gBAAA,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;AACtE,gBAAA,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;AAC9D,aAAA,CAAC;AACF,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAGC,sBAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;AAEA;AACA,SAAS,aAAa,CAAmB,GAAM,EAAA;IAC7C,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CACzC;AACjB;;;;;"}
+{"version":3,"file":"firecrawl.cjs","sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport { processContent } from './content';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private version: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private maxAge?: number;\n  private mobile?: boolean;\n  private skipTlsVerification?: boolean;\n  private blockAds?: boolean;\n  private removeBase64Images?: boolean;\n  private parsePDF?: boolean;\n  private storeInCache?: boolean;\n  private zeroDataRetention?: boolean;\n  private headers?: Record<string, string>;\n  private location?: { country?: string; languages?: string[] };\n  private onlyMainContent?: boolean;\n  private changeTrackingOptions?: object;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    this.version = config.version ?? 'v2';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.maxAge = config.maxAge;\n    this.mobile = config.mobile;\n    this.skipTlsVerification = config.skipTlsVerification;\n    this.blockAds = config.blockAds;\n    this.removeBase64Images = config.removeBase64Images;\n    this.parsePDF = config.parsePDF;\n    this.storeInCache = config.storeInCache;\n    this.zeroDataRetention = config.zeroDataRetention;\n    this.headers = config.headers;\n    this.location = config.location;\n    this.onlyMainContent = config.onlyMainContent;\n    this.changeTrackingOptions = config.changeTrackingOptions;\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        headers: options.headers ?? this.headers,\n        waitFor: options.waitFor ?? this.waitFor,\n        timeout: options.timeout ?? this.timeout,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        maxAge: options.maxAge ?? this.maxAge,\n        mobile: options.mobile ?? this.mobile,\n        skipTlsVerification:\n          options.skipTlsVerification ?? this.skipTlsVerification,\n        parsePDF: options.parsePDF ?? this.parsePDF,\n        location: options.location ?? this.location,\n        removeBase64Images:\n          options.removeBase64Images ?? this.removeBase64Images,\n        blockAds: options.blockAds ?? this.blockAds,\n        storeInCache: options.storeInCache ?? this.storeInCache,\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n        changeTrackingOptions:\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\n      });\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'Content-Type': 'application/json',\n          Authorization: `Bearer ${this.apiKey}`,\n        },\n        timeout: this.timeout,\n      });\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"names":["createDefaultLogger","processContent"],"mappings":";;;;;;AAKA;;;AAGG;MACU,gBAAgB,CAAA;AACnB,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,OAAO;AACP,IAAA,cAAc;AACd,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,WAAW;AACX,IAAA,WAAW;AACX,IAAA,OAAO;AACP,IAAA,MAAM;AACN,IAAA,MAAM;AACN,IAAA,mBAAmB;AACnB,IAAA,QAAQ;AACR,IAAA,kBAAkB;AAClB,IAAA,QAAQ;AACR,IAAA,YAAY;AACZ,IAAA,iBAAiB;AACjB,IAAA,OAAO;AACP,IAAA,QAAQ;AACR,IAAA,eAAe;AACf,IAAA,qBAAqB;AAE7B,IAAA,WAAA,CAAY,SAAmC,EAAE,EAAA;AAC/C,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE;QAElE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;AAErC,QAAA,MAAM,OAAO,GACX,MAAM,CAAC,MAAM;YACb,OAAO,CAAC,GAAG,CAAC,kBAAkB;AAC9B,YAAA,2BAA2B;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,CAAG,EAAA,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA,CAAA,EAAI,IAAI,CAAC,OAAO,SAAS;AAErE,QAAA,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAIA,yBAAmB,EAAE;AAEpD,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW;AACrC,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;AAC3B,QAAA,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB;AACrD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB;AACnD,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY;AACvC,QAAA,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB;AACjD,QAAA,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO;AAC7B,QAAA,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ;AAC/B,QAAA,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe;AAC7C,QAAA,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB;AAEzD,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;AAChB,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC;;QAG3E,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAA+C,4CAAA,EAAA,IAAI,CAAC,MAAM,CAAE,CAAA,CAC7D;;AAGH;;;;;AAKG;AACH,IAAA,MAAM,SAAS,CACb,GAAW,EACX,UAAoC,EAAE,EAAA;AAEtC,QAAA,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;AACd,oBAAA,KAAK,EAAE,8BAA8B;AACtC,iBAAA;aACF;;AAGH,QAAA,IAAI;YACF,MAAM,OAAO,GAAG,aAAa,CAAC;gBAC5B,GAAG;AACH,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc;AAC/C,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW;AACpD,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;AACxC,gBAAA,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,eAAe;AAChE,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;AACrC,gBAAA,mBAAmB,EACjB,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,mBAAmB;AACzD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB;AACvD,gBAAA,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;AAC3C,gBAAA,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY;AACvD,gBAAA,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAC,iBAAiB;AACtE,gBAAA,qBAAqB,EACnB,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC,qBAAqB;AAC9D,aAAA,CAAC;AACF,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;AACtD,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;gBACD,OAAO,EAAE,IAAI,CAAC,OAAO;AACtB,aAAA,CAAC;AAEF,YAAA,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC;;QAC3B,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;gBACL,GAAG;AACH,gBAAA;AACE,oBAAA,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,CAAiC,8BAAA,EAAA,YAAY,CAAE,CAAA;AACvD,iBAAA;aACF;;;AAIL;;;;AAIG;AACH,IAAA,cAAc,CACZ,QAAmC,EAAA;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;AACvC,YAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB,QAAA,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;AAChE,YAAA,IAAI;gBACF,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAGC,sBAAc,CAC1C,QAAQ,CAAC,IAAI,CAAC,IAAI,EAClB,QAAQ,CAAC,IAAI,CAAC,QAAQ,CACvB;AACD,gBAAA,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;YACvB,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC;gBACrD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;aAEvC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;;;QAI5C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE;YAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC;;;QAIxC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;YACjC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;AAG3C,QAAA,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC;;AAGxB;;;;AAIG;AACH,IAAA,eAAe,CAAC,QAAmC,EAAA;AACjD,QAAA,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE;AAClE,YAAA,OAAO,EAAE;;AAGX,QAAA,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ;;AAEhC;AAED;;;;AAIG;MACU,sBAAsB,GAAG,CACpC,MAAmC,GAAA,EAAE,KACjB;AACpB,IAAA,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC;AACrC;AAEA;AACA,SAAS,aAAa,CAAmB,GAAM,EAAA;IAC7C,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CACzC;AACjB;;;;;"}

package/dist/cjs/tools/search/format.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\r\nimport { getDomainName, fileExtRegex } from './utils';\r\n\r\nfunction addHighlightSection(): string[] {\r\n  return ['\\n## Highlights', ''];\r\n}\r\n\r\n// Helper function to format a source (organic or top story)\r\nfunction formatSource(\r\n  source: t.ValidSource,\r\n  index: number,\r\n  turn: number,\r\n  sourceType: 'search' | 'news',\r\n  references: t.ResultReference[]\r\n): string {\r\n  /** Array of all lines to include in the output */\r\n  const outputLines: string[] = [];\r\n\r\n  // Add the title\r\n  outputLines.push(\r\n    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `\"${source.title}\"` : '(no title)'}`\r\n  );\r\n  outputLines.push(`\\nAnchor: \\\\ue202turn${turn}${sourceType}${index}`);\r\n  outputLines.push(`URL: ${source.link}`);\r\n\r\n  // Add optional fields\r\n  if ('snippet' in source && source.snippet != null) {\r\n    outputLines.push(`Summary: ${source.snippet}`);\r\n  }\r\n\r\n  if (source.date != null) {\r\n    outputLines.push(`Date: ${source.date}`);\r\n  }\r\n\r\n  if (source.attribution != null) {\r\n    outputLines.push(`Source: ${source.attribution}`);\r\n  }\r\n\r\n  // Add highlight section or empty line\r\n  if ((source.highlights?.length ?? 0) > 0) {\r\n    outputLines.push(...addHighlightSection());\r\n  } else {\r\n    outputLines.push('');\r\n  }\r\n\r\n  // Process highlights if they exist\r\n  (source.highlights ?? [])\r\n    .filter((h) => h.text.trim().length > 0)\r\n    .forEach((h, hIndex) => {\r\n      outputLines.push(\r\n        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`\r\n      );\r\n      outputLines.push('');\r\n      outputLines.push('```text');\r\n      outputLines.push(h.text.trim());\r\n      outputLines.push('```');\r\n      outputLines.push('');\r\n\r\n      if (h.references != null && h.references.length) {\r\n        let hasHeader = false;\r\n        const refLines: string[] = [];\r\n\r\n        for (let j = 0; j < h.references.length; j++) {\r\n          const ref = h.references[j];\r\n          if (ref.reference.originalUrl.includes('mailto:')) {\r\n            continue;\r\n          }\r\n          if (ref.type !== 'link') {\r\n            continue;\r\n          }\r\n          if (fileExtRegex.test(ref.reference.originalUrl)) {\r\n            continue;\r\n          }\r\n          references.push({\r\n            type: ref.type,\r\n            link: ref.reference.originalUrl,\r\n            attribution: getDomainName(ref.reference.originalUrl),\r\n            title: (\r\n              ((ref.reference.title ?? '') || ref.reference.text) ??\r\n              ''\r\n            ).split('\\n')[0],\r\n          });\r\n\r\n          if (!hasHeader) {\r\n            refLines.push('Core References:');\r\n            hasHeader = true;\r\n          }\r\n\r\n          refLines.push(\r\n            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`\r\n          );\r\n          refLines.push(\r\n            `\\t- Anchor: \\\\ue202turn${turn}ref${references.length - 1}`\r\n          );\r\n        }\r\n\r\n        if (hasHeader) {\r\n          outputLines.push(...refLines);\r\n          outputLines.push('');\r\n        }\r\n      }\r\n\r\n      if (hIndex < (source.highlights?.length ?? 0) - 1) {\r\n        outputLines.push('---');\r\n        outputLines.push('');\r\n      }\r\n    });\r\n\r\n  outputLines.push('');\r\n  return outputLines.join('\\n');\r\n}\r\n\r\nexport function formatResultsForLLM(\r\n  turn: number,\r\n  results: t.SearchResultData\r\n): { output: string; references: t.ResultReference[] } {\r\n  /** Array to collect all output lines */\r\n  const outputLines: string[] = [];\r\n\r\n  const addSection = (title: string): void => {\r\n    outputLines.push('');\r\n    outputLines.push(`=== ${title} ===`);\r\n    outputLines.push('');\r\n  };\r\n\r\n  const references: t.ResultReference[] = [];\r\n\r\n  // Organic (web) results\r\n  if (results.organic?.length != null && results.organic.length > 0) {\r\n    addSection(`Web Results, Turn ${turn}`);\r\n    for (let i = 0; i < results.organic.length; i++) {\r\n      const r = results.organic[i];\r\n      outputLines.push(formatSource(r, i, turn, 'search', references));\r\n      delete results.organic[i].highlights;\r\n    }\r\n  }\r\n\r\n  // Top stories (news)\r\n  const topStories = results.topStories ?? [];\r\n  if (topStories.length) {\r\n    addSection('News Results');\r\n    for (let i = 0; i < topStories.length; i++) {\r\n      const r = topStories[i];\r\n      outputLines.push(formatSource(r, i, turn, 'news', references));\r\n      if (results.topStories?.[i]?.highlights) {\r\n        delete results.topStories[i].highlights;\r\n      }\r\n    }\r\n  }\r\n\r\n  // // Images\r\n  // const images = results.images ?? [];\r\n  // if (images.length) {\r\n  //   addSection('Image Results');\r\n  //   const imageLines = images.map((img, i) => [\r\n  //     `Anchor: \\ue202turn0image${i}`,\r\n  //     `Title: ${img.title ?? '(no title)'}`,\r\n  //     `Image URL: ${img.imageUrl}`,\r\n  //     ''\r\n  //   ].join('\\n'));\r\n  //   outputLines.push(imageLines.join('\\n'));\r\n  // }\r\n\r\n  // Knowledge Graph\r\n  if (results.knowledgeGraph != null) {\r\n    addSection('Knowledge Graph');\r\n    const kgLines = [\r\n      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,\r\n      results.knowledgeGraph.type != null\r\n        ? `**Type:** ${results.knowledgeGraph.type}`\r\n        : '',\r\n      results.knowledgeGraph.description != null\r\n        ? `**Description:** ${results.knowledgeGraph.description}`\r\n        : '',\r\n      results.knowledgeGraph.descriptionSource != null\r\n        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`\r\n        : '',\r\n      results.knowledgeGraph.descriptionLink != null\r\n        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`\r\n        : '',\r\n      results.knowledgeGraph.imageUrl != null\r\n        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`\r\n        : '',\r\n      results.knowledgeGraph.website != null\r\n        ? `**Website:** ${results.knowledgeGraph.website}`\r\n        : '',\r\n      results.knowledgeGraph.attributes != null\r\n        ? `**Attributes:**\\n\\`\\`\\`json\\n${JSON.stringify(\r\n          results.knowledgeGraph.attributes,\r\n          null,\r\n          2\r\n        )}\\n\\`\\`\\``\r\n        : '',\r\n      '',\r\n    ].filter(Boolean);\r\n\r\n    outputLines.push(kgLines.join('\\n\\n'));\r\n  }\r\n\r\n  // Answer Box\r\n  if (results.answerBox != null) {\r\n    addSection('Answer Box');\r\n    const abLines = [\r\n      results.answerBox.title != null\r\n        ? `**Title:** ${results.answerBox.title}`\r\n        : '',\r\n      results.answerBox.snippet != null\r\n        ? `**Snippet:** ${results.answerBox.snippet}`\r\n        : '',\r\n      results.answerBox.snippetHighlighted != null\r\n        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted\r\n          .map((s) => `\\`${s}\\``)\r\n          .join(' ')}`\r\n        : '',\r\n      results.answerBox.link != null\r\n        ? `**Link:** ${results.answerBox.link}`\r\n        : '',\r\n      '',\r\n    ].filter(Boolean);\r\n\r\n    outputLines.push(abLines.join('\\n\\n'));\r\n  }\r\n\r\n  // People also ask\r\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\r\n  if (peopleAlsoAsk.length) {\r\n    addSection('People Also Ask');\r\n\r\n    const paaLines: string[] = [];\r\n    peopleAlsoAsk.forEach((p, i) => {\r\n      const questionLines = [\r\n        `### Question ${i + 1}:`,\r\n        `\"${p.question}\"`,\r\n        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,\r\n        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,\r\n        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,\r\n        '',\r\n      ].filter(Boolean);\r\n\r\n      paaLines.push(questionLines.join('\\n\\n'));\r\n    });\r\n\r\n    outputLines.push(paaLines.join(''));\r\n  }\r\n\r\n  return {\r\n    output: outputLines.join('\\n').trim(),\r\n    references,\r\n  };\r\n}\r\n"],"names":["fileExtRegex","getDomainName"],"mappings":";;;;AAGA,SAAS,mBAAmB,GAAA;AAC1B,IAAA,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;AAChC;AAEA;AACA,SAAS,YAAY,CACnB,MAAqB,EACrB,KAAa,EACb,IAAY,EACZ,UAA6B,EAC7B,UAA+B,EAAA;;IAG/B,MAAM,WAAW,GAAa,EAAE;;IAGhC,WAAW,CAAC,IAAI,CACd,CAAA,EAAA,EAAK,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA,CAAA,EAAI,KAAK,CAAA,EAAA,EAAK,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAG,CAAA,CAAA,GAAG,YAAY,CAAE,CAAA,CACvJ;IACD,WAAW,CAAC,IAAI,CAAC,CAAwB,qBAAA,EAAA,IAAI,CAAG,EAAA,UAAU,CAAG,EAAA,KAAK,CAAE,CAAA,CAAC;IACrE,WAAW,CAAC,IAAI,CAAC,CAAA,KAAA,EAAQ,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;IAGvC,IAAI,SAAS,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE;QACjD,WAAW,CAAC,IAAI,CAAC,CAAA,SAAA,EAAY,MAAM,CAAC,OAAO,CAAE,CAAA,CAAC;;AAGhD,IAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;QACvB,WAAW,CAAC,IAAI,CAAC,CAAA,MAAA,EAAS,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;AAG1C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE;QAC9B,WAAW,CAAC,IAAI,CAAC,CAAA,QAAA,EAAW,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;;AAInD,IAAA,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACxC,QAAA,WAAW,CAAC,IAAI,CAAC,GAAG,mBAAmB,EAAE,CAAC;;SACrC;AACL,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAItB,IAAA,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE;AACrB,SAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;AACtC,SAAA,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,KAAI;AACrB,QAAA,WAAW,CAAC,IAAI,CACd,iBAAiB,MAAM,GAAG,CAAC,CAAgB,aAAA,EAAA,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CACjE;AACD,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC;QAC3B,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AAC/B,QAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AAEpB,QAAA,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE;YAC/C,IAAI,SAAS,GAAG,KAAK;YACrB,MAAM,QAAQ,GAAa,EAAE;AAE7B,YAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC3B,IAAI,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;oBACjD;;AAEF,gBAAA,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE;oBACvB;;gBAEF,IAAIA,kBAAY,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE;oBAChD;;gBAEF,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;AACd,oBAAA,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW;oBAC/B,WAAW,EAAEC,mBAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC;AACrD,oBAAA,KAAK,EAAE,CACL,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,GAAG,CAAC,SAAS,CAAC,IAAI;wBAClD,EAAE,EACF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,iBAAA,CAAC;gBAEF,IAAI,CAAC,SAAS,EAAE;AACd,oBAAA,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC;oBACjC,SAAS,GAAG,IAAI;;gBAGlB,QAAQ,CAAC,IAAI,CACX,CAAA,EAAA,EAAK,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC,KAAK,GAAG,CAAC,SAAS,CAAC,WAAW,CAAE,CAAA,CACvE;AACD,gBAAA,QAAQ,CAAC,IAAI,CACX,CAAA,uBAAA,EAA0B,IAAI,CAAA,GAAA,EAAM,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA,CAAE,CAC5D;;YAGH,IAAI,SAAS,EAAE;AACb,gBAAA,WAAW,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;AAC7B,gBAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAIxB,QAAA,IAAI,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACjD,YAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;AAExB,KAAC,CAAC;AAEJ,IAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,IAAA,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;AAC/B;AAEgB,SAAA,mBAAmB,CACjC,IAAY,EACZ,OAA2B,EAAA;;IAG3B,MAAM,WAAW,GAAa,EAAE;AAEhC,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,OAAO,KAAK,CAAA,IAAA,CAAM,CAAC;AACpC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,KAAC;IAED,MAAM,UAAU,GAAwB,EAAE;;AAG1C,IAAA,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,QAAA,UAAU,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAA,CAAE,CAAC;AACvC,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/C,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5B,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAChE,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU;;;;AAKxC,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE;AAC3C,IAAA,IAAI,UAAU,CAAC,MAAM,EAAE;QACrB,UAAU,CAAC,cAAc,CAAC;AAC1B,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AAC1C,YAAA,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9D,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,UAAU,EAAE;gBACvC,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU;;;;;;;;;;;;;;;;;AAmB7C,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,CAAA,WAAA,EAAc,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AAC5D,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AAC5C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAoB,iBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AAC1D,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,IAAI;AAC1C,kBAAE,CAA2B,wBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAE;AACvE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,eAAe,IAAI;AACxC,kBAAE,CAAyB,sBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,eAAe,CAAE;AACnE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAkB,eAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACrD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI;AAChC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,OAAO,CAAE;AAClD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAgC,6BAAA,EAAA,IAAI,CAAC,SAAS,CAC9C,OAAO,CAAC,cAAc,CAAC,UAAU,EACjC,IAAI,EACJ,CAAC,CACF,CAAU,QAAA;AACX,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AAC7C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,kBAAkB,IAAI;AACtC,kBAAE,CAA4B,yBAAA,EAAA,OAAO,CAAC,SAAS,CAAC;qBAC7C,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,EAAA,EAAK,CAAC,CAAA,EAAA,CAAI;qBACrB,IAAI,CAAC,GAAG,CAAC,CAAE;AACd,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI;AACxB,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,SAAS,CAAC,IAAI,CAAE;AACvC,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAE7B,MAAM,QAAQ,GAAa,EAAE;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AAC7B,YAAA,MAAM,aAAa,GAAG;gBACpB,CAAgB,aAAA,EAAA,CAAC,GAAG,CAAC,CAAG,CAAA,CAAA;gBACxB,CAAI,CAAA,EAAA,CAAC,CAAC,QAAQ,CAAG,CAAA,CAAA;gBACjB,CAAG,EAAA,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,CAAA,SAAA,EAAY,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAE,CAAA;gBAClE,CAAG,EAAA,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAA,OAAA,EAAU,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAE,CAAA;gBAC1D,CAAG,EAAA,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAA,MAAA,EAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAE,CAAA;gBACtD,EAAE;AACH,aAAA,CAAC,MAAM,CAAC,OAAO,CAAC;YAEjB,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC3C,SAAC,CAAC;QAEF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;;IAGrC,OAAO;QACL,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QACrC,UAAU;KACX;AACH;;;;"}
+{"version":3,"file":"format.cjs","sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\nimport { getDomainName, fileExtRegex } from './utils';\n\nfunction addHighlightSection(): string[] {\n  return ['\\n## Highlights', ''];\n}\n\n// Helper function to format a source (organic or top story)\nfunction formatSource(\n  source: t.ValidSource,\n  index: number,\n  turn: number,\n  sourceType: 'search' | 'news',\n  references: t.ResultReference[]\n): string {\n  /** Array of all lines to include in the output */\n  const outputLines: string[] = [];\n\n  // Add the title\n  outputLines.push(\n    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `\"${source.title}\"` : '(no title)'}`\n  );\n  outputLines.push(`\\nAnchor: \\\\ue202turn${turn}${sourceType}${index}`);\n  outputLines.push(`URL: ${source.link}`);\n\n  // Add optional fields\n  if ('snippet' in source && source.snippet != null) {\n    outputLines.push(`Summary: ${source.snippet}`);\n  }\n\n  if (source.date != null) {\n    outputLines.push(`Date: ${source.date}`);\n  }\n\n  if (source.attribution != null) {\n    outputLines.push(`Source: ${source.attribution}`);\n  }\n\n  // Add highlight section or empty line\n  if ((source.highlights?.length ?? 0) > 0) {\n    outputLines.push(...addHighlightSection());\n  } else {\n    outputLines.push('');\n  }\n\n  // Process highlights if they exist\n  (source.highlights ?? [])\n    .filter((h) => h.text.trim().length > 0)\n    .forEach((h, hIndex) => {\n      outputLines.push(\n        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`\n      );\n      outputLines.push('');\n      outputLines.push('```text');\n      outputLines.push(h.text.trim());\n      outputLines.push('```');\n      outputLines.push('');\n\n      if (h.references != null && h.references.length) {\n        let hasHeader = false;\n        const refLines: string[] = [];\n\n        for (let j = 0; j < h.references.length; j++) {\n          const ref = h.references[j];\n          if (ref.reference.originalUrl.includes('mailto:')) {\n            continue;\n          }\n          if (ref.type !== 'link') {\n            continue;\n          }\n          if (fileExtRegex.test(ref.reference.originalUrl)) {\n            continue;\n          }\n          references.push({\n            type: ref.type,\n            link: ref.reference.originalUrl,\n            attribution: getDomainName(ref.reference.originalUrl),\n            title: (\n              ((ref.reference.title ?? '') || ref.reference.text) ??\n              ''\n            ).split('\\n')[0],\n          });\n\n          if (!hasHeader) {\n            refLines.push('Core References:');\n            hasHeader = true;\n          }\n\n          refLines.push(\n            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`\n          );\n          refLines.push(\n            `\\t- Anchor: \\\\ue202turn${turn}ref${references.length - 1}`\n          );\n        }\n\n        if (hasHeader) {\n          outputLines.push(...refLines);\n          outputLines.push('');\n        }\n      }\n\n      if (hIndex < (source.highlights?.length ?? 0) - 1) {\n        outputLines.push('---');\n        outputLines.push('');\n      }\n    });\n\n  outputLines.push('');\n  return outputLines.join('\\n');\n}\n\nexport function formatResultsForLLM(\n  turn: number,\n  results: t.SearchResultData\n): { output: string; references: t.ResultReference[] } {\n  /** Array to collect all output lines */\n  const outputLines: string[] = [];\n\n  const addSection = (title: string): void => {\n    outputLines.push('');\n    outputLines.push(`=== ${title} ===`);\n    outputLines.push('');\n  };\n\n  const references: t.ResultReference[] = [];\n\n  // Organic (web) results\n  if (results.organic?.length != null && results.organic.length > 0) {\n    addSection(`Web Results, Turn ${turn}`);\n    for (let i = 0; i < results.organic.length; i++) {\n      const r = results.organic[i];\n      outputLines.push(formatSource(r, i, turn, 'search', references));\n      delete results.organic[i].highlights;\n    }\n  }\n\n  // Top stories (news)\n  const topStories = results.topStories ?? [];\n  if (topStories.length) {\n    addSection('News Results');\n    for (let i = 0; i < topStories.length; i++) {\n      const r = topStories[i];\n      outputLines.push(formatSource(r, i, turn, 'news', references));\n      if (results.topStories?.[i]?.highlights) {\n        delete results.topStories[i].highlights;\n      }\n    }\n  }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   const imageLines = images.map((img, i) => [\n  //     `Anchor: \\ue202turn0image${i}`,\n  //     `Title: ${img.title ?? '(no title)'}`,\n  //     `Image URL: ${img.imageUrl}`,\n  //     ''\n  //   ].join('\\n'));\n  //   outputLines.push(imageLines.join('\\n'));\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    const kgLines = [\n      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.type != null\n        ? `**Type:** ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.description != null\n        ? `**Description:** ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.descriptionSource != null\n        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`\n        : '',\n      results.knowledgeGraph.descriptionLink != null\n        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.website != null\n        ? `**Website:** ${results.knowledgeGraph.website}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `**Attributes:**\\n\\`\\`\\`json\\n${JSON.stringify(\n          results.knowledgeGraph.attributes,\n          null,\n          2\n        )}\\n\\`\\`\\``\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(kgLines.join('\\n\\n'));\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    const abLines = [\n      results.answerBox.title != null\n        ? `**Title:** ${results.answerBox.title}`\n        : '',\n      results.answerBox.snippet != null\n        ? `**Snippet:** ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.snippetHighlighted != null\n        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted\n          .map((s) => `\\`${s}\\``)\n          .join(' ')}`\n        : '',\n      results.answerBox.link != null\n        ? `**Link:** ${results.answerBox.link}`\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(abLines.join('\\n\\n'));\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n\n    const paaLines: string[] = [];\n    peopleAlsoAsk.forEach((p, i) => {\n      const questionLines = [\n        `### Question ${i + 1}:`,\n        `\"${p.question}\"`,\n        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,\n        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,\n        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,\n        '',\n      ].filter(Boolean);\n\n      paaLines.push(questionLines.join('\\n\\n'));\n    });\n\n    outputLines.push(paaLines.join(''));\n  }\n\n  return {\n    output: outputLines.join('\\n').trim(),\n    references,\n  };\n}\n"],"names":["fileExtRegex","getDomainName"],"mappings":";;;;AAGA,SAAS,mBAAmB,GAAA;AAC1B,IAAA,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;AAChC;AAEA;AACA,SAAS,YAAY,CACnB,MAAqB,EACrB,KAAa,EACb,IAAY,EACZ,UAA6B,EAC7B,UAA+B,EAAA;;IAG/B,MAAM,WAAW,GAAa,EAAE;;IAGhC,WAAW,CAAC,IAAI,CACd,CAAA,EAAA,EAAK,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA,CAAA,EAAI,KAAK,CAAA,EAAA,EAAK,MAAM,CAAC,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAG,CAAA,CAAA,GAAG,YAAY,CAAE,CAAA,CACvJ;IACD,WAAW,CAAC,IAAI,CAAC,CAAwB,qBAAA,EAAA,IAAI,CAAG,EAAA,UAAU,CAAG,EAAA,KAAK,CAAE,CAAA,CAAC;IACrE,WAAW,CAAC,IAAI,CAAC,CAAA,KAAA,EAAQ,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;IAGvC,IAAI,SAAS,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE;QACjD,WAAW,CAAC,IAAI,CAAC,CAAA,SAAA,EAAY,MAAM,CAAC,OAAO,CAAE,CAAA,CAAC;;AAGhD,IAAA,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE;QACvB,WAAW,CAAC,IAAI,CAAC,CAAA,MAAA,EAAS,MAAM,CAAC,IAAI,CAAE,CAAA,CAAC;;AAG1C,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,EAAE;QAC9B,WAAW,CAAC,IAAI,CAAC,CAAA,QAAA,EAAW,MAAM,CAAC,WAAW,CAAE,CAAA,CAAC;;;AAInD,IAAA,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACxC,QAAA,WAAW,CAAC,IAAI,CAAC,GAAG,mBAAmB,EAAE,CAAC;;SACrC;AACL,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAItB,IAAA,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE;AACrB,SAAA,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;AACtC,SAAA,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,KAAI;AACrB,QAAA,WAAW,CAAC,IAAI,CACd,iBAAiB,MAAM,GAAG,CAAC,CAAgB,aAAA,EAAA,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CACjE;AACD,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC;QAC3B,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;AAC/B,QAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AAEpB,QAAA,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE;YAC/C,IAAI,SAAS,GAAG,KAAK;YACrB,MAAM,QAAQ,GAAa,EAAE;AAE7B,YAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC3B,IAAI,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;oBACjD;;AAEF,gBAAA,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE;oBACvB;;gBAEF,IAAIA,kBAAY,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE;oBAChD;;gBAEF,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;AACd,oBAAA,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW;oBAC/B,WAAW,EAAEC,mBAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC;AACrD,oBAAA,KAAK,EAAE,CACL,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,GAAG,CAAC,SAAS,CAAC,IAAI;wBAClD,EAAE,EACF,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,iBAAA,CAAC;gBAEF,IAAI,CAAC,SAAS,EAAE;AACd,oBAAA,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC;oBACjC,SAAS,GAAG,IAAI;;gBAGlB,QAAQ,CAAC,IAAI,CACX,CAAA,EAAA,EAAK,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC,KAAK,GAAG,CAAC,SAAS,CAAC,WAAW,CAAE,CAAA,CACvE;AACD,gBAAA,QAAQ,CAAC,IAAI,CACX,CAAA,uBAAA,EAA0B,IAAI,CAAA,GAAA,EAAM,UAAU,CAAC,MAAM,GAAG,CAAC,CAAA,CAAE,CAC5D;;YAGH,IAAI,SAAS,EAAE;AACb,gBAAA,WAAW,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;AAC7B,gBAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;;AAIxB,QAAA,IAAI,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;AACjD,YAAA,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;;AAExB,KAAC,CAAC;AAEJ,IAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,IAAA,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;AAC/B;AAEgB,SAAA,mBAAmB,CACjC,IAAY,EACZ,OAA2B,EAAA;;IAG3B,MAAM,WAAW,GAAa,EAAE;AAEhC,IAAA,MAAM,UAAU,GAAG,CAAC,KAAa,KAAU;AACzC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACpB,QAAA,WAAW,CAAC,IAAI,CAAC,OAAO,KAAK,CAAA,IAAA,CAAM,CAAC;AACpC,QAAA,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;AACtB,KAAC;IAED,MAAM,UAAU,GAAwB,EAAE;;AAG1C,IAAA,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;AACjE,QAAA,UAAU,CAAC,CAAA,kBAAA,EAAqB,IAAI,CAAA,CAAE,CAAC;AACvC,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/C,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5B,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAChE,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU;;;;AAKxC,IAAA,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE;AAC3C,IAAA,IAAI,UAAU,CAAC,MAAM,EAAE;QACrB,UAAU,CAAC,cAAc,CAAC;AAC1B,QAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AAC1C,YAAA,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;AACvB,YAAA,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC9D,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,UAAU,EAAE;gBACvC,OAAO,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU;;;;;;;;;;;;;;;;;AAmB7C,IAAA,IAAI,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE;QAClC,UAAU,CAAC,iBAAiB,CAAC;AAC7B,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,CAAA,WAAA,EAAc,OAAO,CAAC,cAAc,CAAC,KAAK,IAAI,YAAY,CAAE,CAAA;AAC5D,YAAA,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;AAC7B,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,cAAc,CAAC,IAAI,CAAE;AAC5C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,WAAW,IAAI;AACpC,kBAAE,CAAoB,iBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,WAAW,CAAE;AAC1D,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,IAAI;AAC1C,kBAAE,CAA2B,wBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAE;AACvE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,eAAe,IAAI;AACxC,kBAAE,CAAyB,sBAAA,EAAA,OAAO,CAAC,cAAc,CAAC,eAAe,CAAE;AACnE,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,IAAI;AACjC,kBAAE,CAAkB,eAAA,EAAA,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAE;AACrD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,OAAO,IAAI;AAChC,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,cAAc,CAAC,OAAO,CAAE;AAClD,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,cAAc,CAAC,UAAU,IAAI;AACnC,kBAAE,CAAgC,6BAAA,EAAA,IAAI,CAAC,SAAS,CAC9C,OAAO,CAAC,cAAc,CAAC,UAAU,EACjC,IAAI,EACJ,CAAC,CACF,CAAU,QAAA;AACX,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,EAAE;QAC7B,UAAU,CAAC,YAAY,CAAC;AACxB,QAAA,MAAM,OAAO,GAAG;AACd,YAAA,OAAO,CAAC,SAAS,CAAC,KAAK,IAAI;AACzB,kBAAE,CAAc,WAAA,EAAA,OAAO,CAAC,SAAS,CAAC,KAAK,CAAE;AACzC,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,OAAO,IAAI;AAC3B,kBAAE,CAAgB,aAAA,EAAA,OAAO,CAAC,SAAS,CAAC,OAAO,CAAE;AAC7C,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,kBAAkB,IAAI;AACtC,kBAAE,CAA4B,yBAAA,EAAA,OAAO,CAAC,SAAS,CAAC;qBAC7C,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA,EAAA,EAAK,CAAC,CAAA,EAAA,CAAI;qBACrB,IAAI,CAAC,GAAG,CAAC,CAAE;AACd,kBAAE,EAAE;AACN,YAAA,OAAO,CAAC,SAAS,CAAC,IAAI,IAAI;AACxB,kBAAE,CAAa,UAAA,EAAA,OAAO,CAAC,SAAS,CAAC,IAAI,CAAE;AACvC,kBAAE,EAAE;YACN,EAAE;AACH,SAAA,CAAC,MAAM,CAAC,OAAO,CAAC;QAEjB,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;;AAIxC,IAAA,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE;AACjD,IAAA,IAAI,aAAa,CAAC,MAAM,EAAE;QACxB,UAAU,CAAC,iBAAiB,CAAC;QAE7B,MAAM,QAAQ,GAAa,EAAE;QAC7B,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,KAAI;AAC7B,YAAA,MAAM,aAAa,GAAG;gBACpB,CAAgB,aAAA,EAAA,CAAC,GAAG,CAAC,CAAG,CAAA,CAAA;gBACxB,CAAI,CAAA,EAAA,CAAC,CAAC,QAAQ,CAAG,CAAA,CAAA;gBACjB,CAAG,EAAA,CAAC,CAAC,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,GAAG,CAAA,SAAA,EAAY,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAE,CAAA;gBAClE,CAAG,EAAA,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,KAAK,GAAG,CAAA,OAAA,EAAU,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAE,CAAA;gBAC1D,CAAG,EAAA,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,GAAG,CAAA,MAAA,EAAS,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAE,CAAA;gBACtD,EAAE;AACH,aAAA,CAAC,MAAM,CAAC,OAAO,CAAC;YAEjB,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC3C,SAAC,CAAC;QAEF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;;IAGrC,OAAO;QACL,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QACrC,UAAU;KACX;AACH;;;;"}

package/dist/cjs/tools/search/highlights.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"highlights.cjs","sources":["../../../../src/tools/search/highlights.ts"],"sourcesContent":["import type * as t from './types';\r\n\r\n// 2. Pre-compile all regular expressions (only do this once)\r\n// Group patterns by priority for early returns\r\nconst priorityPatterns = [\r\n  // High priority patterns (structural)\r\n  [\r\n    { regex: /\\n\\n/g }, // Double newline (paragraph break)\r\n    { regex: /\\n/g }, // Single newline\r\n    { regex: /={3,}\\s*\\n|-{3,}\\s*\\n/g }, // Section separators\r\n  ],\r\n  // Medium priority (semantic)\r\n  [\r\n    { regex: /[.!?][\")\\]]?\\s/g }, // End of sentence\r\n    { regex: /;\\s/g }, // Semicolon\r\n    { regex: /:\\s/g }, // Colon\r\n  ],\r\n  // Low priority (any breaks)\r\n  [\r\n    { regex: /,\\s/g }, // Comma\r\n    { regex: /\\s-\\s/g }, // Dash surrounded by spaces\r\n    { regex: /\\s/g }, // Any space\r\n  ],\r\n];\r\n\r\nfunction findFirstMatch(text: string, regex: RegExp): number {\r\n  // Reset regex\r\n  regex.lastIndex = 0;\r\n\r\n  // For very long texts, try chunking\r\n  if (text.length > 10000) {\r\n    const chunkSize = 2000;\r\n    let position = 0;\r\n\r\n    while (position < text.length) {\r\n      const chunk = text.substring(position, position + chunkSize);\r\n      regex.lastIndex = 0;\r\n\r\n      const match = regex.exec(chunk);\r\n      if (match) {\r\n        return position + match.index;\r\n      }\r\n\r\n      // Move to next chunk with some overlap\r\n      position += chunkSize - 100;\r\n      if (position >= text.length) break;\r\n    }\r\n    return -1;\r\n  }\r\n\r\n  // For shorter texts, normal regex search\r\n  const match = regex.exec(text);\r\n  return match ? match.index : -1;\r\n}\r\n\r\n// 3. Optimized boundary finding functions\r\nfunction findLastMatch(text: string, regex: RegExp): number {\r\n  // Reset regex state\r\n  regex.lastIndex = 0;\r\n\r\n  let lastIndex = -1;\r\n  let lastLength = 0;\r\n  let match;\r\n\r\n  // For very long texts, use a different approach to avoid regex engine slowdowns\r\n  if (text.length > 10000) {\r\n    // Try dividing the text into chunks for faster processing\r\n    const chunkSize = 2000;\r\n    let startPosition = Math.max(0, text.length - chunkSize);\r\n\r\n    while (startPosition >= 0) {\r\n      const chunk = text.substring(startPosition, startPosition + chunkSize);\r\n      regex.lastIndex = 0;\r\n\r\n      let chunkLastIndex = -1;\r\n      let chunkLastLength = 0;\r\n\r\n      while ((match = regex.exec(chunk)) !== null) {\r\n        chunkLastIndex = match.index;\r\n        chunkLastLength = match[0].length;\r\n      }\r\n\r\n      if (chunkLastIndex !== -1) {\r\n        return startPosition + chunkLastIndex + chunkLastLength;\r\n      }\r\n\r\n      // Move to previous chunk with some overlap\r\n      startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;\r\n      if (startPosition <= 0) break;\r\n    }\r\n    return -1;\r\n  }\r\n\r\n  // For shorter texts, normal regex search\r\n  while ((match = regex.exec(text)) !== null) {\r\n    lastIndex = match.index;\r\n    lastLength = match[0].length;\r\n  }\r\n\r\n  return lastIndex === -1 ? -1 : lastIndex + lastLength;\r\n}\r\n\r\n// 4. Find the best boundary with priority groups\r\nfunction findBestBoundary(text: string, direction = 'backward'): number {\r\n  if (!text || text.length === 0) return 0;\r\n\r\n  // Try each priority group\r\n  for (const patternGroup of priorityPatterns) {\r\n    for (const pattern of patternGroup) {\r\n      const position =\r\n        direction === 'backward'\r\n          ? findLastMatch(text, pattern.regex)\r\n          : findFirstMatch(text, pattern.regex);\r\n\r\n      if (position !== -1) {\r\n        return position;\r\n      }\r\n    }\r\n  }\r\n\r\n  // No match found, use character boundary\r\n  return direction === 'backward' ? text.length : 0;\r\n}\r\n\r\n/**\r\n * Tracks references used in a highlight without changing their numbers\r\n */\r\nfunction trackReferencesInHighlight(\r\n  text: string,\r\n  sourceResult: t.ValidSource // Source containing the original references\r\n): {\r\n  references: {\r\n    type: 'link' | 'image' | 'video';\r\n    originalIndex: number;\r\n    reference: t.MediaReference; // Original reference object\r\n  }[];\r\n} {\r\n  // Track used references\r\n  const references: {\r\n    type: 'link' | 'image' | 'video';\r\n    originalIndex: number;\r\n    reference: t.MediaReference;\r\n  }[] = [];\r\n\r\n  if (!text || text.length === 0 || !text.includes('#')) {\r\n    return { references }; // Early return\r\n  }\r\n\r\n  // Quick check for reference markers\r\n  if (\r\n    !text.includes('link#') &&\r\n    !text.includes('image#') &&\r\n    !text.includes('video#')\r\n  ) {\r\n    return { references };\r\n  }\r\n\r\n  // Get references from the source if available\r\n  const sourceRefs = sourceResult.references || {\r\n    links: [],\r\n    images: [],\r\n    videos: [],\r\n  };\r\n\r\n  // Find references but don't modify text\r\n  const refRegex = /\\((link|image|video)#(\\d+)(?:\\s+\"([^\"]*)\")?\\)/g;\r\n  let match;\r\n\r\n  while ((match = refRegex.exec(text)) !== null) {\r\n    const [, type, indexStr] = match;\r\n    const originalIndex = parseInt(indexStr, 10) - 1; // Convert to 0-based\r\n\r\n    // Get the source array for this type\r\n    const refType = type as 'link' | 'image' | 'video';\r\n    const sourceArray = sourceRefs[`${refType}s`] as\r\n      | t.MediaReference[]\r\n      | undefined;\r\n\r\n    // Skip if invalid reference\r\n    if (\r\n      !sourceArray ||\r\n      originalIndex < 0 ||\r\n      originalIndex >= sourceArray.length\r\n    ) {\r\n      continue; // Skip invalid references\r\n    }\r\n\r\n    // Get original reference\r\n    const reference = sourceArray[originalIndex];\r\n\r\n    // Track if not already tracked\r\n    const alreadyTracked = references.some(\r\n      (ref) => ref.type === refType && ref.originalIndex === originalIndex\r\n    );\r\n\r\n    if (!alreadyTracked) {\r\n      references.push({\r\n        type: refType,\r\n        originalIndex,\r\n        reference,\r\n      });\r\n    }\r\n  }\r\n\r\n  return { references };\r\n}\r\n\r\n/**\r\n * Expand highlights in search results using smart boundary detection.\r\n *\r\n * This implementation finds natural text boundaries like paragraphs, sentences,\r\n * and phrases to provide context while maintaining readability.\r\n *\r\n * @param searchResults - Search results object\r\n * @param mainExpandBy - Primary expansion size on each side (default: 300)\r\n * @param separatorExpandBy - Additional range to look for separators (default: 150)\r\n * @returns Copy of search results with expanded highlights and tracked references\r\n */\r\nexport function expandHighlights(\r\n  searchResults: t.SearchResultData,\r\n  mainExpandBy = 300,\r\n  separatorExpandBy = 150\r\n): t.SearchResultData {\r\n  // Avoid deep copy - only copy what we modify\r\n  const resultCopy = { ...searchResults };\r\n  if (resultCopy.organic) resultCopy.organic = [...resultCopy.organic];\r\n  if (resultCopy.topStories) resultCopy.topStories = [...resultCopy.topStories];\r\n\r\n  // Process the results efficiently\r\n  const processResultTypes = ['organic', 'topStories'] as const;\r\n\r\n  for (const resultType of processResultTypes) {\r\n    if (!resultCopy[resultType as 'organic' | 'topStories']) continue;\r\n\r\n    // Map results to new array with modified highlights\r\n    resultCopy[resultType] = resultCopy[resultType]?.map((result) => {\r\n      if (\r\n        result.content == null ||\r\n        result.content === '' ||\r\n        !result.highlights ||\r\n        result.highlights.length === 0\r\n      ) {\r\n        return result; // No modification needed\r\n      }\r\n\r\n      // Create a shallow copy with expanded highlights\r\n      const resultCopy = { ...result };\r\n      const content = result.content;\r\n      const highlights = [];\r\n      // Process each highlight\r\n      for (const highlight of result.highlights) {\r\n        const { references } = trackReferencesInHighlight(\r\n          highlight.text,\r\n          result\r\n        );\r\n\r\n        let startPos = content.indexOf(highlight.text);\r\n        let highlightLen = highlight.text.length;\r\n\r\n        if (startPos === -1) {\r\n          // Try with stripped whitespace\r\n          const strippedHighlight = highlight.text.trim();\r\n          startPos = content.indexOf(strippedHighlight);\r\n\r\n          if (startPos === -1) {\r\n            highlights.push({\r\n              text: highlight.text,\r\n              score: highlight.score,\r\n              references,\r\n            });\r\n            continue;\r\n          }\r\n          highlightLen = strippedHighlight.length;\r\n        }\r\n\r\n        // Calculate boundaries\r\n        const mainStart = Math.max(0, startPos - mainExpandBy);\r\n        const mainEnd = Math.min(\r\n          content.length,\r\n          startPos + highlightLen + mainExpandBy\r\n        );\r\n\r\n        const separatorStart = Math.max(0, mainStart - separatorExpandBy);\r\n        const separatorEnd = Math.min(\r\n          content.length,\r\n          mainEnd + separatorExpandBy\r\n        );\r\n\r\n        // Extract text segments\r\n        const headText = content.substring(separatorStart, mainStart);\r\n        const tailText = content.substring(mainEnd, separatorEnd);\r\n\r\n        // Find natural boundaries\r\n        const bestHeadBoundary = findBestBoundary(headText, 'backward');\r\n        const bestTailBoundary = findBestBoundary(tailText, 'forward');\r\n\r\n        // Calculate final positions\r\n        const finalStart = separatorStart + bestHeadBoundary;\r\n        const finalEnd = mainEnd + bestTailBoundary;\r\n\r\n        // Extract the expanded highlight\r\n        const expandedHighlightText = content\r\n          .substring(finalStart, finalEnd)\r\n          .trim();\r\n        highlights.push({\r\n          text: expandedHighlightText,\r\n          score: highlight.score,\r\n          references,\r\n        });\r\n      }\r\n\r\n      resultCopy.highlights = highlights;\r\n      delete resultCopy.content;\r\n      delete resultCopy.references;\r\n      return resultCopy;\r\n    });\r\n  }\r\n\r\n  return resultCopy;\r\n}\r\n"],"names":[],"mappings":";;AAEA;AACA;AACA,MAAM,gBAAgB,GAAG;;AAEvB,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,OAAO,EAAE;AAClB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AAChB,QAAA,EAAE,KAAK,EAAE,wBAAwB,EAAE;AACpC,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,iBAAiB,EAAE;AAC5B,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AAClB,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,QAAQ,EAAE;AACnB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AACjB,KAAA;CACF;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEjD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;;AAGnB,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;QACvB,MAAM,SAAS,GAAG,IAAI;QACtB,IAAI,QAAQ,GAAG,CAAC;AAEhB,QAAA,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE;AAC7B,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAC;AAC5D,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;YAEnB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAC/B,IAAI,KAAK,EAAE;AACT,gBAAA,OAAO,QAAQ,GAAG,KAAK,CAAC,KAAK;;;AAI/B,YAAA,QAAQ,IAAI,SAAS,GAAG,GAAG;AAC3B,YAAA,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM;gBAAE;;QAE/B,OAAO,EAAE;;;IAIX,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE;AACjC;AAEA;AACA,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEhD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,IAAA,IAAI,SAAS,GAAG,EAAE;IAClB,IAAI,UAAU,GAAG,CAAC;AAClB,IAAA,IAAI,KAAK;;AAGT,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;;QAEvB,MAAM,SAAS,GAAG,IAAI;AACtB,QAAA,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;AAExD,QAAA,OAAO,aAAa,IAAI,CAAC,EAAE;AACzB,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS,CAAC;AACtE,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,YAAA,IAAI,cAAc,GAAG,EAAE;YACvB,IAAI,eAAe,GAAG,CAAC;AAEvB,YAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE;AAC3C,gBAAA,cAAc,GAAG,KAAK,CAAC,KAAK;AAC5B,gBAAA,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAGnC,YAAA,IAAI,cAAc,KAAK,EAAE,EAAE;AACzB,gBAAA,OAAO,aAAa,GAAG,cAAc,GAAG,eAAe;;;AAIzD,YAAA,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC;YAChE,IAAI,aAAa,IAAI,CAAC;gBAAE;;QAE1B,OAAO,EAAE;;;AAIX,IAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAC1C,QAAA,SAAS,GAAG,KAAK,CAAC,KAAK;AACvB,QAAA,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAG9B,IAAA,OAAO,SAAS,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,GAAG,UAAU;AACvD;AAEA;AACA,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAS,GAAG,UAAU,EAAA;AAC5D,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,CAAC;;AAGxC,IAAA,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;AAC3C,QAAA,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE;AAClC,YAAA,MAAM,QAAQ,GACZ,SAAS,KAAK;kBACV,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK;kBACjC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC;AAEzC,YAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;AACnB,gBAAA,OAAO,QAAQ;;;;;AAMrB,IAAA,OAAO,SAAS,KAAK,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;AACnD;AAEA;;AAEG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,YAA2B;;;IAS3B,MAAM,UAAU,GAIV,EAAE;AAER,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;AACrD,QAAA,OAAO,EAAE,UAAU,EAAE,CAAC;;;AAIxB,IAAA,IACE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;AACvB,QAAA,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACxB,QAAA,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACxB;QACA,OAAO,EAAE,UAAU,EAAE;;;AAIvB,IAAA,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,IAAI;AAC5C,QAAA,KAAK,EAAE,EAAE;AACT,QAAA,MAAM,EAAE,EAAE;AACV,QAAA,MAAM,EAAE,EAAE;KACX;;IAGD,MAAM,QAAQ,GAAG,gDAAgD;AACjE,IAAA,IAAI,KAAK;AAET,IAAA,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;QAC7C,MAAM,GAAG,IAAI,EAAE,QAAQ,CAAC,GAAG,KAAK;AAChC,QAAA,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;;QAGjD,MAAM,OAAO,GAAG,IAAkC;QAClD,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,OAAO,CAAA,CAAA,CAAG,CAE/B;;AAGb,QAAA,IACE,CAAC,WAAW;AACZ,YAAA,aAAa,GAAG,CAAC;AACjB,YAAA,aAAa,IAAI,WAAW,CAAC,MAAM,EACnC;AACA,YAAA,SAAS;;;AAIX,QAAA,MAAM,SAAS,GAAG,WAAW,CAAC,aAAa,CAAC;;QAG5C,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CACpC,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,aAAa,KAAK,aAAa,CACrE;QAED,IAAI,CAAC,cAAc,EAAE;YACnB,UAAU,CAAC,IAAI,CAAC;AACd,gBAAA,IAAI,EAAE,OAAO;gBACb,aAAa;gBACb,SAAS;AACV,aAAA,CAAC;;;IAIN,OAAO,EAAE,UAAU,EAAE;AACvB;AAEA;;;;;;;;;;AAUG;AACG,SAAU,gBAAgB,CAC9B,aAAiC,EACjC,YAAY,GAAG,GAAG,EAClB,iBAAiB,GAAG,GAAG,EAAA;;AAGvB,IAAA,MAAM,UAAU,GAAG,EAAE,GAAG,aAAa,EAAE;IACvC,IAAI,UAAU,CAAC,OAAO;QAAE,UAAU,CAAC,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;IACpE,IAAI,UAAU,CAAC,UAAU;QAAE,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC;;AAG7E,IAAA,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,YAAY,CAAU;AAE7D,IAAA,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE;AAC3C,QAAA,IAAI,CAAC,UAAU,CAAC,UAAsC,CAAC;YAAE;;AAGzD,QAAA,UAAU,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,KAAI;AAC9D,YAAA,IACE,MAAM,CAAC,OAAO,IAAI,IAAI;gBACtB,MAAM,CAAC,OAAO,KAAK,EAAE;gBACrB,CAAC,MAAM,CAAC,UAAU;AAClB,gBAAA,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAC9B;gBACA,OAAO,MAAM,CAAC;;;AAIhB,YAAA,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE;AAChC,YAAA,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;YAC9B,MAAM,UAAU,GAAG,EAAE;;AAErB,YAAA,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE;AACzC,gBAAA,MAAM,EAAE,UAAU,EAAE,GAAG,0BAA0B,CAC/C,SAAS,CAAC,IAAI,EACd,MAAM,CACP;gBAED,IAAI,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;AAC9C,gBAAA,IAAI,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM;AAExC,gBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;;oBAEnB,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;AAC/C,oBAAA,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;AAE7C,oBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;wBACnB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,SAAS,CAAC,IAAI;4BACpB,KAAK,EAAE,SAAS,CAAC,KAAK;4BACtB,UAAU;AACX,yBAAA,CAAC;wBACF;;AAEF,oBAAA,YAAY,GAAG,iBAAiB,CAAC,MAAM;;;AAIzC,gBAAA,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC;AACtD,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACtB,OAAO,CAAC,MAAM,EACd,QAAQ,GAAG,YAAY,GAAG,YAAY,CACvC;AAED,gBAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,iBAAiB,CAAC;AACjE,gBAAA,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,OAAO,CAAC,MAAM,EACd,OAAO,GAAG,iBAAiB,CAC5B;;gBAGD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC;gBAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;;gBAGzD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC;gBAC/D,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC;;AAG9D,gBAAA,MAAM,UAAU,GAAG,cAAc,GAAG,gBAAgB;AACpD,gBAAA,MAAM,QAAQ,GAAG,OAAO,GAAG,gBAAgB;;gBAG3C,MAAM,qBAAqB,GAAG;AAC3B,qBAAA,SAAS,CAAC,UAAU,EAAE,QAAQ;AAC9B,qBAAA,IAAI,EAAE;gBACT,UAAU,CAAC,IAAI,CAAC;AACd,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;oBACtB,UAAU;AACX,iBAAA,CAAC;;AAGJ,YAAA,UAAU,CAAC,UAAU,GAAG,UAAU;YAClC,OAAO,UAAU,CAAC,OAAO;YACzB,OAAO,UAAU,CAAC,UAAU;AAC5B,YAAA,OAAO,UAAU;AACnB,SAAC,CAAC;;AAGJ,IAAA,OAAO,UAAU;AACnB;;;;"}
+{"version":3,"file":"highlights.cjs","sources":["../../../../src/tools/search/highlights.ts"],"sourcesContent":["import type * as t from './types';\n\n// 2. Pre-compile all regular expressions (only do this once)\n// Group patterns by priority for early returns\nconst priorityPatterns = [\n  // High priority patterns (structural)\n  [\n    { regex: /\\n\\n/g }, // Double newline (paragraph break)\n    { regex: /\\n/g }, // Single newline\n    { regex: /={3,}\\s*\\n|-{3,}\\s*\\n/g }, // Section separators\n  ],\n  // Medium priority (semantic)\n  [\n    { regex: /[.!?][\")\\]]?\\s/g }, // End of sentence\n    { regex: /;\\s/g }, // Semicolon\n    { regex: /:\\s/g }, // Colon\n  ],\n  // Low priority (any breaks)\n  [\n    { regex: /,\\s/g }, // Comma\n    { regex: /\\s-\\s/g }, // Dash surrounded by spaces\n    { regex: /\\s/g }, // Any space\n  ],\n];\n\nfunction findFirstMatch(text: string, regex: RegExp): number {\n  // Reset regex\n  regex.lastIndex = 0;\n\n  // For very long texts, try chunking\n  if (text.length > 10000) {\n    const chunkSize = 2000;\n    let position = 0;\n\n    while (position < text.length) {\n      const chunk = text.substring(position, position + chunkSize);\n      regex.lastIndex = 0;\n\n      const match = regex.exec(chunk);\n      if (match) {\n        return position + match.index;\n      }\n\n      // Move to next chunk with some overlap\n      position += chunkSize - 100;\n      if (position >= text.length) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  const match = regex.exec(text);\n  return match ? match.index : -1;\n}\n\n// 3. Optimized boundary finding functions\nfunction findLastMatch(text: string, regex: RegExp): number {\n  // Reset regex state\n  regex.lastIndex = 0;\n\n  let lastIndex = -1;\n  let lastLength = 0;\n  let match;\n\n  // For very long texts, use a different approach to avoid regex engine slowdowns\n  if (text.length > 10000) {\n    // Try dividing the text into chunks for faster processing\n    const chunkSize = 2000;\n    let startPosition = Math.max(0, text.length - chunkSize);\n\n    while (startPosition >= 0) {\n      const chunk = text.substring(startPosition, startPosition + chunkSize);\n      regex.lastIndex = 0;\n\n      let chunkLastIndex = -1;\n      let chunkLastLength = 0;\n\n      while ((match = regex.exec(chunk)) !== null) {\n        chunkLastIndex = match.index;\n        chunkLastLength = match[0].length;\n      }\n\n      if (chunkLastIndex !== -1) {\n        return startPosition + chunkLastIndex + chunkLastLength;\n      }\n\n      // Move to previous chunk with some overlap\n      startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;\n      if (startPosition <= 0) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  while ((match = regex.exec(text)) !== null) {\n    lastIndex = match.index;\n    lastLength = match[0].length;\n  }\n\n  return lastIndex === -1 ? -1 : lastIndex + lastLength;\n}\n\n// 4. Find the best boundary with priority groups\nfunction findBestBoundary(text: string, direction = 'backward'): number {\n  if (!text || text.length === 0) return 0;\n\n  // Try each priority group\n  for (const patternGroup of priorityPatterns) {\n    for (const pattern of patternGroup) {\n      const position =\n        direction === 'backward'\n          ? findLastMatch(text, pattern.regex)\n          : findFirstMatch(text, pattern.regex);\n\n      if (position !== -1) {\n        return position;\n      }\n    }\n  }\n\n  // No match found, use character boundary\n  return direction === 'backward' ? text.length : 0;\n}\n\n/**\n * Tracks references used in a highlight without changing their numbers\n */\nfunction trackReferencesInHighlight(\n  text: string,\n  sourceResult: t.ValidSource // Source containing the original references\n): {\n  references: {\n    type: 'link' | 'image' | 'video';\n    originalIndex: number;\n    reference: t.MediaReference; // Original reference object\n  }[];\n} {\n  // Track used references\n  const references: {\n    type: 'link' | 'image' | 'video';\n    originalIndex: number;\n    reference: t.MediaReference;\n  }[] = [];\n\n  if (!text || text.length === 0 || !text.includes('#')) {\n    return { references }; // Early return\n  }\n\n  // Quick check for reference markers\n  if (\n    !text.includes('link#') &&\n    !text.includes('image#') &&\n    !text.includes('video#')\n  ) {\n    return { references };\n  }\n\n  // Get references from the source if available\n  const sourceRefs = sourceResult.references || {\n    links: [],\n    images: [],\n    videos: [],\n  };\n\n  // Find references but don't modify text\n  const refRegex = /\\((link|image|video)#(\\d+)(?:\\s+\"([^\"]*)\")?\\)/g;\n  let match;\n\n  while ((match = refRegex.exec(text)) !== null) {\n    const [, type, indexStr] = match;\n    const originalIndex = parseInt(indexStr, 10) - 1; // Convert to 0-based\n\n    // Get the source array for this type\n    const refType = type as 'link' | 'image' | 'video';\n    const sourceArray = sourceRefs[`${refType}s`] as\n      | t.MediaReference[]\n      | undefined;\n\n    // Skip if invalid reference\n    if (\n      !sourceArray ||\n      originalIndex < 0 ||\n      originalIndex >= sourceArray.length\n    ) {\n      continue; // Skip invalid references\n    }\n\n    // Get original reference\n    const reference = sourceArray[originalIndex];\n\n    // Track if not already tracked\n    const alreadyTracked = references.some(\n      (ref) => ref.type === refType && ref.originalIndex === originalIndex\n    );\n\n    if (!alreadyTracked) {\n      references.push({\n        type: refType,\n        originalIndex,\n        reference,\n      });\n    }\n  }\n\n  return { references };\n}\n\n/**\n * Expand highlights in search results using smart boundary detection.\n *\n * This implementation finds natural text boundaries like paragraphs, sentences,\n * and phrases to provide context while maintaining readability.\n *\n * @param searchResults - Search results object\n * @param mainExpandBy - Primary expansion size on each side (default: 300)\n * @param separatorExpandBy - Additional range to look for separators (default: 150)\n * @returns Copy of search results with expanded highlights and tracked references\n */\nexport function expandHighlights(\n  searchResults: t.SearchResultData,\n  mainExpandBy = 300,\n  separatorExpandBy = 150\n): t.SearchResultData {\n  // Avoid deep copy - only copy what we modify\n  const resultCopy = { ...searchResults };\n  if (resultCopy.organic) resultCopy.organic = [...resultCopy.organic];\n  if (resultCopy.topStories) resultCopy.topStories = [...resultCopy.topStories];\n\n  // Process the results efficiently\n  const processResultTypes = ['organic', 'topStories'] as const;\n\n  for (const resultType of processResultTypes) {\n    if (!resultCopy[resultType as 'organic' | 'topStories']) continue;\n\n    // Map results to new array with modified highlights\n    resultCopy[resultType] = resultCopy[resultType]?.map((result) => {\n      if (\n        result.content == null ||\n        result.content === '' ||\n        !result.highlights ||\n        result.highlights.length === 0\n      ) {\n        return result; // No modification needed\n      }\n\n      // Create a shallow copy with expanded highlights\n      const resultCopy = { ...result };\n      const content = result.content;\n      const highlights = [];\n      // Process each highlight\n      for (const highlight of result.highlights) {\n        const { references } = trackReferencesInHighlight(\n          highlight.text,\n          result\n        );\n\n        let startPos = content.indexOf(highlight.text);\n        let highlightLen = highlight.text.length;\n\n        if (startPos === -1) {\n          // Try with stripped whitespace\n          const strippedHighlight = highlight.text.trim();\n          startPos = content.indexOf(strippedHighlight);\n\n          if (startPos === -1) {\n            highlights.push({\n              text: highlight.text,\n              score: highlight.score,\n              references,\n            });\n            continue;\n          }\n          highlightLen = strippedHighlight.length;\n        }\n\n        // Calculate boundaries\n        const mainStart = Math.max(0, startPos - mainExpandBy);\n        const mainEnd = Math.min(\n          content.length,\n          startPos + highlightLen + mainExpandBy\n        );\n\n        const separatorStart = Math.max(0, mainStart - separatorExpandBy);\n        const separatorEnd = Math.min(\n          content.length,\n          mainEnd + separatorExpandBy\n        );\n\n        // Extract text segments\n        const headText = content.substring(separatorStart, mainStart);\n        const tailText = content.substring(mainEnd, separatorEnd);\n\n        // Find natural boundaries\n        const bestHeadBoundary = findBestBoundary(headText, 'backward');\n        const bestTailBoundary = findBestBoundary(tailText, 'forward');\n\n        // Calculate final positions\n        const finalStart = separatorStart + bestHeadBoundary;\n        const finalEnd = mainEnd + bestTailBoundary;\n\n        // Extract the expanded highlight\n        const expandedHighlightText = content\n          .substring(finalStart, finalEnd)\n          .trim();\n        highlights.push({\n          text: expandedHighlightText,\n          score: highlight.score,\n          references,\n        });\n      }\n\n      resultCopy.highlights = highlights;\n      delete resultCopy.content;\n      delete resultCopy.references;\n      return resultCopy;\n    });\n  }\n\n  return resultCopy;\n}\n"],"names":[],"mappings":";;AAEA;AACA;AACA,MAAM,gBAAgB,GAAG;;AAEvB,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,OAAO,EAAE;AAClB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AAChB,QAAA,EAAE,KAAK,EAAE,wBAAwB,EAAE;AACpC,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,iBAAiB,EAAE;AAC5B,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AAClB,KAAA;;AAED,IAAA;AACE,QAAA,EAAE,KAAK,EAAE,MAAM,EAAE;AACjB,QAAA,EAAE,KAAK,EAAE,QAAQ,EAAE;AACnB,QAAA,EAAE,KAAK,EAAE,KAAK,EAAE;AACjB,KAAA;CACF;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEjD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;;AAGnB,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;QACvB,MAAM,SAAS,GAAG,IAAI;QACtB,IAAI,QAAQ,GAAG,CAAC;AAEhB,QAAA,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE;AAC7B,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,CAAC;AAC5D,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;YAEnB,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAC/B,IAAI,KAAK,EAAE;AACT,gBAAA,OAAO,QAAQ,GAAG,KAAK,CAAC,KAAK;;;AAI/B,YAAA,QAAQ,IAAI,SAAS,GAAG,GAAG;AAC3B,YAAA,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM;gBAAE;;QAE/B,OAAO,EAAE;;;IAIX,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AAC9B,IAAA,OAAO,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,EAAE;AACjC;AAEA;AACA,SAAS,aAAa,CAAC,IAAY,EAAE,KAAa,EAAA;;AAEhD,IAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,IAAA,IAAI,SAAS,GAAG,EAAE;IAClB,IAAI,UAAU,GAAG,CAAC;AAClB,IAAA,IAAI,KAAK;;AAGT,IAAA,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,EAAE;;QAEvB,MAAM,SAAS,GAAG,IAAI;AACtB,QAAA,IAAI,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;AAExD,QAAA,OAAO,aAAa,IAAI,CAAC,EAAE;AACzB,YAAA,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,aAAa,GAAG,SAAS,CAAC;AACtE,YAAA,KAAK,CAAC,SAAS,GAAG,CAAC;AAEnB,YAAA,IAAI,cAAc,GAAG,EAAE;YACvB,IAAI,eAAe,GAAG,CAAC;AAEvB,YAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE;AAC3C,gBAAA,cAAc,GAAG,KAAK,CAAC,KAAK;AAC5B,gBAAA,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAGnC,YAAA,IAAI,cAAc,KAAK,EAAE,EAAE;AACzB,gBAAA,OAAO,aAAa,GAAG,cAAc,GAAG,eAAe;;;AAIzD,YAAA,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC;YAChE,IAAI,aAAa,IAAI,CAAC;gBAAE;;QAE1B,OAAO,EAAE;;;AAIX,IAAA,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;AAC1C,QAAA,SAAS,GAAG,KAAK,CAAC,KAAK;AACvB,QAAA,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;;AAG9B,IAAA,OAAO,SAAS,KAAK,EAAE,GAAG,EAAE,GAAG,SAAS,GAAG,UAAU;AACvD;AAEA;AACA,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAS,GAAG,UAAU,EAAA;AAC5D,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,CAAC;;AAGxC,IAAA,KAAK,MAAM,YAAY,IAAI,gBAAgB,EAAE;AAC3C,QAAA,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE;AAClC,YAAA,MAAM,QAAQ,GACZ,SAAS,KAAK;kBACV,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK;kBACjC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC;AAEzC,YAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;AACnB,gBAAA,OAAO,QAAQ;;;;;AAMrB,IAAA,OAAO,SAAS,KAAK,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;AACnD;AAEA;;AAEG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,YAA2B;;;IAS3B,MAAM,UAAU,GAIV,EAAE;AAER,IAAA,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;AACrD,QAAA,OAAO,EAAE,UAAU,EAAE,CAAC;;;AAIxB,IAAA,IACE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;AACvB,QAAA,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACxB,QAAA,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACxB;QACA,OAAO,EAAE,UAAU,EAAE;;;AAIvB,IAAA,MAAM,UAAU,GAAG,YAAY,CAAC,UAAU,IAAI;AAC5C,QAAA,KAAK,EAAE,EAAE;AACT,QAAA,MAAM,EAAE,EAAE;AACV,QAAA,MAAM,EAAE,EAAE;KACX;;IAGD,MAAM,QAAQ,GAAG,gDAAgD;AACjE,IAAA,IAAI,KAAK;AAET,IAAA,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE;QAC7C,MAAM,GAAG,IAAI,EAAE,QAAQ,CAAC,GAAG,KAAK;AAChC,QAAA,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;;QAGjD,MAAM,OAAO,GAAG,IAAkC;QAClD,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,OAAO,CAAA,CAAA,CAAG,CAE/B;;AAGb,QAAA,IACE,CAAC,WAAW;AACZ,YAAA,aAAa,GAAG,CAAC;AACjB,YAAA,aAAa,IAAI,WAAW,CAAC,MAAM,EACnC;AACA,YAAA,SAAS;;;AAIX,QAAA,MAAM,SAAS,GAAG,WAAW,CAAC,aAAa,CAAC;;QAG5C,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CACpC,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,aAAa,KAAK,aAAa,CACrE;QAED,IAAI,CAAC,cAAc,EAAE;YACnB,UAAU,CAAC,IAAI,CAAC;AACd,gBAAA,IAAI,EAAE,OAAO;gBACb,aAAa;gBACb,SAAS;AACV,aAAA,CAAC;;;IAIN,OAAO,EAAE,UAAU,EAAE;AACvB;AAEA;;;;;;;;;;AAUG;AACG,SAAU,gBAAgB,CAC9B,aAAiC,EACjC,YAAY,GAAG,GAAG,EAClB,iBAAiB,GAAG,GAAG,EAAA;;AAGvB,IAAA,MAAM,UAAU,GAAG,EAAE,GAAG,aAAa,EAAE;IACvC,IAAI,UAAU,CAAC,OAAO;QAAE,UAAU,CAAC,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;IACpE,IAAI,UAAU,CAAC,UAAU;QAAE,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC;;AAG7E,IAAA,MAAM,kBAAkB,GAAG,CAAC,SAAS,EAAE,YAAY,CAAU;AAE7D,IAAA,KAAK,MAAM,UAAU,IAAI,kBAAkB,EAAE;AAC3C,QAAA,IAAI,CAAC,UAAU,CAAC,UAAsC,CAAC;YAAE;;AAGzD,QAAA,UAAU,CAAC,UAAU,CAAC,GAAG,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,KAAI;AAC9D,YAAA,IACE,MAAM,CAAC,OAAO,IAAI,IAAI;gBACtB,MAAM,CAAC,OAAO,KAAK,EAAE;gBACrB,CAAC,MAAM,CAAC,UAAU;AAClB,gBAAA,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAC9B;gBACA,OAAO,MAAM,CAAC;;;AAIhB,YAAA,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE;AAChC,YAAA,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;YAC9B,MAAM,UAAU,GAAG,EAAE;;AAErB,YAAA,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE;AACzC,gBAAA,MAAM,EAAE,UAAU,EAAE,GAAG,0BAA0B,CAC/C,SAAS,CAAC,IAAI,EACd,MAAM,CACP;gBAED,IAAI,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;AAC9C,gBAAA,IAAI,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM;AAExC,gBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;;oBAEnB,MAAM,iBAAiB,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE;AAC/C,oBAAA,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC;AAE7C,oBAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;wBACnB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,SAAS,CAAC,IAAI;4BACpB,KAAK,EAAE,SAAS,CAAC,KAAK;4BACtB,UAAU;AACX,yBAAA,CAAC;wBACF;;AAEF,oBAAA,YAAY,GAAG,iBAAiB,CAAC,MAAM;;;AAIzC,gBAAA,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC;AACtD,gBAAA,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACtB,OAAO,CAAC,MAAM,EACd,QAAQ,GAAG,YAAY,GAAG,YAAY,CACvC;AAED,gBAAA,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,iBAAiB,CAAC;AACjE,gBAAA,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAC3B,OAAO,CAAC,MAAM,EACd,OAAO,GAAG,iBAAiB,CAC5B;;gBAGD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC;gBAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;;gBAGzD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC;gBAC/D,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC;;AAG9D,gBAAA,MAAM,UAAU,GAAG,cAAc,GAAG,gBAAgB;AACpD,gBAAA,MAAM,QAAQ,GAAG,OAAO,GAAG,gBAAgB;;gBAG3C,MAAM,qBAAqB,GAAG;AAC3B,qBAAA,SAAS,CAAC,UAAU,EAAE,QAAQ;AAC9B,qBAAA,IAAI,EAAE;gBACT,UAAU,CAAC,IAAI,CAAC;AACd,oBAAA,IAAI,EAAE,qBAAqB;oBAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;oBACtB,UAAU;AACX,iBAAA,CAAC;;AAGJ,YAAA,UAAU,CAAC,UAAU,GAAG,UAAU;YAClC,OAAO,UAAU,CAAC,OAAO;YACzB,OAAO,UAAU,CAAC,UAAU;AAC5B,YAAA,OAAO,UAAU;AACnB,SAAC,CAAC;;AAGJ,IAAA,OAAO,UAAU;AACnB;;;;"}

package/dist/cjs/tools/search/rerankers.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"rerankers.cjs","sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\r\nimport type * as t from './types';\r\nimport { createDefaultLogger } from './utils';\r\n\r\nexport abstract class BaseReranker {\r\n  protected apiKey: string | undefined;\r\n  protected logger: t.Logger;\r\n\r\n  constructor(logger?: t.Logger) {\r\n    // Each specific reranker will set its API key\r\n    this.logger = logger || createDefaultLogger();\r\n  }\r\n\r\n  abstract rerank(\r\n    query: string,\r\n    documents: string[],\r\n    topK?: number\r\n  ): Promise<t.Highlight[]>;\r\n\r\n  protected getDefaultRanking(\r\n    documents: string[],\r\n    topK: number\r\n  ): t.Highlight[] {\r\n    return documents\r\n      .slice(0, Math.min(topK, documents.length))\r\n      .map((doc) => ({ text: doc, score: 0 }));\r\n  }\r\n}\r\n\r\nexport class JinaReranker extends BaseReranker {\r\n  private apiUrl: string;\r\n\r\n  constructor({\r\n    apiKey = process.env.JINA_API_KEY,\r\n    apiUrl = process.env.JINA_API_URL || 'https://api.jina.ai/v1/rerank',\r\n    logger,\r\n  }: {\r\n    apiKey?: string;\r\n    apiUrl?: string;\r\n    logger?: t.Logger;\r\n  }) {\r\n    super(logger);\r\n    this.apiKey = apiKey;\r\n    this.apiUrl = apiUrl;\r\n  }\r\n\r\n  async rerank(\r\n    query: string,\r\n    documents: string[],\r\n    topK: number = 5\r\n  ): Promise<t.Highlight[]> {\r\n    this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);\r\n\r\n    try {\r\n      if (this.apiKey == null || this.apiKey === '') {\r\n        this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\r\n        return this.getDefaultRanking(documents, topK);\r\n      }\r\n\r\n      const requestData = {\r\n        model: 'jina-reranker-v2-base-multilingual',\r\n        query: query,\r\n        top_n: topK,\r\n        documents: documents,\r\n        return_documents: true,\r\n      };\r\n\r\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\r\n        this.apiUrl,\r\n        requestData,\r\n        {\r\n          headers: {\r\n            'Content-Type': 'application/json',\r\n            Authorization: `Bearer ${this.apiKey}`,\r\n          },\r\n        }\r\n      );\r\n\r\n      this.logger.debug('Jina API Model:', response.data?.model);\r\n      this.logger.debug('Jina API Usage:', response.data?.usage);\r\n\r\n      if (response.data && response.data.results.length) {\r\n        return response.data.results.map((result) => {\r\n          const docIndex = result.index;\r\n          const score = result.relevance_score;\r\n          let text = '';\r\n\r\n          // If return_documents is true, the document field will be present\r\n          if (result.document != null) {\r\n            const doc = result.document;\r\n            if (typeof doc === 'object' && 'text' in doc) {\r\n              text = doc.text;\r\n            } else if (typeof doc === 'string') {\r\n              text = doc;\r\n            }\r\n          } else {\r\n            // Otherwise, use the index to get the document\r\n            text = documents[docIndex];\r\n          }\r\n\r\n          return { text, score };\r\n        });\r\n      } else {\r\n        this.logger.warn(\r\n          'Unexpected response format from Jina API. Using default ranking.'\r\n        );\r\n        return this.getDefaultRanking(documents, topK);\r\n      }\r\n    } catch (error) {\r\n      this.logger.error('Error using Jina reranker:', error);\r\n      // Fallback to default ranking on error\r\n      return this.getDefaultRanking(documents, topK);\r\n    }\r\n  }\r\n}\r\n\r\nexport class CohereReranker extends BaseReranker {\r\n  constructor({\r\n    apiKey = process.env.COHERE_API_KEY,\r\n    logger,\r\n  }: {\r\n    apiKey?: string;\r\n    logger?: t.Logger;\r\n  }) {\r\n    super(logger);\r\n    this.apiKey = apiKey;\r\n  }\r\n\r\n  async rerank(\r\n    query: string,\r\n    documents: string[],\r\n    topK: number = 5\r\n  ): Promise<t.Highlight[]> {\r\n    this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\r\n\r\n    try {\r\n      if (this.apiKey == null || this.apiKey === '') {\r\n        this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\r\n        return this.getDefaultRanking(documents, topK);\r\n      }\r\n\r\n      const requestData = {\r\n        model: 'rerank-v3.5',\r\n        query: query,\r\n        top_n: topK,\r\n        documents: documents,\r\n      };\r\n\r\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\r\n        'https://api.cohere.com/v2/rerank',\r\n        requestData,\r\n        {\r\n          headers: {\r\n            'Content-Type': 'application/json',\r\n            Authorization: `Bearer ${this.apiKey}`,\r\n          },\r\n        }\r\n      );\r\n\r\n      this.logger.debug('Cohere API ID:', response.data?.id);\r\n      this.logger.debug('Cohere API Meta:', response.data?.meta);\r\n\r\n      if (response.data && response.data.results.length) {\r\n        return response.data.results.map((result) => {\r\n          const docIndex = result.index;\r\n          const score = result.relevance_score;\r\n          const text = documents[docIndex];\r\n          return { text, score };\r\n        });\r\n      } else {\r\n        this.logger.warn(\r\n          'Unexpected response format from Cohere API. Using default ranking.'\r\n        );\r\n        return this.getDefaultRanking(documents, topK);\r\n      }\r\n    } catch (error) {\r\n      this.logger.error('Error using Cohere reranker:', error);\r\n      // Fallback to default ranking on error\r\n      return this.getDefaultRanking(documents, topK);\r\n    }\r\n  }\r\n}\r\n\r\nexport class InfinityReranker extends BaseReranker {\r\n  constructor(logger?: t.Logger) {\r\n    super(logger);\r\n    // No API key needed for the placeholder implementation\r\n  }\r\n\r\n  async rerank(\r\n    query: string,\r\n    documents: string[],\r\n    topK: number = 5\r\n  ): Promise<t.Highlight[]> {\r\n    this.logger.debug(\r\n      `Reranking ${documents.length} chunks with Infinity (placeholder)`\r\n    );\r\n    // This would be replaced with actual Infinity reranker implementation\r\n    return this.getDefaultRanking(documents, topK);\r\n  }\r\n}\r\n\r\n/**\r\n * Creates the appropriate reranker based on type and configuration\r\n */\r\nexport const createReranker = (config: {\r\n  rerankerType: t.RerankerType;\r\n  jinaApiKey?: string;\r\n  jinaApiUrl?: string;\r\n  cohereApiKey?: string;\r\n  logger?: t.Logger;\r\n}): BaseReranker | undefined => {\r\n  const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, logger } = config;\r\n\r\n  // Create a default logger if none is provided\r\n  const defaultLogger = logger || createDefaultLogger();\r\n\r\n  switch (rerankerType.toLowerCase()) {\r\n  case 'jina':\r\n    return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\r\n  case 'cohere':\r\n    return new CohereReranker({\r\n      apiKey: cohereApiKey,\r\n      logger: defaultLogger,\r\n    });\r\n  case 'infinity':\r\n    return new InfinityReranker(defaultLogger);\r\n  case 'none':\r\n    defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\r\n    return undefined;\r\n  default:\r\n    defaultLogger.warn(\r\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\r\n    );\r\n    return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\r\n  }\r\n};\r\n\r\n// Example usage:\r\n// const jinaReranker = new JinaReranker();\r\n// const cohereReranker = new CohereReranker();\r\n// const infinityReranker = new InfinityReranker();\r\n"],"names":["createDefaultLogger"],"mappings":";;;;;MAIsB,YAAY,CAAA;AACtB,IAAA,MAAM;AACN,IAAA,MAAM;AAEhB,IAAA,WAAA,CAAY,MAAiB,EAAA;;AAE3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,IAAIA,yBAAmB,EAAE;;IASrC,iBAAiB,CACzB,SAAmB,EACnB,IAAY,EAAA;AAEZ,QAAA,OAAO;AACJ,aAAA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC;AACzC,aAAA,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;;AAE7C;AAEK,MAAO,YAAa,SAAQ,YAAY,CAAA;AACpC,IAAA,MAAM;IAEd,WAAY,CAAA,EACV,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EACjC,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,+BAA+B,EACpE,MAAM,GAKP,EAAA;QACC,KAAK,CAAC,MAAM,CAAC;AACb,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;AACpB,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;AAEhB,QAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,oCAAoC,IAAI,CAAC,MAAM,CAAA,CAAE,CAAC;AAEjG,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC;gBACnE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,oCAAoC;AAC3C,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;AACpB,gBAAA,gBAAgB,EAAE,IAAI;aACvB;AAED,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;AAED,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AAC1D,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AAE1D,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;oBACpC,IAAI,IAAI,GAAG,EAAE;;AAGb,oBAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,EAAE;AAC3B,wBAAA,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ;wBAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,EAAE;AAC5C,4BAAA,IAAI,GAAG,GAAG,CAAC,IAAI;;AACV,6BAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;4BAClC,IAAI,GAAG,GAAG;;;yBAEP;;AAEL,wBAAA,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;;AAG5B,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,kEAAkE,CACnE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC;;YAEtD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,cAAe,SAAQ,YAAY,CAAA;IAC9C,WAAY,CAAA,EACV,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EACnC,MAAM,GAIP,EAAA;QACC,KAAK,CAAC,MAAM,CAAC;AACb,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,CAAqB,mBAAA,CAAA,CAAC;AAErE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mDAAmD,CAAC;gBACrE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,aAAa;AACpB,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,kCAAkC,EAClC,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;AAED,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;AACtD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;AAE1D,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;AACpC,oBAAA,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;AAChC,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,oEAAoE,CACrE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;;YAExD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,gBAAiB,SAAQ,YAAY,CAAA;AAChD,IAAA,WAAA,CAAY,MAAiB,EAAA;QAC3B,KAAK,CAAC,MAAM,CAAC;;;IAIf,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,CAAqC,mCAAA,CAAA,CACnE;;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAEjD;AAED;;AAEG;AACU,MAAA,cAAc,GAAG,CAAC,MAM9B,KAA8B;AAC7B,IAAA,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM;;AAG7E,IAAA,MAAM,aAAa,GAAG,MAAM,IAAIA,yBAAmB,EAAE;AAErD,IAAA,QAAQ,YAAY,CAAC,WAAW,EAAE;AAClC,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;AAC5F,QAAA,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC;AACxB,gBAAA,MAAM,EAAE,YAAY;AACpB,gBAAA,MAAM,EAAE,aAAa;AACtB,aAAA,CAAC;AACJ,QAAA,KAAK,UAAU;AACb,YAAA,OAAO,IAAI,gBAAgB,CAAC,aAAa,CAAC;AAC5C,QAAA,KAAK,MAAM;AACT,YAAA,aAAa,CAAC,KAAK,CAAC,iDAAiD,CAAC;AACtE,YAAA,OAAO,SAAS;AAClB,QAAA;AACE,YAAA,aAAa,CAAC,IAAI,CAChB,0BAA0B,YAAY,CAAA,iCAAA,CAAmC,CAC1E;AACD,YAAA,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;;AAE9F;AAEA;AACA;AACA;AACA;;;;;;;;"}
+{"version":3,"file":"rerankers.cjs","sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n  protected logger: t.Logger;\n\n  constructor(logger?: t.Logger) {\n    // Each specific reranker will set its API key\n    this.logger = logger || createDefaultLogger();\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  private apiUrl: string;\n\n  constructor({\n    apiKey = process.env.JINA_API_KEY,\n    apiUrl = process.env.JINA_API_URL || 'https://api.jina.ai/v1/rerank',\n    logger,\n  }: {\n    apiKey?: string;\n    apiUrl?: string;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.apiUrl = apiUrl;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        this.apiUrl,\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      this.logger.debug('Jina API Model:', response.data?.model);\n      this.logger.debug('Jina API Usage:', response.data?.usage);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Jina reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  constructor({\n    apiKey = process.env.COHERE_API_KEY,\n    logger,\n  }: {\n    apiKey?: string;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      this.logger.debug('Cohere API ID:', response.data?.id);\n      this.logger.debug('Cohere API Meta:', response.data?.meta);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Cohere reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor(logger?: t.Logger) {\n    super(logger);\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n  rerankerType: t.RerankerType;\n  jinaApiKey?: string;\n  jinaApiUrl?: string;\n  cohereApiKey?: string;\n  logger?: t.Logger;\n}): BaseReranker | undefined => {\n  const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, logger } = config;\n\n  // Create a default logger if none is provided\n  const defaultLogger = logger || createDefaultLogger();\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\n  case 'cohere':\n    return new CohereReranker({\n      apiKey: cohereApiKey,\n      logger: defaultLogger,\n    });\n  case 'infinity':\n    return new InfinityReranker(defaultLogger);\n  case 'none':\n    defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    defaultLogger.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"names":["createDefaultLogger"],"mappings":";;;;;MAIsB,YAAY,CAAA;AACtB,IAAA,MAAM;AACN,IAAA,MAAM;AAEhB,IAAA,WAAA,CAAY,MAAiB,EAAA;;AAE3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,IAAIA,yBAAmB,EAAE;;IASrC,iBAAiB,CACzB,SAAmB,EACnB,IAAY,EAAA;AAEZ,QAAA,OAAO;AACJ,aAAA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC;AACzC,aAAA,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;;AAE7C;AAEK,MAAO,YAAa,SAAQ,YAAY,CAAA;AACpC,IAAA,MAAM;IAEd,WAAY,CAAA,EACV,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EACjC,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,+BAA+B,EACpE,MAAM,GAKP,EAAA;QACC,KAAK,CAAC,MAAM,CAAC;AACb,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;AACpB,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;AAEhB,QAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,oCAAoC,IAAI,CAAC,MAAM,CAAA,CAAE,CAAC;AAEjG,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC;gBACnE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,oCAAoC;AAC3C,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;AACpB,gBAAA,gBAAgB,EAAE,IAAI;aACvB;AAED,YAAA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,MAAM,EACX,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;AAED,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AAC1D,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AAE1D,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;oBACpC,IAAI,IAAI,GAAG,EAAE;;AAGb,oBAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,EAAE;AAC3B,wBAAA,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ;wBAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,EAAE;AAC5C,4BAAA,IAAI,GAAG,GAAG,CAAC,IAAI;;AACV,6BAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;4BAClC,IAAI,GAAG,GAAG;;;yBAEP;;AAEL,wBAAA,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;;AAG5B,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,kEAAkE,CACnE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC;;YAEtD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,cAAe,SAAQ,YAAY,CAAA;IAC9C,WAAY,CAAA,EACV,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EACnC,MAAM,GAIP,EAAA;QACC,KAAK,CAAC,MAAM,CAAC;AACb,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,CAAqB,mBAAA,CAAA,CAAC;AAErE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mDAAmD,CAAC;gBACrE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,aAAa;AACpB,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,kCAAkC,EAClC,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;AAED,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;AACtD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;AAE1D,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;AACpC,oBAAA,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;AAChC,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,oEAAoE,CACrE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;;YAExD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,gBAAiB,SAAQ,YAAY,CAAA;AAChD,IAAA,WAAA,CAAY,MAAiB,EAAA;QAC3B,KAAK,CAAC,MAAM,CAAC;;;IAIf,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,CAAqC,mCAAA,CAAA,CACnE;;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAEjD;AAED;;AAEG;AACU,MAAA,cAAc,GAAG,CAAC,MAM9B,KAA8B;AAC7B,IAAA,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM;;AAG7E,IAAA,MAAM,aAAa,GAAG,MAAM,IAAIA,yBAAmB,EAAE;AAErD,IAAA,QAAQ,YAAY,CAAC,WAAW,EAAE;AAClC,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;AAC5F,QAAA,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC;AACxB,gBAAA,MAAM,EAAE,YAAY;AACpB,gBAAA,MAAM,EAAE,aAAa;AACtB,aAAA,CAAC;AACJ,QAAA,KAAK,UAAU;AACb,YAAA,OAAO,IAAI,gBAAgB,CAAC,aAAa,CAAC;AAC5C,QAAA,KAAK,MAAM;AACT,YAAA,aAAa,CAAC,KAAK,CAAC,iDAAiD,CAAC;AACtE,YAAA,OAAO,SAAS;AAClB,QAAA;AACE,YAAA,aAAa,CAAC,IAAI,CAChB,0BAA0B,YAAY,CAAA,iCAAA,CAAmC,CAC1E;AACD,YAAA,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;;AAE9F;AAEA;AACA;AACA;AACA;;;;;;;;"}

package/dist/cjs/tools/search/schema.cjs

@@ -10,32 +10,32 @@ exports.DATE_RANGE = void 0;
     DATE_RANGE["PAST_MONTH"] = "m";
     DATE_RANGE["PAST_YEAR"] = "y";
 })(exports.DATE_RANGE || (exports.DATE_RANGE = {}));
-const DEFAULT_QUERY_DESCRIPTION = `
-GUIDELINES:
-- Start broad, then narrow: Begin with key concepts, then refine with specifics
-- Think like sources: Use terminology experts would use in the field
-- Consider perspective: Frame queries from different viewpoints for better results
-- Quality over quantity: A precise 3-4 word query often beats lengthy sentences
-
-TECHNIQUES (combine for power searches):
-- EXACT PHRASES: Use quotes ("climate change report")
-- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)
-- SITE-SPECIFIC: Restrict to websites (site:edu research)
-- FILETYPE: Find specific documents (filetype:pdf study)
-- OR OPERATOR: Find alternatives (electric OR hybrid cars)
-- DATE RANGE: Recent information (data after:2020)
-- WILDCARDS: Use * for unknown terms (how to * bread)
-- SPECIFIC QUESTIONS: Use who/what/when/where/why/how
-- DOMAIN TERMS: Include technical terminology for specialized topics
-- CONCISE TERMS: Prioritize keywords over sentences
+const DEFAULT_QUERY_DESCRIPTION = `
+GUIDELINES:
+- Start broad, then narrow: Begin with key concepts, then refine with specifics
+- Think like sources: Use terminology experts would use in the field
+- Consider perspective: Frame queries from different viewpoints for better results
+- Quality over quantity: A precise 3-4 word query often beats lengthy sentences
+
+TECHNIQUES (combine for power searches):
+- EXACT PHRASES: Use quotes ("climate change report")
+- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)
+- SITE-SPECIFIC: Restrict to websites (site:edu research)
+- FILETYPE: Find specific documents (filetype:pdf study)
+- OR OPERATOR: Find alternatives (electric OR hybrid cars)
+- DATE RANGE: Recent information (data after:2020)
+- WILDCARDS: Use * for unknown terms (how to * bread)
+- SPECIFIC QUESTIONS: Use who/what/when/where/why/how
+- DOMAIN TERMS: Include technical terminology for specialized topics
+- CONCISE TERMS: Prioritize keywords over sentences
 `.trim();
-const DEFAULT_COUNTRY_DESCRIPTION = `Country code to localize search results.
-Use standard 2-letter country codes: "us", "uk", "ca", "de", "fr", "jp", "br", etc.
-Provide this when the search should return results specific to a particular country.
-Examples:
-- "us" for United States (default)
-- "de" for Germany
-- "in" for India
+const DEFAULT_COUNTRY_DESCRIPTION = `Country code to localize search results.
+Use standard 2-letter country codes: "us", "uk", "ca", "de", "fr", "jp", "br", etc.
+Provide this when the search should return results specific to a particular country.
+Examples:
+- "us" for United States (default)
+- "de" for Germany
+- "in" for India
 `.trim();
 const querySchema = zod.z.string().describe(DEFAULT_QUERY_DESCRIPTION);
 const dateSchema = zod.z

package/dist/cjs/tools/search/schema.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"schema.cjs","sources":["../../../../src/tools/search/schema.ts"],"sourcesContent":["import { z } from 'zod';\r\n\r\nexport enum DATE_RANGE {\r\n  PAST_HOUR = 'h',\r\n  PAST_24_HOURS = 'd',\r\n  PAST_WEEK = 'w',\r\n  PAST_MONTH = 'm',\r\n  PAST_YEAR = 'y',\r\n}\r\n\r\nexport const DEFAULT_QUERY_DESCRIPTION = `\r\nGUIDELINES:\r\n- Start broad, then narrow: Begin with key concepts, then refine with specifics\r\n- Think like sources: Use terminology experts would use in the field\r\n- Consider perspective: Frame queries from different viewpoints for better results\r\n- Quality over quantity: A precise 3-4 word query often beats lengthy sentences\r\n\r\nTECHNIQUES (combine for power searches):\r\n- EXACT PHRASES: Use quotes (\"climate change report\")\r\n- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)\r\n- SITE-SPECIFIC: Restrict to websites (site:edu research)\r\n- FILETYPE: Find specific documents (filetype:pdf study)\r\n- OR OPERATOR: Find alternatives (electric OR hybrid cars)\r\n- DATE RANGE: Recent information (data after:2020)\r\n- WILDCARDS: Use * for unknown terms (how to * bread)\r\n- SPECIFIC QUESTIONS: Use who/what/when/where/why/how\r\n- DOMAIN TERMS: Include technical terminology for specialized topics\r\n- CONCISE TERMS: Prioritize keywords over sentences\r\n`.trim();\r\n\r\nexport const DEFAULT_COUNTRY_DESCRIPTION =\r\n  `Country code to localize search results.\r\nUse standard 2-letter country codes: \"us\", \"uk\", \"ca\", \"de\", \"fr\", \"jp\", \"br\", etc.\r\nProvide this when the search should return results specific to a particular country.\r\nExamples:\r\n- \"us\" for United States (default)\r\n- \"de\" for Germany\r\n- \"in\" for India\r\n`.trim();\r\n\r\nexport const querySchema = z.string().describe(DEFAULT_QUERY_DESCRIPTION);\r\nexport const dateSchema = z\r\n  .nativeEnum(DATE_RANGE)\r\n  .optional()\r\n  .describe('Date range for search results.');\r\nexport const countrySchema = z\r\n  .string()\r\n  .optional()\r\n  .describe(DEFAULT_COUNTRY_DESCRIPTION);\r\nexport const imagesSchema = z\r\n  .boolean()\r\n  .optional()\r\n  .describe('Whether to also run an image search.');\r\n\r\nexport const videosSchema = z\r\n  .boolean()\r\n  .optional()\r\n  .describe('Whether to also run a video search.');\r\n\r\nexport const newsSchema = z\r\n  .boolean()\r\n  .optional()\r\n  .describe('Whether to also run a news search.');\r\n"],"names":["DATE_RANGE","z"],"mappings":";;;;AAEYA;AAAZ,CAAA,UAAY,UAAU,EAAA;AACpB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACf,IAAA,UAAA,CAAA,eAAA,CAAA,GAAA,GAAmB;AACnB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACf,IAAA,UAAA,CAAA,YAAA,CAAA,GAAA,GAAgB;AAChB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACjB,CAAC,EANWA,kBAAU,KAAVA,kBAAU,GAMrB,EAAA,CAAA,CAAA;AAEY,MAAA,yBAAyB,GAAG,CAAA;;;;;;;;;;;;;;;;;;CAkBxC,CAAC,IAAI;AAEO,MAAA,2BAA2B,GACtC,CAAA;;;;;;;CAOD,CAAC,IAAI;AAEC,MAAM,WAAW,GAAGC,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yBAAyB;AACjE,MAAM,UAAU,GAAGA;KACvB,UAAU,CAACD,kBAAU;AACrB,KAAA,QAAQ;KACR,QAAQ,CAAC,gCAAgC;AACrC,MAAM,aAAa,GAAGC;AAC1B,KAAA,MAAM;AACN,KAAA,QAAQ;KACR,QAAQ,CAAC,2BAA2B;AAChC,MAAM,YAAY,GAAGA;AACzB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,sCAAsC;AAE3C,MAAM,YAAY,GAAGA;AACzB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,qCAAqC;AAE1C,MAAM,UAAU,GAAGA;AACvB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,oCAAoC;;;;;;;;;;;"}
+{"version":3,"file":"schema.cjs","sources":["../../../../src/tools/search/schema.ts"],"sourcesContent":["import { z } from 'zod';\n\nexport enum DATE_RANGE {\n  PAST_HOUR = 'h',\n  PAST_24_HOURS = 'd',\n  PAST_WEEK = 'w',\n  PAST_MONTH = 'm',\n  PAST_YEAR = 'y',\n}\n\nexport const DEFAULT_QUERY_DESCRIPTION = `\nGUIDELINES:\n- Start broad, then narrow: Begin with key concepts, then refine with specifics\n- Think like sources: Use terminology experts would use in the field\n- Consider perspective: Frame queries from different viewpoints for better results\n- Quality over quantity: A precise 3-4 word query often beats lengthy sentences\n\nTECHNIQUES (combine for power searches):\n- EXACT PHRASES: Use quotes (\"climate change report\")\n- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)\n- SITE-SPECIFIC: Restrict to websites (site:edu research)\n- FILETYPE: Find specific documents (filetype:pdf study)\n- OR OPERATOR: Find alternatives (electric OR hybrid cars)\n- DATE RANGE: Recent information (data after:2020)\n- WILDCARDS: Use * for unknown terms (how to * bread)\n- SPECIFIC QUESTIONS: Use who/what/when/where/why/how\n- DOMAIN TERMS: Include technical terminology for specialized topics\n- CONCISE TERMS: Prioritize keywords over sentences\n`.trim();\n\nexport const DEFAULT_COUNTRY_DESCRIPTION =\n  `Country code to localize search results.\nUse standard 2-letter country codes: \"us\", \"uk\", \"ca\", \"de\", \"fr\", \"jp\", \"br\", etc.\nProvide this when the search should return results specific to a particular country.\nExamples:\n- \"us\" for United States (default)\n- \"de\" for Germany\n- \"in\" for India\n`.trim();\n\nexport const querySchema = z.string().describe(DEFAULT_QUERY_DESCRIPTION);\nexport const dateSchema = z\n  .nativeEnum(DATE_RANGE)\n  .optional()\n  .describe('Date range for search results.');\nexport const countrySchema = z\n  .string()\n  .optional()\n  .describe(DEFAULT_COUNTRY_DESCRIPTION);\nexport const imagesSchema = z\n  .boolean()\n  .optional()\n  .describe('Whether to also run an image search.');\n\nexport const videosSchema = z\n  .boolean()\n  .optional()\n  .describe('Whether to also run a video search.');\n\nexport const newsSchema = z\n  .boolean()\n  .optional()\n  .describe('Whether to also run a news search.');\n"],"names":["DATE_RANGE","z"],"mappings":";;;;AAEYA;AAAZ,CAAA,UAAY,UAAU,EAAA;AACpB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACf,IAAA,UAAA,CAAA,eAAA,CAAA,GAAA,GAAmB;AACnB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACf,IAAA,UAAA,CAAA,YAAA,CAAA,GAAA,GAAgB;AAChB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACjB,CAAC,EANWA,kBAAU,KAAVA,kBAAU,GAMrB,EAAA,CAAA,CAAA;AAEY,MAAA,yBAAyB,GAAG;;;;;;;;;;;;;;;;;;CAkBxC,CAAC,IAAI;AAEO,MAAA,2BAA2B,GACtC,CAAA;;;;;;;CAOD,CAAC,IAAI;AAEC,MAAM,WAAW,GAAGC,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yBAAyB;AACjE,MAAM,UAAU,GAAGA;KACvB,UAAU,CAACD,kBAAU;AACrB,KAAA,QAAQ;KACR,QAAQ,CAAC,gCAAgC;AACrC,MAAM,aAAa,GAAGC;AAC1B,KAAA,MAAM;AACN,KAAA,QAAQ;KACR,QAAQ,CAAC,2BAA2B;AAChC,MAAM,YAAY,GAAGA;AACzB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,sCAAsC;AAE3C,MAAM,YAAY,GAAGA;AACzB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,qCAAqC;AAE1C,MAAM,UAAU,GAAGA;AACvB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,oCAAoC;;;;;;;;;;;"}

package/dist/cjs/tools/search/search.cjs

@@ -426,7 +426,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
             });
         }
     };
-    const processSources = async ({ result, numElements, query, news, proMode = true, onGetHighlights, }) => {
+    const processSources = async ({ result, numElements, query, news, proMode = true, onGetHighlights, skipScraping = false, }) => {
         try {
             if (!result.data) {
                 return {
@@ -439,6 +439,11 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
             else if (!result.data.organic) {
                 return result.data;
             }
+            // If content was already extracted directly (e.g., direct URL extraction), skip scraping
+            if (skipScraping) {
+                logger_.debug('Skipping additional scraping - content already extracted');
+                return result.data;
+            }
             if (!proMode) {
                 const wikiSources = result.data.organic.filter((source) => source.link.includes('wikipedia.org'));
                 if (!wikiSources.length) {

package/dist/cjs/tools/search/search.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"search.cjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\r\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\r\nimport type * as t from './types';\r\nimport { getAttribution, createDefaultLogger } from './utils';\r\nimport { BaseReranker } from './rerankers';\r\n\r\nconst chunker = {\r\n  cleanText: (text: string): string => {\r\n    if (!text) return '';\r\n\r\n    /** Normalized all line endings to '\\n' */\r\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\r\n\r\n    /** Handle multiple backslashes followed by newlines\r\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\r\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\r\n\r\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\r\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\r\n\r\n    /** Cleaned up excessive spaces and tabs */\r\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\r\n\r\n    return cleanedSpaces.trim();\r\n  },\r\n  splitText: async (\r\n    text: string,\r\n    options?: {\r\n      chunkSize?: number;\r\n      chunkOverlap?: number;\r\n      separators?: string[];\r\n    }\r\n  ): Promise<string[]> => {\r\n    const chunkSize = options?.chunkSize ?? 150;\r\n    const chunkOverlap = options?.chunkOverlap ?? 50;\r\n    const separators = options?.separators || ['\\n\\n', '\\n'];\r\n\r\n    const splitter = new RecursiveCharacterTextSplitter({\r\n      separators,\r\n      chunkSize,\r\n      chunkOverlap,\r\n    });\r\n\r\n    return await splitter.splitText(text);\r\n  },\r\n\r\n  splitTexts: async (\r\n    texts: string[],\r\n    options?: {\r\n      chunkSize?: number;\r\n      chunkOverlap?: number;\r\n      separators?: string[];\r\n    },\r\n    logger?: t.Logger\r\n  ): Promise<string[][]> => {\r\n    // Split multiple texts\r\n    const logger_ = logger || createDefaultLogger();\r\n    const promises = texts.map((text) =>\r\n      chunker.splitText(text, options).catch((error) => {\r\n        logger_.error('Error splitting text:', error);\r\n        return [text];\r\n      })\r\n    );\r\n    return Promise.all(promises);\r\n  },\r\n};\r\n\r\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\r\n  return (link: string, update?: Partial<t.ValidSource>): void => {\r\n    const source = sourceMap.get(link);\r\n    if (source) {\r\n      sourceMap.set(link, {\r\n        ...source,\r\n        ...update,\r\n      });\r\n    }\r\n  };\r\n}\r\n\r\nconst getHighlights = async ({\r\n  query,\r\n  content,\r\n  reranker,\r\n  topResults = 5,\r\n  logger,\r\n}: {\r\n  content: string;\r\n  query: string;\r\n  reranker?: BaseReranker;\r\n  topResults?: number;\r\n  logger?: t.Logger;\r\n}): Promise<t.Highlight[] | undefined> => {\r\n  const logger_ = logger || createDefaultLogger();\r\n\r\n  if (!content) {\r\n    logger_.warn('No content provided for highlights');\r\n    return;\r\n  }\r\n  if (!reranker) {\r\n    logger_.warn('No reranker provided for highlights');\r\n    return;\r\n  }\r\n\r\n  try {\r\n    const documents = await chunker.splitText(content);\r\n    if (Array.isArray(documents)) {\r\n      return await reranker.rerank(query, documents, topResults);\r\n    } else {\r\n      logger_.error(\r\n        'Expected documents to be an array, got:',\r\n        typeof documents\r\n      );\r\n      return;\r\n    }\r\n  } catch (error) {\r\n    logger_.error('Error in content processing:', error);\r\n    return;\r\n  }\r\n};\r\n\r\nconst createSerperAPI = (\r\n  apiKey?: string\r\n): {\r\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\r\n} => {\r\n  const config = {\r\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\r\n    apiUrl: 'https://google.serper.dev/search',\r\n    timeout: 10000,\r\n  };\r\n\r\n  if (config.apiKey == null || config.apiKey === '') {\r\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\r\n  }\r\n\r\n  const getSources = async ({\r\n    query,\r\n    date,\r\n    country,\r\n    safeSearch,\r\n    numResults = 8,\r\n    type,\r\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\r\n    if (!query.trim()) {\r\n      return { success: false, error: 'Query cannot be empty' };\r\n    }\r\n\r\n    try {\r\n      const safe = ['off', 'moderate', 'active'] as const;\r\n      const payload: t.SerperSearchPayload = {\r\n        q: query,\r\n        safe: safe[safeSearch ?? 1],\r\n        num: Math.min(Math.max(1, numResults), 10),\r\n      };\r\n\r\n      // Set the search type if provided\r\n      if (type) {\r\n        payload.type = type;\r\n      }\r\n\r\n      if (date != null) {\r\n        payload.tbs = `qdr:${date}`;\r\n      }\r\n\r\n      if (country != null && country !== '') {\r\n        payload['gl'] = country.toLowerCase();\r\n      }\r\n\r\n      // Determine the API endpoint based on the search type\r\n      let apiEndpoint = config.apiUrl;\r\n      if (type === 'images') {\r\n        apiEndpoint = 'https://google.serper.dev/images';\r\n      } else if (type === 'videos') {\r\n        apiEndpoint = 'https://google.serper.dev/videos';\r\n      } else if (type === 'news') {\r\n        apiEndpoint = 'https://google.serper.dev/news';\r\n      }\r\n\r\n      const response = await axios.post<t.SerperResultData>(\r\n        apiEndpoint,\r\n        payload,\r\n        {\r\n          headers: {\r\n            'X-API-KEY': config.apiKey,\r\n            'Content-Type': 'application/json',\r\n          },\r\n          timeout: config.timeout,\r\n        }\r\n      );\r\n\r\n      const data = response.data;\r\n      const results: t.SearchResultData = {\r\n        organic: data.organic,\r\n        images: data.images ?? [],\r\n        answerBox: data.answerBox,\r\n        topStories: data.topStories ?? [],\r\n        peopleAlsoAsk: data.peopleAlsoAsk,\r\n        knowledgeGraph: data.knowledgeGraph,\r\n        relatedSearches: data.relatedSearches,\r\n        videos: data.videos ?? [],\r\n        news: data.news ?? [],\r\n      };\r\n\r\n      return { success: true, data: results };\r\n    } catch (error) {\r\n      const errorMessage =\r\n        error instanceof Error ? error.message : String(error);\r\n      return { success: false, error: `API request failed: ${errorMessage}` };\r\n    }\r\n  };\r\n\r\n  return { getSources };\r\n};\r\n\r\nconst createSearXNGAPI = (\r\n  instanceUrl?: string,\r\n  apiKey?: string\r\n): {\r\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\r\n} => {\r\n  const config = {\r\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\r\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\r\n    defaultLocation: 'all',\r\n    timeout: 10000,\r\n  };\r\n\r\n  if (config.instanceUrl == null || config.instanceUrl === '') {\r\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\r\n  }\r\n\r\n  const getSources = async ({\r\n    query,\r\n    numResults = 8,\r\n    safeSearch,\r\n    type,\r\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\r\n    if (!query.trim()) {\r\n      return { success: false, error: 'Query cannot be empty' };\r\n    }\r\n\r\n    try {\r\n      // Ensure the instance URL ends with /search\r\n      if (config.instanceUrl == null || config.instanceUrl === '') {\r\n        return { success: false, error: 'Instance URL is not defined' };\r\n      }\r\n\r\n      let searchUrl = config.instanceUrl;\r\n      if (!searchUrl.endsWith('/search')) {\r\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\r\n      }\r\n\r\n      // Determine the search category based on the type\r\n      let category = 'general';\r\n      if (type === 'images') {\r\n        category = 'images';\r\n      } else if (type === 'videos') {\r\n        category = 'videos';\r\n      } else if (type === 'news') {\r\n        category = 'news';\r\n      }\r\n\r\n      // Prepare parameters for SearXNG\r\n      const params: t.SearxNGSearchPayload = {\r\n        q: query,\r\n        format: 'json',\r\n        pageno: 1,\r\n        categories: category,\r\n        language: 'all',\r\n        safesearch: safeSearch,\r\n        engines: 'google,bing,duckduckgo',\r\n      };\r\n\r\n      const headers: Record<string, string> = {\r\n        'Content-Type': 'application/json',\r\n      };\r\n\r\n      if (config.apiKey != null && config.apiKey !== '') {\r\n        headers['X-API-Key'] = config.apiKey;\r\n      }\r\n\r\n      const response = await axios.get(searchUrl, {\r\n        headers,\r\n        params,\r\n        timeout: config.timeout,\r\n      });\r\n\r\n      const data = response.data;\r\n\r\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\r\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\r\n        const url = result.url?.toLowerCase() ?? '';\r\n        const title = result.title?.toLowerCase() ?? '';\r\n\r\n        // News-related keywords in title/content\r\n        const newsKeywords = [\r\n          'breaking news',\r\n          'latest news',\r\n          'top stories',\r\n          'news today',\r\n          'developing story',\r\n          'trending news',\r\n          'news',\r\n        ];\r\n\r\n        // Check if title/content contains news keywords\r\n        const hasNewsKeywords = newsKeywords.some(\r\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\r\n        );\r\n\r\n        // Check if URL contains news-related paths\r\n        const hasNewsPath =\r\n          url.includes('/news/') ||\r\n          url.includes('/world/') ||\r\n          url.includes('/politics/') ||\r\n          url.includes('/breaking/');\r\n\r\n        return hasNewsKeywords || hasNewsPath;\r\n      };\r\n\r\n      // Transform SearXNG results to match SerperAPI format\r\n      const organicResults = (data.results ?? [])\r\n        .slice(0, numResults)\r\n        .map((result: t.SearXNGResult, index: number) => {\r\n          let attribution = '';\r\n          try {\r\n            attribution = new URL(result.url ?? '').hostname;\r\n          } catch {\r\n            attribution = '';\r\n          }\r\n\r\n          return {\r\n            position: index + 1,\r\n            title: result.title ?? '',\r\n            link: result.url ?? '',\r\n            snippet: result.content ?? '',\r\n            date: result.publishedDate ?? '',\r\n            attribution,\r\n          };\r\n        });\r\n\r\n      const imageResults = (data.results ?? [])\r\n        .filter((result: t.SearXNGResult) => result.img_src)\r\n        .slice(0, 6)\r\n        .map((result: t.SearXNGResult, index: number) => ({\r\n          title: result.title ?? '',\r\n          imageUrl: result.img_src ?? '',\r\n          position: index + 1,\r\n          source: new URL(result.url ?? '').hostname,\r\n          domain: new URL(result.url ?? '').hostname,\r\n          link: result.url ?? '',\r\n        }));\r\n\r\n      // Extract news results from organic results\r\n      const newsResults = (data.results ?? [])\r\n        .filter(isNewsResult)\r\n        .map((result: t.SearXNGResult, index: number) => {\r\n          let attribution = '';\r\n          try {\r\n            attribution = new URL(result.url ?? '').hostname;\r\n          } catch {\r\n            attribution = '';\r\n          }\r\n\r\n          return {\r\n            title: result.title ?? '',\r\n            link: result.url ?? '',\r\n            snippet: result.content ?? '',\r\n            date: result.publishedDate ?? '',\r\n            source: attribution,\r\n            imageUrl: result.img_src ?? '',\r\n            position: index + 1,\r\n          };\r\n        });\r\n\r\n      const topStories = newsResults.slice(0, 5);\r\n\r\n      const relatedSearches = Array.isArray(data.suggestions)\r\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\r\n        : [];\r\n\r\n      const results: t.SearchResultData = {\r\n        organic: organicResults,\r\n        images: imageResults,\r\n        topStories: topStories, // Use first 5 extracted news as top stories\r\n        relatedSearches,\r\n        videos: [],\r\n        news: newsResults,\r\n        // Add empty arrays for other Serper fields to maintain parity\r\n        places: [],\r\n        shopping: [],\r\n        peopleAlsoAsk: [],\r\n        knowledgeGraph: undefined,\r\n        answerBox: undefined,\r\n      };\r\n\r\n      return { success: true, data: results };\r\n    } catch (error) {\r\n      const errorMessage =\r\n        error instanceof Error ? error.message : String(error);\r\n      return {\r\n        success: false,\r\n        error: `SearXNG API request failed: ${errorMessage}`,\r\n      };\r\n    }\r\n  };\r\n\r\n  return { getSources };\r\n};\r\n\r\nexport const createSearchAPI = (\r\n  config: t.SearchConfig\r\n): {\r\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\r\n} => {\r\n  const {\r\n    searchProvider = 'serper',\r\n    serperApiKey,\r\n    searxngInstanceUrl,\r\n    searxngApiKey,\r\n  } = config;\r\n\r\n  if (searchProvider.toLowerCase() === 'serper') {\r\n    return createSerperAPI(serperApiKey);\r\n  } else if (searchProvider.toLowerCase() === 'searxng') {\r\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\r\n  } else {\r\n    throw new Error(\r\n      `Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`\r\n    );\r\n  }\r\n};\r\n\r\nexport const createSourceProcessor = (\r\n  config: t.ProcessSourcesConfig = {},\r\n  scraperInstance?: t.BaseScraper\r\n): {\r\n  processSources: (\r\n    fields: t.ProcessSourcesFields\r\n  ) => Promise<t.SearchResultData>;\r\n  topResults: number;\r\n} => {\r\n  if (!scraperInstance) {\r\n    throw new Error('Scraper instance is required');\r\n  }\r\n  const {\r\n    topResults = 5,\r\n    // strategies = ['no_extraction'],\r\n    // filterContent = true,\r\n    reranker,\r\n    logger,\r\n  } = config;\r\n\r\n  const logger_ = logger || createDefaultLogger();\r\n  const scraper = scraperInstance;\r\n\r\n  const webScraper = {\r\n    scrapeMany: async ({\r\n      query,\r\n      links,\r\n      onGetHighlights,\r\n    }: {\r\n      query: string;\r\n      links: string[];\r\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\r\n    }): Promise<Array<t.ScrapeResult>> => {\r\n      logger_.debug(`Scraping ${links.length} links`);\r\n      const promises: Array<Promise<t.ScrapeResult>> = [];\r\n      try {\r\n        for (let i = 0; i < links.length; i++) {\r\n          const currentLink = links[i];\r\n          const promise: Promise<t.ScrapeResult> = scraper\r\n            .scrapeUrl(currentLink, {})\r\n            .then(([url, response]) => {\r\n              const attribution = getAttribution(\r\n                url,\r\n                response.data?.metadata,\r\n                logger_\r\n              );\r\n              if (response.success && response.data) {\r\n                const [content, references] = scraper.extractContent(response);\r\n                return {\r\n                  url,\r\n                  references,\r\n                  attribution,\r\n                  content: chunker.cleanText(content),\r\n                } as t.ScrapeResult;\r\n              } else {\r\n                logger_.error(\r\n                  `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\r\n                );\r\n              }\r\n\r\n              return {\r\n                url,\r\n                attribution,\r\n                error: true,\r\n                content: '',\r\n              } as t.ScrapeResult;\r\n            })\r\n            .then(async (result) => {\r\n              try {\r\n                if (result.error != null) {\r\n                  logger_.error(\r\n                    `Error scraping ${result.url}: ${result.content}`\r\n                  );\r\n                  return {\r\n                    ...result,\r\n                  };\r\n                }\r\n                const highlights = await getHighlights({\r\n                  query,\r\n                  reranker,\r\n                  content: result.content,\r\n                  logger: logger_,\r\n                });\r\n                if (onGetHighlights) {\r\n                  onGetHighlights(result.url);\r\n                }\r\n                return {\r\n                  ...result,\r\n                  highlights,\r\n                };\r\n              } catch (error) {\r\n                logger_.error('Error processing scraped content:', error);\r\n                return {\r\n                  ...result,\r\n                };\r\n              }\r\n            })\r\n            .catch((error) => {\r\n              logger_.error(`Error scraping ${currentLink}:`, error);\r\n              return {\r\n                url: currentLink,\r\n                error: true,\r\n                content: '',\r\n              };\r\n            });\r\n          promises.push(promise);\r\n        }\r\n        return await Promise.all(promises);\r\n      } catch (error) {\r\n        logger_.error('Error in scrapeMany:', error);\r\n        return [];\r\n      }\r\n    },\r\n  };\r\n\r\n  const fetchContents = async ({\r\n    links,\r\n    query,\r\n    target,\r\n    onGetHighlights,\r\n    onContentScraped,\r\n  }: {\r\n    links: string[];\r\n    query: string;\r\n    target: number;\r\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\r\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\r\n  }): Promise<void> => {\r\n    const initialLinks = links.slice(0, target);\r\n    // const remainingLinks = links.slice(target).reverse();\r\n    const results = await webScraper.scrapeMany({\r\n      query,\r\n      links: initialLinks,\r\n      onGetHighlights,\r\n    });\r\n    for (const result of results) {\r\n      if (result.error === true) {\r\n        continue;\r\n      }\r\n      const { url, content, attribution, references, highlights } = result;\r\n      onContentScraped?.(url, {\r\n        content,\r\n        attribution,\r\n        references,\r\n        highlights,\r\n      });\r\n    }\r\n  };\r\n\r\n  const processSources = async ({\r\n    result,\r\n    numElements,\r\n    query,\r\n    news,\r\n    proMode = true,\r\n    onGetHighlights,\r\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\r\n    try {\r\n      if (!result.data) {\r\n        return {\r\n          organic: [],\r\n          topStories: [],\r\n          images: [],\r\n          relatedSearches: [],\r\n        };\r\n      } else if (!result.data.organic) {\r\n        return result.data;\r\n      }\r\n\r\n      if (!proMode) {\r\n        const wikiSources = result.data.organic.filter((source) =>\r\n          source.link.includes('wikipedia.org')\r\n        );\r\n\r\n        if (!wikiSources.length) {\r\n          return result.data;\r\n        }\r\n\r\n        const wikiSourceMap = new Map<string, t.ValidSource>();\r\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\r\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\r\n        await fetchContents({\r\n          query,\r\n          target: 1,\r\n          onGetHighlights,\r\n          onContentScraped,\r\n          links: [wikiSources[0].link],\r\n        });\r\n\r\n        for (let i = 0; i < result.data.organic.length; i++) {\r\n          const source = result.data.organic[i];\r\n          const updatedSource = wikiSourceMap.get(source.link);\r\n          if (updatedSource) {\r\n            result.data.organic[i] = {\r\n              ...source,\r\n              ...updatedSource,\r\n            };\r\n          }\r\n        }\r\n\r\n        return result.data;\r\n      }\r\n\r\n      const sourceMap = new Map<string, t.ValidSource>();\r\n      const organicLinksSet = new Set<string>();\r\n\r\n      // Collect organic links\r\n      const organicLinks = collectLinks(\r\n        result.data.organic,\r\n        sourceMap,\r\n        organicLinksSet\r\n      );\r\n\r\n      // Collect top story links, excluding any that are already in organic links\r\n      const topStories = result.data.topStories ?? [];\r\n      const topStoryLinks = collectLinks(\r\n        topStories,\r\n        sourceMap,\r\n        organicLinksSet\r\n      );\r\n\r\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\r\n        return result.data;\r\n      }\r\n\r\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\r\n      const promises: Promise<void>[] = [];\r\n\r\n      // Process organic links\r\n      if (organicLinks.length > 0) {\r\n        promises.push(\r\n          fetchContents({\r\n            query,\r\n            onGetHighlights,\r\n            onContentScraped,\r\n            links: organicLinks,\r\n            target: numElements,\r\n          })\r\n        );\r\n      }\r\n\r\n      // Process top story links\r\n      if (news && topStoryLinks.length > 0) {\r\n        promises.push(\r\n          fetchContents({\r\n            query,\r\n            onGetHighlights,\r\n            onContentScraped,\r\n            links: topStoryLinks,\r\n            target: numElements,\r\n          })\r\n        );\r\n      }\r\n\r\n      await Promise.all(promises);\r\n\r\n      if (result.data.organic.length > 0) {\r\n        updateSourcesWithContent(result.data.organic, sourceMap);\r\n      }\r\n\r\n      if (news && topStories.length > 0) {\r\n        updateSourcesWithContent(topStories, sourceMap);\r\n      }\r\n\r\n      return result.data;\r\n    } catch (error) {\r\n      logger_.error('Error in processSources:', error);\r\n      return {\r\n        organic: [],\r\n        topStories: [],\r\n        images: [],\r\n        relatedSearches: [],\r\n        ...result.data,\r\n        error: error instanceof Error ? error.message : String(error),\r\n      };\r\n    }\r\n  };\r\n\r\n  return {\r\n    processSources,\r\n    topResults,\r\n  };\r\n};\r\n\r\n/** Helper function to collect links and update sourceMap */\r\nfunction collectLinks(\r\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\r\n  sourceMap: Map<string, t.ValidSource>,\r\n  existingLinksSet?: Set<string>\r\n): string[] {\r\n  const links: string[] = [];\r\n\r\n  for (const source of sources) {\r\n    if (source.link) {\r\n      // For topStories, only add if not already in organic links\r\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\r\n        continue;\r\n      }\r\n\r\n      links.push(source.link);\r\n      if (existingLinksSet) {\r\n        existingLinksSet.add(source.link);\r\n      }\r\n      sourceMap.set(source.link, source as t.ValidSource);\r\n    }\r\n  }\r\n\r\n  return links;\r\n}\r\n\r\n/** Helper function to update sources with scraped content */\r\nfunction updateSourcesWithContent<T extends t.ValidSource>(\r\n  sources: T[],\r\n  sourceMap: Map<string, t.ValidSource>\r\n): void {\r\n  for (let i = 0; i < sources.length; i++) {\r\n    const source = sources[i];\r\n    const updatedSource = sourceMap.get(source.link);\r\n    if (updatedSource) {\r\n      sources[i] = {\r\n        ...source,\r\n        ...updatedSource,\r\n      } as T;\r\n    }\r\n  }\r\n}\r\n"],"names":["RecursiveCharacterTextSplitter","createDefaultLogger","getAttribution"],"mappings":";;;;;;AAMA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;KAC5B;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAIA,4CAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;KACtC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAIC,yBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;SACd,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;KAC7B;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;;AAEN,KAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;;IAEF,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;;AAGF,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;;aACrD;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;;;IAEF,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;;AAEJ,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;;IAG7D,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;AACnD,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;;AAGrB,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAO,IAAA,EAAA,IAAI,EAAE;;YAG7B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;;;AAIvC,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;;YAGhD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAuB,oBAAA,EAAA,YAAY,CAAE,CAAA,EAAE;;AAE3E,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QACA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;;AAGrE,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;;AAGjE,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;;;YAItD,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;;;AAInB,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;;YAGtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,aAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,aAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,aAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAA+B,4BAAA,EAAA,YAAY,CAAE,CAAA;aACrD;;AAEL,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEY,MAAA,eAAe,GAAG,CAC7B,MAAsB,KAGpB;AACF,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,GACd,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;AAC7C,QAAA,OAAO,eAAe,CAAC,YAAY,CAAC;;AAC/B,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;;SACrD;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,+BAAA,CAAiC,CAC5E;;AAEL;AAEa,MAAA,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;;IAEjD,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAQ,MAAA,CAAA,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB,wBAAA,MAAM,WAAW,GAAGC,oBAAc,CAChC,GAAG,EACH,QAAQ,CAAC,IAAI,EAAE,QAAQ,EACvB,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;;6BACd;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;;wBAGH,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,qBAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;;AAEH,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;;4BAE7B,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;;wBACD,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;;AAEL,qBAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAG,CAAA,CAAA,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,qBAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;;AAExB,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;;YAClC,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;;SAEZ;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;;AAEF,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;;AAEN,KAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,GACQ,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;;AACI,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;;YAGpB,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;;AAGpB,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;;;gBAIL,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;;YAIH,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;AAGH,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;YAG1D,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;;YAGjD,OAAO,MAAM,CAAC,IAAI;;QAClB,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;;AAGF,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;;YAEnC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;;;AAIvD,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;;;AAGZ;;;;;"}
+{"version":3,"file":"search.cjs","sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(content);\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n  } = config;\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      const promises: Array<Promise<t.ScrapeResult>> = [];\n      try {\n        for (let i = 0; i < links.length; i++) {\n          const currentLink = links[i];\n          const promise: Promise<t.ScrapeResult> = scraper\n            .scrapeUrl(currentLink, {})\n            .then(([url, response]) => {\n              const attribution = getAttribution(\n                url,\n                response.data?.metadata,\n                logger_\n              );\n              if (response.success && response.data) {\n                const [content, references] = scraper.extractContent(response);\n                return {\n                  url,\n                  references,\n                  attribution,\n                  content: chunker.cleanText(content),\n                } as t.ScrapeResult;\n              } else {\n                logger_.error(\n                  `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n                );\n              }\n\n              return {\n                url,\n                attribution,\n                error: true,\n                content: '',\n              } as t.ScrapeResult;\n            })\n            .then(async (result) => {\n              try {\n                if (result.error != null) {\n                  logger_.error(\n                    `Error scraping ${result.url}: ${result.content}`\n                  );\n                  return {\n                    ...result,\n                  };\n                }\n                const highlights = await getHighlights({\n                  query,\n                  reranker,\n                  content: result.content,\n                  logger: logger_,\n                });\n                if (onGetHighlights) {\n                  onGetHighlights(result.url);\n                }\n                return {\n                  ...result,\n                  highlights,\n                };\n              } catch (error) {\n                logger_.error('Error processing scraped content:', error);\n                return {\n                  ...result,\n                };\n              }\n            })\n            .catch((error) => {\n              logger_.error(`Error scraping ${currentLink}:`, error);\n              return {\n                url: currentLink,\n                error: true,\n                content: '',\n              };\n            });\n          promises.push(promise);\n        }\n        return await Promise.all(promises);\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n    skipScraping = false,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      } else if (!result.data.organic) {\n        return result.data;\n      }\n\n      // If content was already extracted directly (e.g., direct URL extraction), skip scraping\n      if (skipScraping) {\n        logger_.debug('Skipping additional scraping - content already extracted');\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"names":["RecursiveCharacterTextSplitter","createDefaultLogger","getAttribution"],"mappings":";;;;;;AAMA,MAAM,OAAO,GAAG;AACd,IAAA,SAAS,EAAE,CAAC,IAAY,KAAY;AAClC,QAAA,IAAI,CAAC,IAAI;AAAE,YAAA,OAAO,EAAE;;AAGpB,QAAA,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;AAEvE;AACiE;QACjE,MAAM,gBAAgB,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;;QAG/D,MAAM,eAAe,GAAG,gBAAgB,CAAC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;;QAG3E,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;AAE7D,QAAA,OAAO,aAAa,CAAC,IAAI,EAAE;KAC5B;AACD,IAAA,SAAS,EAAE,OACT,IAAY,EACZ,OAIC,KACoB;AACrB,QAAA,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG;AAC3C,QAAA,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE;QAChD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC;AAExD,QAAA,MAAM,QAAQ,GAAG,IAAIA,4CAA8B,CAAC;YAClD,UAAU;YACV,SAAS;YACT,YAAY;AACb,SAAA,CAAC;AAEF,QAAA,OAAO,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC;KACtC;IAED,UAAU,EAAE,OACV,KAAe,EACf,OAIC,EACD,MAAiB,KACM;;AAEvB,QAAA,MAAM,OAAO,GAAG,MAAM,IAAIC,yBAAmB,EAAE;QAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAI;AAC/C,YAAA,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC;SACd,CAAC,CACH;AACD,QAAA,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;KAC7B;CACF;AAED,SAAS,0BAA0B,CAAC,SAAqC,EAAA;AACvE,IAAA,OAAO,CAAC,IAAY,EAAE,MAA+B,KAAU;QAC7D,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QAClC,IAAI,MAAM,EAAE;AACV,YAAA,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;AAClB,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,MAAM;AACV,aAAA,CAAC;;AAEN,KAAC;AACH;AAEA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,OAAO,EACP,QAAQ,EACR,UAAU,GAAG,CAAC,EACd,MAAM,GAOP,KAAwC;AACvC,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAE/C,IAAI,CAAC,OAAO,EAAE;AACZ,QAAA,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC;QAClD;;IAEF,IAAI,CAAC,QAAQ,EAAE;AACb,QAAA,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC;QACnD;;AAGF,IAAA,IAAI;QACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;AAClD,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;YAC5B,OAAO,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC;;aACrD;YACL,OAAO,CAAC,KAAK,CACX,yCAAyC,EACzC,OAAO,SAAS,CACjB;YACD;;;IAEF,OAAO,KAAK,EAAE;AACd,QAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;QACpD;;AAEJ,CAAC;AAED,MAAM,eAAe,GAAG,CACtB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;AAC5C,QAAA,MAAM,EAAE,kCAAkC;AAC1C,QAAA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,QAAA,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC;;IAG7D,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,IAAI,EACJ,OAAO,EACP,UAAU,EACV,UAAU,GAAG,CAAC,EACd,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;YACF,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,CAAU;AACnD,YAAA,MAAM,OAAO,GAA0B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;AAC3B,gBAAA,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC3C;;YAGD,IAAI,IAAI,EAAE;AACR,gBAAA,OAAO,CAAC,IAAI,GAAG,IAAI;;AAGrB,YAAA,IAAI,IAAI,IAAI,IAAI,EAAE;AAChB,gBAAA,OAAO,CAAC,GAAG,GAAG,CAAO,IAAA,EAAA,IAAI,EAAE;;YAG7B,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE,EAAE;gBACrC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE;;;AAIvC,YAAA,IAAI,WAAW,GAAG,MAAM,CAAC,MAAM;AAC/B,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,WAAW,GAAG,kCAAkC;;AAC3C,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,WAAW,GAAG,gCAAgC;;YAGhD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,WAAW,EACX,OAAO,EACP;AACE,gBAAA,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM,CAAC,MAAM;AAC1B,oBAAA,cAAc,EAAE,kBAAkB;AACnC,iBAAA;gBACD,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CACF;AAED,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;AAC1B,YAAA,MAAM,OAAO,GAAuB;gBAClC,OAAO,EAAE,IAAI,CAAC,OAAO;AACrB,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;AACzB,gBAAA,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,eAAe,EAAE,IAAI,CAAC,eAAe;AACrC,gBAAA,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;AACzB,gBAAA,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAuB,oBAAA,EAAA,YAAY,CAAE,CAAA,EAAE;;AAE3E,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAED,MAAM,gBAAgB,GAAG,CACvB,WAAoB,EACpB,MAAe,KAGb;AACF,IAAA,MAAM,MAAM,GAAG;AACb,QAAA,WAAW,EAAE,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB;AAC5D,QAAA,MAAM,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe;AAC7C,QACA,OAAO,EAAE,KAAK;KACf;AAED,IAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;AAC3D,QAAA,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC;;AAGrE,IAAA,MAAM,UAAU,GAAG,OAAO,EACxB,KAAK,EACL,UAAU,GAAG,CAAC,EACd,UAAU,EACV,IAAI,GACe,KAA6B;AAChD,QAAA,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;YACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,EAAE;;AAG3D,QAAA,IAAI;;AAEF,YAAA,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI,IAAI,MAAM,CAAC,WAAW,KAAK,EAAE,EAAE;gBAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,EAAE;;AAGjE,YAAA,IAAI,SAAS,GAAG,MAAM,CAAC,WAAW;YAClC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;gBAClC,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,SAAS;;;YAItD,IAAI,QAAQ,GAAG,SAAS;AACxB,YAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBACrB,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,QAAQ,EAAE;gBAC5B,QAAQ,GAAG,QAAQ;;AACd,iBAAA,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1B,QAAQ,GAAG,MAAM;;;AAInB,YAAA,MAAM,MAAM,GAA2B;AACrC,gBAAA,CAAC,EAAE,KAAK;AACR,gBAAA,MAAM,EAAE,MAAM;AACd,gBAAA,MAAM,EAAE,CAAC;AACT,gBAAA,UAAU,EAAE,QAAQ;AACpB,gBAAA,QAAQ,EAAE,KAAK;AACf,gBAAA,UAAU,EAAE,UAAU;AACtB,gBAAA,OAAO,EAAE,wBAAwB;aAClC;AAED,YAAA,MAAM,OAAO,GAA2B;AACtC,gBAAA,cAAc,EAAE,kBAAkB;aACnC;AAED,YAAA,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,EAAE,EAAE;AACjD,gBAAA,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM;;YAGtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC1C,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,OAAO;AACxB,aAAA,CAAC;AAEF,YAAA,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI;;AAG1B,YAAA,MAAM,YAAY,GAAG,CAAC,MAAuB,KAAa;gBACxD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG/C,gBAAA,MAAM,YAAY,GAAG;oBACnB,eAAe;oBACf,aAAa;oBACb,aAAa;oBACb,YAAY;oBACZ,kBAAkB;oBAClB,eAAe;oBACf,MAAM;iBACP;;gBAGD,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI,CACvC,CAAC,OAAO,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;iBACnD;;AAGD,gBAAA,MAAM,WAAW,GACf,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;AACtB,oBAAA,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;AACvB,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;AAC1B,oBAAA,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAE5B,OAAO,eAAe,IAAI,WAAW;AACvC,aAAC;;YAGD,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;AACvC,iBAAA,KAAK,CAAC,CAAC,EAAE,UAAU;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;oBACL,QAAQ,EAAE,KAAK,GAAG,CAAC;AACnB,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;oBAChC,WAAW;iBACZ;AACH,aAAC,CAAC;YAEJ,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACrC,MAAM,CAAC,CAAC,MAAuB,KAAK,MAAM,CAAC,OAAO;AAClD,iBAAA,KAAK,CAAC,CAAC,EAAE,CAAC;iBACV,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,MAAM;AAChD,gBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,gBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;gBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;gBAC1C,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;AAC1C,gBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACvB,aAAA,CAAC,CAAC;;YAGL,MAAM,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE;iBACpC,MAAM,CAAC,YAAY;AACnB,iBAAA,GAAG,CAAC,CAAC,MAAuB,EAAE,KAAa,KAAI;gBAC9C,IAAI,WAAW,GAAG,EAAE;AACpB,gBAAA,IAAI;AACF,oBAAA,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,QAAQ;;AAChD,gBAAA,MAAM;oBACN,WAAW,GAAG,EAAE;;gBAGlB,OAAO;AACL,oBAAA,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;AACzB,oBAAA,IAAI,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;AACtB,oBAAA,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;AAC7B,oBAAA,IAAI,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;AAChC,oBAAA,MAAM,EAAE,WAAW;AACnB,oBAAA,QAAQ,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;oBAC9B,QAAQ,EAAE,KAAK,GAAG,CAAC;iBACpB;AACH,aAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAE1C,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW;AACpD,kBAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAkB,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;kBACpE,EAAE;AAEN,YAAA,MAAM,OAAO,GAAuB;AAClC,gBAAA,OAAO,EAAE,cAAc;AACvB,gBAAA,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,UAAU;gBACtB,eAAe;AACf,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,IAAI,EAAE,WAAW;;AAEjB,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,QAAQ,EAAE,EAAE;AACZ,gBAAA,aAAa,EAAE,EAAE;AACjB,gBAAA,cAAc,EAAE,SAAS;AACzB,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;;QACvC,OAAO,KAAK,EAAE;AACd,YAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;YACxD,OAAO;AACL,gBAAA,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,CAA+B,4BAAA,EAAA,YAAY,CAAE,CAAA;aACrD;;AAEL,KAAC;IAED,OAAO,EAAE,UAAU,EAAE;AACvB,CAAC;AAEY,MAAA,eAAe,GAAG,CAC7B,MAAsB,KAGpB;AACF,IAAA,MAAM,EACJ,cAAc,GAAG,QAAQ,EACzB,YAAY,EACZ,kBAAkB,EAClB,aAAa,GACd,GAAG,MAAM;AAEV,IAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,QAAQ,EAAE;AAC7C,QAAA,OAAO,eAAe,CAAC,YAAY,CAAC;;AAC/B,SAAA,IAAI,cAAc,CAAC,WAAW,EAAE,KAAK,SAAS,EAAE;AACrD,QAAA,OAAO,gBAAgB,CAAC,kBAAkB,EAAE,aAAa,CAAC;;SACrD;AACL,QAAA,MAAM,IAAI,KAAK,CACb,4BAA4B,cAAc,CAAA,+BAAA,CAAiC,CAC5E;;AAEL;AAEa,MAAA,qBAAqB,GAAG,CACnC,SAAiC,EAAE,EACnC,eAA+B,KAM7B;IACF,IAAI,CAAC,eAAe,EAAE;AACpB,QAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;;IAEjD,MAAM,EACJ,UAAU,GAAG,CAAC;;;AAGd,IAAA,QAAQ,EACR,MAAM,GACP,GAAG,MAAM;AAEV,IAAA,MAAM,OAAO,GAAG,MAAM,IAAIA,yBAAmB,EAAE;IAC/C,MAAM,OAAO,GAAG,eAAe;AAE/B,IAAA,MAAM,UAAU,GAAG;QACjB,UAAU,EAAE,OAAO,EACjB,KAAK,EACL,KAAK,EACL,eAAe,GAKhB,KAAoC;YACnC,OAAO,CAAC,KAAK,CAAC,CAAA,SAAA,EAAY,KAAK,CAAC,MAAM,CAAQ,MAAA,CAAA,CAAC;YAC/C,MAAM,QAAQ,GAAmC,EAAE;AACnD,YAAA,IAAI;AACF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACrC,oBAAA,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;oBAC5B,MAAM,OAAO,GAA4B;AACtC,yBAAA,SAAS,CAAC,WAAW,EAAE,EAAE;yBACzB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAI;AACxB,wBAAA,MAAM,WAAW,GAAGC,oBAAc,CAChC,GAAG,EACH,QAAQ,CAAC,IAAI,EAAE,QAAQ,EACvB,OAAO,CACR;wBACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE;AACrC,4BAAA,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC;4BAC9D,OAAO;gCACL,GAAG;gCACH,UAAU;gCACV,WAAW;AACX,gCAAA,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;6BAClB;;6BACd;AACL,4BAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,GAAG,CAAA,EAAA,EAAK,QAAQ,CAAC,KAAK,IAAI,eAAe,CAAA,CAAE,CAC9D;;wBAGH,OAAO;4BACL,GAAG;4BACH,WAAW;AACX,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACM;AACrB,qBAAC;AACA,yBAAA,IAAI,CAAC,OAAO,MAAM,KAAI;AACrB,wBAAA,IAAI;AACF,4BAAA,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE;AACxB,gCAAA,OAAO,CAAC,KAAK,CACX,CAAA,eAAA,EAAkB,MAAM,CAAC,GAAG,CAAA,EAAA,EAAK,MAAM,CAAC,OAAO,CAAA,CAAE,CAClD;gCACD,OAAO;AACL,oCAAA,GAAG,MAAM;iCACV;;AAEH,4BAAA,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC;gCACrC,KAAK;gCACL,QAAQ;gCACR,OAAO,EAAE,MAAM,CAAC,OAAO;AACvB,gCAAA,MAAM,EAAE,OAAO;AAChB,6BAAA,CAAC;4BACF,IAAI,eAAe,EAAE;AACnB,gCAAA,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC;;4BAE7B,OAAO;AACL,gCAAA,GAAG,MAAM;gCACT,UAAU;6BACX;;wBACD,OAAO,KAAK,EAAE;AACd,4BAAA,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC;4BACzD,OAAO;AACL,gCAAA,GAAG,MAAM;6BACV;;AAEL,qBAAC;AACA,yBAAA,KAAK,CAAC,CAAC,KAAK,KAAI;wBACf,OAAO,CAAC,KAAK,CAAC,CAAA,eAAA,EAAkB,WAAW,CAAG,CAAA,CAAA,EAAE,KAAK,CAAC;wBACtD,OAAO;AACL,4BAAA,GAAG,EAAE,WAAW;AAChB,4BAAA,KAAK,EAAE,IAAI;AACX,4BAAA,OAAO,EAAE,EAAE;yBACZ;AACH,qBAAC,CAAC;AACJ,oBAAA,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;;AAExB,gBAAA,OAAO,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;;YAClC,OAAO,KAAK,EAAE;AACd,gBAAA,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC;AAC5C,gBAAA,OAAO,EAAE;;SAEZ;KACF;AAED,IAAA,MAAM,aAAa,GAAG,OAAO,EAC3B,KAAK,EACL,KAAK,EACL,MAAM,EACN,eAAe,EACf,gBAAgB,GAOjB,KAAmB;QAClB,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;;AAE3C,QAAA,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC;YAC1C,KAAK;AACL,YAAA,KAAK,EAAE,YAAY;YACnB,eAAe;AAChB,SAAA,CAAC;AACF,QAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,YAAA,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBACzB;;AAEF,YAAA,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM;YACpE,gBAAgB,GAAG,GAAG,EAAE;gBACtB,OAAO;gBACP,WAAW;gBACX,UAAU;gBACV,UAAU;AACX,aAAA,CAAC;;AAEN,KAAC;IAED,MAAM,cAAc,GAAG,OAAO,EAC5B,MAAM,EACN,WAAW,EACX,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,IAAI,EACd,eAAe,EACf,YAAY,GAAG,KAAK,GACG,KAAiC;AACxD,QAAA,IAAI;AACF,YAAA,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBAChB,OAAO;AACL,oBAAA,OAAO,EAAE,EAAE;AACX,oBAAA,UAAU,EAAE,EAAE;AACd,oBAAA,MAAM,EAAE,EAAE;AACV,oBAAA,eAAe,EAAE,EAAE;iBACpB;;AACI,iBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC/B,OAAO,MAAM,CAAC,IAAI;;;YAIpB,IAAI,YAAY,EAAE;AAChB,gBAAA,OAAO,CAAC,KAAK,CAAC,0DAA0D,CAAC;gBACzE,OAAO,MAAM,CAAC,IAAI;;YAGpB,IAAI,CAAC,OAAO,EAAE;gBACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,KACpD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CACtC;AAED,gBAAA,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;oBACvB,OAAO,MAAM,CAAC,IAAI;;AAGpB,gBAAA,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB;AACtD,gBAAA,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;AACtD,gBAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,aAAa,CAAC;AAClE,gBAAA,MAAM,aAAa,CAAC;oBAClB,KAAK;AACL,oBAAA,MAAM,EAAE,CAAC;oBACT,eAAe;oBACf,gBAAgB;oBAChB,KAAK,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7B,iBAAA,CAAC;AAEF,gBAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACnD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBACrC,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;oBACpD,IAAI,aAAa,EAAE;AACjB,wBAAA,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;AACvB,4BAAA,GAAG,MAAM;AACT,4BAAA,GAAG,aAAa;yBACjB;;;gBAIL,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB;AAClD,YAAA,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU;;AAGzC,YAAA,MAAM,YAAY,GAAG,YAAY,CAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EACnB,SAAS,EACT,eAAe,CAChB;;YAGD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC/C,MAAM,aAAa,GAAG,YAAY,CAChC,UAAU,EACV,SAAS,EACT,eAAe,CAChB;AAED,YAAA,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,KAAK,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,OAAO,MAAM,CAAC,IAAI;;AAGpB,YAAA,MAAM,gBAAgB,GAAG,0BAA0B,CAAC,SAAS,CAAC;YAC9D,MAAM,QAAQ,GAAoB,EAAE;;AAGpC,YAAA,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;AAC3B,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,YAAY;AACnB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;;YAIH,IAAI,IAAI,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;AACpC,gBAAA,QAAQ,CAAC,IAAI,CACX,aAAa,CAAC;oBACZ,KAAK;oBACL,eAAe;oBACf,gBAAgB;AAChB,oBAAA,KAAK,EAAE,aAAa;AACpB,oBAAA,MAAM,EAAE,WAAW;AACpB,iBAAA,CAAC,CACH;;AAGH,YAAA,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;YAE3B,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;gBAClC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;;YAG1D,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;AACjC,gBAAA,wBAAwB,CAAC,UAAU,EAAE,SAAS,CAAC;;YAGjD,OAAO,MAAM,CAAC,IAAI;;QAClB,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC;YAChD,OAAO;AACL,gBAAA,OAAO,EAAE,EAAE;AACX,gBAAA,UAAU,EAAE,EAAE;AACd,gBAAA,MAAM,EAAE,EAAE;AACV,gBAAA,eAAe,EAAE,EAAE;gBACnB,GAAG,MAAM,CAAC,IAAI;AACd,gBAAA,KAAK,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;aAC9D;;AAEL,KAAC;IAED,OAAO;QACL,cAAc;QACd,UAAU;KACX;AACH;AAEA;AACA,SAAS,YAAY,CACnB,OAAkD,EAClD,SAAqC,EACrC,gBAA8B,EAAA;IAE9B,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;AAC5B,QAAA,IAAI,MAAM,CAAC,IAAI,EAAE;;YAEf,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACzD;;AAGF,YAAA,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YACvB,IAAI,gBAAgB,EAAE;AACpB,gBAAA,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;;YAEnC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAuB,CAAC;;;AAIvD,IAAA,OAAO,KAAK;AACd;AAEA;AACA,SAAS,wBAAwB,CAC/B,OAAY,EACZ,SAAqC,EAAA;AAErC,IAAA,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AACvC,QAAA,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,aAAa,EAAE;YACjB,OAAO,CAAC,CAAC,CAAC,GAAG;AACX,gBAAA,GAAG,MAAM;AACT,gBAAA,GAAG,aAAa;aACZ;;;AAGZ;;;;;"}