@librechat/agents 3.4.6 → 3.5.0

package/dist/esm/tools/schema.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"schema.mjs","names":[],"sources":["../../../src/tools/schema.ts"],"sourcesContent":["import { tool, type StructuredToolInterface } from '@langchain/core/tools';\nimport type { LCTool } from '@/types';\n\n/**\n * Creates a schema-only tool for LLM binding in event-driven mode.\n * These tools have valid schemas for the LLM to understand but should\n * never be invoked directly - ToolNode handles execution via events.\n */\nexport function createSchemaOnlyTool(\n  definition: LCTool\n): StructuredToolInterface {\n  const { name, description, parameters, responseFormat } = definition;\n\n  return tool(\n    async () => {\n      throw new Error(\n        `Tool \"${name}\" should not be invoked directly in event-driven mode. ` +\n          'ToolNode should dispatch ON_TOOL_EXECUTE events instead.'\n      );\n    },\n    {\n      name,\n      description: description ?? '',\n      schema: parameters ?? { type: 'object', properties: {} },\n      responseFormat: responseFormat ?? 'content_and_artifact',\n    }\n  );\n}\n\n/**\n * Creates schema-only tools for all definitions in an array.\n */\nexport function createSchemaOnlyTools(\n  definitions: LCTool[]\n): StructuredToolInterface[] {\n  return definitions.map((def) => createSchemaOnlyTool(def));\n}\n"],"mappings":";;;;;;;AAQA,SAAgB,qBACd,YACyB;CACzB,MAAM,EAAE,MAAM,aAAa,YAAY,mBAAmB;CAE1D,OAAO,KACL,YAAY;EACV,MAAM,IAAI,MACR,SAAS,KAAK,gHAEhB;CACF,GACA;EACE;EACA,aAAa,eAAe;EAC5B,QAAQ,cAAc;GAAE,MAAM;GAAU,YAAY,CAAC;EAAE;EACvD,gBAAgB,kBAAkB;CACpC,CACF;AACF;;;;AAKA,SAAgB,sBACd,aAC2B;CAC3B,OAAO,YAAY,KAAK,QAAQ,qBAAqB,GAAG,CAAC;AAC3D"}
+{"version":3,"file":"schema.mjs","names":[],"sources":["../../../src/tools/schema.ts"],"mappings":";;AAQA,SAAgB,qBACd,YACyB;CACzB,MAAM,EAAE,MAAM,aAAa,YAAY,mBAAmB;CAE1D,OAAO,KACL,YAAY;EACV,MAAM,IAAI,MACR,SAAS,KAAK,gHAEhB;CACF,GACA;EACE;EACA,aAAa,eAAe;EAC5B,QAAQ,cAAc;GAAE,MAAM;GAAU,YAAY,CAAC;EAAE;EACvD,gBAAgB,kBAAkB;CACpC,CACF;AACF;AAKA,SAAgB,sBACd,aAC2B;CAC3B,OAAO,YAAY,KAAK,QAAQ,qBAAqB,GAAG,CAAC;AAC3D"}

package/dist/esm/tools/search/anthropic.mjs

@@ -1,11 +1,5 @@
 import { getAttribution } from "./utils.mjs";
 //#region src/tools/search/anthropic.ts
-/**
-* Coerces Anthropic web search results to the SearchResultData format
-* @param results - Array of Anthropic web search results
-* @param turn - The turn number to associate with these results
-* @returns SearchResultData with minimal ProcessedOrganic items
-*/
 function coerceAnthropicSearchResults({ results, turn = 0 }) {
 	return {
 		turn,
@@ -18,9 +12,6 @@ function coerceAnthropicSearchResults({ results, turn = 0 }) {
 		}))
 	};
 }
-/**
-* Helper function to check if an object is an Anthropic web search result
-*/
 function isAnthropicWebSearchResult(obj) {
 	return typeof obj === "object" && obj !== null && "type" in obj && obj.type === "web_search_result" && "url" in obj && typeof obj.url === "string";
 }

package/dist/esm/tools/search/anthropic.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"anthropic.mjs","names":[],"sources":["../../../../src/tools/search/anthropic.ts"],"sourcesContent":["import type {\n  AnthropicTextBlockParam,\n  AnthropicWebSearchResultBlockParam,\n} from '@/llm/anthropic/types';\nimport type { SearchResultData, ProcessedOrganic } from './types';\nimport { getAttribution } from './utils';\n\n/**\n * Coerces Anthropic web search results to the SearchResultData format\n * @param results - Array of Anthropic web search results\n * @param turn - The turn number to associate with these results\n * @returns SearchResultData with minimal ProcessedOrganic items\n */\nexport function coerceAnthropicSearchResults({\n  results,\n  turn = 0,\n}: {\n  results: (AnthropicTextBlockParam | AnthropicWebSearchResultBlockParam)[];\n  turn?: number;\n}): SearchResultData {\n  const organic: ProcessedOrganic[] = results\n    .filter((result) => result.type === 'web_search_result')\n    .map((result, index) => ({\n      link: result.url,\n      position: index + 1,\n      title: result.title,\n      date: result.page_age ?? undefined,\n      attribution: getAttribution(result.url),\n    }));\n\n  return {\n    turn,\n    organic,\n  };\n}\n\n/**\n * Helper function to check if an object is an Anthropic web search result\n */\nexport function isAnthropicWebSearchResult(\n  obj: unknown\n): obj is AnthropicWebSearchResultBlockParam {\n  return (\n    typeof obj === 'object' &&\n    obj !== null &&\n    'type' in obj &&\n    obj.type === 'web_search_result' &&\n    'url' in obj &&\n    typeof (obj as Record<string, unknown>).url === 'string'\n  );\n}\n"],"mappings":";;;;;;;;AAaA,SAAgB,6BAA6B,EAC3C,SACA,OAAO,KAIY;CAWnB,OAAO;EACL;EACA,SAZkC,QACjC,QAAQ,WAAW,OAAO,SAAS,mBAAmB,CAAC,CACvD,KAAK,QAAQ,WAAW;GACvB,MAAM,OAAO;GACb,UAAU,QAAQ;GAClB,OAAO,OAAO;GACd,MAAM,OAAO,YAAY,KAAA;GACzB,aAAa,eAAe,OAAO,GAAG;EACxC,EAIM;CACR;AACF;;;;AAKA,SAAgB,2BACd,KAC2C;CAC3C,OACE,OAAO,QAAQ,YACf,QAAQ,QACR,UAAU,OACV,IAAI,SAAS,uBACb,SAAS,OACT,OAAQ,IAAgC,QAAQ;AAEpD"}
+{"version":3,"file":"anthropic.mjs","names":[],"sources":["../../../../src/tools/search/anthropic.ts"],"mappings":";;AAaA,SAAgB,6BAA6B,EAC3C,SACA,OAAO,KAIY;CAWnB,OAAO;EACL;EACA,SAZkC,QACjC,QAAQ,WAAW,OAAO,SAAS,mBAAmB,CAAC,CACvD,KAAK,QAAQ,WAAW;GACvB,MAAM,OAAO;GACb,UAAU,QAAQ;GAClB,OAAO,OAAO;GACd,MAAM,OAAO,YAAY,KAAA;GACzB,aAAa,eAAe,OAAO,GAAG;EACxC,EAIM;CACR;AACF;AAKA,SAAgB,2BACd,KAC2C;CAC3C,OACE,OAAO,QAAQ,YACf,QAAQ,QACR,UAAU,OACV,IAAI,SAAS,uBACb,SAAS,OACT,OAAQ,IAAgC,QAAQ;AAEpD"}

package/dist/esm/tools/search/content.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"content.mjs","names":[],"sources":["../../../../src/tools/search/content.ts"],"sourcesContent":["import * as cheerio from 'cheerio';\nimport type { References, MediaReference } from './types';\n\nexport function processContent(\n  html: string,\n  markdown: string\n): {\n  markdown: string;\n} & References {\n  const linkMap = new Map<string, MediaReference>();\n  const imageMap = new Map<string, MediaReference>();\n  const videoMap = new Map<string, MediaReference>();\n  const iframeMap = new Map<string, MediaReference>();\n\n  const $ = cheerio.load(html, {\n    xmlMode: false,\n  });\n\n  // Extract all media references\n  $('a[href]').each((_, el) => {\n    const href = $(el).attr('href');\n    if (href != null && href) {\n      linkMap.set(href, {\n        originalUrl: href,\n        title: $(el).attr('title'),\n        text: $(el).text().trim(),\n      });\n    }\n  });\n\n  $('img[src]').each((_, el) => {\n    const src = $(el).attr('src');\n    if (src != null && src) {\n      imageMap.set(src, {\n        originalUrl: src,\n        title: $(el).attr('alt') ?? $(el).attr('title'),\n      });\n    }\n  });\n\n  // Handle videos (dedicated video elements and video platforms in iframes)\n  $('video[src], iframe[src*=\"youtube\"], iframe[src*=\"vimeo\"]').each(\n    (_, el) => {\n      const src = $(el).attr('src');\n      if (src != null && src) {\n        videoMap.set(src, {\n          originalUrl: src,\n          title: $(el).attr('title'),\n        });\n      }\n    }\n  );\n\n  // Handle all other generic iframes that aren't already captured as videos\n  $('iframe').each((_, el) => {\n    const src = $(el).attr('src');\n    if (\n      src != null &&\n      src &&\n      !src.includes('youtube') &&\n      !src.includes('vimeo')\n    ) {\n      iframeMap.set(src, {\n        originalUrl: src,\n        title: $(el).attr('title'),\n      });\n    }\n  });\n\n  // Create lookup maps with indices\n  const linkIndexMap = new Map<string, number>();\n  const imageIndexMap = new Map<string, number>();\n  const videoIndexMap = new Map<string, number>();\n  const iframeIndexMap = new Map<string, number>();\n\n  Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));\n  Array.from(imageMap.keys()).forEach((url, i) =>\n    imageIndexMap.set(url, i + 1)\n  );\n  Array.from(videoMap.keys()).forEach((url, i) =>\n    videoIndexMap.set(url, i + 1)\n  );\n  Array.from(iframeMap.keys()).forEach((url, i) =>\n    iframeIndexMap.set(url, i + 1)\n  );\n\n  // Process the markdown\n  let result = markdown;\n\n  // Replace each URL one by one, starting with the longest URLs first to avoid partial matches\n  const allUrls = [\n    ...Array.from(imageMap.keys()).map((url) => ({\n      url,\n      type: 'image',\n      idx: imageIndexMap.get(url),\n    })),\n    ...Array.from(videoMap.keys()).map((url) => ({\n      url,\n      type: 'video',\n      idx: videoIndexMap.get(url),\n    })),\n    ...Array.from(iframeMap.keys()).map((url) => ({\n      url,\n      type: 'iframe',\n      idx: iframeIndexMap.get(url),\n    })),\n    ...Array.from(linkMap.keys()).map((url) => ({\n      url,\n      type: 'link',\n      idx: linkIndexMap.get(url),\n    })),\n  ].sort((a, b) => b.url.length - a.url.length);\n\n  // Create a function to escape special characters in URLs for regex\n  function escapeRegex(string: string): string {\n    return string.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n  }\n\n  // Replace each URL in the markdown\n  for (const { url, type, idx } of allUrls) {\n    // Create a regex that captures URLs in markdown links\n    const regex = new RegExp(`\\\\(${escapeRegex(url)}(?:\\\\s+\"[^\"]*\")?\\\\)`, 'g');\n\n    result = result.replace(regex, (match) => {\n      // Keep any title attribute that might exist\n      const titleMatch = match.match(/\\s+\"([^\"]*)\"/);\n      const titlePart = titleMatch ? ` \"${titleMatch[1]}\"` : '';\n\n      return `(${type}#${idx}${titlePart})`;\n    });\n  }\n\n  iframeMap.clear();\n  const links = Array.from(linkMap.values());\n  linkMap.clear();\n  const images = Array.from(imageMap.values());\n  imageMap.clear();\n  const videos = Array.from(videoMap.values());\n  videoMap.clear();\n\n  return {\n    markdown: result,\n    links,\n    images,\n    videos,\n  };\n}\n"],"mappings":";;AAGA,SAAgB,eACd,MACA,UAGa;CACb,MAAM,0BAAU,IAAI,IAA4B;CAChD,MAAM,2BAAW,IAAI,IAA4B;CACjD,MAAM,2BAAW,IAAI,IAA4B;CACjD,MAAM,4BAAY,IAAI,IAA4B;CAElD,MAAM,IAAI,QAAQ,KAAK,MAAM,EAC3B,SAAS,MACX,CAAC;CAGD,EAAE,SAAS,CAAC,CAAC,MAAM,GAAG,OAAO;EAC3B,MAAM,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM;EAC9B,IAAI,QAAQ,QAAQ,MAClB,QAAQ,IAAI,MAAM;GAChB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;GACzB,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK;EAC1B,CAAC;CAEL,CAAC;CAED,EAAE,UAAU,CAAC,CAAC,MAAM,GAAG,OAAO;EAC5B,MAAM,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK;EAC5B,IAAI,OAAO,QAAQ,KACjB,SAAS,IAAI,KAAK;GAChB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;EAChD,CAAC;CAEL,CAAC;CAGD,EAAE,8DAA0D,CAAC,CAAC,MAC3D,GAAG,OAAO;EACT,MAAM,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK;EAC5B,IAAI,OAAO,QAAQ,KACjB,SAAS,IAAI,KAAK;GAChB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;EAC3B,CAAC;CAEL,CACF;CAGA,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,OAAO;EAC1B,MAAM,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK;EAC5B,IACE,OAAO,QACP,OACA,CAAC,IAAI,SAAS,SAAS,KACvB,CAAC,IAAI,SAAS,OAAO,GAErB,UAAU,IAAI,KAAK;GACjB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;EAC3B,CAAC;CAEL,CAAC;CAGD,MAAM,+BAAe,IAAI,IAAoB;CAC7C,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,MAAM,iCAAiB,IAAI,IAAoB;CAE/C,MAAM,KAAK,QAAQ,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,aAAa,IAAI,KAAK,IAAI,CAAC,CAAC;CAC3E,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MACxC,cAAc,IAAI,KAAK,IAAI,CAAC,CAC9B;CACA,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MACxC,cAAc,IAAI,KAAK,IAAI,CAAC,CAC9B;CACA,MAAM,KAAK,UAAU,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MACzC,eAAe,IAAI,KAAK,IAAI,CAAC,CAC/B;CAGA,IAAI,SAAS;CAGb,MAAM,UAAU;EACd,GAAG,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC3C;GACA,MAAM;GACN,KAAK,cAAc,IAAI,GAAG;EAC5B,EAAE;EACF,GAAG,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC3C;GACA,MAAM;GACN,KAAK,cAAc,IAAI,GAAG;EAC5B,EAAE;EACF,GAAG,MAAM,KAAK,UAAU,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC5C;GACA,MAAM;GACN,KAAK,eAAe,IAAI,GAAG;EAC7B,EAAE;EACF,GAAG,MAAM,KAAK,QAAQ,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC1C;GACA,MAAM;GACN,KAAK,aAAa,IAAI,GAAG;EAC3B,EAAE;CACJ,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,IAAI,SAAS,EAAE,IAAI,MAAM;CAG5C,SAAS,YAAY,QAAwB;EAC3C,OAAO,OAAO,QAAQ,uBAAuB,MAAM;CACrD;CAGA,KAAK,MAAM,EAAE,KAAK,MAAM,SAAS,SAAS;EAExC,MAAM,QAAQ,IAAI,OAAO,MAAM,YAAY,GAAG,EAAE,sBAAsB,GAAG;EAEzE,SAAS,OAAO,QAAQ,QAAQ,UAAU;GAExC,MAAM,aAAa,MAAM,MAAM,cAAc;GAG7C,OAAO,IAAI,KAAK,GAAG,MAFD,aAAa,KAAK,WAAW,GAAG,KAAK,GAEpB;EACrC,CAAC;CACH;CAEA,UAAU,MAAM;CAChB,MAAM,QAAQ,MAAM,KAAK,QAAQ,OAAO,CAAC;CACzC,QAAQ,MAAM;CACd,MAAM,SAAS,MAAM,KAAK,SAAS,OAAO,CAAC;CAC3C,SAAS,MAAM;CACf,MAAM,SAAS,MAAM,KAAK,SAAS,OAAO,CAAC;CAC3C,SAAS,MAAM;CAEf,OAAO;EACL,UAAU;EACV;EACA;EACA;CACF;AACF"}
+{"version":3,"file":"content.mjs","names":[],"sources":["../../../../src/tools/search/content.ts"],"mappings":";;AAGA,SAAgB,eACd,MACA,UAGa;CACb,MAAM,0BAAU,IAAI,IAA4B;CAChD,MAAM,2BAAW,IAAI,IAA4B;CACjD,MAAM,2BAAW,IAAI,IAA4B;CACjD,MAAM,4BAAY,IAAI,IAA4B;CAElD,MAAM,IAAI,QAAQ,KAAK,MAAM,EAC3B,SAAS,MACX,CAAC;CAGD,EAAE,SAAS,CAAC,CAAC,MAAM,GAAG,OAAO;EAC3B,MAAM,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM;EAC9B,IAAI,QAAQ,QAAQ,MAClB,QAAQ,IAAI,MAAM;GAChB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;GACzB,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK;EAC1B,CAAC;CAEL,CAAC;CAED,EAAE,UAAU,CAAC,CAAC,MAAM,GAAG,OAAO;EAC5B,MAAM,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK;EAC5B,IAAI,OAAO,QAAQ,KACjB,SAAS,IAAI,KAAK;GAChB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;EAChD,CAAC;CAEL,CAAC;CAGD,EAAE,8DAA0D,CAAC,CAAC,MAC3D,GAAG,OAAO;EACT,MAAM,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK;EAC5B,IAAI,OAAO,QAAQ,KACjB,SAAS,IAAI,KAAK;GAChB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;EAC3B,CAAC;CAEL,CACF;CAGA,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,OAAO;EAC1B,MAAM,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK;EAC5B,IACE,OAAO,QACP,OACA,CAAC,IAAI,SAAS,SAAS,KACvB,CAAC,IAAI,SAAS,OAAO,GAErB,UAAU,IAAI,KAAK;GACjB,aAAa;GACb,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO;EAC3B,CAAC;CAEL,CAAC;CAGD,MAAM,+BAAe,IAAI,IAAoB;CAC7C,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,MAAM,gCAAgB,IAAI,IAAoB;CAC9C,MAAM,iCAAiB,IAAI,IAAoB;CAE/C,MAAM,KAAK,QAAQ,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,aAAa,IAAI,KAAK,IAAI,CAAC,CAAC;CAC3E,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MACxC,cAAc,IAAI,KAAK,IAAI,CAAC,CAC9B;CACA,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MACxC,cAAc,IAAI,KAAK,IAAI,CAAC,CAC9B;CACA,MAAM,KAAK,UAAU,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,MACzC,eAAe,IAAI,KAAK,IAAI,CAAC,CAC/B;CAGA,IAAI,SAAS;CAGb,MAAM,UAAU;EACd,GAAG,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC3C;GACA,MAAM;GACN,KAAK,cAAc,IAAI,GAAG;EAC5B,EAAE;EACF,GAAG,MAAM,KAAK,SAAS,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC3C;GACA,MAAM;GACN,KAAK,cAAc,IAAI,GAAG;EAC5B,EAAE;EACF,GAAG,MAAM,KAAK,UAAU,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC5C;GACA,MAAM;GACN,KAAK,eAAe,IAAI,GAAG;EAC7B,EAAE;EACF,GAAG,MAAM,KAAK,QAAQ,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS;GAC1C;GACA,MAAM;GACN,KAAK,aAAa,IAAI,GAAG;EAC3B,EAAE;CACJ,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,IAAI,SAAS,EAAE,IAAI,MAAM;CAG5C,SAAS,YAAY,QAAwB;EAC3C,OAAO,OAAO,QAAQ,uBAAuB,MAAM;CACrD;CAGA,KAAK,MAAM,EAAE,KAAK,MAAM,SAAS,SAAS;EAExC,MAAM,QAAQ,IAAI,OAAO,MAAM,YAAY,GAAG,EAAE,sBAAsB,GAAG;EAEzE,SAAS,OAAO,QAAQ,QAAQ,UAAU;GAExC,MAAM,aAAa,MAAM,MAAM,cAAc;GAG7C,OAAO,IAAI,KAAK,GAAG,MAFD,aAAa,KAAK,WAAW,GAAG,KAAK,GAEpB;EACrC,CAAC;CACH;CAEA,UAAU,MAAM;CAChB,MAAM,QAAQ,MAAM,KAAK,QAAQ,OAAO,CAAC;CACzC,QAAQ,MAAM;CACd,MAAM,SAAS,MAAM,KAAK,SAAS,OAAO,CAAC;CAC3C,SAAS,MAAM;CACf,MAAM,SAAS,MAAM,KAAK,SAAS,OAAO,CAAC;CAC3C,SAAS,MAAM;CAEf,OAAO;EACL,UAAU;EACV;EACA;EACA;CACF;AACF"}

package/dist/esm/tools/search/crw-scraper.mjs

@@ -2,13 +2,7 @@ import { createDefaultLogger } from "./utils.mjs";
 import { processContent } from "./content.mjs";
 import axios from "axios";
 //#region src/tools/search/crw-scraper.ts
-/** HTTP headroom over the payload render budget: fastCRW's queue/verification
-* overhead is not counted against `timeout`, so the client must wait longer. */
 const CRW_TIMEOUT_BUFFER = 5e3;
-/**
-* fastCRW scraper. Firecrawl-compatible web scraper; single binary;
-* self-host or cloud. Posts to {base}/v1/scrape.
-*/
 var CrwScraper = class {
 	apiKey;
 	apiUrl;
@@ -83,15 +77,6 @@ var CrwScraper = class {
 			}];
 		}
 	}
-	/**
-	* Extract content from scrape response. Mirrors FirecrawlScraper — reads
-	* response.data.*, which normalizeCrwResponse guarantees, preserving the
-	* processContent ref-markers used by the reranker. Parameter is typed as the
-	* NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:
-	* TS class-method bivariance accepts this against BaseScraper's
-	* AnyScraperResponse, AND it lets us read response.data.plainText (a
-	* CrwScrapeResponse-only field) with no cast.
-	*/
 	extractContent(response) {
 		if (!response.success || !response.data) return ["", void 0];
 		const htmlSource = response.data.html ?? response.data.rawHtml;
@@ -113,17 +98,7 @@ var CrwScraper = class {
 		return response.data.metadata;
 	}
 };
-/**
-* Create a fastCRW scraper instance
-* @param config Scraper configuration
-* @returns fastCRW scraper instance
-*/
 const createCrwScraper = (config = {}) => new CrwScraper(config);
-/**
-* fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,
-* ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested
-* container and fall back to top-level fields for self-host/legacy responses.
-*/
 function normalizeCrwResponse(raw) {
 	if (raw == null) return {
 		success: false,
@@ -148,14 +123,6 @@ function normalizeCrwResponse(raw) {
 		}
 	};
 }
-/**
-* Replace inline base64 data-URI payloads (typically images) in scraped text
-* with a short placeholder. Such payloads can be hundreds of KB; the content
-* processor builds a per-link RegExp from each URL, and one that large overflows
-* the engine's pattern-size limit ("Invalid regular expression"). Firecrawl
-* drops them via removeBase64Images — mirror that so image-heavy pages stay
-* processable (and don't bloat the LLM context with base64 noise).
-*/
 function stripBase64DataUris(text) {
 	if (text == null) return text;
 	return text.replace(/data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]+/g, "data:base64-content-removed");

package/dist/esm/tools/search/crw-scraper.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"crw-scraper.mjs","names":[],"sources":["../../../../src/tools/search/crw-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/** HTTP headroom over the payload render budget: fastCRW's queue/verification\n * overhead is not counted against `timeout`, so the client must wait longer. */\nconst CRW_TIMEOUT_BUFFER = 5000;\n\n/**\n * fastCRW scraper. Firecrawl-compatible web scraper; single binary;\n * self-host or cloud. Posts to {base}/v1/scrape.\n */\nexport class CrwScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private onlyMainContent?: boolean;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private headers?: Record<string, string>;\n  private renderJs?: boolean | null;\n  private cssSelector?: string;\n  private xpath?: string;\n  private proxy?: string;\n  private stealth?: boolean;\n  private httpAgent?: t.HttpAgent;\n  private httpsAgent?: t.HttpsAgent;\n\n  constructor(config: t.CrwScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? '';\n\n    const baseUrl =\n      config.apiUrl ?? process.env.CRW_API_URL ?? 'https://api.fastcrw.com';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'html'];\n    this.timeout = config.timeout ?? 7500;\n    this.logger = config.logger || createDefaultLogger();\n\n    this.onlyMainContent = config.onlyMainContent;\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.headers = config.headers;\n    this.renderJs = config.renderJs;\n    this.cssSelector = config.cssSelector;\n    this.xpath = config.xpath;\n    this.proxy = config.proxy;\n    this.stealth = config.stealth;\n    this.httpAgent = config.httpAgent;\n    this.httpsAgent = config.httpsAgent;\n\n    // Self-host fastCRW may run without auth, so a missing key is only a\n    // warning — unlike Firecrawl/Tavily, scrapeUrl does NOT early-return on it.\n    if (!this.apiKey) {\n      this.logger.warn('CRW_API_KEY is not set. Scraping will not work.');\n    }\n    this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);\n  }\n\n  async scrapeUrl(\n    url: string,\n    options: t.CrwScrapeOptions = {}\n  ): Promise<[string, t.CrwScrapeResponse]> {\n    try {\n      const payloadTimeout = options.timeout ?? this.timeout;\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        waitFor: options.waitFor ?? this.waitFor,\n        headers: options.headers ?? this.headers,\n        renderJs: options.renderJs ?? this.renderJs,\n        cssSelector: options.cssSelector ?? this.cssSelector,\n        xpath: options.xpath ?? this.xpath,\n        proxy: options.proxy ?? this.proxy,\n        stealth: options.stealth ?? this.stealth,\n        // Cloud honors `timeout` (live-verified); the published OpenAPI\n        // documents `deadlineMs` (1..60000) instead. Send both.\n        timeout: payloadTimeout,\n        deadlineMs: Math.max(1, Math.min(payloadTimeout, 60000)),\n      });\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n      if (this.apiKey) {\n        headers.Authorization = `Bearer ${this.apiKey}`;\n      }\n\n      const response = await axios.post<t.CrwRawScrapeResponse>(\n        this.apiUrl,\n        payload,\n        {\n          headers,\n          timeout: payloadTimeout + CRW_TIMEOUT_BUFFER,\n          httpAgent: this.httpAgent,\n          httpsAgent: this.httpsAgent,\n        }\n      );\n\n      return [url, normalizeCrwResponse(response.data)];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `fastCRW API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response. Mirrors FirecrawlScraper — reads\n   * response.data.*, which normalizeCrwResponse guarantees, preserving the\n   * processContent ref-markers used by the reranker. Parameter is typed as the\n   * NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:\n   * TS class-method bivariance accepts this against BaseScraper's\n   * AnyScraperResponse, AND it lets us read response.data.plainText (a\n   * CrwScrapeResponse-only field) with no cast.\n   */\n  extractContent(\n    response: t.CrwScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    const htmlSource = response.data.html ?? response.data.rawHtml;\n    if (response.data.markdown != null && htmlSource != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          htmlSource,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    // CRW-only fallback (no Firecrawl equivalent): plain-text body.\n    if (response.data.plainText != null) {\n      return [response.data.plainText, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  extractMetadata(response: t.CrwScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a fastCRW scraper instance\n * @param config Scraper configuration\n * @returns fastCRW scraper instance\n */\nexport const createCrwScraper = (config: t.CrwScraperConfig = {}): CrwScraper =>\n  new CrwScraper(config);\n\n/**\n * fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,\n * ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested\n * container and fall back to top-level fields for self-host/legacy responses.\n */\nfunction normalizeCrwResponse(\n  raw: t.CrwRawScrapeResponse | null | undefined\n): t.CrwScrapeResponse {\n  if (raw == null) {\n    return { success: false, error: 'Empty fastCRW response' };\n  }\n  if (raw.success === false) {\n    return {\n      success: false,\n      error:\n        raw.error_code != null\n          ? `[${raw.error_code}] ${raw.error ?? 'Unknown error'}`\n          : (raw.error ?? 'fastCRW scrape failed'),\n      error_code: raw.error_code,\n    };\n  }\n  const data = raw.data ?? raw;\n  return {\n    success: true,\n    data: {\n      // Strip inline base64 image payloads from text content (not `screenshot`,\n      // which is base64 by design and never reaches the content processor).\n      markdown: stripBase64DataUris(data.markdown),\n      html: stripBase64DataUris(data.html),\n      rawHtml: stripBase64DataUris(data.rawHtml),\n      plainText: stripBase64DataUris(data.plainText),\n      screenshot: data.screenshot,\n      links: data.links,\n      metadata: data.metadata,\n    },\n  };\n}\n\n/**\n * Replace inline base64 data-URI payloads (typically images) in scraped text\n * with a short placeholder. Such payloads can be hundreds of KB; the content\n * processor builds a per-link RegExp from each URL, and one that large overflows\n * the engine's pattern-size limit (\"Invalid regular expression\"). Firecrawl\n * drops them via removeBase64Images — mirror that so image-heavy pages stay\n * processable (and don't bloat the LLM context with base64 noise).\n */\nfunction stripBase64DataUris(text: string | undefined): string | undefined {\n  if (text == null) {\n    return text;\n  }\n  return text.replace(\n    /data:[\\w.+-]+\\/[\\w.+-]+;base64,[A-Za-z0-9+/=]+/g,\n    'data:base64-content-removed'\n  );\n}\n\n// Helper function to clean up payload for fastCRW\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"mappings":";;;;;;AAOA,MAAM,qBAAqB;;;;;AAM3B,IAAa,aAAb,MAAiD;CAC/C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA6B,CAAC,GAAG;EAC3C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,eAAe;EAE1D,MAAM,UACJ,OAAO,UAAU,QAAQ,IAAI,eAAe;EAC9C,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE;EAE7C,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,MAAM;EAC3D,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,kBAAkB,OAAO;EAC9B,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,cAAc,OAAO;EAC1B,KAAK,QAAQ,OAAO;EACpB,KAAK,QAAQ,OAAO;EACpB,KAAK,UAAU,OAAO;EACtB,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EAIzB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,iDAAiD;EAEpE,KAAK,OAAO,MAAM,yCAAyC,KAAK,QAAQ;CAC1E;CAEA,MAAM,UACJ,KACA,UAA8B,CAAC,GACS;EACxC,IAAI;GACF,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,UAAU,QAAQ,YAAY,KAAK;IACnC,aAAa,QAAQ,eAAe,KAAK;IACzC,OAAO,QAAQ,SAAS,KAAK;IAC7B,OAAO,QAAQ,SAAS,KAAK;IAC7B,SAAS,QAAQ,WAAW,KAAK;IAGjC,SAAS;IACT,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,GAAK,CAAC;GACzD,CAAC;GAED,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,KAAK,QACP,QAAQ,gBAAgB,UAAU,KAAK;GAczC,OAAO,CAAC,KAAK,sBAAqB,MAXX,MAAM,KAC3B,KAAK,QACL,SACA;IACE;IACA,SAAS,iBAAiB;IAC1B,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CACF,EAAA,CAE2C,IAAI,CAAC;EAClD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,+BALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;;;;;CAWA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,aAAa,SAAS,KAAK,QAAQ,SAAS,KAAK;EACvD,IAAI,SAAS,KAAK,YAAY,QAAQ,cAAc,MAClD,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,YACA,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAI1C,IAAI,SAAS,KAAK,aAAa,MAC7B,OAAO,CAAC,SAAS,KAAK,WAAW,KAAA,CAAS;EAG5C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;CAEA,gBAAgB,UAAiD;EAC/D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,oBAAoB,SAA6B,CAAC,MAC7D,IAAI,WAAW,MAAM;;;;;;AAOvB,SAAS,qBACP,KACqB;CACrB,IAAI,OAAO,MACT,OAAO;EAAE,SAAS;EAAO,OAAO;CAAyB;CAE3D,IAAI,IAAI,YAAY,OAClB,OAAO;EACL,SAAS;EACT,OACE,IAAI,cAAc,OACd,IAAI,IAAI,WAAW,IAAI,IAAI,SAAS,oBACnC,IAAI,SAAS;EACpB,YAAY,IAAI;CAClB;CAEF,MAAM,OAAO,IAAI,QAAQ;CACzB,OAAO;EACL,SAAS;EACT,MAAM;GAGJ,UAAU,oBAAoB,KAAK,QAAQ;GAC3C,MAAM,oBAAoB,KAAK,IAAI;GACnC,SAAS,oBAAoB,KAAK,OAAO;GACzC,WAAW,oBAAoB,KAAK,SAAS;GAC7C,YAAY,KAAK;GACjB,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;CACF;AACF;;;;;;;;;AAUA,SAAS,oBAAoB,MAA8C;CACzE,IAAI,QAAQ,MACV,OAAO;CAET,OAAO,KAAK,QACV,mDACA,6BACF;AACF;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}
+{"version":3,"file":"crw-scraper.mjs","names":[],"sources":["../../../../src/tools/search/crw-scraper.ts"],"mappings":";;;;AAOA,MAAM,qBAAqB;AAM3B,IAAa,aAAb,MAAiD;CAC/C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA6B,CAAC,GAAG;EAC3C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,eAAe;EAE1D,MAAM,UACJ,OAAO,UAAU,QAAQ,IAAI,eAAe;EAC9C,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE;EAE7C,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,MAAM;EAC3D,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,kBAAkB,OAAO;EAC9B,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,cAAc,OAAO;EAC1B,KAAK,QAAQ,OAAO;EACpB,KAAK,QAAQ,OAAO;EACpB,KAAK,UAAU,OAAO;EACtB,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EAIzB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,iDAAiD;EAEpE,KAAK,OAAO,MAAM,yCAAyC,KAAK,QAAQ;CAC1E;CAEA,MAAM,UACJ,KACA,UAA8B,CAAC,GACS;EACxC,IAAI;GACF,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,UAAU,QAAQ,YAAY,KAAK;IACnC,aAAa,QAAQ,eAAe,KAAK;IACzC,OAAO,QAAQ,SAAS,KAAK;IAC7B,OAAO,QAAQ,SAAS,KAAK;IAC7B,SAAS,QAAQ,WAAW,KAAK;IAGjC,SAAS;IACT,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,GAAK,CAAC;GACzD,CAAC;GAED,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,KAAK,QACP,QAAQ,gBAAgB,UAAU,KAAK;GAczC,OAAO,CAAC,KAAK,sBAAqB,MAXX,MAAM,KAC3B,KAAK,QACL,SACA;IACE;IACA,SAAS,iBAAiB;IAC1B,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CACF,EAAA,CAE2C,IAAI,CAAC;EAClD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,+BALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;CAWA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,aAAa,SAAS,KAAK,QAAQ,SAAS,KAAK;EACvD,IAAI,SAAS,KAAK,YAAY,QAAQ,cAAc,MAClD,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,YACA,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAI1C,IAAI,SAAS,KAAK,aAAa,MAC7B,OAAO,CAAC,SAAS,KAAK,WAAW,KAAA,CAAS;EAG5C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;CAEA,gBAAgB,UAAiD;EAC/D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;AAOA,MAAa,oBAAoB,SAA6B,CAAC,MAC7D,IAAI,WAAW,MAAM;AAOvB,SAAS,qBACP,KACqB;CACrB,IAAI,OAAO,MACT,OAAO;EAAE,SAAS;EAAO,OAAO;CAAyB;CAE3D,IAAI,IAAI,YAAY,OAClB,OAAO;EACL,SAAS;EACT,OACE,IAAI,cAAc,OACd,IAAI,IAAI,WAAW,IAAI,IAAI,SAAS,oBACnC,IAAI,SAAS;EACpB,YAAY,IAAI;CAClB;CAEF,MAAM,OAAO,IAAI,QAAQ;CACzB,OAAO;EACL,SAAS;EACT,MAAM;GAGJ,UAAU,oBAAoB,KAAK,QAAQ;GAC3C,MAAM,oBAAoB,KAAK,IAAI;GACnC,SAAS,oBAAoB,KAAK,OAAO;GACzC,WAAW,oBAAoB,KAAK,SAAS;GAC7C,YAAY,KAAK;GACjB,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;CACF;AACF;AAUA,SAAS,oBAAoB,MAA8C;CACzE,IAAI,QAAQ,MACV,OAAO;CAET,OAAO,KAAK,QACV,mDACA,6BACF;AACF;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}

package/dist/esm/tools/search/crw-search.mjs

@@ -8,8 +8,6 @@ const getHostname = (link) => {
 		return link;
 	}
 };
-/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat
-* SearchResult[] with a `category` field; route rows into source groups. */
 const splitByCategory = (rows) => {
 	const groups = {
 		web: [],

package/dist/esm/tools/search/crw-search.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"crw-search.mjs","names":[],"sources":["../../../../src/tools/search/crw-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\n\nconst DEFAULT_CRW_TIMEOUT = 15000;\n\nconst getHostname = (link: string): string => {\n  try {\n    return new URL(link).hostname;\n  } catch {\n    return link;\n  }\n};\n\n/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat\n * SearchResult[] with a `category` field; route rows into source groups. */\nconst splitByCategory = (rows: t.CrwSearchResult[]): t.CrwSearchGroups => {\n  const groups: Required<t.CrwSearchGroups> = { web: [], images: [], news: [] };\n  for (const row of rows) {\n    if (row.category === 'news') {\n      groups.news.push(row);\n    } else if (row.category === 'images') {\n      groups.images.push({ ...row, imageUrl: row.url });\n    } else {\n      groups.web.push(row);\n    }\n  }\n  return groups;\n};\n\nexport const createCrwAPI = (\n  apiKey?: string,\n  apiUrl?: string,\n  options?: t.CrwSearchOptions\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  // NOTE: fastCRW /v1/search is cloud-only, but we still allow a base-URL\n  // override for parity with the scraper. Self-host may have no auth, so —\n  // unlike Tavily (createTavilyAPI throws on missing key) — we do NOT throw\n  // here; we only attach the Authorization header when a key is present.\n  const base = (\n    apiUrl ??\n    process.env.CRW_API_URL ??\n    'https://api.fastcrw.com'\n  ).replace(/\\/+$/, '');\n  const config = {\n    apiKey: apiKey ?? process.env.CRW_API_KEY,\n    apiUrl: `${base}/v1/search`,\n    timeout: options?.timeout ?? DEFAULT_CRW_TIMEOUT,\n  };\n\n  const getSources = async ({\n    query,\n    date,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const limit = Math.min(\n        Math.max(1, options?.maxResults ?? numResults),\n        20\n      );\n      // Mirror Serper's verticals: image/news requests hit only their native\n      // fastCRW source; plain web optionally adds images via includeImages.\n      let sources: t.CrwSearchSource[];\n      if (type === 'images') {\n        sources = ['images'];\n      } else if (type === 'news') {\n        sources = ['news'];\n      } else {\n        sources = options?.includeImages === true ? ['web', 'images'] : ['web'];\n      }\n\n      const payload: t.CrwSearchPayload = { query, limit, sources };\n      if (date != null) {\n        // Serper-style qdr filter; live-verified to constrain results.\n        payload.tbs = `qdr:${date}`;\n      }\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers.Authorization = `Bearer ${config.apiKey}`;\n      }\n\n      const response = await axios.post<t.CrwSearchResponse>(\n        config.apiUrl,\n        payload,\n        {\n          headers,\n          timeout: config.timeout,\n          httpAgent: options?.httpAgent,\n          httpsAgent: options?.httpsAgent,\n        }\n      );\n\n      const body = response.data;\n      if (body.success === false) {\n        return {\n          success: false,\n          error: `fastCRW search failed: ${\n            body.error_code != null ? `[${body.error_code}] ` : ''\n          }${body.error ?? 'Unknown error'}`,\n        };\n      }\n\n      // fastCRW cloud keys results by source: {data: {web: [...], images:\n      // [...], news: [...]}} (live-verified 2026-07-02); the published\n      // OpenAPI documents a flat SearchResult[] instead (self-host), and a\n      // {data: {results: {...groups}}} wrapper also exists. Accept all three.\n      // OrganicResult.link is a REQUIRED string (types.ts), so defend against\n      // null/empty urls to never feed a broken '' link into the scraper.\n      const container = body.data ?? {};\n      const data: t.CrwSearchGroups = Array.isArray(container)\n        ? splitByCategory(container)\n        : (container.results ?? container);\n      const organicResults: t.OrganicResult[] = (data.web ?? [])\n        .filter((r) => r.url != null && r.url !== '')\n        .map((r) => ({\n          title: r.title ?? '',\n          link: r.url as string,\n          snippet: r.description ?? r.snippet ?? '',\n          position: r.position,\n        }));\n\n      const imageResults: t.ImageResult[] = (data.images ?? [])\n        .filter((r) => r.imageUrl != null && r.imageUrl !== '')\n        .map((r) => ({\n          title: r.title,\n          imageUrl: r.imageUrl,\n          link: r.url,\n          position: r.position,\n        }));\n\n      const newsResults: t.NewsResult[] = (data.news ?? [])\n        .filter((r) => r.url != null && r.url !== '')\n        .map((r) => ({\n          title: r.title ?? '',\n          link: r.url as string,\n          snippet: r.description ?? r.snippet ?? '',\n          date: r.publishedDate,\n          source: getHostname(r.url as string),\n          position: r.position,\n        }));\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: [],\n        videos: [],\n        news: newsResults,\n        relatedSearches: [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `fastCRW search request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n"],"mappings":";;AAGA,MAAM,sBAAsB;AAE5B,MAAM,eAAe,SAAyB;CAC5C,IAAI;EACF,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC;CACvB,QAAQ;EACN,OAAO;CACT;AACF;;;AAIA,MAAM,mBAAmB,SAAiD;CACxE,MAAM,SAAsC;EAAE,KAAK,CAAC;EAAG,QAAQ,CAAC;EAAG,MAAM,CAAC;CAAE;CAC5E,KAAK,MAAM,OAAO,MAChB,IAAI,IAAI,aAAa,QACnB,OAAO,KAAK,KAAK,GAAG;MACf,IAAI,IAAI,aAAa,UAC1B,OAAO,OAAO,KAAK;EAAE,GAAG;EAAK,UAAU,IAAI;CAAI,CAAC;MAEhD,OAAO,IAAI,KAAK,GAAG;CAGvB,OAAO;AACT;AAEA,MAAa,gBACX,QACA,QACA,YAGG;CAKH,MAAM,QACJ,UACA,QAAQ,IAAI,eACZ,0BAAA,CACA,QAAQ,QAAQ,EAAE;CACpB,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ,GAAG,KAAK;EAChB,SAAS,SAAS,WAAW;CAC/B;CAEA,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GACF,MAAM,QAAQ,KAAK,IACjB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAGA,IAAI;GACJ,IAAI,SAAS,UACX,UAAU,CAAC,QAAQ;QACd,IAAI,SAAS,QAClB,UAAU,CAAC,MAAM;QAEjB,UAAU,SAAS,kBAAkB,OAAO,CAAC,OAAO,QAAQ,IAAI,CAAC,KAAK;GAGxE,MAAM,UAA8B;IAAE;IAAO;IAAO;GAAQ;GAC5D,IAAI,QAAQ,MAEV,QAAQ,MAAM,OAAO;GAGvB,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,gBAAgB,UAAU,OAAO;GAc3C,MAAM,QAAO,MAXU,MAAM,KAC3B,OAAO,QACP,SACA;IACE;IACA,SAAS,OAAO;IAChB,WAAW,SAAS;IACpB,YAAY,SAAS;GACvB,CACF,EAAA,CAEsB;GACtB,IAAI,KAAK,YAAY,OACnB,OAAO;IACL,SAAS;IACT,OAAO,0BACL,KAAK,cAAc,OAAO,IAAI,KAAK,WAAW,MAAM,KACnD,KAAK,SAAS;GACnB;GASF,MAAM,YAAY,KAAK,QAAQ,CAAC;GAChC,MAAM,OAA0B,MAAM,QAAQ,SAAS,IACnD,gBAAgB,SAAS,IACxB,UAAU,WAAW;GAuC1B,OAAO;IAAE,SAAS;IAAM,MAAM;KAR5B,UA9ByC,KAAK,OAAO,CAAC,EAAA,CACrD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,UAAU,EAAE;KACd,EAuBsB;KACtB,SAtBqC,KAAK,UAAU,CAAC,EAAA,CACpD,QAAQ,MAAM,EAAE,YAAY,QAAQ,EAAE,aAAa,EAAE,CAAC,CACtD,KAAK,OAAO;MACX,OAAO,EAAE;MACT,UAAU,EAAE;MACZ,MAAM,EAAE;MACR,UAAU,EAAE;KACd,EAemB;KACnB,YAAY,CAAC;KACb,QAAQ,CAAC;KACT,OAhBmC,KAAK,QAAQ,CAAC,EAAA,CAChD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,MAAM,EAAE;MACR,QAAQ,YAAY,EAAE,GAAa;MACnC,UAAU,EAAE;KACd,EAOgB;KAChB,iBAAiB,CAAC;IAGgB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,kCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}
+{"version":3,"file":"crw-search.mjs","names":[],"sources":["../../../../src/tools/search/crw-search.ts"],"mappings":";;AAGA,MAAM,sBAAsB;AAE5B,MAAM,eAAe,SAAyB;CAC5C,IAAI;EACF,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC;CACvB,QAAQ;EACN,OAAO;CACT;AACF;AAIA,MAAM,mBAAmB,SAAiD;CACxE,MAAM,SAAsC;EAAE,KAAK,CAAC;EAAG,QAAQ,CAAC;EAAG,MAAM,CAAC;CAAE;CAC5E,KAAK,MAAM,OAAO,MAChB,IAAI,IAAI,aAAa,QACnB,OAAO,KAAK,KAAK,GAAG;MACf,IAAI,IAAI,aAAa,UAC1B,OAAO,OAAO,KAAK;EAAE,GAAG;EAAK,UAAU,IAAI;CAAI,CAAC;MAEhD,OAAO,IAAI,KAAK,GAAG;CAGvB,OAAO;AACT;AAEA,MAAa,gBACX,QACA,QACA,YAGG;CAKH,MAAM,QACJ,UACA,QAAQ,IAAI,eACZ,0BAAA,CACA,QAAQ,QAAQ,EAAE;CACpB,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ,GAAG,KAAK;EAChB,SAAS,SAAS,WAAW;CAC/B;CAEA,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GACF,MAAM,QAAQ,KAAK,IACjB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAGA,IAAI;GACJ,IAAI,SAAS,UACX,UAAU,CAAC,QAAQ;QACd,IAAI,SAAS,QAClB,UAAU,CAAC,MAAM;QAEjB,UAAU,SAAS,kBAAkB,OAAO,CAAC,OAAO,QAAQ,IAAI,CAAC,KAAK;GAGxE,MAAM,UAA8B;IAAE;IAAO;IAAO;GAAQ;GAC5D,IAAI,QAAQ,MAEV,QAAQ,MAAM,OAAO;GAGvB,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,gBAAgB,UAAU,OAAO;GAc3C,MAAM,QAAO,MAXU,MAAM,KAC3B,OAAO,QACP,SACA;IACE;IACA,SAAS,OAAO;IAChB,WAAW,SAAS;IACpB,YAAY,SAAS;GACvB,CACF,EAAA,CAEsB;GACtB,IAAI,KAAK,YAAY,OACnB,OAAO;IACL,SAAS;IACT,OAAO,0BACL,KAAK,cAAc,OAAO,IAAI,KAAK,WAAW,MAAM,KACnD,KAAK,SAAS;GACnB;GASF,MAAM,YAAY,KAAK,QAAQ,CAAC;GAChC,MAAM,OAA0B,MAAM,QAAQ,SAAS,IACnD,gBAAgB,SAAS,IACxB,UAAU,WAAW;GAuC1B,OAAO;IAAE,SAAS;IAAM,MAAM;KAR5B,UA9ByC,KAAK,OAAO,CAAC,EAAA,CACrD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,UAAU,EAAE;KACd,EAuBsB;KACtB,SAtBqC,KAAK,UAAU,CAAC,EAAA,CACpD,QAAQ,MAAM,EAAE,YAAY,QAAQ,EAAE,aAAa,EAAE,CAAC,CACtD,KAAK,OAAO;MACX,OAAO,EAAE;MACT,UAAU,EAAE;MACZ,MAAM,EAAE;MACR,UAAU,EAAE;KACd,EAemB;KACnB,YAAY,CAAC;KACb,QAAQ,CAAC;KACT,OAhBmC,KAAK,QAAQ,CAAC,EAAA,CAChD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,MAAM,EAAE;MACR,QAAQ,YAAY,EAAE,GAAa;MACnC,UAAU,EAAE;KACd,EAOgB;KAChB,iBAAiB,CAAC;IAGgB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,kCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}

package/dist/esm/tools/search/firecrawl.mjs

@@ -2,10 +2,6 @@ import { createDefaultLogger } from "./utils.mjs";
 import { processContent } from "./content.mjs";
 import axios from "axios";
 //#region src/tools/search/firecrawl.ts
-/**
-* Firecrawl scraper implementation
-* Uses the Firecrawl API to scrape web pages
-*/
 var FirecrawlScraper = class {
 	apiKey;
 	apiUrl;
@@ -58,12 +54,6 @@ var FirecrawlScraper = class {
 		if (!this.apiKey) this.logger.warn("FIRECRAWL_API_KEY is not set. Scraping will not work.");
 		this.logger.debug(`Firecrawl scraper initialized with API URL: ${this.apiUrl}`);
 	}
-	/**
-	* Scrape a single URL
-	* @param url URL to scrape
-	* @param options Scrape options
-	* @returns Scrape response
-	*/
 	async scrapeUrl(url, options = {}) {
 		if (!this.apiKey) return [url, {
 			success: false,
@@ -106,11 +96,6 @@ var FirecrawlScraper = class {
 			}];
 		}
 	}
-	/**
-	* Extract content from scrape response
-	* @param response Scrape response
-	* @returns Extracted content or empty string if not available
-	*/
 	extractContent(response) {
 		if (!response.success || !response.data) return ["", void 0];
 		if (response.data.markdown != null && response.data.html != null) try {
@@ -125,21 +110,11 @@ var FirecrawlScraper = class {
 		if (response.data.rawHtml != null) return [response.data.rawHtml, void 0];
 		return ["", void 0];
 	}
-	/**
-	* Extract metadata from scrape response
-	* @param response Scrape response
-	* @returns Metadata object
-	*/
 	extractMetadata(response) {
 		if (!response.success || !response.data || !response.data.metadata) return {};
 		return response.data.metadata;
 	}
 };
-/**
-* Create a Firecrawl scraper instance
-* @param config Scraper configuration
-* @returns Firecrawl scraper instance
-*/
 const createFirecrawlScraper = (config = {}) => {
 	return new FirecrawlScraper(config);
 };

package/dist/esm/tools/search/firecrawl.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"firecrawl.mjs","names":[],"sources":["../../../../src/tools/search/firecrawl.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/**\n * Firecrawl scraper implementation\n * Uses the Firecrawl API to scrape web pages\n */\nexport class FirecrawlScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private version: string;\n  private defaultFormats: string[];\n  private timeout: number;\n  private logger: t.Logger;\n  private includeTags?: string[];\n  private excludeTags?: string[];\n  private waitFor?: number;\n  private maxAge?: number;\n  private mobile?: boolean;\n  private skipTlsVerification?: boolean;\n  private blockAds?: boolean;\n  private removeBase64Images?: boolean;\n  private parsePDF?: boolean;\n  private storeInCache?: boolean;\n  private zeroDataRetention?: boolean;\n  private headers?: Record<string, string>;\n  private location?: { country?: string; languages?: string[] };\n  private onlyMainContent?: boolean;\n  private changeTrackingOptions?: object;\n  private httpAgent?: t.HttpAgent;\n  private httpsAgent?: t.HttpsAgent;\n\n  constructor(config: t.FirecrawlScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';\n\n    this.version = config.version ?? 'v2';\n\n    const baseUrl =\n      config.apiUrl ??\n      process.env.FIRECRAWL_BASE_URL ??\n      'https://api.firecrawl.dev';\n    this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/${this.version}/scrape`;\n\n    this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];\n    this.timeout = config.timeout ?? 7500;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    this.includeTags = config.includeTags;\n    this.excludeTags = config.excludeTags;\n    this.waitFor = config.waitFor;\n    this.maxAge = config.maxAge;\n    this.mobile = config.mobile;\n    this.skipTlsVerification = config.skipTlsVerification;\n    this.blockAds = config.blockAds;\n    this.removeBase64Images = config.removeBase64Images;\n    this.parsePDF = config.parsePDF;\n    this.storeInCache = config.storeInCache;\n    this.zeroDataRetention = config.zeroDataRetention;\n    this.headers = config.headers;\n    this.location = config.location;\n    this.onlyMainContent = config.onlyMainContent;\n    this.changeTrackingOptions = config.changeTrackingOptions;\n    this.httpAgent = config.httpAgent;\n    this.httpsAgent = config.httpsAgent;\n\n    if (!this.apiKey) {\n      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.FirecrawlScrapeOptions = {}\n  ): Promise<[string, t.FirecrawlScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'FIRECRAWL_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = omitUndefined({\n        url,\n        formats: options.formats ?? this.defaultFormats,\n        includeTags: options.includeTags ?? this.includeTags,\n        excludeTags: options.excludeTags ?? this.excludeTags,\n        headers: options.headers ?? this.headers,\n        waitFor: options.waitFor ?? this.waitFor,\n        timeout: options.timeout ?? this.timeout,\n        onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n        maxAge: options.maxAge ?? this.maxAge,\n        mobile: options.mobile ?? this.mobile,\n        skipTlsVerification:\n          options.skipTlsVerification ?? this.skipTlsVerification,\n        parsePDF: options.parsePDF ?? this.parsePDF,\n        location: options.location ?? this.location,\n        removeBase64Images:\n          options.removeBase64Images ?? this.removeBase64Images,\n        blockAds: options.blockAds ?? this.blockAds,\n        storeInCache: options.storeInCache ?? this.storeInCache,\n        zeroDataRetention: options.zeroDataRetention ?? this.zeroDataRetention,\n        changeTrackingOptions:\n          options.changeTrackingOptions ?? this.changeTrackingOptions,\n      });\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'Content-Type': 'application/json',\n          Authorization: `Bearer ${this.apiKey}`,\n        },\n        timeout: this.timeout,\n        httpAgent: this.httpAgent,\n        httpsAgent: this.httpsAgent,\n      });\n\n      return [url, response.data];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Firecrawl API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.FirecrawlScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null && response.data.html != null) {\n      try {\n        const { markdown, ...rest } = processContent(\n          response.data.html,\n          response.data.markdown\n        );\n        return [markdown, rest];\n      } catch (error) {\n        this.logger.error('Error processing content:', error);\n        return [response.data.markdown, undefined];\n      }\n    } else if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    // Fall back to HTML content\n    if (response.data.html != null) {\n      return [response.data.html, undefined];\n    }\n\n    // Fall back to raw HTML content\n    if (response.data.rawHtml != null) {\n      return [response.data.rawHtml, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Firecrawl scraper instance\n * @param config Scraper configuration\n * @returns Firecrawl scraper instance\n */\nexport const createFirecrawlScraper = (\n  config: t.FirecrawlScraperConfig = {}\n): FirecrawlScraper => {\n  return new FirecrawlScraper(config);\n};\n\n// Helper function to clean up payload for firecrawl\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n  return Object.fromEntries(\n    Object.entries(obj).filter(([, v]) => v !== undefined)\n  ) as Partial<T>;\n}\n"],"mappings":";;;;;;;;AASA,IAAa,mBAAb,MAAuD;CACrD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAmC,CAAC,GAAG;EACjD,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,qBAAqB;EAEhE,KAAK,UAAU,OAAO,WAAW;EAEjC,MAAM,UACJ,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE,GAAG,KAAK,QAAQ;EAE7D,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,SAAS;EAC9D,KAAK,UAAU,OAAO,WAAW;EAEjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO;EACrB,KAAK,sBAAsB,OAAO;EAClC,KAAK,WAAW,OAAO;EACvB,KAAK,qBAAqB,OAAO;EACjC,KAAK,WAAW,OAAO;EACvB,KAAK,eAAe,OAAO;EAC3B,KAAK,oBAAoB,OAAO;EAChC,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,kBAAkB,OAAO;EAC9B,KAAK,wBAAwB,OAAO;EACpC,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EAEzB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,uDAAuD;EAG1E,KAAK,OAAO,MACV,+CAA+C,KAAK,QACtD;CACF;;;;;;;CAQA,MAAM,UACJ,KACA,UAAoC,CAAC,GACS;EAC9C,IAAI,CAAC,KAAK,QACR,OAAO,CACL,KACA;GACE,SAAS;GACT,OAAO;EACT,CACF;EAGF,IAAI;GACF,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,QAAQ,QAAQ,UAAU,KAAK;IAC/B,QAAQ,QAAQ,UAAU,KAAK;IAC/B,qBACE,QAAQ,uBAAuB,KAAK;IACtC,UAAU,QAAQ,YAAY,KAAK;IACnC,UAAU,QAAQ,YAAY,KAAK;IACnC,oBACE,QAAQ,sBAAsB,KAAK;IACrC,UAAU,QAAQ,YAAY,KAAK;IACnC,cAAc,QAAQ,gBAAgB,KAAK;IAC3C,mBAAmB,QAAQ,qBAAqB,KAAK;IACrD,uBACE,QAAQ,yBAAyB,KAAK;GAC1C,CAAC;GAWD,OAAO,CAAC,MAAK,MAVU,MAAM,KAAK,KAAK,QAAQ,SAAS;IACtD,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;IACd,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CAAC,EAAA,CAEqB,IAAI;EAC5B,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,iCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;CAOA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,IAAI,SAAS,KAAK,YAAY,QAAQ,SAAS,KAAK,QAAQ,MAC1D,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,SAAS,KAAK,MACd,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAG1C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;;;;;;CAOA,gBAAgB,UAAuD;EACrE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,0BACX,SAAmC,CAAC,MACf;CACrB,OAAO,IAAI,iBAAiB,MAAM;AACpC;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}
+{"version":3,"file":"firecrawl.mjs","names":[],"sources":["../../../../src/tools/search/firecrawl.ts"],"mappings":";;;;AASA,IAAa,mBAAb,MAAuD;CACrD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAmC,CAAC,GAAG;EACjD,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,qBAAqB;EAEhE,KAAK,UAAU,OAAO,WAAW;EAEjC,MAAM,UACJ,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE,GAAG,KAAK,QAAQ;EAE7D,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,SAAS;EAC9D,KAAK,UAAU,OAAO,WAAW;EAEjC,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO;EACrB,KAAK,sBAAsB,OAAO;EAClC,KAAK,WAAW,OAAO;EACvB,KAAK,qBAAqB,OAAO;EACjC,KAAK,WAAW,OAAO;EACvB,KAAK,eAAe,OAAO;EAC3B,KAAK,oBAAoB,OAAO;EAChC,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,kBAAkB,OAAO;EAC9B,KAAK,wBAAwB,OAAO;EACpC,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EAEzB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,uDAAuD;EAG1E,KAAK,OAAO,MACV,+CAA+C,KAAK,QACtD;CACF;CAQA,MAAM,UACJ,KACA,UAAoC,CAAC,GACS;EAC9C,IAAI,CAAC,KAAK,QACR,OAAO,CACL,KACA;GACE,SAAS;GACT,OAAO;EACT,CACF;EAGF,IAAI;GACF,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,QAAQ,QAAQ,UAAU,KAAK;IAC/B,QAAQ,QAAQ,UAAU,KAAK;IAC/B,qBACE,QAAQ,uBAAuB,KAAK;IACtC,UAAU,QAAQ,YAAY,KAAK;IACnC,UAAU,QAAQ,YAAY,KAAK;IACnC,oBACE,QAAQ,sBAAsB,KAAK;IACrC,UAAU,QAAQ,YAAY,KAAK;IACnC,cAAc,QAAQ,gBAAgB,KAAK;IAC3C,mBAAmB,QAAQ,qBAAqB,KAAK;IACrD,uBACE,QAAQ,yBAAyB,KAAK;GAC1C,CAAC;GAWD,OAAO,CAAC,MAAK,MAVU,MAAM,KAAK,KAAK,QAAQ,SAAS;IACtD,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;IACd,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CAAC,EAAA,CAEqB,IAAI;EAC5B,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,iCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;CAOA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,IAAI,SAAS,KAAK,YAAY,QAAQ,SAAS,KAAK,QAAQ,MAC1D,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAAS,eAC5B,SAAS,KAAK,MACd,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAG1C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;CAOA,gBAAgB,UAAuD;EACrE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;AAOA,MAAa,0BACX,SAAmC,CAAC,MACf;CACrB,OAAO,IAAI,iBAAiB,MAAM;AACpC;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}

package/dist/esm/tools/search/format.mjs

@@ -1,27 +1,14 @@
 import { fileExtRegex, getDomainName } from "./utils.mjs";
 //#region src/tools/search/format.ts
-/** Default per-search budget for model-facing highlight content (chars). Hosts
-*  that know the context window (e.g. LibreChat) pass a window-relative value;
-*  this fixed fallback keeps standalone consumers bounded instead of dumping the
-*  full reranked content of every source into the prompt. */
 const DEFAULT_MAX_LLM_OUTPUT_CHARS = 5e4;
-/** Minimum room (chars) worth filling with a truncated boundary highlight; below
-*  this we drop it whole rather than emit a useless sliver. */
 const MIN_PARTIAL_HIGHLIGHT_CHARS = 200;
-/** Resolves the per-search highlight budget from config, the
-*  `SEARCH_MAX_LLM_OUTPUT_CHARS` env var, or the default (50,000 chars). */
 function resolveMaxLLMOutputChars(maxOutputChars) {
 	if (maxOutputChars != null && maxOutputChars > 0) return maxOutputChars;
 	const envValue = Number(process.env.SEARCH_MAX_LLM_OUTPUT_CHARS);
 	if (Number.isFinite(envValue) && envValue > 0) return envValue;
 	return DEFAULT_MAX_LLM_OUTPUT_CHARS;
 }
-/** Inline citation markers embedded in highlight text, e.g. `(link#2 "Title")`.
-*  Mirrors the matcher in `highlights.ts` so truncation can tell which citations
-*  survive in a sliced prefix. */
 const REFERENCE_MARKER_REGEX = /\((link|image|video)#(\d+)(?:\s+"[^"]*")?\)/g;
-/** Builds the set of `type#originalIndex` keys whose complete citation marker
-*  appears in `text`, so references can be filtered to those still visible. */
 function visibleReferenceKeys(text) {
 	const keys = /* @__PURE__ */ new Set();
 	if (!text.includes("#")) return keys;
@@ -30,10 +17,6 @@ function visibleReferenceKeys(text) {
 	while ((match = regex.exec(text)) !== null) keys.add(`${match[1]}#${parseInt(match[2], 10) - 1}`);
 	return keys;
 }
-/** Truncates a highlight to `maxLen` chars of (already-trimmed) text, keeping
-*  only the references whose markers survive in the kept prefix — markers in the
-*  cut tail would otherwise emit Core References for citations the model can no
-*  longer see, while a blanket drop would lose still-visible ones. */
 function truncateHighlight(highlight, text, maxLen) {
 	const prefix = text.slice(0, maxLen);
 	const truncated = {
@@ -47,16 +30,6 @@ function truncateHighlight(highlight, text, maxLen) {
 	}
 	return truncated;
 }
-/** Bounds the highlight chunks — the dominant, unbounded part of search output —
-*  to `maxChars`, walking sources in relevance order (organic first, then news;
-*  highlights in their reranked order). Whole highlights are kept until the
-*  budget is hit, the boundary one is truncated if meaningful room remains, and
-*  every later highlight is dropped (relevance-ordered prefix). Blank highlights
-*  are skipped (never rendered, so never charged); a truncated highlight keeps
-*  only references whose markers survive in the kept prefix. Snippets/titles/URLs
-*  are left untouched (small, high-signal) and per-source `content` stays in the
-*  `WEB_SEARCH` artifact for citations. Mutates `results` in place; returns how
-*  many highlights were dropped or truncated (0 when everything fit). */
 function trimHighlightsToBudget(results, maxChars) {
 	let used = 0;
 	let trimmed = 0;
@@ -89,7 +62,6 @@ function addHighlightSection() {
 	return ["\n## Highlights", ""];
 }
 function formatSource(source, index, turn, sourceType, references) {
-	/** Array of all lines to include in the output */
 	const outputLines = [];
 	outputLines.push(`# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `"${source.title}"` : "(no title)"}`);
 	outputLines.push(`\nAnchor: \\ue202turn${turn}${sourceType}${index}`);
@@ -141,9 +113,7 @@ function formatSource(source, index, turn, sourceType, references) {
 	return outputLines.join("\n");
 }
 function formatResultsForLLM(turn, results, maxOutputChars) {
-	/** Bound highlight content to the per-search budget before formatting */
 	const trimmedHighlights = trimHighlightsToBudget(results, resolveMaxLLMOutputChars(maxOutputChars));
-	/** Array to collect all output lines */
 	const outputLines = [];
 	const addSection = (title) => {
 		outputLines.push("");

package/dist/esm/tools/search/format.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"format.mjs","names":[],"sources":["../../../../src/tools/search/format.ts"],"sourcesContent":["import type * as t from './types';\nimport { getDomainName, fileExtRegex } from './utils';\n\n/** Default per-search budget for model-facing highlight content (chars). Hosts\n *  that know the context window (e.g. LibreChat) pass a window-relative value;\n *  this fixed fallback keeps standalone consumers bounded instead of dumping the\n *  full reranked content of every source into the prompt. */\nconst DEFAULT_MAX_LLM_OUTPUT_CHARS = 50000;\n\n/** Minimum room (chars) worth filling with a truncated boundary highlight; below\n *  this we drop it whole rather than emit a useless sliver. */\nconst MIN_PARTIAL_HIGHLIGHT_CHARS = 200;\n\n/** Resolves the per-search highlight budget from config, the\n *  `SEARCH_MAX_LLM_OUTPUT_CHARS` env var, or the default (50,000 chars). */\nexport function resolveMaxLLMOutputChars(maxOutputChars?: number): number {\n  if (maxOutputChars != null && maxOutputChars > 0) {\n    return maxOutputChars;\n  }\n  const envValue = Number(process.env.SEARCH_MAX_LLM_OUTPUT_CHARS);\n  if (Number.isFinite(envValue) && envValue > 0) {\n    return envValue;\n  }\n  return DEFAULT_MAX_LLM_OUTPUT_CHARS;\n}\n\n/** Inline citation markers embedded in highlight text, e.g. `(link#2 \"Title\")`.\n *  Mirrors the matcher in `highlights.ts` so truncation can tell which citations\n *  survive in a sliced prefix. */\nconst REFERENCE_MARKER_REGEX = /\\((link|image|video)#(\\d+)(?:\\s+\"[^\"]*\")?\\)/g;\n\n/** Builds the set of `type#originalIndex` keys whose complete citation marker\n *  appears in `text`, so references can be filtered to those still visible. */\nfunction visibleReferenceKeys(text: string): Set<string> {\n  const keys = new Set<string>();\n  if (!text.includes('#')) {\n    return keys;\n  }\n  const regex = new RegExp(REFERENCE_MARKER_REGEX);\n  let match: RegExpExecArray | null;\n  while ((match = regex.exec(text)) !== null) {\n    keys.add(`${match[1]}#${parseInt(match[2], 10) - 1}`);\n  }\n  return keys;\n}\n\n/** Truncates a highlight to `maxLen` chars of (already-trimmed) text, keeping\n *  only the references whose markers survive in the kept prefix — markers in the\n *  cut tail would otherwise emit Core References for citations the model can no\n *  longer see, while a blanket drop would lose still-visible ones. */\nfunction truncateHighlight(highlight: t.Highlight, text: string, maxLen: number): t.Highlight {\n  const prefix = text.slice(0, maxLen);\n  const truncated: t.Highlight = { score: highlight.score, text: `${prefix}\\n…[truncated]` };\n  if (highlight.references != null && highlight.references.length > 0) {\n    const keys = visibleReferenceKeys(prefix);\n    const visible = highlight.references.filter((ref) => keys.has(`${ref.type}#${ref.originalIndex}`));\n    if (visible.length > 0) {\n      truncated.references = visible;\n    }\n  }\n  return truncated;\n}\n\n/** Bounds the highlight chunks — the dominant, unbounded part of search output —\n *  to `maxChars`, walking sources in relevance order (organic first, then news;\n *  highlights in their reranked order). Whole highlights are kept until the\n *  budget is hit, the boundary one is truncated if meaningful room remains, and\n *  every later highlight is dropped (relevance-ordered prefix). Blank highlights\n *  are skipped (never rendered, so never charged); a truncated highlight keeps\n *  only references whose markers survive in the kept prefix. Snippets/titles/URLs\n *  are left untouched (small, high-signal) and per-source `content` stays in the\n *  `WEB_SEARCH` artifact for citations. Mutates `results` in place; returns how\n *  many highlights were dropped or truncated (0 when everything fit). */\nfunction trimHighlightsToBudget(results: t.SearchResultData, maxChars: number): number {\n  let used = 0;\n  let trimmed = 0;\n  const sections: (t.ValidSource[] | undefined)[] = [results.organic, results.topStories];\n  for (const sources of sections) {\n    if (sources == null) {\n      continue;\n    }\n    for (const source of sources) {\n      const highlights = source.highlights;\n      if (highlights == null || highlights.length === 0) {\n        continue;\n      }\n      const kept: t.Highlight[] = [];\n      for (const highlight of highlights) {\n        const text = highlight.text.trim();\n        if (text.length === 0) {\n          continue;\n        }\n        if (used + text.length <= maxChars) {\n          kept.push(highlight);\n          used += text.length;\n          continue;\n        }\n        const remaining = maxChars - used;\n        if (remaining >= MIN_PARTIAL_HIGHLIGHT_CHARS) {\n          kept.push(truncateHighlight(highlight, text, remaining));\n        }\n        used = maxChars;\n        trimmed++;\n      }\n      source.highlights = kept;\n    }\n  }\n  return trimmed;\n}\n\nfunction addHighlightSection(): string[] {\n  return ['\\n## Highlights', ''];\n}\n\n// Helper function to format a source (organic or top story)\nfunction formatSource(\n  source: t.ValidSource,\n  index: number,\n  turn: number,\n  sourceType: 'search' | 'news',\n  references: t.ResultReference[]\n): string {\n  /** Array of all lines to include in the output */\n  const outputLines: string[] = [];\n\n  // Add the title\n  outputLines.push(\n    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `\"${source.title}\"` : '(no title)'}`\n  );\n  outputLines.push(`\\nAnchor: \\\\ue202turn${turn}${sourceType}${index}`);\n  outputLines.push(`URL: ${source.link}`);\n\n  // Add optional fields\n  if ('snippet' in source && source.snippet != null) {\n    outputLines.push(`Summary: ${source.snippet}`);\n  }\n\n  if (source.date != null) {\n    outputLines.push(`Date: ${source.date}`);\n  }\n\n  if (source.attribution != null) {\n    outputLines.push(`Source: ${source.attribution}`);\n  }\n\n  // Add highlight section or empty line\n  if ((source.highlights?.length ?? 0) > 0) {\n    outputLines.push(...addHighlightSection());\n  } else {\n    outputLines.push('');\n  }\n\n  // Process highlights if they exist\n  (source.highlights ?? [])\n    .filter((h) => h.text.trim().length > 0)\n    .forEach((h, hIndex) => {\n      outputLines.push(\n        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`\n      );\n      outputLines.push('');\n      outputLines.push('```text');\n      outputLines.push(h.text.trim());\n      outputLines.push('```');\n      outputLines.push('');\n\n      if (h.references != null && h.references.length) {\n        let hasHeader = false;\n        const refLines: string[] = [];\n\n        for (let j = 0; j < h.references.length; j++) {\n          const ref = h.references[j];\n          if (ref.reference.originalUrl.includes('mailto:')) {\n            continue;\n          }\n          if (ref.type !== 'link') {\n            continue;\n          }\n          if (fileExtRegex.test(ref.reference.originalUrl)) {\n            continue;\n          }\n          references.push({\n            type: ref.type,\n            link: ref.reference.originalUrl,\n            attribution: getDomainName(ref.reference.originalUrl),\n            title: (\n              ((ref.reference.title ?? '') || ref.reference.text) ??\n              ''\n            ).split('\\n')[0],\n          });\n\n          if (!hasHeader) {\n            refLines.push('Core References:');\n            hasHeader = true;\n          }\n\n          refLines.push(\n            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`\n          );\n          refLines.push(\n            `\\t- Anchor: \\\\ue202turn${turn}ref${references.length - 1}`\n          );\n        }\n\n        if (hasHeader) {\n          outputLines.push(...refLines);\n          outputLines.push('');\n        }\n      }\n\n      if (hIndex < (source.highlights?.length ?? 0) - 1) {\n        outputLines.push('---');\n        outputLines.push('');\n      }\n    });\n\n  outputLines.push('');\n  return outputLines.join('\\n');\n}\n\nexport function formatResultsForLLM(\n  turn: number,\n  results: t.SearchResultData,\n  maxOutputChars?: number\n): { output: string; references: t.ResultReference[] } {\n  /** Bound highlight content to the per-search budget before formatting */\n  const trimmedHighlights = trimHighlightsToBudget(\n    results,\n    resolveMaxLLMOutputChars(maxOutputChars)\n  );\n\n  /** Array to collect all output lines */\n  const outputLines: string[] = [];\n\n  const addSection = (title: string): void => {\n    outputLines.push('');\n    outputLines.push(`=== ${title} ===`);\n    outputLines.push('');\n  };\n\n  const references: t.ResultReference[] = [];\n\n  // Organic (web) results\n  if (results.organic?.length != null && results.organic.length > 0) {\n    addSection(`Web Results, Turn ${turn}`);\n    for (let i = 0; i < results.organic.length; i++) {\n      const r = results.organic[i];\n      outputLines.push(formatSource(r, i, turn, 'search', references));\n      delete results.organic[i].highlights;\n    }\n  }\n\n  // Top stories (news)\n  const topStories = results.topStories ?? [];\n  if (topStories.length) {\n    addSection('News Results');\n    for (let i = 0; i < topStories.length; i++) {\n      const r = topStories[i];\n      outputLines.push(formatSource(r, i, turn, 'news', references));\n      if (results.topStories?.[i]?.highlights) {\n        delete results.topStories[i].highlights;\n      }\n    }\n  }\n\n  // // Images\n  // const images = results.images ?? [];\n  // if (images.length) {\n  //   addSection('Image Results');\n  //   const imageLines = images.map((img, i) => [\n  //     `Anchor: \\ue202turn0image${i}`,\n  //     `Title: ${img.title ?? '(no title)'}`,\n  //     `Image URL: ${img.imageUrl}`,\n  //     ''\n  //   ].join('\\n'));\n  //   outputLines.push(imageLines.join('\\n'));\n  // }\n\n  // Knowledge Graph\n  if (results.knowledgeGraph != null) {\n    addSection('Knowledge Graph');\n    const kgLines = [\n      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,\n      results.knowledgeGraph.type != null\n        ? `**Type:** ${results.knowledgeGraph.type}`\n        : '',\n      results.knowledgeGraph.description != null\n        ? `**Description:** ${results.knowledgeGraph.description}`\n        : '',\n      results.knowledgeGraph.descriptionSource != null\n        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`\n        : '',\n      results.knowledgeGraph.descriptionLink != null\n        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`\n        : '',\n      results.knowledgeGraph.imageUrl != null\n        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`\n        : '',\n      results.knowledgeGraph.website != null\n        ? `**Website:** ${results.knowledgeGraph.website}`\n        : '',\n      results.knowledgeGraph.attributes != null\n        ? `**Attributes:**\\n\\`\\`\\`json\\n${JSON.stringify(\n          results.knowledgeGraph.attributes,\n          null,\n          2\n        )}\\n\\`\\`\\``\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(kgLines.join('\\n\\n'));\n  }\n\n  // Answer Box\n  if (results.answerBox != null) {\n    addSection('Answer Box');\n    const abLines = [\n      results.answerBox.title != null\n        ? `**Title:** ${results.answerBox.title}`\n        : '',\n      results.answerBox.snippet != null\n        ? `**Snippet:** ${results.answerBox.snippet}`\n        : '',\n      results.answerBox.snippetHighlighted != null\n        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted\n          .map((s) => `\\`${s}\\``)\n          .join(' ')}`\n        : '',\n      results.answerBox.link != null\n        ? `**Link:** ${results.answerBox.link}`\n        : '',\n      '',\n    ].filter(Boolean);\n\n    outputLines.push(abLines.join('\\n\\n'));\n  }\n\n  // People also ask\n  const peopleAlsoAsk = results.peopleAlsoAsk ?? [];\n  if (peopleAlsoAsk.length) {\n    addSection('People Also Ask');\n\n    const paaLines: string[] = [];\n    peopleAlsoAsk.forEach((p, i) => {\n      const questionLines = [\n        `### Question ${i + 1}:`,\n        `\"${p.question}\"`,\n        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,\n        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,\n        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,\n        '',\n      ].filter(Boolean);\n\n      paaLines.push(questionLines.join('\\n\\n'));\n    });\n\n    outputLines.push(paaLines.join(''));\n  }\n\n  let output = outputLines.join('\\n').trim();\n  if (trimmedHighlights > 0) {\n    output += `\\n\\n_[${trimmedHighlights} additional highlight${\n      trimmedHighlights === 1 ? '' : 's'\n    } omitted to fit the context budget; the cited sources contain the full content.]_`;\n  }\n  return { output, references };\n}\n"],"mappings":";;;;;;AAOA,MAAM,+BAA+B;;;AAIrC,MAAM,8BAA8B;;;AAIpC,SAAgB,yBAAyB,gBAAiC;CACxE,IAAI,kBAAkB,QAAQ,iBAAiB,GAC7C,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,2BAA2B;CAC/D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;;;;AAKA,MAAM,yBAAyB;;;AAI/B,SAAS,qBAAqB,MAA2B;CACvD,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,CAAC,KAAK,SAAS,GAAG,GACpB,OAAO;CAET,MAAM,QAAQ,IAAI,OAAO,sBAAsB;CAC/C,IAAI;CACJ,QAAQ,QAAQ,MAAM,KAAK,IAAI,OAAO,MACpC,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG,SAAS,MAAM,IAAI,EAAE,IAAI,GAAG;CAEtD,OAAO;AACT;;;;;AAMA,SAAS,kBAAkB,WAAwB,MAAc,QAA6B;CAC5F,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM;CACnC,MAAM,YAAyB;EAAE,OAAO,UAAU;EAAO,MAAM,GAAG,OAAO;CAAgB;CACzF,IAAI,UAAU,cAAc,QAAQ,UAAU,WAAW,SAAS,GAAG;EACnE,MAAM,OAAO,qBAAqB,MAAM;EACxC,MAAM,UAAU,UAAU,WAAW,QAAQ,QAAQ,KAAK,IAAI,GAAG,IAAI,KAAK,GAAG,IAAI,eAAe,CAAC;EACjG,IAAI,QAAQ,SAAS,GACnB,UAAU,aAAa;CAE3B;CACA,OAAO;AACT;;;;;;;;;;;AAYA,SAAS,uBAAuB,SAA6B,UAA0B;CACrF,IAAI,OAAO;CACX,IAAI,UAAU;CACd,MAAM,WAA4C,CAAC,QAAQ,SAAS,QAAQ,UAAU;CACtF,KAAK,MAAM,WAAW,UAAU;EAC9B,IAAI,WAAW,MACb;EAEF,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,aAAa,OAAO;GAC1B,IAAI,cAAc,QAAQ,WAAW,WAAW,GAC9C;GAEF,MAAM,OAAsB,CAAC;GAC7B,KAAK,MAAM,aAAa,YAAY;IAClC,MAAM,OAAO,UAAU,KAAK,KAAK;IACjC,IAAI,KAAK,WAAW,GAClB;IAEF,IAAI,OAAO,KAAK,UAAU,UAAU;KAClC,KAAK,KAAK,SAAS;KACnB,QAAQ,KAAK;KACb;IACF;IACA,MAAM,YAAY,WAAW;IAC7B,IAAI,aAAa,6BACf,KAAK,KAAK,kBAAkB,WAAW,MAAM,SAAS,CAAC;IAEzD,OAAO;IACP;GACF;GACA,OAAO,aAAa;EACtB;CACF;CACA,OAAO;AACT;AAEA,SAAS,sBAAgC;CACvC,OAAO,CAAC,mBAAmB,EAAE;AAC/B;AAGA,SAAS,aACP,QACA,OACA,MACA,YACA,YACQ;;CAER,MAAM,cAAwB,CAAC;CAG/B,YAAY,KACV,KAAK,WAAW,OAAO,CAAC,CAAC,CAAC,YAAY,IAAI,WAAW,MAAM,CAAC,EAAE,GAAG,MAAM,IAAI,OAAO,SAAS,QAAQ,OAAO,QAAQ,IAAI,OAAO,MAAM,KAAK,cAC1I;CACA,YAAY,KAAK,wBAAwB,OAAO,aAAa,OAAO;CACpE,YAAY,KAAK,QAAQ,OAAO,MAAM;CAGtC,IAAI,aAAa,UAAU,OAAO,WAAW,MAC3C,YAAY,KAAK,YAAY,OAAO,SAAS;CAG/C,IAAI,OAAO,QAAQ,MACjB,YAAY,KAAK,SAAS,OAAO,MAAM;CAGzC,IAAI,OAAO,eAAe,MACxB,YAAY,KAAK,WAAW,OAAO,aAAa;CAIlD,KAAK,OAAO,YAAY,UAAU,KAAK,GACrC,YAAY,KAAK,GAAG,oBAAoB,CAAC;MAEzC,YAAY,KAAK,EAAE;CAIrB,CAAC,OAAO,cAAc,CAAC,EAAA,CACpB,QAAQ,MAAM,EAAE,KAAK,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,CACvC,SAAS,GAAG,WAAW;EACtB,YAAY,KACV,iBAAiB,SAAS,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC,EAAE,EAChE;EACA,YAAY,KAAK,EAAE;EACnB,YAAY,KAAK,SAAS;EAC1B,YAAY,KAAK,EAAE,KAAK,KAAK,CAAC;EAC9B,YAAY,KAAK,KAAK;EACtB,YAAY,KAAK,EAAE;EAEnB,IAAI,EAAE,cAAc,QAAQ,EAAE,WAAW,QAAQ;GAC/C,IAAI,YAAY;GAChB,MAAM,WAAqB,CAAC;GAE5B,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,WAAW,QAAQ,KAAK;IAC5C,MAAM,MAAM,EAAE,WAAW;IACzB,IAAI,IAAI,UAAU,YAAY,SAAS,SAAS,GAC9C;IAEF,IAAI,IAAI,SAAS,QACf;IAEF,IAAI,aAAa,KAAK,IAAI,UAAU,WAAW,GAC7C;IAEF,WAAW,KAAK;KACd,MAAM,IAAI;KACV,MAAM,IAAI,UAAU;KACpB,aAAa,cAAc,IAAI,UAAU,WAAW;KACpD,UACI,IAAI,UAAU,SAAS,OAAO,IAAI,UAAU,SAC9C,GAAA,CACA,MAAM,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,IAAI,CAAC,WAAW;KACd,SAAS,KAAK,kBAAkB;KAChC,YAAY;IACd;IAEA,SAAS,KACP,KAAK,IAAI,KAAK,GAAG,IAAI,gBAAgB,EAAE,IAAI,IAAI,UAAU,aAC3D;IACA,SAAS,KACP,0BAA0B,KAAK,KAAK,WAAW,SAAS,GAC1D;GACF;GAEA,IAAI,WAAW;IACb,YAAY,KAAK,GAAG,QAAQ;IAC5B,YAAY,KAAK,EAAE;GACrB;EACF;EAEA,IAAI,UAAU,OAAO,YAAY,UAAU,KAAK,GAAG;GACjD,YAAY,KAAK,KAAK;GACtB,YAAY,KAAK,EAAE;EACrB;CACF,CAAC;CAEH,YAAY,KAAK,EAAE;CACnB,OAAO,YAAY,KAAK,IAAI;AAC9B;AAEA,SAAgB,oBACd,MACA,SACA,gBACqD;;CAErD,MAAM,oBAAoB,uBACxB,SACA,yBAAyB,cAAc,CACzC;;CAGA,MAAM,cAAwB,CAAC;CAE/B,MAAM,cAAc,UAAwB;EAC1C,YAAY,KAAK,EAAE;EACnB,YAAY,KAAK,OAAO,MAAM,KAAK;EACnC,YAAY,KAAK,EAAE;CACrB;CAEA,MAAM,aAAkC,CAAC;CAGzC,IAAI,QAAQ,SAAS,UAAU,QAAQ,QAAQ,QAAQ,SAAS,GAAG;EACjE,WAAW,qBAAqB,MAAM;EACtC,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,QAAQ,KAAK;GAC/C,MAAM,IAAI,QAAQ,QAAQ;GAC1B,YAAY,KAAK,aAAa,GAAG,GAAG,MAAM,UAAU,UAAU,CAAC;GAC/D,OAAO,QAAQ,QAAQ,EAAE,CAAC;EAC5B;CACF;CAGA,MAAM,aAAa,QAAQ,cAAc,CAAC;CAC1C,IAAI,WAAW,QAAQ;EACrB,WAAW,cAAc;EACzB,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;GAC1C,MAAM,IAAI,WAAW;GACrB,YAAY,KAAK,aAAa,GAAG,GAAG,MAAM,QAAQ,UAAU,CAAC;GAC7D,IAAI,QAAQ,aAAa,EAAE,EAAE,YAC3B,OAAO,QAAQ,WAAW,EAAE,CAAC;EAEjC;CACF;CAgBA,IAAI,QAAQ,kBAAkB,MAAM;EAClC,WAAW,iBAAiB;EAC5B,MAAM,UAAU;GACd,cAAc,QAAQ,eAAe,SAAS;GAC9C,QAAQ,eAAe,QAAQ,OAC3B,aAAa,QAAQ,eAAe,SACpC;GACJ,QAAQ,eAAe,eAAe,OAClC,oBAAoB,QAAQ,eAAe,gBAC3C;GACJ,QAAQ,eAAe,qBAAqB,OACxC,2BAA2B,QAAQ,eAAe,sBAClD;GACJ,QAAQ,eAAe,mBAAmB,OACtC,yBAAyB,QAAQ,eAAe,oBAChD;GACJ,QAAQ,eAAe,YAAY,OAC/B,kBAAkB,QAAQ,eAAe,aACzC;GACJ,QAAQ,eAAe,WAAW,OAC9B,gBAAgB,QAAQ,eAAe,YACvC;GACJ,QAAQ,eAAe,cAAc,OACjC,gCAAgC,KAAK,UACrC,QAAQ,eAAe,YACvB,MACA,CACF,EAAE,YACA;GACJ;EACF,CAAC,CAAC,OAAO,OAAO;EAEhB,YAAY,KAAK,QAAQ,KAAK,MAAM,CAAC;CACvC;CAGA,IAAI,QAAQ,aAAa,MAAM;EAC7B,WAAW,YAAY;EACvB,MAAM,UAAU;GACd,QAAQ,UAAU,SAAS,OACvB,cAAc,QAAQ,UAAU,UAChC;GACJ,QAAQ,UAAU,WAAW,OACzB,gBAAgB,QAAQ,UAAU,YAClC;GACJ,QAAQ,UAAU,sBAAsB,OACpC,4BAA4B,QAAQ,UAAU,mBAC7C,KAAK,MAAM,KAAK,EAAE,GAAG,CAAC,CACtB,KAAK,GAAG,MACT;GACJ,QAAQ,UAAU,QAAQ,OACtB,aAAa,QAAQ,UAAU,SAC/B;GACJ;EACF,CAAC,CAAC,OAAO,OAAO;EAEhB,YAAY,KAAK,QAAQ,KAAK,MAAM,CAAC;CACvC;CAGA,MAAM,gBAAgB,QAAQ,iBAAiB,CAAC;CAChD,IAAI,cAAc,QAAQ;EACxB,WAAW,iBAAiB;EAE5B,MAAM,WAAqB,CAAC;EAC5B,cAAc,SAAS,GAAG,MAAM;GAC9B,MAAM,gBAAgB;IACpB,gBAAgB,IAAI,EAAE;IACtB,IAAI,EAAE,SAAS;IACf,GAAG,EAAE,WAAW,QAAQ,EAAE,UAAU,YAAY,EAAE,YAAY;IAC9D,GAAG,EAAE,SAAS,QAAQ,EAAE,QAAQ,UAAU,EAAE,UAAU;IACtD,GAAG,EAAE,QAAQ,QAAQ,EAAE,OAAO,SAAS,EAAE,SAAS;IAClD;GACF,CAAC,CAAC,OAAO,OAAO;GAEhB,SAAS,KAAK,cAAc,KAAK,MAAM,CAAC;EAC1C,CAAC;EAED,YAAY,KAAK,SAAS,KAAK,EAAE,CAAC;CACpC;CAEA,IAAI,SAAS,YAAY,KAAK,IAAI,CAAC,CAAC,KAAK;CACzC,IAAI,oBAAoB,GACtB,UAAU,SAAS,kBAAkB,uBACnC,sBAAsB,IAAI,KAAK,IAChC;CAEH,OAAO;EAAE;EAAQ;CAAW;AAC9B"}
+{"version":3,"file":"format.mjs","names":[],"sources":["../../../../src/tools/search/format.ts"],"mappings":";;AAOA,MAAM,+BAA+B;AAIrC,MAAM,8BAA8B;AAIpC,SAAgB,yBAAyB,gBAAiC;CACxE,IAAI,kBAAkB,QAAQ,iBAAiB,GAC7C,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,2BAA2B;CAC/D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAKA,MAAM,yBAAyB;AAI/B,SAAS,qBAAqB,MAA2B;CACvD,MAAM,uBAAO,IAAI,IAAY;CAC7B,IAAI,CAAC,KAAK,SAAS,GAAG,GACpB,OAAO;CAET,MAAM,QAAQ,IAAI,OAAO,sBAAsB;CAC/C,IAAI;CACJ,QAAQ,QAAQ,MAAM,KAAK,IAAI,OAAO,MACpC,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG,SAAS,MAAM,IAAI,EAAE,IAAI,GAAG;CAEtD,OAAO;AACT;AAMA,SAAS,kBAAkB,WAAwB,MAAc,QAA6B;CAC5F,MAAM,SAAS,KAAK,MAAM,GAAG,MAAM;CACnC,MAAM,YAAyB;EAAE,OAAO,UAAU;EAAO,MAAM,GAAG,OAAO;CAAgB;CACzF,IAAI,UAAU,cAAc,QAAQ,UAAU,WAAW,SAAS,GAAG;EACnE,MAAM,OAAO,qBAAqB,MAAM;EACxC,MAAM,UAAU,UAAU,WAAW,QAAQ,QAAQ,KAAK,IAAI,GAAG,IAAI,KAAK,GAAG,IAAI,eAAe,CAAC;EACjG,IAAI,QAAQ,SAAS,GACnB,UAAU,aAAa;CAE3B;CACA,OAAO;AACT;AAYA,SAAS,uBAAuB,SAA6B,UAA0B;CACrF,IAAI,OAAO;CACX,IAAI,UAAU;CACd,MAAM,WAA4C,CAAC,QAAQ,SAAS,QAAQ,UAAU;CACtF,KAAK,MAAM,WAAW,UAAU;EAC9B,IAAI,WAAW,MACb;EAEF,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,aAAa,OAAO;GAC1B,IAAI,cAAc,QAAQ,WAAW,WAAW,GAC9C;GAEF,MAAM,OAAsB,CAAC;GAC7B,KAAK,MAAM,aAAa,YAAY;IAClC,MAAM,OAAO,UAAU,KAAK,KAAK;IACjC,IAAI,KAAK,WAAW,GAClB;IAEF,IAAI,OAAO,KAAK,UAAU,UAAU;KAClC,KAAK,KAAK,SAAS;KACnB,QAAQ,KAAK;KACb;IACF;IACA,MAAM,YAAY,WAAW;IAC7B,IAAI,aAAa,6BACf,KAAK,KAAK,kBAAkB,WAAW,MAAM,SAAS,CAAC;IAEzD,OAAO;IACP;GACF;GACA,OAAO,aAAa;EACtB;CACF;CACA,OAAO;AACT;AAEA,SAAS,sBAAgC;CACvC,OAAO,CAAC,mBAAmB,EAAE;AAC/B;AAGA,SAAS,aACP,QACA,OACA,MACA,YACA,YACQ;CAER,MAAM,cAAwB,CAAC;CAG/B,YAAY,KACV,KAAK,WAAW,OAAO,CAAC,CAAC,CAAC,YAAY,IAAI,WAAW,MAAM,CAAC,EAAE,GAAG,MAAM,IAAI,OAAO,SAAS,QAAQ,OAAO,QAAQ,IAAI,OAAO,MAAM,KAAK,cAC1I;CACA,YAAY,KAAK,wBAAwB,OAAO,aAAa,OAAO;CACpE,YAAY,KAAK,QAAQ,OAAO,MAAM;CAGtC,IAAI,aAAa,UAAU,OAAO,WAAW,MAC3C,YAAY,KAAK,YAAY,OAAO,SAAS;CAG/C,IAAI,OAAO,QAAQ,MACjB,YAAY,KAAK,SAAS,OAAO,MAAM;CAGzC,IAAI,OAAO,eAAe,MACxB,YAAY,KAAK,WAAW,OAAO,aAAa;CAIlD,KAAK,OAAO,YAAY,UAAU,KAAK,GACrC,YAAY,KAAK,GAAG,oBAAoB,CAAC;MAEzC,YAAY,KAAK,EAAE;CAIrB,CAAC,OAAO,cAAc,CAAC,EAAA,CACpB,QAAQ,MAAM,EAAE,KAAK,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,CACvC,SAAS,GAAG,WAAW;EACtB,YAAY,KACV,iBAAiB,SAAS,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC,EAAE,EAChE;EACA,YAAY,KAAK,EAAE;EACnB,YAAY,KAAK,SAAS;EAC1B,YAAY,KAAK,EAAE,KAAK,KAAK,CAAC;EAC9B,YAAY,KAAK,KAAK;EACtB,YAAY,KAAK,EAAE;EAEnB,IAAI,EAAE,cAAc,QAAQ,EAAE,WAAW,QAAQ;GAC/C,IAAI,YAAY;GAChB,MAAM,WAAqB,CAAC;GAE5B,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,WAAW,QAAQ,KAAK;IAC5C,MAAM,MAAM,EAAE,WAAW;IACzB,IAAI,IAAI,UAAU,YAAY,SAAS,SAAS,GAC9C;IAEF,IAAI,IAAI,SAAS,QACf;IAEF,IAAI,aAAa,KAAK,IAAI,UAAU,WAAW,GAC7C;IAEF,WAAW,KAAK;KACd,MAAM,IAAI;KACV,MAAM,IAAI,UAAU;KACpB,aAAa,cAAc,IAAI,UAAU,WAAW;KACpD,UACI,IAAI,UAAU,SAAS,OAAO,IAAI,UAAU,SAC9C,GAAA,CACA,MAAM,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,IAAI,CAAC,WAAW;KACd,SAAS,KAAK,kBAAkB;KAChC,YAAY;IACd;IAEA,SAAS,KACP,KAAK,IAAI,KAAK,GAAG,IAAI,gBAAgB,EAAE,IAAI,IAAI,UAAU,aAC3D;IACA,SAAS,KACP,0BAA0B,KAAK,KAAK,WAAW,SAAS,GAC1D;GACF;GAEA,IAAI,WAAW;IACb,YAAY,KAAK,GAAG,QAAQ;IAC5B,YAAY,KAAK,EAAE;GACrB;EACF;EAEA,IAAI,UAAU,OAAO,YAAY,UAAU,KAAK,GAAG;GACjD,YAAY,KAAK,KAAK;GACtB,YAAY,KAAK,EAAE;EACrB;CACF,CAAC;CAEH,YAAY,KAAK,EAAE;CACnB,OAAO,YAAY,KAAK,IAAI;AAC9B;AAEA,SAAgB,oBACd,MACA,SACA,gBACqD;CAErD,MAAM,oBAAoB,uBACxB,SACA,yBAAyB,cAAc,CACzC;CAGA,MAAM,cAAwB,CAAC;CAE/B,MAAM,cAAc,UAAwB;EAC1C,YAAY,KAAK,EAAE;EACnB,YAAY,KAAK,OAAO,MAAM,KAAK;EACnC,YAAY,KAAK,EAAE;CACrB;CAEA,MAAM,aAAkC,CAAC;CAGzC,IAAI,QAAQ,SAAS,UAAU,QAAQ,QAAQ,QAAQ,SAAS,GAAG;EACjE,WAAW,qBAAqB,MAAM;EACtC,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,QAAQ,KAAK;GAC/C,MAAM,IAAI,QAAQ,QAAQ;GAC1B,YAAY,KAAK,aAAa,GAAG,GAAG,MAAM,UAAU,UAAU,CAAC;GAC/D,OAAO,QAAQ,QAAQ,EAAE,CAAC;EAC5B;CACF;CAGA,MAAM,aAAa,QAAQ,cAAc,CAAC;CAC1C,IAAI,WAAW,QAAQ;EACrB,WAAW,cAAc;EACzB,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;GAC1C,MAAM,IAAI,WAAW;GACrB,YAAY,KAAK,aAAa,GAAG,GAAG,MAAM,QAAQ,UAAU,CAAC;GAC7D,IAAI,QAAQ,aAAa,EAAE,EAAE,YAC3B,OAAO,QAAQ,WAAW,EAAE,CAAC;EAEjC;CACF;CAgBA,IAAI,QAAQ,kBAAkB,MAAM;EAClC,WAAW,iBAAiB;EAC5B,MAAM,UAAU;GACd,cAAc,QAAQ,eAAe,SAAS;GAC9C,QAAQ,eAAe,QAAQ,OAC3B,aAAa,QAAQ,eAAe,SACpC;GACJ,QAAQ,eAAe,eAAe,OAClC,oBAAoB,QAAQ,eAAe,gBAC3C;GACJ,QAAQ,eAAe,qBAAqB,OACxC,2BAA2B,QAAQ,eAAe,sBAClD;GACJ,QAAQ,eAAe,mBAAmB,OACtC,yBAAyB,QAAQ,eAAe,oBAChD;GACJ,QAAQ,eAAe,YAAY,OAC/B,kBAAkB,QAAQ,eAAe,aACzC;GACJ,QAAQ,eAAe,WAAW,OAC9B,gBAAgB,QAAQ,eAAe,YACvC;GACJ,QAAQ,eAAe,cAAc,OACjC,gCAAgC,KAAK,UACrC,QAAQ,eAAe,YACvB,MACA,CACF,EAAE,YACA;GACJ;EACF,CAAC,CAAC,OAAO,OAAO;EAEhB,YAAY,KAAK,QAAQ,KAAK,MAAM,CAAC;CACvC;CAGA,IAAI,QAAQ,aAAa,MAAM;EAC7B,WAAW,YAAY;EACvB,MAAM,UAAU;GACd,QAAQ,UAAU,SAAS,OACvB,cAAc,QAAQ,UAAU,UAChC;GACJ,QAAQ,UAAU,WAAW,OACzB,gBAAgB,QAAQ,UAAU,YAClC;GACJ,QAAQ,UAAU,sBAAsB,OACpC,4BAA4B,QAAQ,UAAU,mBAC7C,KAAK,MAAM,KAAK,EAAE,GAAG,CAAC,CACtB,KAAK,GAAG,MACT;GACJ,QAAQ,UAAU,QAAQ,OACtB,aAAa,QAAQ,UAAU,SAC/B;GACJ;EACF,CAAC,CAAC,OAAO,OAAO;EAEhB,YAAY,KAAK,QAAQ,KAAK,MAAM,CAAC;CACvC;CAGA,MAAM,gBAAgB,QAAQ,iBAAiB,CAAC;CAChD,IAAI,cAAc,QAAQ;EACxB,WAAW,iBAAiB;EAE5B,MAAM,WAAqB,CAAC;EAC5B,cAAc,SAAS,GAAG,MAAM;GAC9B,MAAM,gBAAgB;IACpB,gBAAgB,IAAI,EAAE;IACtB,IAAI,EAAE,SAAS;IACf,GAAG,EAAE,WAAW,QAAQ,EAAE,UAAU,YAAY,EAAE,YAAY;IAC9D,GAAG,EAAE,SAAS,QAAQ,EAAE,QAAQ,UAAU,EAAE,UAAU;IACtD,GAAG,EAAE,QAAQ,QAAQ,EAAE,OAAO,SAAS,EAAE,SAAS;IAClD;GACF,CAAC,CAAC,OAAO,OAAO;GAEhB,SAAS,KAAK,cAAc,KAAK,MAAM,CAAC;EAC1C,CAAC;EAED,YAAY,KAAK,SAAS,KAAK,EAAE,CAAC;CACpC;CAEA,IAAI,SAAS,YAAY,KAAK,IAAI,CAAC,CAAC,KAAK;CACzC,IAAI,oBAAoB,GACtB,UAAU,SAAS,kBAAkB,uBACnC,sBAAsB,IAAI,KAAK,IAChC;CAEH,OAAO;EAAE;EAAQ;CAAW;AAC9B"}

package/dist/esm/tools/search/highlights.mjs

@@ -71,9 +71,6 @@ function findBestBoundary(text, direction = "backward") {
 	}
 	return direction === "backward" ? text.length : 0;
 }
-/**
-* Tracks references used in a highlight without changing their numbers
-*/
 function trackReferencesInHighlight(text, sourceResult) {
 	const references = [];
 	if (!text || text.length === 0 || !text.includes("#")) return { references };
@@ -100,17 +97,6 @@ function trackReferencesInHighlight(text, sourceResult) {
 	}
 	return { references };
 }
-/**
-* Expand highlights in search results using smart boundary detection.
-*
-* This implementation finds natural text boundaries like paragraphs, sentences,
-* and phrases to provide context while maintaining readability.
-*
-* @param searchResults - Search results object
-* @param mainExpandBy - Primary expansion size on each side (default: 300)
-* @param separatorExpandBy - Additional range to look for separators (default: 150)
-* @returns Copy of search results with expanded highlights and tracked references
-*/
 function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy = 150) {
 	const resultCopy = { ...searchResults };
 	if (resultCopy.organic) resultCopy.organic = [...resultCopy.organic];
@@ -120,8 +106,6 @@ function expandHighlights(searchResults, mainExpandBy = 300, separatorExpandBy =
 		resultCopy[resultType] = resultCopy[resultType]?.map((result) => {
 			if (result.content == null || result.content === "" || !result.highlights || result.highlights.length === 0) {
 				if (result.content == null && result.references == null) return result;
-				/** Raw scraped content must never leave this function — without
-				* highlights to expand, strip it instead of passing it downstream */
 				const strippedResult = { ...result };
 				delete strippedResult.content;
 				delete strippedResult.references;

package/dist/esm/tools/search/highlights.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"highlights.mjs","names":[],"sources":["../../../../src/tools/search/highlights.ts"],"sourcesContent":["import type * as t from './types';\n\n// 2. Pre-compile all regular expressions (only do this once)\n// Group patterns by priority for early returns\nconst priorityPatterns = [\n  // High priority patterns (structural)\n  [\n    { regex: /\\n\\n/g }, // Double newline (paragraph break)\n    { regex: /\\n/g }, // Single newline\n    { regex: /={3,}\\s*\\n|-{3,}\\s*\\n/g }, // Section separators\n  ],\n  // Medium priority (semantic)\n  [\n    { regex: /[.!?][\")\\]]?\\s/g }, // End of sentence\n    { regex: /;\\s/g }, // Semicolon\n    { regex: /:\\s/g }, // Colon\n  ],\n  // Low priority (any breaks)\n  [\n    { regex: /,\\s/g }, // Comma\n    { regex: /\\s-\\s/g }, // Dash surrounded by spaces\n    { regex: /\\s/g }, // Any space\n  ],\n];\n\nfunction findFirstMatch(text: string, regex: RegExp): number {\n  // Reset regex\n  regex.lastIndex = 0;\n\n  // For very long texts, try chunking\n  if (text.length > 10000) {\n    const chunkSize = 2000;\n    let position = 0;\n\n    while (position < text.length) {\n      const chunk = text.substring(position, position + chunkSize);\n      regex.lastIndex = 0;\n\n      const match = regex.exec(chunk);\n      if (match) {\n        return position + match.index;\n      }\n\n      // Move to next chunk with some overlap\n      position += chunkSize - 100;\n      if (position >= text.length) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  const match = regex.exec(text);\n  return match ? match.index : -1;\n}\n\n// 3. Optimized boundary finding functions\nfunction findLastMatch(text: string, regex: RegExp): number {\n  // Reset regex state\n  regex.lastIndex = 0;\n\n  let lastIndex = -1;\n  let lastLength = 0;\n  let match;\n\n  // For very long texts, use a different approach to avoid regex engine slowdowns\n  if (text.length > 10000) {\n    // Try dividing the text into chunks for faster processing\n    const chunkSize = 2000;\n    let startPosition = Math.max(0, text.length - chunkSize);\n\n    while (startPosition >= 0) {\n      const chunk = text.substring(startPosition, startPosition + chunkSize);\n      regex.lastIndex = 0;\n\n      let chunkLastIndex = -1;\n      let chunkLastLength = 0;\n\n      while ((match = regex.exec(chunk)) !== null) {\n        chunkLastIndex = match.index;\n        chunkLastLength = match[0].length;\n      }\n\n      if (chunkLastIndex !== -1) {\n        return startPosition + chunkLastIndex + chunkLastLength;\n      }\n\n      // Move to previous chunk with some overlap\n      startPosition = Math.max(0, startPosition - chunkSize + 100) - 1;\n      if (startPosition <= 0) break;\n    }\n    return -1;\n  }\n\n  // For shorter texts, normal regex search\n  while ((match = regex.exec(text)) !== null) {\n    lastIndex = match.index;\n    lastLength = match[0].length;\n  }\n\n  return lastIndex === -1 ? -1 : lastIndex + lastLength;\n}\n\n// 4. Find the best boundary with priority groups\nfunction findBestBoundary(text: string, direction = 'backward'): number {\n  if (!text || text.length === 0) return 0;\n\n  // Try each priority group\n  for (const patternGroup of priorityPatterns) {\n    for (const pattern of patternGroup) {\n      const position =\n        direction === 'backward'\n          ? findLastMatch(text, pattern.regex)\n          : findFirstMatch(text, pattern.regex);\n\n      if (position !== -1) {\n        return position;\n      }\n    }\n  }\n\n  // No match found, use character boundary\n  return direction === 'backward' ? text.length : 0;\n}\n\n/**\n * Tracks references used in a highlight without changing their numbers\n */\nfunction trackReferencesInHighlight(\n  text: string,\n  sourceResult: t.ValidSource // Source containing the original references\n): {\n  references: {\n    type: 'link' | 'image' | 'video';\n    originalIndex: number;\n    reference: t.MediaReference; // Original reference object\n  }[];\n} {\n  // Track used references\n  const references: {\n    type: 'link' | 'image' | 'video';\n    originalIndex: number;\n    reference: t.MediaReference;\n  }[] = [];\n\n  if (!text || text.length === 0 || !text.includes('#')) {\n    return { references }; // Early return\n  }\n\n  // Quick check for reference markers\n  if (\n    !text.includes('link#') &&\n    !text.includes('image#') &&\n    !text.includes('video#')\n  ) {\n    return { references };\n  }\n\n  // Get references from the source if available\n  const sourceRefs = sourceResult.references || {\n    links: [],\n    images: [],\n    videos: [],\n  };\n\n  // Find references but don't modify text\n  const refRegex = /\\((link|image|video)#(\\d+)(?:\\s+\"([^\"]*)\")?\\)/g;\n  let match;\n\n  while ((match = refRegex.exec(text)) !== null) {\n    const [, type, indexStr] = match;\n    const originalIndex = parseInt(indexStr, 10) - 1; // Convert to 0-based\n\n    // Get the source array for this type\n    const refType = type as 'link' | 'image' | 'video';\n    const sourceArray = sourceRefs[`${refType}s`] as\n      | t.MediaReference[]\n      | undefined;\n\n    // Skip if invalid reference\n    if (\n      !sourceArray ||\n      originalIndex < 0 ||\n      originalIndex >= sourceArray.length\n    ) {\n      continue; // Skip invalid references\n    }\n\n    // Get original reference\n    const reference = sourceArray[originalIndex];\n\n    // Track if not already tracked\n    const alreadyTracked = references.some(\n      (ref) => ref.type === refType && ref.originalIndex === originalIndex\n    );\n\n    if (!alreadyTracked) {\n      references.push({\n        type: refType,\n        originalIndex,\n        reference,\n      });\n    }\n  }\n\n  return { references };\n}\n\n/**\n * Expand highlights in search results using smart boundary detection.\n *\n * This implementation finds natural text boundaries like paragraphs, sentences,\n * and phrases to provide context while maintaining readability.\n *\n * @param searchResults - Search results object\n * @param mainExpandBy - Primary expansion size on each side (default: 300)\n * @param separatorExpandBy - Additional range to look for separators (default: 150)\n * @returns Copy of search results with expanded highlights and tracked references\n */\nexport function expandHighlights(\n  searchResults: t.SearchResultData,\n  mainExpandBy = 300,\n  separatorExpandBy = 150\n): t.SearchResultData {\n  // Avoid deep copy - only copy what we modify\n  const resultCopy = { ...searchResults };\n  if (resultCopy.organic) resultCopy.organic = [...resultCopy.organic];\n  if (resultCopy.topStories) resultCopy.topStories = [...resultCopy.topStories];\n\n  // Process the results efficiently\n  const processResultTypes = ['organic', 'topStories'] as const;\n\n  for (const resultType of processResultTypes) {\n    if (!resultCopy[resultType as 'organic' | 'topStories']) continue;\n\n    // Map results to new array with modified highlights\n    resultCopy[resultType] = resultCopy[resultType]?.map((result) => {\n      if (\n        result.content == null ||\n        result.content === '' ||\n        !result.highlights ||\n        result.highlights.length === 0\n      ) {\n        if (result.content == null && result.references == null) {\n          return result;\n        }\n        /** Raw scraped content must never leave this function — without\n         * highlights to expand, strip it instead of passing it downstream */\n        const strippedResult = { ...result };\n        delete strippedResult.content;\n        delete strippedResult.references;\n        return strippedResult;\n      }\n\n      // Create a shallow copy with expanded highlights\n      const resultCopy = { ...result };\n      const content = result.content;\n      const highlights = [];\n      // Process each highlight\n      for (const highlight of result.highlights) {\n        const { references } = trackReferencesInHighlight(\n          highlight.text,\n          result\n        );\n\n        let startPos = content.indexOf(highlight.text);\n        let highlightLen = highlight.text.length;\n\n        if (startPos === -1) {\n          // Try with stripped whitespace\n          const strippedHighlight = highlight.text.trim();\n          startPos = content.indexOf(strippedHighlight);\n\n          if (startPos === -1) {\n            highlights.push({\n              text: highlight.text,\n              score: highlight.score,\n              references,\n            });\n            continue;\n          }\n          highlightLen = strippedHighlight.length;\n        }\n\n        // Calculate boundaries\n        const mainStart = Math.max(0, startPos - mainExpandBy);\n        const mainEnd = Math.min(\n          content.length,\n          startPos + highlightLen + mainExpandBy\n        );\n\n        const separatorStart = Math.max(0, mainStart - separatorExpandBy);\n        const separatorEnd = Math.min(\n          content.length,\n          mainEnd + separatorExpandBy\n        );\n\n        // Extract text segments\n        const headText = content.substring(separatorStart, mainStart);\n        const tailText = content.substring(mainEnd, separatorEnd);\n\n        // Find natural boundaries\n        const bestHeadBoundary = findBestBoundary(headText, 'backward');\n        const bestTailBoundary = findBestBoundary(tailText, 'forward');\n\n        // Calculate final positions\n        const finalStart = separatorStart + bestHeadBoundary;\n        const finalEnd = mainEnd + bestTailBoundary;\n\n        // Extract the expanded highlight\n        const expandedHighlightText = content\n          .substring(finalStart, finalEnd)\n          .trim();\n        highlights.push({\n          text: expandedHighlightText,\n          score: highlight.score,\n          references,\n        });\n      }\n\n      resultCopy.highlights = highlights;\n      delete resultCopy.content;\n      delete resultCopy.references;\n      return resultCopy;\n    });\n  }\n\n  return resultCopy;\n}\n"],"mappings":";AAIA,MAAM,mBAAmB;CAEvB;EACE,EAAE,OAAO,QAAQ;EACjB,EAAE,OAAO,MAAM;EACf,EAAE,OAAO,yBAAyB;CACpC;CAEA;EACE,EAAE,OAAO,kBAAkB;EAC3B,EAAE,OAAO,OAAO;EAChB,EAAE,OAAO,OAAO;CAClB;CAEA;EACE,EAAE,OAAO,OAAO;EAChB,EAAE,OAAO,SAAS;EAClB,EAAE,OAAO,MAAM;CACjB;AACF;AAEA,SAAS,eAAe,MAAc,OAAuB;CAE3D,MAAM,YAAY;CAGlB,IAAI,KAAK,SAAS,KAAO;EACvB,MAAM,YAAY;EAClB,IAAI,WAAW;EAEf,OAAO,WAAW,KAAK,QAAQ;GAC7B,MAAM,QAAQ,KAAK,UAAU,UAAU,WAAW,SAAS;GAC3D,MAAM,YAAY;GAElB,MAAM,QAAQ,MAAM,KAAK,KAAK;GAC9B,IAAI,OACF,OAAO,WAAW,MAAM;GAI1B,YAAY,YAAY;GACxB,IAAI,YAAY,KAAK,QAAQ;EAC/B;EACA,OAAO;CACT;CAGA,MAAM,QAAQ,MAAM,KAAK,IAAI;CAC7B,OAAO,QAAQ,MAAM,QAAQ;AAC/B;AAGA,SAAS,cAAc,MAAc,OAAuB;CAE1D,MAAM,YAAY;CAElB,IAAI,YAAY;CAChB,IAAI,aAAa;CACjB,IAAI;CAGJ,IAAI,KAAK,SAAS,KAAO;EAEvB,MAAM,YAAY;EAClB,IAAI,gBAAgB,KAAK,IAAI,GAAG,KAAK,SAAS,SAAS;EAEvD,OAAO,iBAAiB,GAAG;GACzB,MAAM,QAAQ,KAAK,UAAU,eAAe,gBAAgB,SAAS;GACrE,MAAM,YAAY;GAElB,IAAI,iBAAiB;GACrB,IAAI,kBAAkB;GAEtB,QAAQ,QAAQ,MAAM,KAAK,KAAK,OAAO,MAAM;IAC3C,iBAAiB,MAAM;IACvB,kBAAkB,MAAM,EAAE,CAAC;GAC7B;GAEA,IAAI,mBAAmB,IACrB,OAAO,gBAAgB,iBAAiB;GAI1C,gBAAgB,KAAK,IAAI,GAAG,gBAAgB,YAAY,GAAG,IAAI;GAC/D,IAAI,iBAAiB,GAAG;EAC1B;EACA,OAAO;CACT;CAGA,QAAQ,QAAQ,MAAM,KAAK,IAAI,OAAO,MAAM;EAC1C,YAAY,MAAM;EAClB,aAAa,MAAM,EAAE,CAAC;CACxB;CAEA,OAAO,cAAc,KAAK,KAAK,YAAY;AAC7C;AAGA,SAAS,iBAAiB,MAAc,YAAY,YAAoB;CACtE,IAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,OAAO;CAGvC,KAAK,MAAM,gBAAgB,kBACzB,KAAK,MAAM,WAAW,cAAc;EAClC,MAAM,WACJ,cAAc,aACV,cAAc,MAAM,QAAQ,KAAK,IACjC,eAAe,MAAM,QAAQ,KAAK;EAExC,IAAI,aAAa,IACf,OAAO;CAEX;CAIF,OAAO,cAAc,aAAa,KAAK,SAAS;AAClD;;;;AAKA,SAAS,2BACP,MACA,cAOA;CAEA,MAAM,aAIA,CAAC;CAEP,IAAI,CAAC,QAAQ,KAAK,WAAW,KAAK,CAAC,KAAK,SAAS,GAAG,GAClD,OAAO,EAAE,WAAW;CAItB,IACE,CAAC,KAAK,SAAS,OAAO,KACtB,CAAC,KAAK,SAAS,QAAQ,KACvB,CAAC,KAAK,SAAS,QAAQ,GAEvB,OAAO,EAAE,WAAW;CAItB,MAAM,aAAa,aAAa,cAAc;EAC5C,OAAO,CAAC;EACR,QAAQ,CAAC;EACT,QAAQ,CAAC;CACX;CAGA,MAAM,WAAW;CACjB,IAAI;CAEJ,QAAQ,QAAQ,SAAS,KAAK,IAAI,OAAO,MAAM;EAC7C,MAAM,GAAG,MAAM,YAAY;EAC3B,MAAM,gBAAgB,SAAS,UAAU,EAAE,IAAI;EAG/C,MAAM,UAAU;EAChB,MAAM,cAAc,WAAW,GAAG,QAAQ;EAK1C,IACE,CAAC,eACD,gBAAgB,KAChB,iBAAiB,YAAY,QAE7B;EAIF,MAAM,YAAY,YAAY;EAO9B,IAAI,CAJmB,WAAW,MAC/B,QAAQ,IAAI,SAAS,WAAW,IAAI,kBAAkB,aAGvC,GAChB,WAAW,KAAK;GACd,MAAM;GACN;GACA;EACF,CAAC;CAEL;CAEA,OAAO,EAAE,WAAW;AACtB;;;;;;;;;;;;AAaA,SAAgB,iBACd,eACA,eAAe,KACf,oBAAoB,KACA;CAEpB,MAAM,aAAa,EAAE,GAAG,cAAc;CACtC,IAAI,WAAW,SAAS,WAAW,UAAU,CAAC,GAAG,WAAW,OAAO;CACnE,IAAI,WAAW,YAAY,WAAW,aAAa,CAAC,GAAG,WAAW,UAAU;CAK5E,KAAK,MAAM,cAAc,CAFG,WAAW,YAEG,GAAG;EAC3C,IAAI,CAAC,WAAW,aAAyC;EAGzD,WAAW,cAAc,WAAW,WAAW,EAAE,KAAK,WAAW;GAC/D,IACE,OAAO,WAAW,QAClB,OAAO,YAAY,MACnB,CAAC,OAAO,cACR,OAAO,WAAW,WAAW,GAC7B;IACA,IAAI,OAAO,WAAW,QAAQ,OAAO,cAAc,MACjD,OAAO;;;IAIT,MAAM,iBAAiB,EAAE,GAAG,OAAO;IACnC,OAAO,eAAe;IACtB,OAAO,eAAe;IACtB,OAAO;GACT;GAGA,MAAM,aAAa,EAAE,GAAG,OAAO;GAC/B,MAAM,UAAU,OAAO;GACvB,MAAM,aAAa,CAAC;GAEpB,KAAK,MAAM,aAAa,OAAO,YAAY;IACzC,MAAM,EAAE,eAAe,2BACrB,UAAU,MACV,MACF;IAEA,IAAI,WAAW,QAAQ,QAAQ,UAAU,IAAI;IAC7C,IAAI,eAAe,UAAU,KAAK;IAElC,IAAI,aAAa,IAAI;KAEnB,MAAM,oBAAoB,UAAU,KAAK,KAAK;KAC9C,WAAW,QAAQ,QAAQ,iBAAiB;KAE5C,IAAI,aAAa,IAAI;MACnB,WAAW,KAAK;OACd,MAAM,UAAU;OAChB,OAAO,UAAU;OACjB;MACF,CAAC;MACD;KACF;KACA,eAAe,kBAAkB;IACnC;IAGA,MAAM,YAAY,KAAK,IAAI,GAAG,WAAW,YAAY;IACrD,MAAM,UAAU,KAAK,IACnB,QAAQ,QACR,WAAW,eAAe,YAC5B;IAEA,MAAM,iBAAiB,KAAK,IAAI,GAAG,YAAY,iBAAiB;IAChE,MAAM,eAAe,KAAK,IACxB,QAAQ,QACR,UAAU,iBACZ;IAGA,MAAM,WAAW,QAAQ,UAAU,gBAAgB,SAAS;IAC5D,MAAM,WAAW,QAAQ,UAAU,SAAS,YAAY;IAGxD,MAAM,mBAAmB,iBAAiB,UAAU,UAAU;IAC9D,MAAM,mBAAmB,iBAAiB,UAAU,SAAS;IAG7D,MAAM,aAAa,iBAAiB;IACpC,MAAM,WAAW,UAAU;IAG3B,MAAM,wBAAwB,QAC3B,UAAU,YAAY,QAAQ,CAAC,CAC/B,KAAK;IACR,WAAW,KAAK;KACd,MAAM;KACN,OAAO,UAAU;KACjB;IACF,CAAC;GACH;GAEA,WAAW,aAAa;GACxB,OAAO,WAAW;GAClB,OAAO,WAAW;GAClB,OAAO;EACT,CAAC;CACH;CAEA,OAAO;AACT"}
+{"version":3,"file":"highlights.mjs","names":[],"sources":["../../../../src/tools/search/highlights.ts"],"mappings":";AAIA,MAAM,mBAAmB;CAEvB;EACE,EAAE,OAAO,QAAQ;EACjB,EAAE,OAAO,MAAM;EACf,EAAE,OAAO,yBAAyB;CACpC;CAEA;EACE,EAAE,OAAO,kBAAkB;EAC3B,EAAE,OAAO,OAAO;EAChB,EAAE,OAAO,OAAO;CAClB;CAEA;EACE,EAAE,OAAO,OAAO;EAChB,EAAE,OAAO,SAAS;EAClB,EAAE,OAAO,MAAM;CACjB;AACF;AAEA,SAAS,eAAe,MAAc,OAAuB;CAE3D,MAAM,YAAY;CAGlB,IAAI,KAAK,SAAS,KAAO;EACvB,MAAM,YAAY;EAClB,IAAI,WAAW;EAEf,OAAO,WAAW,KAAK,QAAQ;GAC7B,MAAM,QAAQ,KAAK,UAAU,UAAU,WAAW,SAAS;GAC3D,MAAM,YAAY;GAElB,MAAM,QAAQ,MAAM,KAAK,KAAK;GAC9B,IAAI,OACF,OAAO,WAAW,MAAM;GAI1B,YAAY,YAAY;GACxB,IAAI,YAAY,KAAK,QAAQ;EAC/B;EACA,OAAO;CACT;CAGA,MAAM,QAAQ,MAAM,KAAK,IAAI;CAC7B,OAAO,QAAQ,MAAM,QAAQ;AAC/B;AAGA,SAAS,cAAc,MAAc,OAAuB;CAE1D,MAAM,YAAY;CAElB,IAAI,YAAY;CAChB,IAAI,aAAa;CACjB,IAAI;CAGJ,IAAI,KAAK,SAAS,KAAO;EAEvB,MAAM,YAAY;EAClB,IAAI,gBAAgB,KAAK,IAAI,GAAG,KAAK,SAAS,SAAS;EAEvD,OAAO,iBAAiB,GAAG;GACzB,MAAM,QAAQ,KAAK,UAAU,eAAe,gBAAgB,SAAS;GACrE,MAAM,YAAY;GAElB,IAAI,iBAAiB;GACrB,IAAI,kBAAkB;GAEtB,QAAQ,QAAQ,MAAM,KAAK,KAAK,OAAO,MAAM;IAC3C,iBAAiB,MAAM;IACvB,kBAAkB,MAAM,EAAE,CAAC;GAC7B;GAEA,IAAI,mBAAmB,IACrB,OAAO,gBAAgB,iBAAiB;GAI1C,gBAAgB,KAAK,IAAI,GAAG,gBAAgB,YAAY,GAAG,IAAI;GAC/D,IAAI,iBAAiB,GAAG;EAC1B;EACA,OAAO;CACT;CAGA,QAAQ,QAAQ,MAAM,KAAK,IAAI,OAAO,MAAM;EAC1C,YAAY,MAAM;EAClB,aAAa,MAAM,EAAE,CAAC;CACxB;CAEA,OAAO,cAAc,KAAK,KAAK,YAAY;AAC7C;AAGA,SAAS,iBAAiB,MAAc,YAAY,YAAoB;CACtE,IAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,OAAO;CAGvC,KAAK,MAAM,gBAAgB,kBACzB,KAAK,MAAM,WAAW,cAAc;EAClC,MAAM,WACJ,cAAc,aACV,cAAc,MAAM,QAAQ,KAAK,IACjC,eAAe,MAAM,QAAQ,KAAK;EAExC,IAAI,aAAa,IACf,OAAO;CAEX;CAIF,OAAO,cAAc,aAAa,KAAK,SAAS;AAClD;AAKA,SAAS,2BACP,MACA,cAOA;CAEA,MAAM,aAIA,CAAC;CAEP,IAAI,CAAC,QAAQ,KAAK,WAAW,KAAK,CAAC,KAAK,SAAS,GAAG,GAClD,OAAO,EAAE,WAAW;CAItB,IACE,CAAC,KAAK,SAAS,OAAO,KACtB,CAAC,KAAK,SAAS,QAAQ,KACvB,CAAC,KAAK,SAAS,QAAQ,GAEvB,OAAO,EAAE,WAAW;CAItB,MAAM,aAAa,aAAa,cAAc;EAC5C,OAAO,CAAC;EACR,QAAQ,CAAC;EACT,QAAQ,CAAC;CACX;CAGA,MAAM,WAAW;CACjB,IAAI;CAEJ,QAAQ,QAAQ,SAAS,KAAK,IAAI,OAAO,MAAM;EAC7C,MAAM,GAAG,MAAM,YAAY;EAC3B,MAAM,gBAAgB,SAAS,UAAU,EAAE,IAAI;EAG/C,MAAM,UAAU;EAChB,MAAM,cAAc,WAAW,GAAG,QAAQ;EAK1C,IACE,CAAC,eACD,gBAAgB,KAChB,iBAAiB,YAAY,QAE7B;EAIF,MAAM,YAAY,YAAY;EAO9B,IAAI,CAJmB,WAAW,MAC/B,QAAQ,IAAI,SAAS,WAAW,IAAI,kBAAkB,aAGvC,GAChB,WAAW,KAAK;GACd,MAAM;GACN;GACA;EACF,CAAC;CAEL;CAEA,OAAO,EAAE,WAAW;AACtB;AAaA,SAAgB,iBACd,eACA,eAAe,KACf,oBAAoB,KACA;CAEpB,MAAM,aAAa,EAAE,GAAG,cAAc;CACtC,IAAI,WAAW,SAAS,WAAW,UAAU,CAAC,GAAG,WAAW,OAAO;CACnE,IAAI,WAAW,YAAY,WAAW,aAAa,CAAC,GAAG,WAAW,UAAU;CAK5E,KAAK,MAAM,cAAc,CAFG,WAAW,YAEG,GAAG;EAC3C,IAAI,CAAC,WAAW,aAAyC;EAGzD,WAAW,cAAc,WAAW,WAAW,EAAE,KAAK,WAAW;GAC/D,IACE,OAAO,WAAW,QAClB,OAAO,YAAY,MACnB,CAAC,OAAO,cACR,OAAO,WAAW,WAAW,GAC7B;IACA,IAAI,OAAO,WAAW,QAAQ,OAAO,cAAc,MACjD,OAAO;IAIT,MAAM,iBAAiB,EAAE,GAAG,OAAO;IACnC,OAAO,eAAe;IACtB,OAAO,eAAe;IACtB,OAAO;GACT;GAGA,MAAM,aAAa,EAAE,GAAG,OAAO;GAC/B,MAAM,UAAU,OAAO;GACvB,MAAM,aAAa,CAAC;GAEpB,KAAK,MAAM,aAAa,OAAO,YAAY;IACzC,MAAM,EAAE,eAAe,2BACrB,UAAU,MACV,MACF;IAEA,IAAI,WAAW,QAAQ,QAAQ,UAAU,IAAI;IAC7C,IAAI,eAAe,UAAU,KAAK;IAElC,IAAI,aAAa,IAAI;KAEnB,MAAM,oBAAoB,UAAU,KAAK,KAAK;KAC9C,WAAW,QAAQ,QAAQ,iBAAiB;KAE5C,IAAI,aAAa,IAAI;MACnB,WAAW,KAAK;OACd,MAAM,UAAU;OAChB,OAAO,UAAU;OACjB;MACF,CAAC;MACD;KACF;KACA,eAAe,kBAAkB;IACnC;IAGA,MAAM,YAAY,KAAK,IAAI,GAAG,WAAW,YAAY;IACrD,MAAM,UAAU,KAAK,IACnB,QAAQ,QACR,WAAW,eAAe,YAC5B;IAEA,MAAM,iBAAiB,KAAK,IAAI,GAAG,YAAY,iBAAiB;IAChE,MAAM,eAAe,KAAK,IACxB,QAAQ,QACR,UAAU,iBACZ;IAGA,MAAM,WAAW,QAAQ,UAAU,gBAAgB,SAAS;IAC5D,MAAM,WAAW,QAAQ,UAAU,SAAS,YAAY;IAGxD,MAAM,mBAAmB,iBAAiB,UAAU,UAAU;IAC9D,MAAM,mBAAmB,iBAAiB,UAAU,SAAS;IAG7D,MAAM,aAAa,iBAAiB;IACpC,MAAM,WAAW,UAAU;IAG3B,MAAM,wBAAwB,QAC3B,UAAU,YAAY,QAAQ,CAAC,CAC/B,KAAK;IACR,WAAW,KAAK;KACd,MAAM;KACN,OAAO,UAAU;KACjB;IACF,CAAC;GACH;GAEA,WAAW,aAAa;GACxB,OAAO,WAAW;GAClB,OAAO,WAAW;GAClB,OAAO;EACT,CAAC;CACH;CAEA,OAAO;AACT"}

package/dist/esm/tools/search/keenable-scraper.mjs

@@ -2,8 +2,6 @@ import { createDefaultLogger } from "./utils.mjs";
 import axios from "axios";
 //#region src/tools/search/keenable-scraper.ts
 const DEFAULT_KEENABLE_SCRAPE_TIMEOUT = 15e3;
-/** Keyed and keyless fetch endpoints. Keenable reads any URL as clean markdown
-* without a key via the public endpoint; a key only lifts rate limits. */
 const KEENABLE_FETCH_API_URL = "https://api.keenable.ai/v1/fetch";
 const KEENABLE_FETCH_PUBLIC_URL = "https://api.keenable.ai/v1/fetch/public";
 var KeenableScraper = class {
@@ -25,9 +23,6 @@ var KeenableScraper = class {
 		this.logger = config.logger || createDefaultLogger();
 	}
 	buildHeaders() {
-		/** X-Keenable-Title is used for traffic attribution and required on the
-		* keyless endpoint; the key only lifts rate limits, so it is sent only when
-		* present. */
 		const headers = { "X-Keenable-Title": this.attributionTitle };
 		if (this.apiKey != null) headers["X-API-Key"] = this.apiKey;
 		return headers;
@@ -67,13 +62,10 @@ var KeenableScraper = class {
 		}
 	}
 	async scrapeUrls(urls, options = {}) {
-		/** Keenable fetch is single-URL; run the batch concurrently. */
 		return Promise.all(urls.map((url) => this.scrapeUrl(url, options)));
 	}
 	extractContent(response) {
 		if (!response.success || !response.data) return ["", void 0];
-		/** Keenable returns clean markdown with no separate media arrays, so there
-		* are no structured references to surface. */
 		return [response.data.content, void 0];
 	}
 	extractMetadata(response) {

package/dist/esm/tools/search/keenable-scraper.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"keenable-scraper.mjs","names":[],"sources":["../../../../src/tools/search/keenable-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_KEENABLE_SCRAPE_TIMEOUT = 15000;\n\n/** Keyed and keyless fetch endpoints. Keenable reads any URL as clean markdown\n * without a key via the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_FETCH_API_URL = 'https://api.keenable.ai/v1/fetch';\nconst KEENABLE_FETCH_PUBLIC_URL = 'https://api.keenable.ai/v1/fetch/public';\n\nexport class KeenableScraper implements t.BaseScraper {\n  private apiKey: string | undefined;\n  private apiUrl: string;\n  private timeout: number;\n  private attributionTitle: string;\n  private logger: t.Logger;\n  private httpAgent?: t.HttpAgent;\n  private httpsAgent?: t.HttpsAgent;\n\n  constructor(config: t.KeenableScraperConfig = {}) {\n    const resolvedKey = config.apiKey ?? process.env.KEENABLE_API_KEY;\n    this.apiKey =\n      resolvedKey != null && resolvedKey !== '' ? resolvedKey : undefined;\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.KEENABLE_FETCH_URL ??\n      (this.apiKey != null\n        ? KEENABLE_FETCH_API_URL\n        : KEENABLE_FETCH_PUBLIC_URL);\n    this.timeout = config.timeout ?? DEFAULT_KEENABLE_SCRAPE_TIMEOUT;\n    this.attributionTitle = config.attributionTitle ?? 'LibreChat';\n    this.httpAgent = config.httpAgent;\n    this.httpsAgent = config.httpsAgent;\n    this.logger = config.logger || createDefaultLogger();\n  }\n\n  private buildHeaders(): Record<string, string> {\n    /** X-Keenable-Title is used for traffic attribution and required on the\n     * keyless endpoint; the key only lifts rate limits, so it is sent only when\n     * present. */\n    const headers: Record<string, string> = {\n      'X-Keenable-Title': this.attributionTitle,\n    };\n    if (this.apiKey != null) {\n      headers['X-API-Key'] = this.apiKey;\n    }\n    return headers;\n  }\n\n  async scrapeUrl(\n    url: string,\n    options: t.KeenableScrapeOptions = {}\n  ): Promise<[string, t.KeenableScrapeResponse]> {\n    if (!url || !url.trim()) {\n      return [url, { success: false, error: 'URL cannot be empty' }];\n    }\n\n    try {\n      const response = await axios.get<t.KeenableFetchResult>(this.apiUrl, {\n        params: { url },\n        headers: this.buildHeaders(),\n        timeout: options.timeout ?? this.timeout,\n        httpAgent: this.httpAgent,\n        httpsAgent: this.httpsAgent,\n      });\n\n      const data = response.data;\n      const content = data.content ?? '';\n      if (!content) {\n        return [\n          url,\n          { success: false, error: 'Keenable Fetch returned no content' },\n        ];\n      }\n\n      return [\n        url,\n        {\n          success: true,\n          data: {\n            content,\n            title: data.title,\n            description: data.description,\n            url: data.url ?? url,\n          },\n        },\n      ];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Keenable Fetch API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  async scrapeUrls(\n    urls: string[],\n    options: t.KeenableScrapeOptions = {}\n  ): Promise<Array<[string, t.KeenableScrapeResponse]>> {\n    /** Keenable fetch is single-URL; run the batch concurrently. */\n    return Promise.all(urls.map((url) => this.scrapeUrl(url, options)));\n  }\n\n  extractContent(\n    response: t.KeenableScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n    /** Keenable returns clean markdown with no separate media arrays, so there\n     * are no structured references to surface. */\n    return [response.data.content, undefined];\n  }\n\n  extractMetadata(response: t.KeenableScrapeResponse): t.GenericScrapeMetadata {\n    if (!response.success || !response.data) {\n      return {};\n    }\n    const metadata: t.GenericScrapeMetadata = {};\n    if (response.data.title != null) {\n      metadata.title = response.data.title;\n    }\n    if (response.data.description != null && response.data.description !== '') {\n      metadata.description = response.data.description;\n    }\n    if (response.data.url != null) {\n      metadata.url = response.data.url;\n    }\n    return metadata;\n  }\n}\n\nexport const createKeenableScraper = (\n  config: t.KeenableScraperConfig = {}\n): KeenableScraper => {\n  return new KeenableScraper(config);\n};\n"],"mappings":";;;AAIA,MAAM,kCAAkC;;;AAIxC,MAAM,yBAAyB;AAC/B,MAAM,4BAA4B;AAElC,IAAa,kBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAkC,CAAC,GAAG;EAChD,MAAM,cAAc,OAAO,UAAU,QAAQ,IAAI;EACjD,KAAK,SACH,eAAe,QAAQ,gBAAgB,KAAK,cAAc,KAAA;EAC5D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,uBACX,KAAK,UAAU,OACZ,yBACA;EACN,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,mBAAmB,OAAO,oBAAoB;EACnD,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EACzB,KAAK,SAAS,OAAO,UAAU,oBAAoB;CACrD;CAEA,eAA+C;;;;EAI7C,MAAM,UAAkC,EACtC,oBAAoB,KAAK,iBAC3B;EACA,IAAI,KAAK,UAAU,MACjB,QAAQ,eAAe,KAAK;EAE9B,OAAO;CACT;CAEA,MAAM,UACJ,KACA,UAAmC,CAAC,GACS;EAC7C,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,GACpB,OAAO,CAAC,KAAK;GAAE,SAAS;GAAO,OAAO;EAAsB,CAAC;EAG/D,IAAI;GASF,MAAM,QAAO,MARU,MAAM,IAA2B,KAAK,QAAQ;IACnE,QAAQ,EAAE,IAAI;IACd,SAAS,KAAK,aAAa;IAC3B,SAAS,QAAQ,WAAW,KAAK;IACjC,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CAAC,EAAA,CAEqB;GACtB,MAAM,UAAU,KAAK,WAAW;GAChC,IAAI,CAAC,SACH,OAAO,CACL,KACA;IAAE,SAAS;IAAO,OAAO;GAAqC,CAChE;GAGF,OAAO,CACL,KACA;IACE,SAAS;IACT,MAAM;KACJ;KACA,OAAO,KAAK;KACZ,aAAa,KAAK;KAClB,KAAK,KAAK,OAAO;IACnB;GACF,CACF;EACF,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,sCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;CAEA,MAAM,WACJ,MACA,UAAmC,CAAC,GACgB;;EAEpD,OAAO,QAAQ,IAAI,KAAK,KAAK,QAAQ,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC;CACpE;CAEA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;;;EAIvB,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;CAC1C;CAEA,gBAAgB,UAA6D;EAC3E,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC;EAEV,MAAM,WAAoC,CAAC;EAC3C,IAAI,SAAS,KAAK,SAAS,MACzB,SAAS,QAAQ,SAAS,KAAK;EAEjC,IAAI,SAAS,KAAK,eAAe,QAAQ,SAAS,KAAK,gBAAgB,IACrE,SAAS,cAAc,SAAS,KAAK;EAEvC,IAAI,SAAS,KAAK,OAAO,MACvB,SAAS,MAAM,SAAS,KAAK;EAE/B,OAAO;CACT;AACF;AAEA,MAAa,yBACX,SAAkC,CAAC,MACf;CACpB,OAAO,IAAI,gBAAgB,MAAM;AACnC"}
+{"version":3,"file":"keenable-scraper.mjs","names":[],"sources":["../../../../src/tools/search/keenable-scraper.ts"],"mappings":";;;AAIA,MAAM,kCAAkC;AAIxC,MAAM,yBAAyB;AAC/B,MAAM,4BAA4B;AAElC,IAAa,kBAAb,MAAsD;CACpD;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAkC,CAAC,GAAG;EAChD,MAAM,cAAc,OAAO,UAAU,QAAQ,IAAI;EACjD,KAAK,SACH,eAAe,QAAQ,gBAAgB,KAAK,cAAc,KAAA;EAC5D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,uBACX,KAAK,UAAU,OACZ,yBACA;EACN,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,mBAAmB,OAAO,oBAAoB;EACnD,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EACzB,KAAK,SAAS,OAAO,UAAU,oBAAoB;CACrD;CAEA,eAA+C;EAI7C,MAAM,UAAkC,EACtC,oBAAoB,KAAK,iBAC3B;EACA,IAAI,KAAK,UAAU,MACjB,QAAQ,eAAe,KAAK;EAE9B,OAAO;CACT;CAEA,MAAM,UACJ,KACA,UAAmC,CAAC,GACS;EAC7C,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,GACpB,OAAO,CAAC,KAAK;GAAE,SAAS;GAAO,OAAO;EAAsB,CAAC;EAG/D,IAAI;GASF,MAAM,QAAO,MARU,MAAM,IAA2B,KAAK,QAAQ;IACnE,QAAQ,EAAE,IAAI;IACd,SAAS,KAAK,aAAa;IAC3B,SAAS,QAAQ,WAAW,KAAK;IACjC,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CAAC,EAAA,CAEqB;GACtB,MAAM,UAAU,KAAK,WAAW;GAChC,IAAI,CAAC,SACH,OAAO,CACL,KACA;IAAE,SAAS;IAAO,OAAO;GAAqC,CAChE;GAGF,OAAO,CACL,KACA;IACE,SAAS;IACT,MAAM;KACJ;KACA,OAAO,KAAK;KACZ,aAAa,KAAK;KAClB,KAAK,KAAK,OAAO;IACnB;GACF,CACF;EACF,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,sCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;CAEA,MAAM,WACJ,MACA,UAAmC,CAAC,GACgB;EAEpD,OAAO,QAAQ,IAAI,KAAK,KAAK,QAAQ,KAAK,UAAU,KAAK,OAAO,CAAC,CAAC;CACpE;CAEA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAIvB,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;CAC1C;CAEA,gBAAgB,UAA6D;EAC3E,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC;EAEV,MAAM,WAAoC,CAAC;EAC3C,IAAI,SAAS,KAAK,SAAS,MACzB,SAAS,QAAQ,SAAS,KAAK;EAEjC,IAAI,SAAS,KAAK,eAAe,QAAQ,SAAS,KAAK,gBAAgB,IACrE,SAAS,cAAc,SAAS,KAAK;EAEvC,IAAI,SAAS,KAAK,OAAO,MACvB,SAAS,MAAM,SAAS,KAAK;EAE/B,OAAO;CACT;AACF;AAEA,MAAa,yBACX,SAAkC,CAAC,MACf;CACpB,OAAO,IAAI,gBAAgB,MAAM;AACnC"}

package/dist/esm/tools/search/keenable-search.mjs

@@ -2,8 +2,6 @@ import "./schema.mjs";
 import axios from "axios";
 //#region src/tools/search/keenable-search.ts
 const DEFAULT_KEENABLE_TIMEOUT = 15e3;
-/** Authenticated and keyless endpoints. Keenable works without an API key by
-* falling back to the public endpoint; a key only lifts rate limits. */
 const KEENABLE_DEFAULT_API_URL = "https://api.keenable.ai/v1/search";
 const KEENABLE_PUBLIC_API_URL = "https://api.keenable.ai/v1/search/public";
 const KEENABLE_DATE_RANGES = {
@@ -18,9 +16,6 @@ const createKeenableAPI = (apiKey, apiUrl, options) => {
 	const hasKey = resolvedKey != null && resolvedKey !== "";
 	const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;
 	const resolvedUrl = apiUrl ?? process.env.KEENABLE_API_URL ?? (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);
-	/** Constant for the provider's lifetime. X-Keenable-Title is required for
-	* keyless requests and used for traffic attribution; the API key only lifts
-	* rate limits, so it is sent only when present. */
 	const headers = {
 		"Content-Type": "application/json",
 		"X-Keenable-Title": options?.attributionTitle ?? "LibreChat"
@@ -32,8 +27,6 @@ const createKeenableAPI = (apiKey, apiUrl, options) => {
 			error: "Query cannot be empty"
 		};
 		try {
-			/** Keenable's endpoint has no result-count parameter; the count is
-			* applied client-side after the response (see slice below). */
 			const payload = { query };
 			if (options?.site != null && options.site !== "") payload.site = options.site;
 			if (date != null) payload.published_after = KEENABLE_DATE_RANGES[date];

package/dist/esm/tools/search/keenable-search.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"keenable-search.mjs","names":[],"sources":["../../../../src/tools/search/keenable-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { DATE_RANGE } from './schema';\n\nconst DEFAULT_KEENABLE_TIMEOUT = 15000;\n\n/** Authenticated and keyless endpoints. Keenable works without an API key by\n * falling back to the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_DEFAULT_API_URL = 'https://api.keenable.ai/v1/search';\nconst KEENABLE_PUBLIC_API_URL = 'https://api.keenable.ai/v1/search/public';\nconst KEENABLE_DATE_RANGES: Record<DATE_RANGE, string> = {\n  [DATE_RANGE.PAST_HOUR]: '1h',\n  [DATE_RANGE.PAST_24_HOURS]: '1d',\n  [DATE_RANGE.PAST_WEEK]: '7d',\n  [DATE_RANGE.PAST_MONTH]: '1mo',\n  [DATE_RANGE.PAST_YEAR]: '1y',\n};\n\nexport const createKeenableAPI = (\n  apiKey?: string,\n  apiUrl?: string,\n  options?: t.KeenableSearchOptions\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const resolvedKey = apiKey ?? process.env.KEENABLE_API_KEY;\n  const hasKey = resolvedKey != null && resolvedKey !== '';\n  const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;\n  const resolvedUrl =\n    apiUrl ??\n    process.env.KEENABLE_API_URL ??\n    (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);\n\n  /** Constant for the provider's lifetime. X-Keenable-Title is required for\n   * keyless requests and used for traffic attribution; the API key only lifts\n   * rate limits, so it is sent only when present. */\n  const headers: Record<string, string> = {\n    'Content-Type': 'application/json',\n    'X-Keenable-Title': options?.attributionTitle ?? 'LibreChat',\n  };\n  if (hasKey) {\n    headers['X-API-Key'] = resolvedKey;\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    numResults = 8,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      /** Keenable's endpoint has no result-count parameter; the count is\n       * applied client-side after the response (see slice below). */\n      const payload: t.KeenableSearchPayload = { query };\n      if (options?.site != null && options.site !== '') {\n        payload.site = options.site;\n      }\n      if (date != null) {\n        payload.published_after = KEENABLE_DATE_RANGES[date];\n      }\n\n      const response = await axios.post<t.KeenableSearchResponse>(\n        resolvedUrl,\n        payload,\n        {\n          headers,\n          timeout,\n          httpAgent: options?.httpAgent,\n          httpsAgent: options?.httpsAgent,\n        }\n      );\n\n      const maxResults = Math.min(\n        Math.max(1, options?.maxResults ?? numResults),\n        20\n      );\n      const rawResults = Array.isArray(response.data.results)\n        ? response.data.results.slice(0, maxResults)\n        : [];\n\n      const organic: t.OrganicResult[] = rawResults.map((result) => ({\n        title: result.title ?? '',\n        link: result.url ?? '',\n        snippet: result.description ?? result.snippet ?? '',\n        date: result.published_at,\n      }));\n\n      return { success: true, data: { organic } };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `Keenable API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n"],"mappings":";;;AAIA,MAAM,2BAA2B;;;AAIjC,MAAM,2BAA2B;AACjC,MAAM,0BAA0B;AAChC,MAAM,uBAAmD;QAC/B;QACI;QACJ;QACC;QACD;AAC1B;AAEA,MAAa,qBACX,QACA,QACA,YAGG;CACH,MAAM,cAAc,UAAU,QAAQ,IAAI;CAC1C,MAAM,SAAS,eAAe,QAAQ,gBAAgB;CACtD,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,cACJ,UACA,QAAQ,IAAI,qBACX,SAAS,2BAA2B;;;;CAKvC,MAAM,UAAkC;EACtC,gBAAgB;EAChB,oBAAoB,SAAS,oBAAoB;CACnD;CACA,IAAI,QACF,QAAQ,eAAe;CAGzB,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,QACoC;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;;;GAGF,MAAM,UAAmC,EAAE,MAAM;GACjD,IAAI,SAAS,QAAQ,QAAQ,QAAQ,SAAS,IAC5C,QAAQ,OAAO,QAAQ;GAEzB,IAAI,QAAQ,MACV,QAAQ,kBAAkB,qBAAqB;GAGjD,MAAM,WAAW,MAAM,MAAM,KAC3B,aACA,SACA;IACE;IACA;IACA,WAAW,SAAS;IACpB,YAAY,SAAS;GACvB,CACF;GAEA,MAAM,aAAa,KAAK,IACtB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAYA,OAAO;IAAE,SAAS;IAAM,MAAM,EAAE,UAXb,MAAM,QAAQ,SAAS,KAAK,OAAO,IAClD,SAAS,KAAK,QAAQ,MAAM,GAAG,UAAU,IACzC,CAAC,EAAA,CAEyC,KAAK,YAAY;KAC7D,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,eAAe,OAAO,WAAW;KACjD,MAAM,OAAO;IACf,EAEsC,EAAE;GAAE;EAC5C,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,gCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}
+{"version":3,"file":"keenable-search.mjs","names":[],"sources":["../../../../src/tools/search/keenable-search.ts"],"mappings":";;;AAIA,MAAM,2BAA2B;AAIjC,MAAM,2BAA2B;AACjC,MAAM,0BAA0B;AAChC,MAAM,uBAAmD;QAC/B;QACI;QACJ;QACC;QACD;AAC1B;AAEA,MAAa,qBACX,QACA,QACA,YAGG;CACH,MAAM,cAAc,UAAU,QAAQ,IAAI;CAC1C,MAAM,SAAS,eAAe,QAAQ,gBAAgB;CACtD,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,cACJ,UACA,QAAQ,IAAI,qBACX,SAAS,2BAA2B;CAKvC,MAAM,UAAkC;EACtC,gBAAgB;EAChB,oBAAoB,SAAS,oBAAoB;CACnD;CACA,IAAI,QACF,QAAQ,eAAe;CAGzB,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,QACoC;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAGF,MAAM,UAAmC,EAAE,MAAM;GACjD,IAAI,SAAS,QAAQ,QAAQ,QAAQ,SAAS,IAC5C,QAAQ,OAAO,QAAQ;GAEzB,IAAI,QAAQ,MACV,QAAQ,kBAAkB,qBAAqB;GAGjD,MAAM,WAAW,MAAM,MAAM,KAC3B,aACA,SACA;IACE;IACA;IACA,WAAW,SAAS;IACpB,YAAY,SAAS;GACvB,CACF;GAEA,MAAM,aAAa,KAAK,IACtB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAYA,OAAO;IAAE,SAAS;IAAM,MAAM,EAAE,UAXb,MAAM,QAAQ,SAAS,KAAK,OAAO,IAClD,SAAS,KAAK,QAAQ,MAAM,GAAG,UAAU,IACzC,CAAC,EAAA,CAEyC,KAAK,YAAY;KAC7D,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,eAAe,OAAO,WAAW;KACjD,MAAM,OAAO;IACf,EAEsC,EAAE;GAAE;EAC5C,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,gCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}