@librechat/agents 3.2.56 → 3.2.58

package/dist/cjs/tools/search/keenable-search.cjs

@@ -0,0 +1,68 @@
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+require("./schema.cjs");
+let axios = require("axios");
+axios = require_runtime.__toESM(axios, 1);
+//#region src/tools/search/keenable-search.ts
+const DEFAULT_KEENABLE_TIMEOUT = 15e3;
+/** Authenticated and keyless endpoints. Keenable works without an API key by
+* falling back to the public endpoint; a key only lifts rate limits. */
+const KEENABLE_DEFAULT_API_URL = "https://api.keenable.ai/v1/search";
+const KEENABLE_PUBLIC_API_URL = "https://api.keenable.ai/v1/search/public";
+const KEENABLE_DATE_RANGES = {
+	["h"]: "1h",
+	["d"]: "1d",
+	["w"]: "7d",
+	["m"]: "1mo",
+	["y"]: "1y"
+};
+const createKeenableAPI = (apiKey, apiUrl, options) => {
+	const resolvedKey = apiKey ?? process.env.KEENABLE_API_KEY;
+	const hasKey = resolvedKey != null && resolvedKey !== "";
+	const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;
+	const resolvedUrl = apiUrl ?? process.env.KEENABLE_API_URL ?? (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);
+	/** Constant for the provider's lifetime. X-Keenable-Title is required for
+	* keyless requests and used for traffic attribution; the API key only lifts
+	* rate limits, so it is sent only when present. */
+	const headers = {
+		"Content-Type": "application/json",
+		"X-Keenable-Title": options?.attributionTitle ?? "LibreChat"
+	};
+	if (hasKey) headers["X-API-Key"] = resolvedKey;
+	const getSources = async ({ query, date, numResults = 8 }) => {
+		if (!query.trim()) return {
+			success: false,
+			error: "Query cannot be empty"
+		};
+		try {
+			/** Keenable's endpoint has no result-count parameter; the count is
+			* applied client-side after the response (see slice below). */
+			const payload = { query };
+			if (options?.site != null && options.site !== "") payload.site = options.site;
+			if (date != null) payload.published_after = KEENABLE_DATE_RANGES[date];
+			const response = await axios.default.post(resolvedUrl, payload, {
+				headers,
+				timeout
+			});
+			const maxResults = Math.min(Math.max(1, options?.maxResults ?? numResults), 20);
+			return {
+				success: true,
+				data: { organic: (Array.isArray(response.data.results) ? response.data.results.slice(0, maxResults) : []).map((result) => ({
+					title: result.title ?? "",
+					link: result.url ?? "",
+					snippet: result.description ?? result.snippet ?? "",
+					date: result.published_at
+				})) }
+			};
+		} catch (error) {
+			return {
+				success: false,
+				error: `Keenable API request failed: ${error instanceof Error ? error.message : String(error)}`
+			};
+		}
+	};
+	return { getSources };
+};
+//#endregion
+exports.createKeenableAPI = createKeenableAPI;
+
+//# sourceMappingURL=keenable-search.cjs.map

package/dist/cjs/tools/search/keenable-search.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"keenable-search.cjs","names":[],"sources":["../../../../src/tools/search/keenable-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { DATE_RANGE } from './schema';\n\nconst DEFAULT_KEENABLE_TIMEOUT = 15000;\n\n/** Authenticated and keyless endpoints. Keenable works without an API key by\n * falling back to the public endpoint; a key only lifts rate limits. */\nconst KEENABLE_DEFAULT_API_URL = 'https://api.keenable.ai/v1/search';\nconst KEENABLE_PUBLIC_API_URL = 'https://api.keenable.ai/v1/search/public';\nconst KEENABLE_DATE_RANGES: Record<DATE_RANGE, string> = {\n  [DATE_RANGE.PAST_HOUR]: '1h',\n  [DATE_RANGE.PAST_24_HOURS]: '1d',\n  [DATE_RANGE.PAST_WEEK]: '7d',\n  [DATE_RANGE.PAST_MONTH]: '1mo',\n  [DATE_RANGE.PAST_YEAR]: '1y',\n};\n\nexport const createKeenableAPI = (\n  apiKey?: string,\n  apiUrl?: string,\n  options?: t.KeenableSearchOptions\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const resolvedKey = apiKey ?? process.env.KEENABLE_API_KEY;\n  const hasKey = resolvedKey != null && resolvedKey !== '';\n  const timeout = options?.timeout ?? DEFAULT_KEENABLE_TIMEOUT;\n  const resolvedUrl =\n    apiUrl ??\n    process.env.KEENABLE_API_URL ??\n    (hasKey ? KEENABLE_DEFAULT_API_URL : KEENABLE_PUBLIC_API_URL);\n\n  /** Constant for the provider's lifetime. X-Keenable-Title is required for\n   * keyless requests and used for traffic attribution; the API key only lifts\n   * rate limits, so it is sent only when present. */\n  const headers: Record<string, string> = {\n    'Content-Type': 'application/json',\n    'X-Keenable-Title': options?.attributionTitle ?? 'LibreChat',\n  };\n  if (hasKey) {\n    headers['X-API-Key'] = resolvedKey;\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    numResults = 8,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      /** Keenable's endpoint has no result-count parameter; the count is\n       * applied client-side after the response (see slice below). */\n      const payload: t.KeenableSearchPayload = { query };\n      if (options?.site != null && options.site !== '') {\n        payload.site = options.site;\n      }\n      if (date != null) {\n        payload.published_after = KEENABLE_DATE_RANGES[date];\n      }\n\n      const response = await axios.post<t.KeenableSearchResponse>(\n        resolvedUrl,\n        payload,\n        { headers, timeout }\n      );\n\n      const maxResults = Math.min(\n        Math.max(1, options?.maxResults ?? numResults),\n        20\n      );\n      const rawResults = Array.isArray(response.data.results)\n        ? response.data.results.slice(0, maxResults)\n        : [];\n\n      const organic: t.OrganicResult[] = rawResults.map((result) => ({\n        title: result.title ?? '',\n        link: result.url ?? '',\n        snippet: result.description ?? result.snippet ?? '',\n        date: result.published_at,\n      }));\n\n      return { success: true, data: { organic } };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `Keenable API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n"],"mappings":";;;;;AAIA,MAAM,2BAA2B;;;AAIjC,MAAM,2BAA2B;AACjC,MAAM,0BAA0B;AAChC,MAAM,uBAAmD;QAC/B;QACI;QACJ;QACC;QACD;AAC1B;AAEA,MAAa,qBACX,QACA,QACA,YAGG;CACH,MAAM,cAAc,UAAU,QAAQ,IAAI;CAC1C,MAAM,SAAS,eAAe,QAAQ,gBAAgB;CACtD,MAAM,UAAU,SAAS,WAAW;CACpC,MAAM,cACJ,UACA,QAAQ,IAAI,qBACX,SAAS,2BAA2B;;;;CAKvC,MAAM,UAAkC;EACtC,gBAAgB;EAChB,oBAAoB,SAAS,oBAAoB;CACnD;CACA,IAAI,QACF,QAAQ,eAAe;CAGzB,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,QACoC;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;;;GAGF,MAAM,UAAmC,EAAE,MAAM;GACjD,IAAI,SAAS,QAAQ,QAAQ,QAAQ,SAAS,IAC5C,QAAQ,OAAO,QAAQ;GAEzB,IAAI,QAAQ,MACV,QAAQ,kBAAkB,qBAAqB;GAGjD,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,aACA,SACA;IAAE;IAAS;GAAQ,CACrB;GAEA,MAAM,aAAa,KAAK,IACtB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAYA,OAAO;IAAE,SAAS;IAAM,MAAM,EAAE,UAXb,MAAM,QAAQ,SAAS,KAAK,OAAO,IAClD,SAAS,KAAK,QAAQ,MAAM,GAAG,UAAU,IACzC,CAAC,EAAA,CAEyC,KAAK,YAAY;KAC7D,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,eAAe,OAAO,WAAW;KACjD,MAAM,OAAO;IACf,EAEsC,EAAE;GAAE;EAC5C,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,gCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}

package/dist/cjs/tools/search/rerankers.cjs

@@ -4,6 +4,10 @@ let axios = require("axios");
 axios = require_runtime.__toESM(axios, 1);
 //#region src/tools/search/rerankers.ts
 const DEFAULT_JINA_API_URL = "https://api.jina.ai/v1/rerank";
+/** Every other network call in the search pipeline is bounded (scrapers,
+* search providers); rerank requests must be too, or a hung rerank API
+* stalls the whole tool. */
+const DEFAULT_RERANKER_TIMEOUT = 1e4;
 const getDefaultJinaApiUrl = () => process.env.JINA_API_URL != null && process.env.JINA_API_URL !== "" ? process.env.JINA_API_URL : DEFAULT_JINA_API_URL;
 var BaseReranker = class {
 	apiKey;
@@ -20,10 +24,12 @@ var BaseReranker = class {
 };
 var JinaReranker = class extends BaseReranker {
 	apiUrl;
-	constructor({ apiKey = process.env.JINA_API_KEY, apiUrl = getDefaultJinaApiUrl(), logger }) {
+	timeout;
+	constructor({ apiKey = process.env.JINA_API_KEY, apiUrl = getDefaultJinaApiUrl(), timeout = DEFAULT_RERANKER_TIMEOUT, logger }) {
 		super(logger);
 		this.apiKey = apiKey;
 		this.apiUrl = apiUrl;
+		this.timeout = timeout;
 	}
 	async rerank(query, documents, topK = 5) {
 		this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);
@@ -39,10 +45,13 @@ var JinaReranker = class extends BaseReranker {
 				documents,
 				return_documents: true
 			};
-			const response = await axios.default.post(this.apiUrl, requestData, { headers: {
-				"Content-Type": "application/json",
-				Authorization: `Bearer ${this.apiKey}`
-			} });
+			const response = await axios.default.post(this.apiUrl, requestData, {
+				headers: {
+					"Content-Type": "application/json",
+					Authorization: `Bearer ${this.apiKey}`
+				},
+				timeout: this.timeout
+			});
 			this.logger.debug("Jina API Model:", response.data?.model);
 			this.logger.debug("Jina API Usage:", response.data?.usage);
 			if (response.data && response.data.results.length) return response.data.results.map((result) => {
@@ -70,9 +79,11 @@ var JinaReranker = class extends BaseReranker {
 	}
 };
 var CohereReranker = class extends BaseReranker {
-	constructor({ apiKey = process.env.COHERE_API_KEY, logger }) {
+	timeout;
+	constructor({ apiKey = process.env.COHERE_API_KEY, timeout = DEFAULT_RERANKER_TIMEOUT, logger }) {
 		super(logger);
 		this.apiKey = apiKey;
+		this.timeout = timeout;
 	}
 	async rerank(query, documents, topK = 5) {
 		this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);
@@ -87,10 +98,13 @@ var CohereReranker = class extends BaseReranker {
 				top_n: topK,
 				documents
 			};
-			const response = await axios.default.post("https://api.cohere.com/v2/rerank", requestData, { headers: {
-				"Content-Type": "application/json",
-				Authorization: `Bearer ${this.apiKey}`
-			} });
+			const response = await axios.default.post("https://api.cohere.com/v2/rerank", requestData, {
+				headers: {
+					"Content-Type": "application/json",
+					Authorization: `Bearer ${this.apiKey}`
+				},
+				timeout: this.timeout
+			});
 			this.logger.debug("Cohere API ID:", response.data?.id);
 			this.logger.debug("Cohere API Meta:", response.data?.meta);
 			if (response.data && response.data.results.length) return response.data.results.map((result) => {
@@ -124,16 +138,18 @@ var InfinityReranker = class extends BaseReranker {
 * Creates the appropriate reranker based on type and configuration
 */
 const createReranker = (config) => {
-	const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, logger } = config;
+	const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, rerankerTimeout, logger } = config;
 	const defaultLogger = logger || require_utils.createDefaultLogger();
 	switch (rerankerType.toLowerCase()) {
 		case "jina": return new JinaReranker({
 			apiKey: jinaApiKey,
 			apiUrl: jinaApiUrl,
+			timeout: rerankerTimeout,
 			logger: defaultLogger
 		});
 		case "cohere": return new CohereReranker({
 			apiKey: cohereApiKey,
+			timeout: rerankerTimeout,
 			logger: defaultLogger
 		});
 		case "infinity": return new InfinityReranker(defaultLogger);
@@ -145,6 +161,7 @@ const createReranker = (config) => {
 			return new JinaReranker({
 				apiKey: jinaApiKey,
 				apiUrl: jinaApiUrl,
+				timeout: rerankerTimeout,
 				logger: defaultLogger
 			});
 	}

package/dist/cjs/tools/search/rerankers.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"rerankers.cjs","names":["createDefaultLogger","formatErrorForLog"],"sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger, formatErrorForLog } from './utils';\n\nconst DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank';\n\nconst getDefaultJinaApiUrl = (): string =>\n  process.env.JINA_API_URL != null && process.env.JINA_API_URL !== ''\n    ? process.env.JINA_API_URL\n    : DEFAULT_JINA_API_URL;\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n  protected logger: t.Logger;\n\n  constructor(logger?: t.Logger) {\n    // Each specific reranker will set its API key\n    this.logger = logger || createDefaultLogger();\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  private apiUrl: string;\n\n  constructor({\n    apiKey = process.env.JINA_API_KEY,\n    apiUrl = getDefaultJinaApiUrl(),\n    logger,\n  }: {\n    apiKey?: string;\n    apiUrl?: string;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.apiUrl = apiUrl;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        this.apiUrl,\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      this.logger.debug('Jina API Model:', response.data?.model);\n      this.logger.debug('Jina API Usage:', response.data?.usage);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Jina reranker', formatErrorForLog(error));\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  constructor({\n    apiKey = process.env.COHERE_API_KEY,\n    logger,\n  }: {\n    apiKey?: string;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      this.logger.debug('Cohere API ID:', response.data?.id);\n      this.logger.debug('Cohere API Meta:', response.data?.meta);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error(\n        'Error using Cohere reranker',\n        formatErrorForLog(error)\n      );\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor(logger?: t.Logger) {\n    super(logger);\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n  rerankerType: t.RerankerType;\n  jinaApiKey?: string;\n  jinaApiUrl?: string;\n  cohereApiKey?: string;\n  logger?: t.Logger;\n}): BaseReranker | undefined => {\n  const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, logger } = config;\n\n  // Create a default logger if none is provided\n  const defaultLogger = logger || createDefaultLogger();\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\n  case 'cohere':\n    return new CohereReranker({\n      apiKey: cohereApiKey,\n      logger: defaultLogger,\n    });\n  case 'infinity':\n    return new InfinityReranker(defaultLogger);\n  case 'none':\n    defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    defaultLogger.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({ apiKey: jinaApiKey, apiUrl: jinaApiUrl, logger: defaultLogger });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"mappings":";;;;;AAIA,MAAM,uBAAuB;AAE7B,MAAM,6BACJ,QAAQ,IAAI,gBAAgB,QAAQ,QAAQ,IAAI,iBAAiB,KAC7D,QAAQ,IAAI,eACZ;AAEN,IAAsB,eAAtB,MAAmC;CACjC;CACA;CAEA,YAAY,QAAmB;EAE7B,KAAK,SAAS,UAAUA,cAAAA,oBAAoB;CAC9C;CAQA,kBACE,WACA,MACe;EACf,OAAO,UACJ,MAAM,GAAG,KAAK,IAAI,MAAM,UAAU,MAAM,CAAC,CAAC,CAC1C,KAAK,SAAS;GAAE,MAAM;GAAK,OAAO;EAAE,EAAE;CAC3C;AACF;AAEA,IAAa,eAAb,cAAkC,aAAa;CAC7C;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,cACrB,SAAS,qBAAqB,GAC9B,UAKC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,SAAS;CAChB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,mCAAmC,KAAK,QAAQ;EAEhG,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,iDAAiD;IAClE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;IACX,kBAAkB;GACpB;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,KAAK,QACL,aACA,EACE,SAAS;IACP,gBAAgB;IAChB,eAAe,UAAU,KAAK;GAChC,EACF,CACF;GAEA,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GACzD,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IACrB,IAAI,OAAO;IAGX,IAAI,OAAO,YAAY,MAAM;KAC3B,MAAM,MAAM,OAAO;KACnB,IAAI,OAAO,QAAQ,YAAY,UAAU,KACvC,OAAO,IAAI;UACN,IAAI,OAAO,QAAQ,UACxB,OAAO;IAEX,OAEE,OAAO,UAAU;IAGnB,OAAO;KAAE;KAAM;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,kEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6BC,cAAAA,kBAAkB,KAAK,CAAC;GAEvE,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,iBAAb,cAAoC,aAAa;CAC/C,YAAY,EACV,SAAS,QAAQ,IAAI,gBACrB,UAIC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;CAChB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,oBAAoB;EAEpE,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,mDAAmD;IACpE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;GACb;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,oCACA,aACA,EACE,SAAS;IACP,gBAAgB;IAChB,eAAe,UAAU,KAAK;GAChC,EACF,CACF;GAEA,KAAK,OAAO,MAAM,kBAAkB,SAAS,MAAM,EAAE;GACrD,KAAK,OAAO,MAAM,oBAAoB,SAAS,MAAM,IAAI;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IAErB,OAAO;KAAE,MADI,UAAU;KACR;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,oEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MACV,+BACAA,cAAAA,kBAAkB,KAAK,CACzB;GAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,mBAAb,cAAsC,aAAa;CACjD,YAAY,QAAmB;EAC7B,MAAM,MAAM;CAEd;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,oCAChC;EAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;CAC/C;AACF;;;;AAKA,MAAa,kBAAkB,WAMC;CAC9B,MAAM,EAAE,cAAc,YAAY,YAAY,cAAc,WAAW;CAGvE,MAAM,gBAAgB,UAAUD,cAAAA,oBAAoB;CAEpD,QAAQ,aAAa,YAAY,GAAjC;EACA,KAAK,QACH,OAAO,IAAI,aAAa;GAAE,QAAQ;GAAY,QAAQ;GAAY,QAAQ;EAAc,CAAC;EAC3F,KAAK,UACH,OAAO,IAAI,eAAe;GACxB,QAAQ;GACR,QAAQ;EACV,CAAC;EACH,KAAK,YACH,OAAO,IAAI,iBAAiB,aAAa;EAC3C,KAAK;GACH,cAAc,MAAM,mDAAiD;GACrE;EACF;GACE,cAAc,KACZ,0BAA0B,aAAa,kCACzC;GACA,OAAO,IAAI,aAAa;IAAE,QAAQ;IAAY,QAAQ;IAAY,QAAQ;GAAc,CAAC;CAC3F;AACF"}
+{"version":3,"file":"rerankers.cjs","names":["createDefaultLogger","formatErrorForLog"],"sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger, formatErrorForLog } from './utils';\n\nconst DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank';\n\n/** Every other network call in the search pipeline is bounded (scrapers,\n * search providers); rerank requests must be too, or a hung rerank API\n * stalls the whole tool. */\nconst DEFAULT_RERANKER_TIMEOUT = 10000;\n\nconst getDefaultJinaApiUrl = (): string =>\n  process.env.JINA_API_URL != null && process.env.JINA_API_URL !== ''\n    ? process.env.JINA_API_URL\n    : DEFAULT_JINA_API_URL;\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n  protected logger: t.Logger;\n\n  constructor(logger?: t.Logger) {\n    // Each specific reranker will set its API key\n    this.logger = logger || createDefaultLogger();\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  private apiUrl: string;\n  private timeout: number;\n\n  constructor({\n    apiKey = process.env.JINA_API_KEY,\n    apiUrl = getDefaultJinaApiUrl(),\n    timeout = DEFAULT_RERANKER_TIMEOUT,\n    logger,\n  }: {\n    apiKey?: string;\n    apiUrl?: string;\n    timeout?: number;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.apiUrl = apiUrl;\n    this.timeout = timeout;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`\n    );\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        this.apiUrl,\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      this.logger.debug('Jina API Model:', response.data?.model);\n      this.logger.debug('Jina API Usage:', response.data?.usage);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Jina reranker', formatErrorForLog(error));\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  private timeout: number;\n\n  constructor({\n    apiKey = process.env.COHERE_API_KEY,\n    timeout = DEFAULT_RERANKER_TIMEOUT,\n    logger,\n  }: {\n    apiKey?: string;\n    timeout?: number;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.timeout = timeout;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      this.logger.debug('Cohere API ID:', response.data?.id);\n      this.logger.debug('Cohere API Meta:', response.data?.meta);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error(\n        'Error using Cohere reranker',\n        formatErrorForLog(error)\n      );\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor(logger?: t.Logger) {\n    super(logger);\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n  rerankerType: t.RerankerType;\n  jinaApiKey?: string;\n  jinaApiUrl?: string;\n  cohereApiKey?: string;\n  rerankerTimeout?: number;\n  logger?: t.Logger;\n}): BaseReranker | undefined => {\n  const {\n    rerankerType,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    rerankerTimeout,\n    logger,\n  } = config;\n\n  // Create a default logger if none is provided\n  const defaultLogger = logger || createDefaultLogger();\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({\n      apiKey: jinaApiKey,\n      apiUrl: jinaApiUrl,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n    });\n  case 'cohere':\n    return new CohereReranker({\n      apiKey: cohereApiKey,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n    });\n  case 'infinity':\n    return new InfinityReranker(defaultLogger);\n  case 'none':\n    defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    defaultLogger.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({\n      apiKey: jinaApiKey,\n      apiUrl: jinaApiUrl,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n    });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"mappings":";;;;;AAIA,MAAM,uBAAuB;;;;AAK7B,MAAM,2BAA2B;AAEjC,MAAM,6BACJ,QAAQ,IAAI,gBAAgB,QAAQ,QAAQ,IAAI,iBAAiB,KAC7D,QAAQ,IAAI,eACZ;AAEN,IAAsB,eAAtB,MAAmC;CACjC;CACA;CAEA,YAAY,QAAmB;EAE7B,KAAK,SAAS,UAAUA,cAAAA,oBAAoB;CAC9C;CAQA,kBACE,WACA,MACe;EACf,OAAO,UACJ,MAAM,GAAG,KAAK,IAAI,MAAM,UAAU,MAAM,CAAC,CAAC,CAC1C,KAAK,SAAS;GAAE,MAAM;GAAK,OAAO;EAAE,EAAE;CAC3C;AACF;AAEA,IAAa,eAAb,cAAkC,aAAa;CAC7C;CACA;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,cACrB,SAAS,qBAAqB,GAC9B,UAAU,0BACV,UAMC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,mCAAmC,KAAK,QACxE;EAEA,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,iDAAiD;IAClE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;IACX,kBAAkB;GACpB;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,KAAK,QACL,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CACF;GAEA,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GACzD,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IACrB,IAAI,OAAO;IAGX,IAAI,OAAO,YAAY,MAAM;KAC3B,MAAM,MAAM,OAAO;KACnB,IAAI,OAAO,QAAQ,YAAY,UAAU,KACvC,OAAO,IAAI;UACN,IAAI,OAAO,QAAQ,UACxB,OAAO;IAEX,OAEE,OAAO,UAAU;IAGnB,OAAO;KAAE;KAAM;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,kEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6BC,cAAAA,kBAAkB,KAAK,CAAC;GAEvE,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,iBAAb,cAAoC,aAAa;CAC/C;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,gBACrB,UAAU,0BACV,UAKC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,oBAAoB;EAEpE,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,mDAAmD;IACpE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;GACb;GAEA,MAAM,WAAW,MAAM,MAAA,QAAM,KAC3B,oCACA,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CACF;GAEA,KAAK,OAAO,MAAM,kBAAkB,SAAS,MAAM,EAAE;GACrD,KAAK,OAAO,MAAM,oBAAoB,SAAS,MAAM,IAAI;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IAErB,OAAO;KAAE,MADI,UAAU;KACR;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,oEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MACV,+BACAA,cAAAA,kBAAkB,KAAK,CACzB;GAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,mBAAb,cAAsC,aAAa;CACjD,YAAY,QAAmB;EAC7B,MAAM,MAAM;CAEd;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,oCAChC;EAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;CAC/C;AACF;;;;AAKA,MAAa,kBAAkB,WAOC;CAC9B,MAAM,EACJ,cACA,YACA,YACA,cACA,iBACA,WACE;CAGJ,MAAM,gBAAgB,UAAUD,cAAAA,oBAAoB;CAEpD,QAAQ,aAAa,YAAY,GAAjC;EACA,KAAK,QACH,OAAO,IAAI,aAAa;GACtB,QAAQ;GACR,QAAQ;GACR,SAAS;GACT,QAAQ;EACV,CAAC;EACH,KAAK,UACH,OAAO,IAAI,eAAe;GACxB,QAAQ;GACR,SAAS;GACT,QAAQ;EACV,CAAC;EACH,KAAK,YACH,OAAO,IAAI,iBAAiB,aAAa;EAC3C,KAAK;GACH,cAAc,MAAM,mDAAiD;GACrE;EACF;GACE,cAAc,KACZ,0BAA0B,aAAa,kCACzC;GACA,OAAO,IAAI,aAAa;IACtB,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,QAAQ;GACV,CAAC;CACH;AACF"}

package/dist/cjs/tools/search/search.cjs

@@ -1,5 +1,6 @@
 const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
 const require_utils = require("./utils.cjs");
+const require_keenable_search = require("./keenable-search.cjs");
 const require_tavily_search = require("./tavily-search.cjs");
 let axios = require("axios");
 axios = require_runtime.__toESM(axios, 1);
@@ -29,6 +30,27 @@ const chunker = {
 	}
 };
 const DEFAULT_MAX_CONTENT_LENGTH = 5e4;
+const DEFAULT_CHUNK_SIZE = 150;
+const DEFAULT_CHUNK_OVERLAP = 50;
+/** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /
+* `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The
+* overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`
+* throws when overlap >= size. */
+function resolveChunkOptions(chunkSize, chunkOverlap) {
+	const resolve = (configValue, envVar, fallback) => {
+		if (configValue != null && configValue > 0) return configValue;
+		const envValue = Number(process.env[envVar]);
+		if (Number.isFinite(envValue) && envValue > 0) return envValue;
+		return fallback;
+	};
+	const size = resolve(chunkSize, "SEARCH_CHUNK_SIZE", DEFAULT_CHUNK_SIZE);
+	let overlap = resolve(chunkOverlap, "SEARCH_CHUNK_OVERLAP", DEFAULT_CHUNK_OVERLAP);
+	if (overlap >= size) overlap = Math.floor(size / 3);
+	return {
+		chunkSize: size,
+		chunkOverlap: overlap
+	};
+}
 /** Resolves the per-source scraped content cap from config, the
 * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */
 function resolveMaxContentLength(maxContentLength) {
@@ -49,7 +71,7 @@ function createSourceUpdateCallback(sourceMap) {
 		});
 	};
 }
-const getHighlights = async ({ query, content, reranker, topResults = 5, maxContentLength = DEFAULT_MAX_CONTENT_LENGTH, logger }) => {
+const getHighlights = async ({ query, content, reranker, topResults = 5, maxContentLength = DEFAULT_MAX_CONTENT_LENGTH, chunkOptions, logger }) => {
 	const logger_ = logger || require_utils.createDefaultLogger();
 	if (!content) {
 		logger_.warn("No content provided for highlights");
@@ -60,7 +82,7 @@ const getHighlights = async ({ query, content, reranker, topResults = 5, maxCont
 		return;
 	}
 	try {
-		const documents = await chunker.splitText(truncateContent(content, maxContentLength));
+		const documents = await chunker.splitText(truncateContent(content, maxContentLength), chunkOptions);
 		if (Array.isArray(documents)) return await reranker.rerank(query, documents, topResults);
 		else {
 			logger_.error("Expected documents to be an array, got:", typeof documents);
@@ -252,16 +274,18 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
 	return { getSources };
 };
 const createSearchAPI = (config) => {
-	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions } = config;
+	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions } = config;
 	if (searchProvider.toLowerCase() === "serper") return createSerperAPI(serperApiKey);
 	else if (searchProvider.toLowerCase() === "searxng") return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
 	else if (searchProvider.toLowerCase() === "tavily") return require_tavily_search.createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
-	else throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`);
+	else if (searchProvider.toLowerCase() === "keenable") return require_keenable_search.createKeenableAPI(keenableApiKey, keenableApiUrl, keenableSearchOptions);
+	else throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', or 'keenable'`);
 };
 const createSourceProcessor = (config = {}, scraperInstance) => {
 	if (!scraperInstance) throw new Error("Scraper instance is required");
 	const { topResults = 5, reranker, logger } = config;
 	const maxContentLength = resolveMaxContentLength(config.maxContentLength);
+	const chunkOptions = resolveChunkOptions(config.chunkSize, config.chunkOverlap);
 	const logger_ = logger || require_utils.createDefaultLogger();
 	const scraper = scraperInstance;
 	const processResponse = (url, response) => {
@@ -290,8 +314,10 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
 			const highlights = await getHighlights({
 				query,
 				reranker,
+				topResults,
 				content: result.content,
 				maxContentLength,
+				chunkOptions,
 				logger: logger_
 			});
 			if (onGetHighlights) onGetHighlights(result.url);

package/dist/cjs/tools/search/search.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"search.cjs","names":["RecursiveCharacterTextSplitter","createDefaultLogger","createTavilyAPI","getAttribution"],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createTavilyAPI } from './tavily-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n  if (maxContentLength != null && maxContentLength > 0) {\n    return maxContentLength;\n  }\n  const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n  if (Number.isFinite(envValue) && envValue > 0) {\n    return envValue;\n  }\n  return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n  return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  maxContentLength?: number;\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(\n      truncateContent(content, maxContentLength)\n    );\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions,\n  } = config;\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n  } else if (searchProvider.toLowerCase() === 'tavily') {\n    return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const processResponse = (\n    url: string,\n    response: t.AnyScraperResponse\n  ): t.ScrapeResult => {\n    const rawMetadata = scraper.extractMetadata(response);\n    const metadata =\n      Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n    const attribution = getAttribution(url, metadata, logger_);\n\n    if (response.success && response.data) {\n      const [content, references] = scraper.extractContent(response);\n      return {\n        url,\n        references,\n        attribution,\n        content: truncateContent(chunker.cleanText(content), maxContentLength),\n      };\n    }\n\n    logger_.error(\n      `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n    );\n    return { url, attribution, error: true, content: '' };\n  };\n\n  const addHighlights = async (\n    result: t.ScrapeResult,\n    query: string,\n    onGetHighlights: t.SearchToolConfig['onGetHighlights']\n  ): Promise<t.ScrapeResult> => {\n    if (result.error != null) {\n      return result;\n    }\n    try {\n      const highlights = await getHighlights({\n        query,\n        reranker,\n        content: result.content,\n        maxContentLength,\n        logger: logger_,\n      });\n      if (onGetHighlights) {\n        onGetHighlights(result.url);\n      }\n      return { ...result, highlights };\n    } catch (error) {\n      logger_.error('Error processing scraped content:', error);\n      return result;\n    }\n  };\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      try {\n        let responses: Array<[string, t.AnyScraperResponse]>;\n\n        if (scraper.scrapeUrls) {\n          responses = await scraper.scrapeUrls(links);\n        } else {\n          responses = await Promise.all(\n            links.map((link) =>\n              scraper\n                .scrapeUrl(link, {})\n                .catch((error): [string, t.AnyScraperResponse] => {\n                  logger_.error(`Error scraping ${link}:`, error);\n                  return [link, { success: false, error: String(error) }];\n                })\n            )\n          );\n        }\n\n        const withHighlights = await Promise.all(\n          responses.map(([url, response]) =>\n            addHighlights(\n              processResponse(url, response),\n              query,\n              onGetHighlights\n            )\n          )\n        );\n        return withHighlights;\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      }\n\n      if (\n        result.data.topStories != null &&\n        result.data.topStories.length > numElements\n      ) {\n        /** Merged news results can far exceed the requested source count;\n         * every entry is formatted into the LLM output, so cap them up\n         * front — before any early return below and before scraping\n         * entries the cap would discard */\n        result.data.topStories = result.data.topStories.slice(0, numElements);\n      }\n\n      if (!result.data.organic) {\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"mappings":";;;;;;;AAOA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQA,yBAAAA,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAUC,cAAAA,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;;;AAInC,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,aAQwC;CACxC,MAAM,UAAU,UAAUA,cAAAA,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,CAC3C;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAehB,MAAM,QAAO,MAZU,MAAA,QAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;GAClB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAShC,MAAM,QAAO,MANU,MAAA,QAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;GAClB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,wBACE;CAEJ,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,YAAY;MAC9B,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,aAAa;MACpD,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAOC,sBAAAA,gBAAgB,cAAc,iBAAiB,mBAAmB;MAEzE,MAAM,IAAI,MACR,4BAA4B,eAAe,2CAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,UAAU,UAAUD,cAAAA,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAcE,cAAAA,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA,SAAS,OAAO;IAChB;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}
+{"version":3,"file":"search.cjs","names":["RecursiveCharacterTextSplitter","createDefaultLogger","createTavilyAPI","createKeenableAPI","getAttribution"],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createKeenableAPI } from './keenable-search';\nimport { createTavilyAPI } from './tavily-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\nconst DEFAULT_CHUNK_SIZE = 150;\nconst DEFAULT_CHUNK_OVERLAP = 50;\n\n/** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /\n * `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The\n * overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`\n * throws when overlap >= size. */\nfunction resolveChunkOptions(\n  chunkSize?: number,\n  chunkOverlap?: number\n): { chunkSize: number; chunkOverlap: number } {\n  const resolve = (\n    configValue: number | undefined,\n    envVar: string,\n    fallback: number\n  ): number => {\n    if (configValue != null && configValue > 0) {\n      return configValue;\n    }\n    const envValue = Number(process.env[envVar]);\n    if (Number.isFinite(envValue) && envValue > 0) {\n      return envValue;\n    }\n    return fallback;\n  };\n\n  const size = resolve(chunkSize, 'SEARCH_CHUNK_SIZE', DEFAULT_CHUNK_SIZE);\n  let overlap = resolve(\n    chunkOverlap,\n    'SEARCH_CHUNK_OVERLAP',\n    DEFAULT_CHUNK_OVERLAP\n  );\n  if (overlap >= size) {\n    overlap = Math.floor(size / 3);\n  }\n  return { chunkSize: size, chunkOverlap: overlap };\n}\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n  if (maxContentLength != null && maxContentLength > 0) {\n    return maxContentLength;\n  }\n  const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n  if (Number.isFinite(envValue) && envValue > 0) {\n    return envValue;\n  }\n  return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n  return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n  chunkOptions,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  maxContentLength?: number;\n  chunkOptions?: { chunkSize: number; chunkOverlap: number };\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(\n      truncateContent(content, maxContentLength),\n      chunkOptions\n    );\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n  } = config;\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n  } else if (searchProvider.toLowerCase() === 'tavily') {\n    return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n  } else if (searchProvider.toLowerCase() === 'keenable') {\n    return createKeenableAPI(\n      keenableApiKey,\n      keenableApiUrl,\n      keenableSearchOptions\n    );\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', or 'keenable'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n  const chunkOptions = resolveChunkOptions(\n    config.chunkSize,\n    config.chunkOverlap\n  );\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const processResponse = (\n    url: string,\n    response: t.AnyScraperResponse\n  ): t.ScrapeResult => {\n    const rawMetadata = scraper.extractMetadata(response);\n    const metadata =\n      Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n    const attribution = getAttribution(url, metadata, logger_);\n\n    if (response.success && response.data) {\n      const [content, references] = scraper.extractContent(response);\n      return {\n        url,\n        references,\n        attribution,\n        content: truncateContent(chunker.cleanText(content), maxContentLength),\n      };\n    }\n\n    logger_.error(\n      `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n    );\n    return { url, attribution, error: true, content: '' };\n  };\n\n  const addHighlights = async (\n    result: t.ScrapeResult,\n    query: string,\n    onGetHighlights: t.SearchToolConfig['onGetHighlights']\n  ): Promise<t.ScrapeResult> => {\n    if (result.error != null) {\n      return result;\n    }\n    try {\n      const highlights = await getHighlights({\n        query,\n        reranker,\n        topResults,\n        content: result.content,\n        maxContentLength,\n        chunkOptions,\n        logger: logger_,\n      });\n      if (onGetHighlights) {\n        onGetHighlights(result.url);\n      }\n      return { ...result, highlights };\n    } catch (error) {\n      logger_.error('Error processing scraped content:', error);\n      return result;\n    }\n  };\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      try {\n        let responses: Array<[string, t.AnyScraperResponse]>;\n\n        if (scraper.scrapeUrls) {\n          responses = await scraper.scrapeUrls(links);\n        } else {\n          responses = await Promise.all(\n            links.map((link) =>\n              scraper\n                .scrapeUrl(link, {})\n                .catch((error): [string, t.AnyScraperResponse] => {\n                  logger_.error(`Error scraping ${link}:`, error);\n                  return [link, { success: false, error: String(error) }];\n                })\n            )\n          );\n        }\n\n        const withHighlights = await Promise.all(\n          responses.map(([url, response]) =>\n            addHighlights(\n              processResponse(url, response),\n              query,\n              onGetHighlights\n            )\n          )\n        );\n        return withHighlights;\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      }\n\n      if (\n        result.data.topStories != null &&\n        result.data.topStories.length > numElements\n      ) {\n        /** Merged news results can far exceed the requested source count;\n         * every entry is formatted into the LLM output, so cap them up\n         * front — before any early return below and before scraping\n         * entries the cap would discard */\n        result.data.topStories = result.data.topStories.slice(0, numElements);\n      }\n\n      if (!result.data.organic) {\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"mappings":";;;;;;;;AAQA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQA,yBAAAA,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAUC,cAAAA,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;AACnC,MAAM,qBAAqB;AAC3B,MAAM,wBAAwB;;;;;AAM9B,SAAS,oBACP,WACA,cAC6C;CAC7C,MAAM,WACJ,aACA,QACA,aACW;EACX,IAAI,eAAe,QAAQ,cAAc,GACvC,OAAO;EAET,MAAM,WAAW,OAAO,QAAQ,IAAI,OAAO;EAC3C,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;EAET,OAAO;CACT;CAEA,MAAM,OAAO,QAAQ,WAAW,qBAAqB,kBAAkB;CACvE,IAAI,UAAU,QACZ,cACA,wBACA,qBACF;CACA,IAAI,WAAW,MACb,UAAU,KAAK,MAAM,OAAO,CAAC;CAE/B,OAAO;EAAE,WAAW;EAAM,cAAc;CAAQ;AAClD;;;AAIA,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,cACA,aASwC;CACxC,MAAM,UAAU,UAAUA,cAAAA,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,GACzC,YACF;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAehB,MAAM,QAAO,MAZU,MAAA,QAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;GAClB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAShC,MAAM,QAAO,MANU,MAAA,QAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;GAClB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,qBACA,gBACA,gBACA,0BACE;CAEJ,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,YAAY;MAC9B,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,aAAa;MACpD,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAOC,sBAAAA,gBAAgB,cAAc,iBAAiB,mBAAmB;MACpE,IAAI,eAAe,YAAY,MAAM,YAC1C,OAAOC,wBAAAA,kBACL,gBACA,gBACA,qBACF;MAEA,MAAM,IAAI,MACR,4BAA4B,eAAe,uDAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,eAAe,oBACnB,OAAO,WACP,OAAO,YACT;CACA,MAAM,UAAU,UAAUF,cAAAA,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAcG,cAAAA,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA;IACA,SAAS,OAAO;IAChB;IACA;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}

package/dist/cjs/tools/search/tool.cjs

@@ -106,7 +106,7 @@ async function executeParallelSearches({ searchAPI, query, date, country, safeSe
 		data: mergedResults
 	};
 }
-function createSearchProcessor({ searchAPI, safeSearch, supportsVideos, sourceProcessor, onGetHighlights, logger }) {
+function createSearchProcessor({ searchAPI, safeSearch, supportsImages, supportsVideos, supportsNews, sourceProcessor, onGetHighlights, logger }) {
 	return async function({ query, date, country, proMode = true, maxSources = 5, onSearchResults, images = false, videos = false, news = false }) {
 		try {
 			const searchResult = await executeParallelSearches({
@@ -115,9 +115,9 @@ function createSearchProcessor({ searchAPI, safeSearch, supportsVideos, sourcePr
 				date,
 				country,
 				safeSearch,
-				images,
+				images: supportsImages && images,
 				videos: supportsVideos && videos,
-				news,
+				news: supportsNews && news,
 				logger
 			});
 			onSearchResults?.(searchResult);
@@ -180,7 +180,7 @@ function createTool({ schema, search, maxOutputChars, onSearchResults: _onSearch
 	});
 }
 const createSearchTool = (config = {}) => {
-	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilyExtractUrl, tavilySearchOptions, rerankerType = "cohere", topResults = 5, maxContentLength, maxOutputChars, strategies = ["no_extraction"], filterContent = true, safeSearch = 1, scraperProvider = "firecrawl", firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, tavilyScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights } = config;
+	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilyExtractUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions, rerankerType = "cohere", rerankerTimeout, topResults = 5, maxContentLength, chunkSize, chunkOverlap, maxOutputChars, strategies = ["no_extraction"], filterContent = true, safeSearch = 1, scraperProvider = "firecrawl", firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, tavilyScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights } = config;
 	const logger = config.logger || require_utils.createDefaultLogger();
 	const effectiveTavilySearchOptions = searchProvider === "tavily" && config.safeSearch != null ? {
 		...tavilySearchOptions,
@@ -206,7 +206,10 @@ const createSearchTool = (config = {}) => {
 		searxngApiKey,
 		tavilyApiKey,
 		tavilySearchUrl,
-		tavilySearchOptions: effectiveTavilySearchOptions
+		tavilySearchOptions: effectiveTavilySearchOptions,
+		keenableApiKey,
+		keenableApiUrl,
+		keenableSearchOptions
 	});
 	/** Create scraper based on scraperProvider */
 	let scraperInstance;
@@ -237,6 +240,7 @@ const createSearchTool = (config = {}) => {
 		jinaApiKey,
 		jinaApiUrl,
 		cohereApiKey,
+		rerankerTimeout,
 		logger
 	});
 	if (!selectedReranker) logger.warn("No reranker selected. Using default ranking.");
@@ -244,6 +248,8 @@ const createSearchTool = (config = {}) => {
 		reranker: selectedReranker,
 		topResults,
 		maxContentLength,
+		chunkSize,
+		chunkOverlap,
 		strategies,
 		filterContent,
 		logger
@@ -252,7 +258,9 @@ const createSearchTool = (config = {}) => {
 		search: createSearchProcessor({
 			searchAPI,
 			safeSearch,
-			supportsVideos: searchProvider !== "tavily",
+			supportsImages: searchProvider !== "keenable",
+			supportsVideos: searchProvider !== "tavily" && searchProvider !== "keenable",
+			supportsNews: searchProvider !== "keenable",
 			sourceProcessor,
 			onGetHighlights,
 			logger