@librechat/agents 3.3.11 → 3.3.13

package/dist/esm/tools/search/rerankers.mjs

@@ -23,11 +23,15 @@ var BaseReranker = class {
 var JinaReranker = class extends BaseReranker {
 	apiUrl;
 	timeout;
-	constructor({ apiKey = process.env.JINA_API_KEY, apiUrl = getDefaultJinaApiUrl(), timeout = DEFAULT_RERANKER_TIMEOUT, logger }) {
+	httpAgent;
+	httpsAgent;
+	constructor({ apiKey = process.env.JINA_API_KEY, apiUrl = getDefaultJinaApiUrl(), timeout = DEFAULT_RERANKER_TIMEOUT, logger, httpAgent, httpsAgent }) {
 		super(logger);
 		this.apiKey = apiKey;
 		this.apiUrl = apiUrl;
 		this.timeout = timeout;
+		this.httpAgent = httpAgent;
+		this.httpsAgent = httpsAgent;
 	}
 	async rerank(query, documents, topK = 5) {
 		this.logger.debug(`Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`);
@@ -48,7 +52,9 @@ var JinaReranker = class extends BaseReranker {
 					"Content-Type": "application/json",
 					Authorization: `Bearer ${this.apiKey}`
 				},
-				timeout: this.timeout
+				timeout: this.timeout,
+				httpAgent: this.httpAgent,
+				httpsAgent: this.httpsAgent
 			});
 			this.logger.debug("Jina API Model:", response.data?.model);
 			this.logger.debug("Jina API Usage:", response.data?.usage);
@@ -78,10 +84,14 @@ var JinaReranker = class extends BaseReranker {
 };
 var CohereReranker = class extends BaseReranker {
 	timeout;
-	constructor({ apiKey = process.env.COHERE_API_KEY, timeout = DEFAULT_RERANKER_TIMEOUT, logger }) {
+	httpAgent;
+	httpsAgent;
+	constructor({ apiKey = process.env.COHERE_API_KEY, timeout = DEFAULT_RERANKER_TIMEOUT, logger, httpAgent, httpsAgent }) {
 		super(logger);
 		this.apiKey = apiKey;
 		this.timeout = timeout;
+		this.httpAgent = httpAgent;
+		this.httpsAgent = httpsAgent;
 	}
 	async rerank(query, documents, topK = 5) {
 		this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);
@@ -101,7 +111,9 @@ var CohereReranker = class extends BaseReranker {
 					"Content-Type": "application/json",
 					Authorization: `Bearer ${this.apiKey}`
 				},
-				timeout: this.timeout
+				timeout: this.timeout,
+				httpAgent: this.httpAgent,
+				httpsAgent: this.httpsAgent
 			});
 			this.logger.debug("Cohere API ID:", response.data?.id);
 			this.logger.debug("Cohere API Meta:", response.data?.meta);
@@ -136,19 +148,23 @@ var InfinityReranker = class extends BaseReranker {
 * Creates the appropriate reranker based on type and configuration
 */
 const createReranker = (config) => {
-	const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, rerankerTimeout, logger } = config;
+	const { rerankerType, jinaApiKey, jinaApiUrl, cohereApiKey, rerankerTimeout, logger, httpAgent, httpsAgent } = config;
 	const defaultLogger = logger || createDefaultLogger();
 	switch (rerankerType.toLowerCase()) {
 		case "jina": return new JinaReranker({
 			apiKey: jinaApiKey,
 			apiUrl: jinaApiUrl,
 			timeout: rerankerTimeout,
-			logger: defaultLogger
+			logger: defaultLogger,
+			httpAgent,
+			httpsAgent
 		});
 		case "cohere": return new CohereReranker({
 			apiKey: cohereApiKey,
 			timeout: rerankerTimeout,
-			logger: defaultLogger
+			logger: defaultLogger,
+			httpAgent,
+			httpsAgent
 		});
 		case "infinity": return new InfinityReranker(defaultLogger);
 		case "none":
@@ -160,7 +176,9 @@ const createReranker = (config) => {
 				apiKey: jinaApiKey,
 				apiUrl: jinaApiUrl,
 				timeout: rerankerTimeout,
-				logger: defaultLogger
+				logger: defaultLogger,
+				httpAgent,
+				httpsAgent
 			});
 	}
 };

package/dist/esm/tools/search/rerankers.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"rerankers.mjs","names":[],"sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger, formatErrorForLog } from './utils';\n\nconst DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank';\n\n/** Every other network call in the search pipeline is bounded (scrapers,\n * search providers); rerank requests must be too, or a hung rerank API\n * stalls the whole tool. */\nconst DEFAULT_RERANKER_TIMEOUT = 10000;\n\nconst getDefaultJinaApiUrl = (): string =>\n  process.env.JINA_API_URL != null && process.env.JINA_API_URL !== ''\n    ? process.env.JINA_API_URL\n    : DEFAULT_JINA_API_URL;\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n  protected logger: t.Logger;\n\n  constructor(logger?: t.Logger) {\n    // Each specific reranker will set its API key\n    this.logger = logger || createDefaultLogger();\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  private apiUrl: string;\n  private timeout: number;\n\n  constructor({\n    apiKey = process.env.JINA_API_KEY,\n    apiUrl = getDefaultJinaApiUrl(),\n    timeout = DEFAULT_RERANKER_TIMEOUT,\n    logger,\n  }: {\n    apiKey?: string;\n    apiUrl?: string;\n    timeout?: number;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.apiUrl = apiUrl;\n    this.timeout = timeout;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`\n    );\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        this.apiUrl,\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      this.logger.debug('Jina API Model:', response.data?.model);\n      this.logger.debug('Jina API Usage:', response.data?.usage);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Jina reranker', formatErrorForLog(error));\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  private timeout: number;\n\n  constructor({\n    apiKey = process.env.COHERE_API_KEY,\n    timeout = DEFAULT_RERANKER_TIMEOUT,\n    logger,\n  }: {\n    apiKey?: string;\n    timeout?: number;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.timeout = timeout;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n        }\n      );\n\n      this.logger.debug('Cohere API ID:', response.data?.id);\n      this.logger.debug('Cohere API Meta:', response.data?.meta);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error(\n        'Error using Cohere reranker',\n        formatErrorForLog(error)\n      );\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor(logger?: t.Logger) {\n    super(logger);\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n  rerankerType: t.RerankerType;\n  jinaApiKey?: string;\n  jinaApiUrl?: string;\n  cohereApiKey?: string;\n  rerankerTimeout?: number;\n  logger?: t.Logger;\n}): BaseReranker | undefined => {\n  const {\n    rerankerType,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    rerankerTimeout,\n    logger,\n  } = config;\n\n  // Create a default logger if none is provided\n  const defaultLogger = logger || createDefaultLogger();\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({\n      apiKey: jinaApiKey,\n      apiUrl: jinaApiUrl,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n    });\n  case 'cohere':\n    return new CohereReranker({\n      apiKey: cohereApiKey,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n    });\n  case 'infinity':\n    return new InfinityReranker(defaultLogger);\n  case 'none':\n    defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    defaultLogger.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({\n      apiKey: jinaApiKey,\n      apiUrl: jinaApiUrl,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n    });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"mappings":";;;AAIA,MAAM,uBAAuB;;;;AAK7B,MAAM,2BAA2B;AAEjC,MAAM,6BACJ,QAAQ,IAAI,gBAAgB,QAAQ,QAAQ,IAAI,iBAAiB,KAC7D,QAAQ,IAAI,eACZ;AAEN,IAAsB,eAAtB,MAAmC;CACjC;CACA;CAEA,YAAY,QAAmB;EAE7B,KAAK,SAAS,UAAU,oBAAoB;CAC9C;CAQA,kBACE,WACA,MACe;EACf,OAAO,UACJ,MAAM,GAAG,KAAK,IAAI,MAAM,UAAU,MAAM,CAAC,CAAC,CAC1C,KAAK,SAAS;GAAE,MAAM;GAAK,OAAO;EAAE,EAAE;CAC3C;AACF;AAEA,IAAa,eAAb,cAAkC,aAAa;CAC7C;CACA;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,cACrB,SAAS,qBAAqB,GAC9B,UAAU,0BACV,UAMC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,mCAAmC,KAAK,QACxE;EAEA,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,iDAAiD;IAClE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;IACX,kBAAkB;GACpB;GAEA,MAAM,WAAW,MAAM,MAAM,KAC3B,KAAK,QACL,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CACF;GAEA,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GACzD,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IACrB,IAAI,OAAO;IAGX,IAAI,OAAO,YAAY,MAAM;KAC3B,MAAM,MAAM,OAAO;KACnB,IAAI,OAAO,QAAQ,YAAY,UAAU,KACvC,OAAO,IAAI;UACN,IAAI,OAAO,QAAQ,UACxB,OAAO;IAEX,OAEE,OAAO,UAAU;IAGnB,OAAO;KAAE;KAAM;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,kEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,kBAAkB,KAAK,CAAC;GAEvE,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,iBAAb,cAAoC,aAAa;CAC/C;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,gBACrB,UAAU,0BACV,UAKC;EACD,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,UAAU;CACjB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,oBAAoB;EAEpE,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,mDAAmD;IACpE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;GACb;GAEA,MAAM,WAAW,MAAM,MAAM,KAC3B,oCACA,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;GAChB,CACF;GAEA,KAAK,OAAO,MAAM,kBAAkB,SAAS,MAAM,EAAE;GACrD,KAAK,OAAO,MAAM,oBAAoB,SAAS,MAAM,IAAI;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IAErB,OAAO;KAAE,MADI,UAAU;KACR;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,oEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MACV,+BACA,kBAAkB,KAAK,CACzB;GAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,mBAAb,cAAsC,aAAa;CACjD,YAAY,QAAmB;EAC7B,MAAM,MAAM;CAEd;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,oCAChC;EAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;CAC/C;AACF;;;;AAKA,MAAa,kBAAkB,WAOC;CAC9B,MAAM,EACJ,cACA,YACA,YACA,cACA,iBACA,WACE;CAGJ,MAAM,gBAAgB,UAAU,oBAAoB;CAEpD,QAAQ,aAAa,YAAY,GAAjC;EACA,KAAK,QACH,OAAO,IAAI,aAAa;GACtB,QAAQ;GACR,QAAQ;GACR,SAAS;GACT,QAAQ;EACV,CAAC;EACH,KAAK,UACH,OAAO,IAAI,eAAe;GACxB,QAAQ;GACR,SAAS;GACT,QAAQ;EACV,CAAC;EACH,KAAK,YACH,OAAO,IAAI,iBAAiB,aAAa;EAC3C,KAAK;GACH,cAAc,MAAM,mDAAiD;GACrE;EACF;GACE,cAAc,KACZ,0BAA0B,aAAa,kCACzC;GACA,OAAO,IAAI,aAAa;IACtB,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,QAAQ;GACV,CAAC;CACH;AACF"}
+{"version":3,"file":"rerankers.mjs","names":[],"sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger, formatErrorForLog } from './utils';\n\nconst DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank';\n\n/** Every other network call in the search pipeline is bounded (scrapers,\n * search providers); rerank requests must be too, or a hung rerank API\n * stalls the whole tool. */\nconst DEFAULT_RERANKER_TIMEOUT = 10000;\n\nconst getDefaultJinaApiUrl = (): string =>\n  process.env.JINA_API_URL != null && process.env.JINA_API_URL !== ''\n    ? process.env.JINA_API_URL\n    : DEFAULT_JINA_API_URL;\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n  protected logger: t.Logger;\n\n  constructor(logger?: t.Logger) {\n    // Each specific reranker will set its API key\n    this.logger = logger || createDefaultLogger();\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  private apiUrl: string;\n  private timeout: number;\n  private httpAgent?: t.HttpAgent;\n  private httpsAgent?: t.HttpsAgent;\n\n  constructor({\n    apiKey = process.env.JINA_API_KEY,\n    apiUrl = getDefaultJinaApiUrl(),\n    timeout = DEFAULT_RERANKER_TIMEOUT,\n    logger,\n    httpAgent,\n    httpsAgent,\n  }: {\n    apiKey?: string;\n    apiUrl?: string;\n    timeout?: number;\n    logger?: t.Logger;\n  } & t.HttpAgentConfig) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.apiUrl = apiUrl;\n    this.timeout = timeout;\n    this.httpAgent = httpAgent;\n    this.httpsAgent = httpsAgent;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Jina using API URL: ${this.apiUrl}`\n    );\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        this.apiUrl,\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n          httpAgent: this.httpAgent,\n          httpsAgent: this.httpsAgent,\n        }\n      );\n\n      this.logger.debug('Jina API Model:', response.data?.model);\n      this.logger.debug('Jina API Usage:', response.data?.usage);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Jina reranker', formatErrorForLog(error));\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  private timeout: number;\n  private httpAgent?: t.HttpAgent;\n  private httpsAgent?: t.HttpsAgent;\n\n  constructor({\n    apiKey = process.env.COHERE_API_KEY,\n    timeout = DEFAULT_RERANKER_TIMEOUT,\n    logger,\n    httpAgent,\n    httpsAgent,\n  }: {\n    apiKey?: string;\n    timeout?: number;\n    logger?: t.Logger;\n  } & t.HttpAgentConfig) {\n    super(logger);\n    this.apiKey = apiKey;\n    this.timeout = timeout;\n    this.httpAgent = httpAgent;\n    this.httpsAgent = httpsAgent;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} chunks with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n          timeout: this.timeout,\n          httpAgent: this.httpAgent,\n          httpsAgent: this.httpsAgent,\n        }\n      );\n\n      this.logger.debug('Cohere API ID:', response.data?.id);\n      this.logger.debug('Cohere API Meta:', response.data?.meta);\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error(\n        'Error using Cohere reranker',\n        formatErrorForLog(error)\n      );\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor(logger?: t.Logger) {\n    super(logger);\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} chunks with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (\n  config: {\n    rerankerType: t.RerankerType;\n    jinaApiKey?: string;\n    jinaApiUrl?: string;\n    cohereApiKey?: string;\n    rerankerTimeout?: number;\n    logger?: t.Logger;\n  } & t.HttpAgentConfig\n): BaseReranker | undefined => {\n  const {\n    rerankerType,\n    jinaApiKey,\n    jinaApiUrl,\n    cohereApiKey,\n    rerankerTimeout,\n    logger,\n    httpAgent,\n    httpsAgent,\n  } = config;\n\n  // Create a default logger if none is provided\n  const defaultLogger = logger || createDefaultLogger();\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({\n      apiKey: jinaApiKey,\n      apiUrl: jinaApiUrl,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n      httpAgent,\n      httpsAgent,\n    });\n  case 'cohere':\n    return new CohereReranker({\n      apiKey: cohereApiKey,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n      httpAgent,\n      httpsAgent,\n    });\n  case 'infinity':\n    return new InfinityReranker(defaultLogger);\n  case 'none':\n    defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    defaultLogger.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({\n      apiKey: jinaApiKey,\n      apiUrl: jinaApiUrl,\n      timeout: rerankerTimeout,\n      logger: defaultLogger,\n      httpAgent,\n      httpsAgent,\n    });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"mappings":";;;AAIA,MAAM,uBAAuB;;;;AAK7B,MAAM,2BAA2B;AAEjC,MAAM,6BACJ,QAAQ,IAAI,gBAAgB,QAAQ,QAAQ,IAAI,iBAAiB,KAC7D,QAAQ,IAAI,eACZ;AAEN,IAAsB,eAAtB,MAAmC;CACjC;CACA;CAEA,YAAY,QAAmB;EAE7B,KAAK,SAAS,UAAU,oBAAoB;CAC9C;CAQA,kBACE,WACA,MACe;EACf,OAAO,UACJ,MAAM,GAAG,KAAK,IAAI,MAAM,UAAU,MAAM,CAAC,CAAC,CAC1C,KAAK,SAAS;GAAE,MAAM;GAAK,OAAO;EAAE,EAAE;CAC3C;AACF;AAEA,IAAa,eAAb,cAAkC,aAAa;CAC7C;CACA;CACA;CACA;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,cACrB,SAAS,qBAAqB,GAC9B,UAAU,0BACV,QACA,WACA,cAMqB;EACrB,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,UAAU;EACf,KAAK,YAAY;EACjB,KAAK,aAAa;CACpB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,mCAAmC,KAAK,QACxE;EAEA,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,iDAAiD;IAClE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;IACX,kBAAkB;GACpB;GAEA,MAAM,WAAW,MAAM,MAAM,KAC3B,KAAK,QACL,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;IACd,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CACF;GAEA,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GACzD,KAAK,OAAO,MAAM,mBAAmB,SAAS,MAAM,KAAK;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IACrB,IAAI,OAAO;IAGX,IAAI,OAAO,YAAY,MAAM;KAC3B,MAAM,MAAM,OAAO;KACnB,IAAI,OAAO,QAAQ,YAAY,UAAU,KACvC,OAAO,IAAI;UACN,IAAI,OAAO,QAAQ,UACxB,OAAO;IAEX,OAEE,OAAO,UAAU;IAGnB,OAAO;KAAE;KAAM;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,kEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,kBAAkB,KAAK,CAAC;GAEvE,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,iBAAb,cAAoC,aAAa;CAC/C;CACA;CACA;CAEA,YAAY,EACV,SAAS,QAAQ,IAAI,gBACrB,UAAU,0BACV,QACA,WACA,cAKqB;EACrB,MAAM,MAAM;EACZ,KAAK,SAAS;EACd,KAAK,UAAU;EACf,KAAK,YAAY;EACjB,KAAK,aAAa;CACpB;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MAAM,aAAa,UAAU,OAAO,oBAAoB;EAEpE,IAAI;GACF,IAAI,KAAK,UAAU,QAAQ,KAAK,WAAW,IAAI;IAC7C,KAAK,OAAO,KAAK,mDAAmD;IACpE,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;GAEA,MAAM,cAAc;IAClB,OAAO;IACA;IACP,OAAO;IACI;GACb;GAEA,MAAM,WAAW,MAAM,MAAM,KAC3B,oCACA,aACA;IACE,SAAS;KACP,gBAAgB;KAChB,eAAe,UAAU,KAAK;IAChC;IACA,SAAS,KAAK;IACd,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CACF;GAEA,KAAK,OAAO,MAAM,kBAAkB,SAAS,MAAM,EAAE;GACrD,KAAK,OAAO,MAAM,oBAAoB,SAAS,MAAM,IAAI;GAEzD,IAAI,SAAS,QAAQ,SAAS,KAAK,QAAQ,QACzC,OAAO,SAAS,KAAK,QAAQ,KAAK,WAAW;IAC3C,MAAM,WAAW,OAAO;IACxB,MAAM,QAAQ,OAAO;IAErB,OAAO;KAAE,MADI,UAAU;KACR;IAAM;GACvB,CAAC;QACI;IACL,KAAK,OAAO,KACV,oEACF;IACA,OAAO,KAAK,kBAAkB,WAAW,IAAI;GAC/C;EACF,SAAS,OAAO;GACd,KAAK,OAAO,MACV,+BACA,kBAAkB,KAAK,CACzB;GAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;EAC/C;CACF;AACF;AAEA,IAAa,mBAAb,cAAsC,aAAa;CACjD,YAAY,QAAmB;EAC7B,MAAM,MAAM;CAEd;CAEA,MAAM,OACJ,OACA,WACA,OAAe,GACS;EACxB,KAAK,OAAO,MACV,aAAa,UAAU,OAAO,oCAChC;EAEA,OAAO,KAAK,kBAAkB,WAAW,IAAI;CAC/C;AACF;;;;AAKA,MAAa,kBACX,WAQ6B;CAC7B,MAAM,EACJ,cACA,YACA,YACA,cACA,iBACA,QACA,WACA,eACE;CAGJ,MAAM,gBAAgB,UAAU,oBAAoB;CAEpD,QAAQ,aAAa,YAAY,GAAjC;EACA,KAAK,QACH,OAAO,IAAI,aAAa;GACtB,QAAQ;GACR,QAAQ;GACR,SAAS;GACT,QAAQ;GACR;GACA;EACF,CAAC;EACH,KAAK,UACH,OAAO,IAAI,eAAe;GACxB,QAAQ;GACR,SAAS;GACT,QAAQ;GACR;GACA;EACF,CAAC;EACH,KAAK,YACH,OAAO,IAAI,iBAAiB,aAAa;EAC3C,KAAK;GACH,cAAc,MAAM,mDAAiD;GACrE;EACF;GACE,cAAc,KACZ,0BAA0B,aAAa,kCACzC;GACA,OAAO,IAAI,aAAa;IACtB,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,QAAQ;IACR;IACA;GACF,CAAC;CACH;AACF"}

package/dist/esm/tools/search/search.mjs

@@ -92,7 +92,7 @@ const getHighlights = async ({ query, content, reranker, topResults = 5, maxCont
 		return;
 	}
 };
-const createSerperAPI = (apiKey) => {
+const createSerperAPI = (apiKey, agents) => {
 	const config = {
 		apiKey: apiKey ?? process.env.SERPER_API_KEY,
 		apiUrl: "https://google.serper.dev/search",
@@ -126,7 +126,9 @@ const createSerperAPI = (apiKey) => {
 					"X-API-KEY": config.apiKey,
 					"Content-Type": "application/json"
 				},
-				timeout: config.timeout
+				timeout: config.timeout,
+				httpAgent: agents?.httpAgent,
+				httpsAgent: agents?.httpsAgent
 			})).data;
 			return {
 				success: true,
@@ -151,7 +153,7 @@ const createSerperAPI = (apiKey) => {
 	};
 	return { getSources };
 };
-const createSearXNGAPI = (instanceUrl, apiKey) => {
+const createSearXNGAPI = (instanceUrl, apiKey, agents) => {
 	const config = {
 		instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,
 		apiKey: apiKey ?? process.env.SEARXNG_API_KEY,
@@ -189,7 +191,9 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
 			const data = (await axios.get(searchUrl, {
 				headers,
 				params,
-				timeout: config.timeout
+				timeout: config.timeout,
+				httpAgent: agents?.httpAgent,
+				httpsAgent: agents?.httpsAgent
 			})).data;
 			const isNewsResult = (result) => {
 				const url = result.url?.toLowerCase() ?? "";
@@ -273,12 +277,28 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
 	return { getSources };
 };
 const createSearchAPI = (config) => {
-	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions, crwApiKey, crwApiUrl, crwSearchOptions } = config;
-	if (searchProvider.toLowerCase() === "serper") return createSerperAPI(serperApiKey);
-	else if (searchProvider.toLowerCase() === "searxng") return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
-	else if (searchProvider.toLowerCase() === "tavily") return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
-	else if (searchProvider.toLowerCase() === "keenable") return createKeenableAPI(keenableApiKey, keenableApiUrl, keenableSearchOptions);
-	else if (searchProvider.toLowerCase() === "crw") return createCrwAPI(crwApiKey, crwApiUrl, crwSearchOptions);
+	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions, crwApiKey, crwApiUrl, crwSearchOptions, httpAgent, httpsAgent } = config;
+	const agents = {
+		httpAgent,
+		httpsAgent
+	};
+	if (searchProvider.toLowerCase() === "serper") return createSerperAPI(serperApiKey, agents);
+	else if (searchProvider.toLowerCase() === "searxng") return createSearXNGAPI(searxngInstanceUrl, searxngApiKey, agents);
+	else if (searchProvider.toLowerCase() === "tavily") return createTavilyAPI(tavilyApiKey, tavilySearchUrl, {
+		...tavilySearchOptions,
+		httpAgent: httpAgent ?? tavilySearchOptions?.httpAgent,
+		httpsAgent: httpsAgent ?? tavilySearchOptions?.httpsAgent
+	});
+	else if (searchProvider.toLowerCase() === "keenable") return createKeenableAPI(keenableApiKey, keenableApiUrl, {
+		...keenableSearchOptions,
+		httpAgent: httpAgent ?? keenableSearchOptions?.httpAgent,
+		httpsAgent: httpsAgent ?? keenableSearchOptions?.httpsAgent
+	});
+	else if (searchProvider.toLowerCase() === "crw") return createCrwAPI(crwApiKey, crwApiUrl, {
+		...crwSearchOptions,
+		httpAgent: httpAgent ?? crwSearchOptions?.httpAgent,
+		httpsAgent: httpsAgent ?? crwSearchOptions?.httpsAgent
+	});
 	else throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', 'keenable', or 'crw'`);
 };
 const createSourceProcessor = (config = {}, scraperInstance) => {

package/dist/esm/tools/search/search.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"search.mjs","names":[],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createKeenableAPI } from './keenable-search';\nimport { createTavilyAPI } from './tavily-search';\nimport { createCrwAPI } from './crw-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\nconst DEFAULT_CHUNK_SIZE = 150;\nconst DEFAULT_CHUNK_OVERLAP = 50;\n\n/** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /\n * `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The\n * overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`\n * throws when overlap >= size. */\nfunction resolveChunkOptions(\n  chunkSize?: number,\n  chunkOverlap?: number\n): { chunkSize: number; chunkOverlap: number } {\n  const resolve = (\n    configValue: number | undefined,\n    envVar: string,\n    fallback: number\n  ): number => {\n    if (configValue != null && configValue > 0) {\n      return configValue;\n    }\n    const envValue = Number(process.env[envVar]);\n    if (Number.isFinite(envValue) && envValue > 0) {\n      return envValue;\n    }\n    return fallback;\n  };\n\n  const size = resolve(chunkSize, 'SEARCH_CHUNK_SIZE', DEFAULT_CHUNK_SIZE);\n  let overlap = resolve(\n    chunkOverlap,\n    'SEARCH_CHUNK_OVERLAP',\n    DEFAULT_CHUNK_OVERLAP\n  );\n  if (overlap >= size) {\n    overlap = Math.floor(size / 3);\n  }\n  return { chunkSize: size, chunkOverlap: overlap };\n}\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n  if (maxContentLength != null && maxContentLength > 0) {\n    return maxContentLength;\n  }\n  const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n  if (Number.isFinite(envValue) && envValue > 0) {\n    return envValue;\n  }\n  return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n  return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n  chunkOptions,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  maxContentLength?: number;\n  chunkOptions?: { chunkSize: number; chunkOverlap: number };\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(\n      truncateContent(content, maxContentLength),\n      chunkOptions\n    );\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n    crwApiKey,\n    crwApiUrl,\n    crwSearchOptions,\n  } = config;\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);\n  } else if (searchProvider.toLowerCase() === 'tavily') {\n    return createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);\n  } else if (searchProvider.toLowerCase() === 'keenable') {\n    return createKeenableAPI(\n      keenableApiKey,\n      keenableApiUrl,\n      keenableSearchOptions\n    );\n  } else if (searchProvider.toLowerCase() === 'crw') {\n    return createCrwAPI(crwApiKey, crwApiUrl, crwSearchOptions);\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', 'keenable', or 'crw'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n  const chunkOptions = resolveChunkOptions(\n    config.chunkSize,\n    config.chunkOverlap\n  );\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const processResponse = (\n    url: string,\n    response: t.AnyScraperResponse\n  ): t.ScrapeResult => {\n    const rawMetadata = scraper.extractMetadata(response);\n    const metadata =\n      Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n    const attribution = getAttribution(url, metadata, logger_);\n\n    if (response.success && response.data) {\n      const [content, references] = scraper.extractContent(response);\n      return {\n        url,\n        references,\n        attribution,\n        content: truncateContent(chunker.cleanText(content), maxContentLength),\n      };\n    }\n\n    logger_.error(\n      `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n    );\n    return { url, attribution, error: true, content: '' };\n  };\n\n  const addHighlights = async (\n    result: t.ScrapeResult,\n    query: string,\n    onGetHighlights: t.SearchToolConfig['onGetHighlights']\n  ): Promise<t.ScrapeResult> => {\n    if (result.error != null) {\n      return result;\n    }\n    try {\n      const highlights = await getHighlights({\n        query,\n        reranker,\n        topResults,\n        content: result.content,\n        maxContentLength,\n        chunkOptions,\n        logger: logger_,\n      });\n      if (onGetHighlights) {\n        onGetHighlights(result.url);\n      }\n      return { ...result, highlights };\n    } catch (error) {\n      logger_.error('Error processing scraped content:', error);\n      return result;\n    }\n  };\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      try {\n        let responses: Array<[string, t.AnyScraperResponse]>;\n\n        if (scraper.scrapeUrls) {\n          responses = await scraper.scrapeUrls(links);\n        } else {\n          responses = await Promise.all(\n            links.map((link) =>\n              scraper\n                .scrapeUrl(link, {})\n                .catch((error): [string, t.AnyScraperResponse] => {\n                  logger_.error(`Error scraping ${link}:`, error);\n                  return [link, { success: false, error: String(error) }];\n                })\n            )\n          );\n        }\n\n        const withHighlights = await Promise.all(\n          responses.map(([url, response]) =>\n            addHighlights(\n              processResponse(url, response),\n              query,\n              onGetHighlights\n            )\n          )\n        );\n        return withHighlights;\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      }\n\n      if (\n        result.data.topStories != null &&\n        result.data.topStories.length > numElements\n      ) {\n        /** Merged news results can far exceed the requested source count;\n         * every entry is formatted into the LLM output, so cap them up\n         * front — before any early return below and before scraping\n         * entries the cap would discard */\n        result.data.topStories = result.data.topStories.slice(0, numElements);\n      }\n\n      if (!result.data.organic) {\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"mappings":";;;;;;;AASA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQ,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAU,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;AACnC,MAAM,qBAAqB;AAC3B,MAAM,wBAAwB;;;;;AAM9B,SAAS,oBACP,WACA,cAC6C;CAC7C,MAAM,WACJ,aACA,QACA,aACW;EACX,IAAI,eAAe,QAAQ,cAAc,GACvC,OAAO;EAET,MAAM,WAAW,OAAO,QAAQ,IAAI,OAAO;EAC3C,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;EAET,OAAO;CACT;CAEA,MAAM,OAAO,QAAQ,WAAW,qBAAqB,kBAAkB;CACvE,IAAI,UAAU,QACZ,cACA,wBACA,qBACF;CACA,IAAI,WAAW,MACb,UAAU,KAAK,MAAM,OAAO,CAAC;CAE/B,OAAO;EAAE,WAAW;EAAM,cAAc;CAAQ;AAClD;;;AAIA,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,cACA,aASwC;CACxC,MAAM,UAAU,UAAU,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,GACzC,YACF;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAehB,MAAM,QAAO,MAZU,MAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;GAClB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAShC,MAAM,QAAO,MANU,MAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;GAClB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,qBACA,gBACA,gBACA,uBACA,WACA,WACA,qBACE;CAEJ,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,YAAY;MAC9B,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,aAAa;MACpD,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAO,gBAAgB,cAAc,iBAAiB,mBAAmB;MACpE,IAAI,eAAe,YAAY,MAAM,YAC1C,OAAO,kBACL,gBACA,gBACA,qBACF;MACK,IAAI,eAAe,YAAY,MAAM,OAC1C,OAAO,aAAa,WAAW,WAAW,gBAAgB;MAE1D,MAAM,IAAI,MACR,4BAA4B,eAAe,8DAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,eAAe,oBACnB,OAAO,WACP,OAAO,YACT;CACA,MAAM,UAAU,UAAU,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAc,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA;IACA,SAAS,OAAO;IAChB;IACA;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}
+{"version":3,"file":"search.mjs","names":[],"sources":["../../../../src/tools/search/search.ts"],"sourcesContent":["import axios from 'axios';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport type * as t from './types';\nimport { getAttribution, createDefaultLogger } from './utils';\nimport { createKeenableAPI } from './keenable-search';\nimport { createTavilyAPI } from './tavily-search';\nimport { createCrwAPI } from './crw-search';\nimport { BaseReranker } from './rerankers';\n\nconst chunker = {\n  cleanText: (text: string): string => {\n    if (!text) return '';\n\n    /** Normalized all line endings to '\\n' */\n    const normalizedText = text.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n');\n\n    /** Handle multiple backslashes followed by newlines\n     * This replaces patterns like '\\\\\\\\\\\\n' with a single newline */\n    const fixedBackslashes = normalizedText.replace(/\\\\+\\n/g, '\\n');\n\n    /** Cleaned up consecutive newlines, tabs, and spaces around newlines */\n    const cleanedNewlines = fixedBackslashes.replace(/[\\t ]*\\n[\\t \\n]*/g, '\\n');\n\n    /** Cleaned up excessive spaces and tabs */\n    const cleanedSpaces = cleanedNewlines.replace(/[ \\t]+/g, ' ');\n\n    return cleanedSpaces.trim();\n  },\n  splitText: async (\n    text: string,\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    }\n  ): Promise<string[]> => {\n    const chunkSize = options?.chunkSize ?? 150;\n    const chunkOverlap = options?.chunkOverlap ?? 50;\n    const separators = options?.separators || ['\\n\\n', '\\n'];\n\n    const splitter = new RecursiveCharacterTextSplitter({\n      separators,\n      chunkSize,\n      chunkOverlap,\n    });\n\n    return await splitter.splitText(text);\n  },\n\n  splitTexts: async (\n    texts: string[],\n    options?: {\n      chunkSize?: number;\n      chunkOverlap?: number;\n      separators?: string[];\n    },\n    logger?: t.Logger\n  ): Promise<string[][]> => {\n    // Split multiple texts\n    const logger_ = logger || createDefaultLogger();\n    const promises = texts.map((text) =>\n      chunker.splitText(text, options).catch((error) => {\n        logger_.error('Error splitting text:', error);\n        return [text];\n      })\n    );\n    return Promise.all(promises);\n  },\n};\n\nconst DEFAULT_MAX_CONTENT_LENGTH = 50000;\nconst DEFAULT_CHUNK_SIZE = 150;\nconst DEFAULT_CHUNK_OVERLAP = 50;\n\n/** Resolves reranker chunking from config, the `SEARCH_CHUNK_SIZE` /\n * `SEARCH_CHUNK_OVERLAP` env vars, or the defaults (150 / 50 chars). The\n * overlap is clamped below the chunk size — `RecursiveCharacterTextSplitter`\n * throws when overlap >= size. */\nfunction resolveChunkOptions(\n  chunkSize?: number,\n  chunkOverlap?: number\n): { chunkSize: number; chunkOverlap: number } {\n  const resolve = (\n    configValue: number | undefined,\n    envVar: string,\n    fallback: number\n  ): number => {\n    if (configValue != null && configValue > 0) {\n      return configValue;\n    }\n    const envValue = Number(process.env[envVar]);\n    if (Number.isFinite(envValue) && envValue > 0) {\n      return envValue;\n    }\n    return fallback;\n  };\n\n  const size = resolve(chunkSize, 'SEARCH_CHUNK_SIZE', DEFAULT_CHUNK_SIZE);\n  let overlap = resolve(\n    chunkOverlap,\n    'SEARCH_CHUNK_OVERLAP',\n    DEFAULT_CHUNK_OVERLAP\n  );\n  if (overlap >= size) {\n    overlap = Math.floor(size / 3);\n  }\n  return { chunkSize: size, chunkOverlap: overlap };\n}\n\n/** Resolves the per-source scraped content cap from config, the\n * `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */\nfunction resolveMaxContentLength(maxContentLength?: number): number {\n  if (maxContentLength != null && maxContentLength > 0) {\n    return maxContentLength;\n  }\n  const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);\n  if (Number.isFinite(envValue) && envValue > 0) {\n    return envValue;\n  }\n  return DEFAULT_MAX_CONTENT_LENGTH;\n}\n\nfunction truncateContent(content: string, maxLength: number): string {\n  return content.length > maxLength ? content.slice(0, maxLength) : content;\n}\n\nfunction createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {\n  return (link: string, update?: Partial<t.ValidSource>): void => {\n    const source = sourceMap.get(link);\n    if (source) {\n      sourceMap.set(link, {\n        ...source,\n        ...update,\n      });\n    }\n  };\n}\n\nconst getHighlights = async ({\n  query,\n  content,\n  reranker,\n  topResults = 5,\n  maxContentLength = DEFAULT_MAX_CONTENT_LENGTH,\n  chunkOptions,\n  logger,\n}: {\n  content: string;\n  query: string;\n  reranker?: BaseReranker;\n  topResults?: number;\n  maxContentLength?: number;\n  chunkOptions?: { chunkSize: number; chunkOverlap: number };\n  logger?: t.Logger;\n}): Promise<t.Highlight[] | undefined> => {\n  const logger_ = logger || createDefaultLogger();\n\n  if (!content) {\n    logger_.warn('No content provided for highlights');\n    return;\n  }\n  if (!reranker) {\n    logger_.warn('No reranker provided for highlights');\n    return;\n  }\n\n  try {\n    const documents = await chunker.splitText(\n      truncateContent(content, maxContentLength),\n      chunkOptions\n    );\n    if (Array.isArray(documents)) {\n      return await reranker.rerank(query, documents, topResults);\n    } else {\n      logger_.error(\n        'Expected documents to be an array, got:',\n        typeof documents\n      );\n      return;\n    }\n  } catch (error) {\n    logger_.error('Error in content processing:', error);\n    return;\n  }\n};\n\nconst createSerperAPI = (\n  apiKey?: string,\n  agents?: t.HttpAgentConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    apiKey: apiKey ?? process.env.SERPER_API_KEY,\n    apiUrl: 'https://google.serper.dev/search',\n    timeout: 10000,\n  };\n\n  if (config.apiKey == null || config.apiKey === '') {\n    throw new Error('SERPER_API_KEY is required for SerperAPI');\n  }\n\n  const getSources = async ({\n    query,\n    date,\n    country,\n    safeSearch,\n    numResults = 8,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      const safe = ['off', 'moderate', 'active'] as const;\n      const payload: t.SerperSearchPayload = {\n        q: query,\n        safe: safe[safeSearch ?? 1],\n        num: Math.min(Math.max(1, numResults), 10),\n      };\n\n      // Set the search type if provided\n      if (type) {\n        payload.type = type;\n      }\n\n      if (date != null) {\n        payload.tbs = `qdr:${date}`;\n      }\n\n      if (country != null && country !== '') {\n        payload['gl'] = country.toLowerCase();\n      }\n\n      // Determine the API endpoint based on the search type\n      let apiEndpoint = config.apiUrl;\n      if (type === 'images') {\n        apiEndpoint = 'https://google.serper.dev/images';\n      } else if (type === 'videos') {\n        apiEndpoint = 'https://google.serper.dev/videos';\n      } else if (type === 'news') {\n        apiEndpoint = 'https://google.serper.dev/news';\n      }\n\n      const response = await axios.post<t.SerperResultData>(\n        apiEndpoint,\n        payload,\n        {\n          headers: {\n            'X-API-KEY': config.apiKey,\n            'Content-Type': 'application/json',\n          },\n          timeout: config.timeout,\n          httpAgent: agents?.httpAgent,\n          httpsAgent: agents?.httpsAgent,\n        }\n      );\n\n      const data = response.data;\n      const results: t.SearchResultData = {\n        organic: data.organic,\n        images: data.images ?? [],\n        answerBox: data.answerBox,\n        topStories: data.topStories ?? [],\n        peopleAlsoAsk: data.peopleAlsoAsk,\n        knowledgeGraph: data.knowledgeGraph,\n        relatedSearches: data.relatedSearches,\n        videos: data.videos ?? [],\n        news: data.news ?? [],\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return { success: false, error: `API request failed: ${errorMessage}` };\n    }\n  };\n\n  return { getSources };\n};\n\nconst createSearXNGAPI = (\n  instanceUrl?: string,\n  apiKey?: string,\n  agents?: t.HttpAgentConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const config = {\n    instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,\n    apiKey: apiKey ?? process.env.SEARXNG_API_KEY,\n    defaultLocation: 'all',\n    timeout: 10000,\n  };\n\n  if (config.instanceUrl == null || config.instanceUrl === '') {\n    throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');\n  }\n\n  const getSources = async ({\n    query,\n    numResults = 8,\n    safeSearch,\n    type,\n  }: t.GetSourcesParams): Promise<t.SearchResult> => {\n    if (!query.trim()) {\n      return { success: false, error: 'Query cannot be empty' };\n    }\n\n    try {\n      // Ensure the instance URL ends with /search\n      if (config.instanceUrl == null || config.instanceUrl === '') {\n        return { success: false, error: 'Instance URL is not defined' };\n      }\n\n      let searchUrl = config.instanceUrl;\n      if (!searchUrl.endsWith('/search')) {\n        searchUrl = searchUrl.replace(/\\/$/, '') + '/search';\n      }\n\n      // Determine the search category based on the type\n      let category = 'general';\n      if (type === 'images') {\n        category = 'images';\n      } else if (type === 'videos') {\n        category = 'videos';\n      } else if (type === 'news') {\n        category = 'news';\n      }\n\n      // Prepare parameters for SearXNG\n      const params: t.SearxNGSearchPayload = {\n        q: query,\n        format: 'json',\n        pageno: 1,\n        categories: category,\n        language: 'all',\n        safesearch: safeSearch,\n        engines: 'google,bing,duckduckgo',\n      };\n\n      const headers: Record<string, string> = {\n        'Content-Type': 'application/json',\n      };\n\n      if (config.apiKey != null && config.apiKey !== '') {\n        headers['X-API-Key'] = config.apiKey;\n      }\n\n      const response = await axios.get(searchUrl, {\n        headers,\n        params,\n        timeout: config.timeout,\n        httpAgent: agents?.httpAgent,\n        httpsAgent: agents?.httpsAgent,\n      });\n\n      const data = response.data;\n\n      // Helper function to identify news results since SearXNG doesn't provide that classification by default\n      const isNewsResult = (result: t.SearXNGResult): boolean => {\n        const url = result.url?.toLowerCase() ?? '';\n        const title = result.title?.toLowerCase() ?? '';\n\n        // News-related keywords in title/content\n        const newsKeywords = [\n          'breaking news',\n          'latest news',\n          'top stories',\n          'news today',\n          'developing story',\n          'trending news',\n          'news',\n        ];\n\n        // Check if title/content contains news keywords\n        const hasNewsKeywords = newsKeywords.some(\n          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)\n        );\n\n        // Check if URL contains news-related paths\n        const hasNewsPath =\n          url.includes('/news/') ||\n          url.includes('/world/') ||\n          url.includes('/politics/') ||\n          url.includes('/breaking/');\n\n        return hasNewsKeywords || hasNewsPath;\n      };\n\n      // Transform SearXNG results to match SerperAPI format\n      const organicResults = (data.results ?? [])\n        .slice(0, numResults)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            position: index + 1,\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            attribution,\n          };\n        });\n\n      const imageResults = (data.results ?? [])\n        .filter((result: t.SearXNGResult) => result.img_src)\n        .slice(0, 6)\n        .map((result: t.SearXNGResult, index: number) => ({\n          title: result.title ?? '',\n          imageUrl: result.img_src ?? '',\n          position: index + 1,\n          source: new URL(result.url ?? '').hostname,\n          domain: new URL(result.url ?? '').hostname,\n          link: result.url ?? '',\n        }));\n\n      // Extract news results from organic results\n      const newsResults = (data.results ?? [])\n        .filter(isNewsResult)\n        .map((result: t.SearXNGResult, index: number) => {\n          let attribution = '';\n          try {\n            attribution = new URL(result.url ?? '').hostname;\n          } catch {\n            attribution = '';\n          }\n\n          return {\n            title: result.title ?? '',\n            link: result.url ?? '',\n            snippet: result.content ?? '',\n            date: result.publishedDate ?? '',\n            source: attribution,\n            imageUrl: result.img_src ?? '',\n            position: index + 1,\n          };\n        });\n\n      const topStories = newsResults.slice(0, 5);\n\n      const relatedSearches = Array.isArray(data.suggestions)\n        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))\n        : [];\n\n      const results: t.SearchResultData = {\n        organic: organicResults,\n        images: imageResults,\n        topStories: topStories, // Use first 5 extracted news as top stories\n        relatedSearches,\n        videos: [],\n        news: newsResults,\n        // Add empty arrays for other Serper fields to maintain parity\n        places: [],\n        shopping: [],\n        peopleAlsoAsk: [],\n        knowledgeGraph: undefined,\n        answerBox: undefined,\n      };\n\n      return { success: true, data: results };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return {\n        success: false,\n        error: `SearXNG API request failed: ${errorMessage}`,\n      };\n    }\n  };\n\n  return { getSources };\n};\n\nexport const createSearchAPI = (\n  config: t.SearchConfig\n): {\n  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n  const {\n    searchProvider = 'serper',\n    serperApiKey,\n    searxngInstanceUrl,\n    searxngApiKey,\n    tavilyApiKey,\n    tavilySearchUrl,\n    tavilySearchOptions,\n    keenableApiKey,\n    keenableApiUrl,\n    keenableSearchOptions,\n    crwApiKey,\n    crwApiUrl,\n    crwSearchOptions,\n    httpAgent,\n    httpsAgent,\n  } = config;\n\n  const agents: t.HttpAgentConfig = { httpAgent, httpsAgent };\n\n  if (searchProvider.toLowerCase() === 'serper') {\n    return createSerperAPI(serperApiKey, agents);\n  } else if (searchProvider.toLowerCase() === 'searxng') {\n    return createSearXNGAPI(searxngInstanceUrl, searxngApiKey, agents);\n  } else if (searchProvider.toLowerCase() === 'tavily') {\n    return createTavilyAPI(tavilyApiKey, tavilySearchUrl, {\n      ...tavilySearchOptions,\n      httpAgent: httpAgent ?? tavilySearchOptions?.httpAgent,\n      httpsAgent: httpsAgent ?? tavilySearchOptions?.httpsAgent,\n    });\n  } else if (searchProvider.toLowerCase() === 'keenable') {\n    return createKeenableAPI(keenableApiKey, keenableApiUrl, {\n      ...keenableSearchOptions,\n      httpAgent: httpAgent ?? keenableSearchOptions?.httpAgent,\n      httpsAgent: httpsAgent ?? keenableSearchOptions?.httpsAgent,\n    });\n  } else if (searchProvider.toLowerCase() === 'crw') {\n    return createCrwAPI(crwApiKey, crwApiUrl, {\n      ...crwSearchOptions,\n      httpAgent: httpAgent ?? crwSearchOptions?.httpAgent,\n      httpsAgent: httpsAgent ?? crwSearchOptions?.httpsAgent,\n    });\n  } else {\n    throw new Error(\n      `Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', 'keenable', or 'crw'`\n    );\n  }\n};\n\nexport const createSourceProcessor = (\n  config: t.ProcessSourcesConfig = {},\n  scraperInstance?: t.BaseScraper\n): {\n  processSources: (\n    fields: t.ProcessSourcesFields\n  ) => Promise<t.SearchResultData>;\n  topResults: number;\n} => {\n  if (!scraperInstance) {\n    throw new Error('Scraper instance is required');\n  }\n  const {\n    topResults = 5,\n    // strategies = ['no_extraction'],\n    // filterContent = true,\n    reranker,\n    logger,\n  } = config;\n\n  const maxContentLength = resolveMaxContentLength(config.maxContentLength);\n  const chunkOptions = resolveChunkOptions(\n    config.chunkSize,\n    config.chunkOverlap\n  );\n  const logger_ = logger || createDefaultLogger();\n  const scraper = scraperInstance;\n\n  const processResponse = (\n    url: string,\n    response: t.AnyScraperResponse\n  ): t.ScrapeResult => {\n    const rawMetadata = scraper.extractMetadata(response);\n    const metadata =\n      Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;\n    const attribution = getAttribution(url, metadata, logger_);\n\n    if (response.success && response.data) {\n      const [content, references] = scraper.extractContent(response);\n      return {\n        url,\n        references,\n        attribution,\n        content: truncateContent(chunker.cleanText(content), maxContentLength),\n      };\n    }\n\n    logger_.error(\n      `Error scraping ${url}: ${response.error ?? 'Unknown error'}`\n    );\n    return { url, attribution, error: true, content: '' };\n  };\n\n  const addHighlights = async (\n    result: t.ScrapeResult,\n    query: string,\n    onGetHighlights: t.SearchToolConfig['onGetHighlights']\n  ): Promise<t.ScrapeResult> => {\n    if (result.error != null) {\n      return result;\n    }\n    try {\n      const highlights = await getHighlights({\n        query,\n        reranker,\n        topResults,\n        content: result.content,\n        maxContentLength,\n        chunkOptions,\n        logger: logger_,\n      });\n      if (onGetHighlights) {\n        onGetHighlights(result.url);\n      }\n      return { ...result, highlights };\n    } catch (error) {\n      logger_.error('Error processing scraped content:', error);\n      return result;\n    }\n  };\n\n  const webScraper = {\n    scrapeMany: async ({\n      query,\n      links,\n      onGetHighlights,\n    }: {\n      query: string;\n      links: string[];\n      onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    }): Promise<Array<t.ScrapeResult>> => {\n      logger_.debug(`Scraping ${links.length} links`);\n      try {\n        let responses: Array<[string, t.AnyScraperResponse]>;\n\n        if (scraper.scrapeUrls) {\n          responses = await scraper.scrapeUrls(links);\n        } else {\n          responses = await Promise.all(\n            links.map((link) =>\n              scraper\n                .scrapeUrl(link, {})\n                .catch((error): [string, t.AnyScraperResponse] => {\n                  logger_.error(`Error scraping ${link}:`, error);\n                  return [link, { success: false, error: String(error) }];\n                })\n            )\n          );\n        }\n\n        const withHighlights = await Promise.all(\n          responses.map(([url, response]) =>\n            addHighlights(\n              processResponse(url, response),\n              query,\n              onGetHighlights\n            )\n          )\n        );\n        return withHighlights;\n      } catch (error) {\n        logger_.error('Error in scrapeMany:', error);\n        return [];\n      }\n    },\n  };\n\n  const fetchContents = async ({\n    links,\n    query,\n    target,\n    onGetHighlights,\n    onContentScraped,\n  }: {\n    links: string[];\n    query: string;\n    target: number;\n    onGetHighlights: t.SearchToolConfig['onGetHighlights'];\n    onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;\n  }): Promise<void> => {\n    const initialLinks = links.slice(0, target);\n    // const remainingLinks = links.slice(target).reverse();\n    const results = await webScraper.scrapeMany({\n      query,\n      links: initialLinks,\n      onGetHighlights,\n    });\n    for (const result of results) {\n      if (result.error === true) {\n        continue;\n      }\n      const { url, content, attribution, references, highlights } = result;\n      onContentScraped?.(url, {\n        content,\n        attribution,\n        references,\n        highlights,\n      });\n    }\n  };\n\n  const processSources = async ({\n    result,\n    numElements,\n    query,\n    news,\n    proMode = true,\n    onGetHighlights,\n  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {\n    try {\n      if (!result.data) {\n        return {\n          organic: [],\n          topStories: [],\n          images: [],\n          relatedSearches: [],\n        };\n      }\n\n      if (\n        result.data.topStories != null &&\n        result.data.topStories.length > numElements\n      ) {\n        /** Merged news results can far exceed the requested source count;\n         * every entry is formatted into the LLM output, so cap them up\n         * front — before any early return below and before scraping\n         * entries the cap would discard */\n        result.data.topStories = result.data.topStories.slice(0, numElements);\n      }\n\n      if (!result.data.organic) {\n        return result.data;\n      }\n\n      if (!proMode) {\n        const wikiSources = result.data.organic.filter((source) =>\n          source.link.includes('wikipedia.org')\n        );\n\n        if (!wikiSources.length) {\n          return result.data;\n        }\n\n        const wikiSourceMap = new Map<string, t.ValidSource>();\n        wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);\n        const onContentScraped = createSourceUpdateCallback(wikiSourceMap);\n        await fetchContents({\n          query,\n          target: 1,\n          onGetHighlights,\n          onContentScraped,\n          links: [wikiSources[0].link],\n        });\n\n        for (let i = 0; i < result.data.organic.length; i++) {\n          const source = result.data.organic[i];\n          const updatedSource = wikiSourceMap.get(source.link);\n          if (updatedSource) {\n            result.data.organic[i] = {\n              ...source,\n              ...updatedSource,\n            };\n          }\n        }\n\n        return result.data;\n      }\n\n      const sourceMap = new Map<string, t.ValidSource>();\n      const organicLinksSet = new Set<string>();\n\n      // Collect organic links\n      const organicLinks = collectLinks(\n        result.data.organic,\n        sourceMap,\n        organicLinksSet\n      );\n\n      // Collect top story links, excluding any that are already in organic links\n      const topStories = result.data.topStories ?? [];\n      const topStoryLinks = collectLinks(\n        topStories,\n        sourceMap,\n        organicLinksSet\n      );\n\n      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {\n        return result.data;\n      }\n\n      const onContentScraped = createSourceUpdateCallback(sourceMap);\n      const promises: Promise<void>[] = [];\n\n      // Process organic links\n      if (organicLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: organicLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      // Process top story links\n      if (news && topStoryLinks.length > 0) {\n        promises.push(\n          fetchContents({\n            query,\n            onGetHighlights,\n            onContentScraped,\n            links: topStoryLinks,\n            target: numElements,\n          })\n        );\n      }\n\n      await Promise.all(promises);\n\n      if (result.data.organic.length > 0) {\n        updateSourcesWithContent(result.data.organic, sourceMap);\n      }\n\n      if (news && topStories.length > 0) {\n        updateSourcesWithContent(topStories, sourceMap);\n      }\n\n      return result.data;\n    } catch (error) {\n      logger_.error('Error in processSources:', error);\n      return {\n        organic: [],\n        topStories: [],\n        images: [],\n        relatedSearches: [],\n        ...result.data,\n        error: error instanceof Error ? error.message : String(error),\n      };\n    }\n  };\n\n  return {\n    processSources,\n    topResults,\n  };\n};\n\n/** Helper function to collect links and update sourceMap */\nfunction collectLinks(\n  sources: Array<t.OrganicResult | t.TopStoryResult>,\n  sourceMap: Map<string, t.ValidSource>,\n  existingLinksSet?: Set<string>\n): string[] {\n  const links: string[] = [];\n\n  for (const source of sources) {\n    if (source.link) {\n      // For topStories, only add if not already in organic links\n      if (existingLinksSet && existingLinksSet.has(source.link)) {\n        continue;\n      }\n\n      links.push(source.link);\n      if (existingLinksSet) {\n        existingLinksSet.add(source.link);\n      }\n      sourceMap.set(source.link, source as t.ValidSource);\n    }\n  }\n\n  return links;\n}\n\n/** Helper function to update sources with scraped content */\nfunction updateSourcesWithContent<T extends t.ValidSource>(\n  sources: T[],\n  sourceMap: Map<string, t.ValidSource>\n): void {\n  for (let i = 0; i < sources.length; i++) {\n    const source = sources[i];\n    const updatedSource = sourceMap.get(source.link);\n    if (updatedSource) {\n      sources[i] = {\n        ...source,\n        ...updatedSource,\n      } as T;\n    }\n  }\n}\n"],"mappings":";;;;;;;AASA,MAAM,UAAU;CACd,YAAY,SAAyB;EACnC,IAAI,CAAC,MAAM,OAAO;EAelB,OAZuB,KAAK,QAAQ,SAAS,IAAI,CAAC,CAAC,QAAQ,OAAO,IAI5B,CAAC,CAAC,QAAQ,UAAU,IAGnB,CAAC,CAAC,QAAQ,qBAAqB,IAGlC,CAAC,CAAC,QAAQ,WAAW,GAEtC,CAAC,CAAC,KAAK;CAC5B;CACA,WAAW,OACT,MACA,YAKsB;EACtB,MAAM,YAAY,SAAS,aAAa;EACxC,MAAM,eAAe,SAAS,gBAAgB;EAS9C,OAAO,MAAM,IANQ,+BAA+B;GAClD,YAHiB,SAAS,cAAc,CAAC,QAAQ,IAAI;GAIrD;GACA;EACF,CAEoB,CAAC,CAAC,UAAU,IAAI;CACtC;CAEA,YAAY,OACV,OACA,SAKA,WACwB;EAExB,MAAM,UAAU,UAAU,oBAAoB;EAC9C,MAAM,WAAW,MAAM,KAAK,SAC1B,QAAQ,UAAU,MAAM,OAAO,CAAC,CAAC,OAAO,UAAU;GAChD,QAAQ,MAAM,yBAAyB,KAAK;GAC5C,OAAO,CAAC,IAAI;EACd,CAAC,CACH;EACA,OAAO,QAAQ,IAAI,QAAQ;CAC7B;AACF;AAEA,MAAM,6BAA6B;AACnC,MAAM,qBAAqB;AAC3B,MAAM,wBAAwB;;;;;AAM9B,SAAS,oBACP,WACA,cAC6C;CAC7C,MAAM,WACJ,aACA,QACA,aACW;EACX,IAAI,eAAe,QAAQ,cAAc,GACvC,OAAO;EAET,MAAM,WAAW,OAAO,QAAQ,IAAI,OAAO;EAC3C,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;EAET,OAAO;CACT;CAEA,MAAM,OAAO,QAAQ,WAAW,qBAAqB,kBAAkB;CACvE,IAAI,UAAU,QACZ,cACA,wBACA,qBACF;CACA,IAAI,WAAW,MACb,UAAU,KAAK,MAAM,OAAO,CAAC;CAE/B,OAAO;EAAE,WAAW;EAAM,cAAc;CAAQ;AAClD;;;AAIA,SAAS,wBAAwB,kBAAmC;CAClE,IAAI,oBAAoB,QAAQ,mBAAmB,GACjD,OAAO;CAET,MAAM,WAAW,OAAO,QAAQ,IAAI,yBAAyB;CAC7D,IAAI,OAAO,SAAS,QAAQ,KAAK,WAAW,GAC1C,OAAO;CAET,OAAO;AACT;AAEA,SAAS,gBAAgB,SAAiB,WAA2B;CACnE,OAAO,QAAQ,SAAS,YAAY,QAAQ,MAAM,GAAG,SAAS,IAAI;AACpE;AAEA,SAAS,2BAA2B,WAAuC;CACzE,QAAQ,MAAc,WAA0C;EAC9D,MAAM,SAAS,UAAU,IAAI,IAAI;EACjC,IAAI,QACF,UAAU,IAAI,MAAM;GAClB,GAAG;GACH,GAAG;EACL,CAAC;CAEL;AACF;AAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,SACA,UACA,aAAa,GACb,mBAAmB,4BACnB,cACA,aASwC;CACxC,MAAM,UAAU,UAAU,oBAAoB;CAE9C,IAAI,CAAC,SAAS;EACZ,QAAQ,KAAK,oCAAoC;EACjD;CACF;CACA,IAAI,CAAC,UAAU;EACb,QAAQ,KAAK,qCAAqC;EAClD;CACF;CAEA,IAAI;EACF,MAAM,YAAY,MAAM,QAAQ,UAC9B,gBAAgB,SAAS,gBAAgB,GACzC,YACF;EACA,IAAI,MAAM,QAAQ,SAAS,GACzB,OAAO,MAAM,SAAS,OAAO,OAAO,WAAW,UAAU;OACpD;GACL,QAAQ,MACN,2CACA,OAAO,SACT;GACA;EACF;CACF,SAAS,OAAO;EACd,QAAQ,MAAM,gCAAgC,KAAK;EACnD;CACF;AACF;AAEA,MAAM,mBACJ,QACA,WAGG;CACH,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ;EACR,SAAS;CACX;CAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,MAAM,IAAI,MAAM,0CAA0C;CAG5D,MAAM,aAAa,OAAO,EACxB,OACA,MACA,SACA,YACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,MAAM,UAAiC;IACrC,GAAG;IACH,MAAM;KAHM;KAAO;KAAY;IAGtB,CAAC,CAAC,cAAc;IACzB,KAAK,KAAK,IAAI,KAAK,IAAI,GAAG,UAAU,GAAG,EAAE;GAC3C;GAGA,IAAI,MACF,QAAQ,OAAO;GAGjB,IAAI,QAAQ,MACV,QAAQ,MAAM,OAAO;GAGvB,IAAI,WAAW,QAAQ,YAAY,IACjC,QAAQ,QAAQ,QAAQ,YAAY;GAItC,IAAI,cAAc,OAAO;GACzB,IAAI,SAAS,UACX,cAAc;QACT,IAAI,SAAS,UAClB,cAAc;QACT,IAAI,SAAS,QAClB,cAAc;GAiBhB,MAAM,QAAO,MAdU,MAAM,KAC3B,aACA,SACA;IACE,SAAS;KACP,aAAa,OAAO;KACpB,gBAAgB;IAClB;IACA,SAAS,OAAO;IAChB,WAAW,QAAQ;IACnB,YAAY,QAAQ;GACtB,CACF,EAAA,CAEsB;GAatB,OAAO;IAAE,SAAS;IAAM,MAAM;KAX5B,SAAS,KAAK;KACd,QAAQ,KAAK,UAAU,CAAC;KACxB,WAAW,KAAK;KAChB,YAAY,KAAK,cAAc,CAAC;KAChC,eAAe,KAAK;KACpB,gBAAgB,KAAK;KACrB,iBAAiB,KAAK;KACtB,QAAQ,KAAK,UAAU,CAAC;KACxB,MAAM,KAAK,QAAQ,CAAC;IAGc;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IAAE,SAAS;IAAO,OAAO,uBAD9B,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACe;EACxE;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAM,oBACJ,aACA,QACA,WAGG;CACH,MAAM,SAAS;EACb,aAAa,eAAe,QAAQ,IAAI;EACxC,QAAQ,UAAU,QAAQ,IAAI;EAC9B,iBAAiB;EACjB,SAAS;CACX;CAEA,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,MAAM,IAAI,MAAM,kDAAkD;CAGpE,MAAM,aAAa,OAAO,EACxB,OACA,aAAa,GACb,YACA,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GAEF,IAAI,OAAO,eAAe,QAAQ,OAAO,gBAAgB,IACvD,OAAO;IAAE,SAAS;IAAO,OAAO;GAA8B;GAGhE,IAAI,YAAY,OAAO;GACvB,IAAI,CAAC,UAAU,SAAS,SAAS,GAC/B,YAAY,UAAU,QAAQ,OAAO,EAAE,IAAI;GAI7C,IAAI,WAAW;GACf,IAAI,SAAS,UACX,WAAW;QACN,IAAI,SAAS,UAClB,WAAW;QACN,IAAI,SAAS,QAClB,WAAW;GAIb,MAAM,SAAiC;IACrC,GAAG;IACH,QAAQ;IACR,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,YAAY;IACZ,SAAS;GACX;GAEA,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GAEA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,eAAe,OAAO;GAWhC,MAAM,QAAO,MARU,MAAM,IAAI,WAAW;IAC1C;IACA;IACA,SAAS,OAAO;IAChB,WAAW,QAAQ;IACnB,YAAY,QAAQ;GACtB,CAAC,EAAA,CAEqB;GAGtB,MAAM,gBAAgB,WAAqC;IACzD,MAAM,MAAM,OAAO,KAAK,YAAY,KAAK;IACzC,MAAM,QAAQ,OAAO,OAAO,YAAY,KAAK;IAc7C,MAAM,kBAAkB;KAVtB;KACA;KACA;KACA;KACA;KACA;KACA;IAIiC,CAAC,CAAC,MAClC,YAAY,MAAM,YAAY,CAAC,CAAC,SAAS,OAAO,CACnD;IAGA,MAAM,cACJ,IAAI,SAAS,QAAQ,KACrB,IAAI,SAAS,SAAS,KACtB,IAAI,SAAS,YAAY,KACzB,IAAI,SAAS,YAAY;IAE3B,OAAO,mBAAmB;GAC5B;GAGA,MAAM,kBAAkB,KAAK,WAAW,CAAC,EAAA,CACtC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,UAAU,QAAQ;KAClB,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B;IACF;GACF,CAAC;GAEH,MAAM,gBAAgB,KAAK,WAAW,CAAC,EAAA,CACpC,QAAQ,WAA4B,OAAO,OAAO,CAAC,CACnD,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,QAAyB,WAAmB;IAChD,OAAO,OAAO,SAAS;IACvB,UAAU,OAAO,WAAW;IAC5B,UAAU,QAAQ;IAClB,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,QAAQ,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAClC,MAAM,OAAO,OAAO;GACtB,EAAE;GAGJ,MAAM,eAAe,KAAK,WAAW,CAAC,EAAA,CACnC,OAAO,YAAY,CAAC,CACpB,KAAK,QAAyB,UAAkB;IAC/C,IAAI,cAAc;IAClB,IAAI;KACF,cAAc,IAAI,IAAI,OAAO,OAAO,EAAE,CAAC,CAAC;IAC1C,QAAQ;KACN,cAAc;IAChB;IAEA,OAAO;KACL,OAAO,OAAO,SAAS;KACvB,MAAM,OAAO,OAAO;KACpB,SAAS,OAAO,WAAW;KAC3B,MAAM,OAAO,iBAAiB;KAC9B,QAAQ;KACR,UAAU,OAAO,WAAW;KAC5B,UAAU,QAAQ;IACpB;GACF,CAAC;GAuBH,OAAO;IAAE,SAAS;IAAM,MAAM;KAd5B,SAAS;KACT,QAAQ;KACR,YATiB,YAAY,MAAM,GAAG,CASjB;KACrB,iBARsB,MAAM,QAAQ,KAAK,WAAW,IAClD,KAAK,YAAY,KAAK,gBAAwB,EAAE,OAAO,WAAW,EAAE,IACpE,CAAC;KAOH,QAAQ,CAAC;KACT,MAAM;KAEN,QAAQ,CAAC;KACT,UAAU,CAAC;KACX,eAAe,CAAC;KAChB,gBAAgB,KAAA;KAChB,WAAW,KAAA;IAGuB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,+BAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB;AAEA,MAAa,mBACX,WAGG;CACH,MAAM,EACJ,iBAAiB,UACjB,cACA,oBACA,eACA,cACA,iBACA,qBACA,gBACA,gBACA,uBACA,WACA,WACA,kBACA,WACA,eACE;CAEJ,MAAM,SAA4B;EAAE;EAAW;CAAW;CAE1D,IAAI,eAAe,YAAY,MAAM,UACnC,OAAO,gBAAgB,cAAc,MAAM;MACtC,IAAI,eAAe,YAAY,MAAM,WAC1C,OAAO,iBAAiB,oBAAoB,eAAe,MAAM;MAC5D,IAAI,eAAe,YAAY,MAAM,UAC1C,OAAO,gBAAgB,cAAc,iBAAiB;EACpD,GAAG;EACH,WAAW,aAAa,qBAAqB;EAC7C,YAAY,cAAc,qBAAqB;CACjD,CAAC;MACI,IAAI,eAAe,YAAY,MAAM,YAC1C,OAAO,kBAAkB,gBAAgB,gBAAgB;EACvD,GAAG;EACH,WAAW,aAAa,uBAAuB;EAC/C,YAAY,cAAc,uBAAuB;CACnD,CAAC;MACI,IAAI,eAAe,YAAY,MAAM,OAC1C,OAAO,aAAa,WAAW,WAAW;EACxC,GAAG;EACH,WAAW,aAAa,kBAAkB;EAC1C,YAAY,cAAc,kBAAkB;CAC9C,CAAC;MAED,MAAM,IAAI,MACR,4BAA4B,eAAe,8DAC7C;AAEJ;AAEA,MAAa,yBACX,SAAiC,CAAC,GAClC,oBAMG;CACH,IAAI,CAAC,iBACH,MAAM,IAAI,MAAM,8BAA8B;CAEhD,MAAM,EACJ,aAAa,GAGb,UACA,WACE;CAEJ,MAAM,mBAAmB,wBAAwB,OAAO,gBAAgB;CACxE,MAAM,eAAe,oBACnB,OAAO,WACP,OAAO,YACT;CACA,MAAM,UAAU,UAAU,oBAAoB;CAC9C,MAAM,UAAU;CAEhB,MAAM,mBACJ,KACA,aACmB;EACnB,MAAM,cAAc,QAAQ,gBAAgB,QAAQ;EAGpD,MAAM,cAAc,eAAe,KADjC,OAAO,KAAK,WAAW,CAAC,CAAC,SAAS,IAAI,cAAc,KAAA,GACJ,OAAO;EAEzD,IAAI,SAAS,WAAW,SAAS,MAAM;GACrC,MAAM,CAAC,SAAS,cAAc,QAAQ,eAAe,QAAQ;GAC7D,OAAO;IACL;IACA;IACA;IACA,SAAS,gBAAgB,QAAQ,UAAU,OAAO,GAAG,gBAAgB;GACvE;EACF;EAEA,QAAQ,MACN,kBAAkB,IAAI,IAAI,SAAS,SAAS,iBAC9C;EACA,OAAO;GAAE;GAAK;GAAa,OAAO;GAAM,SAAS;EAAG;CACtD;CAEA,MAAM,gBAAgB,OACpB,QACA,OACA,oBAC4B;EAC5B,IAAI,OAAO,SAAS,MAClB,OAAO;EAET,IAAI;GACF,MAAM,aAAa,MAAM,cAAc;IACrC;IACA;IACA;IACA,SAAS,OAAO;IAChB;IACA;IACA,QAAQ;GACV,CAAC;GACD,IAAI,iBACF,gBAAgB,OAAO,GAAG;GAE5B,OAAO;IAAE,GAAG;IAAQ;GAAW;EACjC,SAAS,OAAO;GACd,QAAQ,MAAM,qCAAqC,KAAK;GACxD,OAAO;EACT;CACF;CAEA,MAAM,aAAa,EACjB,YAAY,OAAO,EACjB,OACA,OACA,sBAKoC;EACpC,QAAQ,MAAM,YAAY,MAAM,OAAO,OAAO;EAC9C,IAAI;GACF,IAAI;GAEJ,IAAI,QAAQ,YACV,YAAY,MAAM,QAAQ,WAAW,KAAK;QAE1C,YAAY,MAAM,QAAQ,IACxB,MAAM,KAAK,SACT,QACG,UAAU,MAAM,CAAC,CAAC,CAAC,CACnB,OAAO,UAA0C;IAChD,QAAQ,MAAM,kBAAkB,KAAK,IAAI,KAAK;IAC9C,OAAO,CAAC,MAAM;KAAE,SAAS;KAAO,OAAO,OAAO,KAAK;IAAE,CAAC;GACxD,CAAC,CACL,CACF;GAYF,OAAO,MATsB,QAAQ,IACnC,UAAU,KAAK,CAAC,KAAK,cACnB,cACE,gBAAgB,KAAK,QAAQ,GAC7B,OACA,eACF,CACF,CACF;EAEF,SAAS,OAAO;GACd,QAAQ,MAAM,wBAAwB,KAAK;GAC3C,OAAO,CAAC;EACV;CACF,EACF;CAEA,MAAM,gBAAgB,OAAO,EAC3B,OACA,OACA,QACA,iBACA,uBAOmB;EACnB,MAAM,eAAe,MAAM,MAAM,GAAG,MAAM;EAE1C,MAAM,UAAU,MAAM,WAAW,WAAW;GAC1C;GACA,OAAO;GACP;EACF,CAAC;EACD,KAAK,MAAM,UAAU,SAAS;GAC5B,IAAI,OAAO,UAAU,MACnB;GAEF,MAAM,EAAE,KAAK,SAAS,aAAa,YAAY,eAAe;GAC9D,mBAAmB,KAAK;IACtB;IACA;IACA;IACA;GACF,CAAC;EACH;CACF;CAEA,MAAM,iBAAiB,OAAO,EAC5B,QACA,aACA,OACA,MACA,UAAU,MACV,sBACyD;EACzD,IAAI;GACF,IAAI,CAAC,OAAO,MACV,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;GACpB;GAGF,IACE,OAAO,KAAK,cAAc,QAC1B,OAAO,KAAK,WAAW,SAAS;;;;;GAMhC,OAAO,KAAK,aAAa,OAAO,KAAK,WAAW,MAAM,GAAG,WAAW;GAGtE,IAAI,CAAC,OAAO,KAAK,SACf,OAAO,OAAO;GAGhB,IAAI,CAAC,SAAS;IACZ,MAAM,cAAc,OAAO,KAAK,QAAQ,QAAQ,WAC9C,OAAO,KAAK,SAAS,eAAe,CACtC;IAEA,IAAI,CAAC,YAAY,QACf,OAAO,OAAO;IAGhB,MAAM,gCAAgB,IAAI,IAA2B;IACrD,cAAc,IAAI,YAAY,EAAE,CAAC,MAAM,YAAY,EAAE;IAErD,MAAM,cAAc;KAClB;KACA,QAAQ;KACR;KACA,kBALuB,2BAA2B,aAKnC;KACf,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;IAC7B,CAAC;IAED,KAAK,IAAI,IAAI,GAAG,IAAI,OAAO,KAAK,QAAQ,QAAQ,KAAK;KACnD,MAAM,SAAS,OAAO,KAAK,QAAQ;KACnC,MAAM,gBAAgB,cAAc,IAAI,OAAO,IAAI;KACnD,IAAI,eACF,OAAO,KAAK,QAAQ,KAAK;MACvB,GAAG;MACH,GAAG;KACL;IAEJ;IAEA,OAAO,OAAO;GAChB;GAEA,MAAM,4BAAY,IAAI,IAA2B;GACjD,MAAM,kCAAkB,IAAI,IAAY;GAGxC,MAAM,eAAe,aACnB,OAAO,KAAK,SACZ,WACA,eACF;GAGA,MAAM,aAAa,OAAO,KAAK,cAAc,CAAC;GAC9C,MAAM,gBAAgB,aACpB,YACA,WACA,eACF;GAEA,IAAI,aAAa,WAAW,MAAM,cAAc,WAAW,KAAK,CAAC,OAC/D,OAAO,OAAO;GAGhB,MAAM,mBAAmB,2BAA2B,SAAS;GAC7D,MAAM,WAA4B,CAAC;GAGnC,IAAI,aAAa,SAAS,GACxB,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAIF,IAAI,QAAQ,cAAc,SAAS,GACjC,SAAS,KACP,cAAc;IACZ;IACA;IACA;IACA,OAAO;IACP,QAAQ;GACV,CAAC,CACH;GAGF,MAAM,QAAQ,IAAI,QAAQ;GAE1B,IAAI,OAAO,KAAK,QAAQ,SAAS,GAC/B,yBAAyB,OAAO,KAAK,SAAS,SAAS;GAGzD,IAAI,QAAQ,WAAW,SAAS,GAC9B,yBAAyB,YAAY,SAAS;GAGhD,OAAO,OAAO;EAChB,SAAS,OAAO;GACd,QAAQ,MAAM,4BAA4B,KAAK;GAC/C,OAAO;IACL,SAAS,CAAC;IACV,YAAY,CAAC;IACb,QAAQ,CAAC;IACT,iBAAiB,CAAC;IAClB,GAAG,OAAO;IACV,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAC9D;EACF;CACF;CAEA,OAAO;EACL;EACA;CACF;AACF;;AAGA,SAAS,aACP,SACA,WACA,kBACU;CACV,MAAM,QAAkB,CAAC;CAEzB,KAAK,MAAM,UAAU,SACnB,IAAI,OAAO,MAAM;EAEf,IAAI,oBAAoB,iBAAiB,IAAI,OAAO,IAAI,GACtD;EAGF,MAAM,KAAK,OAAO,IAAI;EACtB,IAAI,kBACF,iBAAiB,IAAI,OAAO,IAAI;EAElC,UAAU,IAAI,OAAO,MAAM,MAAuB;CACpD;CAGF,OAAO;AACT;;AAGA,SAAS,yBACP,SACA,WACM;CACN,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;EACvC,MAAM,SAAS,QAAQ;EACvB,MAAM,gBAAgB,UAAU,IAAI,OAAO,IAAI;EAC/C,IAAI,eACF,QAAQ,KAAK;GACX,GAAG;GACH,GAAG;EACL;CAEJ;AACF"}

package/dist/esm/tools/search/serper-scraper.mjs

@@ -32,11 +32,15 @@ var SerperScraper = class {
 	timeout;
 	logger;
 	includeMarkdown;
+	httpAgent;
+	httpsAgent;
 	constructor(config = {}) {
 		this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? "";
 		this.apiUrl = config.apiUrl ?? process.env.SERPER_SCRAPE_URL ?? "https://scrape.serper.dev";
 		this.timeout = config.timeout ?? 7500;
 		this.includeMarkdown = config.includeMarkdown ?? true;
+		this.httpAgent = config.httpAgent;
+		this.httpsAgent = config.httpsAgent;
 		this.logger = config.logger || createDefaultLogger();
 		if (!this.apiKey) this.logger.warn("SERPER_API_KEY is not set. Scraping will not work.");
 		this.logger.debug(`Serper scraper initialized with API URL: ${this.apiUrl}`);
@@ -64,7 +68,9 @@ var SerperScraper = class {
 						"X-API-KEY": this.apiKey,
 						"Content-Type": "application/json"
 					},
-					timeout: options.timeout ?? this.timeout
+					timeout: options.timeout ?? this.timeout,
+					httpAgent: this.httpAgent,
+					httpsAgent: this.httpsAgent
 				})).data
 			}];
 		} catch (error) {

package/dist/esm/tools/search/serper-scraper.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"serper-scraper.mjs","names":[],"sources":["../../../../src/tools/search/serper-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Serper scraper implementation\n * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages\n *\n * Features:\n * - Simple API with single endpoint\n * - Returns both text and markdown content\n * - Includes metadata from scraped pages\n * - Credits-based pricing model\n *\n * @example\n * ```typescript\n * const scraper = createSerperScraper({\n *   apiKey: 'your-serper-api-key',\n *   includeMarkdown: true,\n *   timeout: 10000\n * });\n *\n * const [url, response] = await scraper.scrapeUrl('https://example.com');\n * if (response.success) {\n *   const [content] = scraper.extractContent(response);\n *   console.log(content);\n * }\n * ```\n */\nexport class SerperScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private logger: t.Logger;\n  private includeMarkdown: boolean;\n\n  constructor(config: t.SerperScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';\n\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.SERPER_SCRAPE_URL ??\n      'https://scrape.serper.dev';\n\n    this.timeout = config.timeout ?? 7500;\n    this.includeMarkdown = config.includeMarkdown ?? true;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Serper scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.SerperScrapeOptions = {}\n  ): Promise<[string, t.SerperScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'SERPER_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = {\n        url,\n        includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,\n      };\n\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'X-API-KEY': this.apiKey,\n          'Content-Type': 'application/json',\n        },\n        timeout: options.timeout ?? this.timeout,\n      });\n\n      return [url, { success: true, data: response.data }];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Serper Scrape API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.SerperScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    if (response.data.text != null) {\n      return [response.data.text, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(\n    response: t.SerperScrapeResponse\n  ): Record<string, string | number | boolean | null | undefined> {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Serper scraper instance\n * @param config Scraper configuration\n * @returns Serper scraper instance\n */\nexport const createSerperScraper = (\n  config: t.SerperScraperConfig = {}\n): SerperScraper => {\n  return new SerperScraper(config);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,IAAa,gBAAb,MAAoD;CAClD;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAgC,CAAC,GAAG;EAC9C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,kBAAkB;EAE7D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,qBACZ;EAEF,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,kBAAkB,OAAO,mBAAmB;EAEjD,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,oDAAoD;EAGvE,KAAK,OAAO,MACV,4CAA4C,KAAK,QACnD;CACF;;;;;;;CAQA,MAAM,UACJ,KACA,UAAiC,CAAC,GACS;EAC3C,IAAI,CAAC,KAAK,QACR,OAAO,CACL,KACA;GACE,SAAS;GACT,OAAO;EACT,CACF;EAGF,IAAI;GACF,MAAM,UAAU;IACd;IACA,iBAAiB,QAAQ,mBAAmB,KAAK;GACnD;GAUA,OAAO,CAAC,KAAK;IAAE,SAAS;IAAM,OAAM,MARb,MAAM,KAAK,KAAK,QAAQ,SAAS;KACtD,SAAS;MACP,aAAa,KAAK;MAClB,gBAAgB;KAClB;KACA,SAAS,QAAQ,WAAW,KAAK;IACnC,CAAC,EAAA,CAE4C;GAAK,CAAC;EACrD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,qCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;CAOA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,IAAI,SAAS,KAAK,YAAY,MAC5B,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAG3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAGvC,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;;;;;;CAOA,gBACE,UAC8D;EAC9D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,uBACX,SAAgC,CAAC,MACf;CAClB,OAAO,IAAI,cAAc,MAAM;AACjC"}
+{"version":3,"file":"serper-scraper.mjs","names":[],"sources":["../../../../src/tools/search/serper-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\n/**\n * Serper scraper implementation\n * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages\n *\n * Features:\n * - Simple API with single endpoint\n * - Returns both text and markdown content\n * - Includes metadata from scraped pages\n * - Credits-based pricing model\n *\n * @example\n * ```typescript\n * const scraper = createSerperScraper({\n *   apiKey: 'your-serper-api-key',\n *   includeMarkdown: true,\n *   timeout: 10000\n * });\n *\n * const [url, response] = await scraper.scrapeUrl('https://example.com');\n * if (response.success) {\n *   const [content] = scraper.extractContent(response);\n *   console.log(content);\n * }\n * ```\n */\nexport class SerperScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private logger: t.Logger;\n  private includeMarkdown: boolean;\n  private httpAgent?: t.HttpAgent;\n  private httpsAgent?: t.HttpsAgent;\n\n  constructor(config: t.SerperScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';\n\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.SERPER_SCRAPE_URL ??\n      'https://scrape.serper.dev';\n\n    this.timeout = config.timeout ?? 7500;\n    this.includeMarkdown = config.includeMarkdown ?? true;\n    this.httpAgent = config.httpAgent;\n    this.httpsAgent = config.httpsAgent;\n\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');\n    }\n\n    this.logger.debug(\n      `Serper scraper initialized with API URL: ${this.apiUrl}`\n    );\n  }\n\n  /**\n   * Scrape a single URL\n   * @param url URL to scrape\n   * @param options Scrape options\n   * @returns Scrape response\n   */\n  async scrapeUrl(\n    url: string,\n    options: t.SerperScrapeOptions = {}\n  ): Promise<[string, t.SerperScrapeResponse]> {\n    if (!this.apiKey) {\n      return [\n        url,\n        {\n          success: false,\n          error: 'SERPER_API_KEY is not set',\n        },\n      ];\n    }\n\n    try {\n      const payload = {\n        url,\n        includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,\n      };\n\n      const response = await axios.post(this.apiUrl, payload, {\n        headers: {\n          'X-API-KEY': this.apiKey,\n          'Content-Type': 'application/json',\n        },\n        timeout: options.timeout ?? this.timeout,\n        httpAgent: this.httpAgent,\n        httpsAgent: this.httpsAgent,\n      });\n\n      return [url, { success: true, data: response.data }];\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return [\n        url,\n        {\n          success: false,\n          error: `Serper Scrape API request failed: ${errorMessage}`,\n        },\n      ];\n    }\n  }\n\n  /**\n   * Extract content from scrape response\n   * @param response Scrape response\n   * @returns Extracted content or empty string if not available\n   */\n  extractContent(\n    response: t.SerperScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    if (response.data.markdown != null) {\n      return [response.data.markdown, undefined];\n    }\n\n    if (response.data.text != null) {\n      return [response.data.text, undefined];\n    }\n\n    return ['', undefined];\n  }\n\n  /**\n   * Extract metadata from scrape response\n   * @param response Scrape response\n   * @returns Metadata object\n   */\n  extractMetadata(\n    response: t.SerperScrapeResponse\n  ): Record<string, string | number | boolean | null | undefined> {\n    if (!response.success || !response.data || !response.data.metadata) {\n      return {};\n    }\n\n    return response.data.metadata;\n  }\n}\n\n/**\n * Create a Serper scraper instance\n * @param config Scraper configuration\n * @returns Serper scraper instance\n */\nexport const createSerperScraper = (\n  config: t.SerperScraperConfig = {}\n): SerperScraper => {\n  return new SerperScraper(config);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,IAAa,gBAAb,MAAoD;CAClD;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAgC,CAAC,GAAG;EAC9C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,kBAAkB;EAE7D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,qBACZ;EAEF,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,kBAAkB,OAAO,mBAAmB;EACjD,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EAEzB,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,oDAAoD;EAGvE,KAAK,OAAO,MACV,4CAA4C,KAAK,QACnD;CACF;;;;;;;CAQA,MAAM,UACJ,KACA,UAAiC,CAAC,GACS;EAC3C,IAAI,CAAC,KAAK,QACR,OAAO,CACL,KACA;GACE,SAAS;GACT,OAAO;EACT,CACF;EAGF,IAAI;GACF,MAAM,UAAU;IACd;IACA,iBAAiB,QAAQ,mBAAmB,KAAK;GACnD;GAYA,OAAO,CAAC,KAAK;IAAE,SAAS;IAAM,OAAM,MAVb,MAAM,KAAK,KAAK,QAAQ,SAAS;KACtD,SAAS;MACP,aAAa,KAAK;MAClB,gBAAgB;KAClB;KACA,SAAS,QAAQ,WAAW,KAAK;KACjC,WAAW,KAAK;KAChB,YAAY,KAAK;IACnB,CAAC,EAAA,CAE4C;GAAK,CAAC;EACrD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,qCALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;CAOA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,IAAI,SAAS,KAAK,YAAY,MAC5B,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAG3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAGvC,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;;;;;;CAOA,gBACE,UAC8D;EAC9D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,uBACX,SAAgC,CAAC,MACf;CAClB,OAAO,IAAI,cAAc,MAAM;AACjC"}

package/dist/esm/tools/search/tavily-scraper.mjs

@@ -30,6 +30,8 @@ var TavilyScraper = class {
 	includeImages;
 	includeFavicon;
 	format;
+	httpAgent;
+	httpsAgent;
 	constructor(config = {}) {
 		this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? "";
 		this.apiUrl = config.apiUrl ?? process.env.TAVILY_EXTRACT_URL ?? "https://api.tavily.com/extract";
@@ -39,6 +41,8 @@ var TavilyScraper = class {
 		this.includeImages = config.includeImages ?? false;
 		this.includeFavicon = config.includeFavicon ?? false;
 		this.format = config.format;
+		this.httpAgent = config.httpAgent;
+		this.httpsAgent = config.httpsAgent;
 		this.logger = config.logger || createDefaultLogger();
 		if (!this.apiKey) this.logger.warn("TAVILY_API_KEY is not set. Scraping will not work.");
 	}
@@ -79,7 +83,9 @@ var TavilyScraper = class {
 					Authorization: `Bearer ${this.apiKey}`,
 					"Content-Type": "application/json"
 				},
-				timeout: effectiveTimeout
+				timeout: effectiveTimeout,
+				httpAgent: this.httpAgent,
+				httpsAgent: this.httpsAgent
 			})).data;
 			const successMap = /* @__PURE__ */ new Map();
 			const failedMap = /* @__PURE__ */ new Map();

package/dist/esm/tools/search/tavily-scraper.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tavily-scraper.mjs","names":[],"sources":["../../../../src/tools/search/tavily-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_BASIC_TIMEOUT = 15000;\nconst DEFAULT_ADVANCED_TIMEOUT = 30000;\nconst MAX_BATCH_SIZE = 20;\n\nconst getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>\n  extractDepth === 'advanced'\n    ? DEFAULT_ADVANCED_TIMEOUT\n    : DEFAULT_BASIC_TIMEOUT;\n\nconst normalizeUrlKey = (url: string): string => {\n  try {\n    const parsedUrl = new URL(url);\n    parsedUrl.hash = '';\n    if (parsedUrl.pathname.length > 1) {\n      parsedUrl.pathname = parsedUrl.pathname.replace(/\\/+$/, '');\n    }\n    return parsedUrl.toString();\n  } catch {\n    return url;\n  }\n};\n\nconst setUrlResult = (\n  map: Map<string, t.TavilyExtractResult>,\n  result: t.TavilyExtractResult\n): void => {\n  map.set(result.url, result);\n  const normalizedUrl = normalizeUrlKey(result.url);\n  if (!map.has(normalizedUrl)) {\n    map.set(normalizedUrl, result);\n  }\n};\n\nexport class TavilyScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private payloadTimeout: number | undefined;\n  private logger: t.Logger;\n  private extractDepth: 'basic' | 'advanced';\n  private includeImages: boolean;\n  private includeFavicon: boolean;\n  private format: 'markdown' | 'text' | undefined;\n\n  constructor(config: t.TavilyScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.TAVILY_EXTRACT_URL ??\n      'https://api.tavily.com/extract';\n    this.payloadTimeout = config.timeout;\n    this.extractDepth = config.extractDepth ?? 'basic';\n    this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);\n    this.includeImages = config.includeImages ?? false;\n    this.includeFavicon = config.includeFavicon ?? false;\n    this.format = config.format;\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');\n    }\n  }\n\n  async scrapeUrl(\n    url: string,\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<[string, t.TavilyScrapeResponse]> {\n    const results = await this.scrapeUrls([url], options);\n    return results[0];\n  }\n\n  async scrapeUrls(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    if (!this.apiKey) {\n      return urls.map((url) => [\n        url,\n        { success: false, error: 'TAVILY_API_KEY is not set' },\n      ]);\n    }\n\n    const batches: string[][] = [];\n    for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {\n      batches.push(urls.slice(i, i + MAX_BATCH_SIZE));\n    }\n\n    const allResults: Array<[string, t.TavilyScrapeResponse]> = [];\n\n    for (const batch of batches) {\n      const batchResults = await this.extractBatch(batch, options);\n      allResults.push(...batchResults);\n    }\n\n    return allResults;\n  }\n\n  private async extractBatch(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    try {\n      const includeFavicon = options.includeFavicon ?? this.includeFavicon;\n      const format = options.format ?? this.format;\n      const extractDepth = options.extractDepth ?? this.extractDepth;\n      const payload: t.TavilyExtractPayload = {\n        urls,\n        extract_depth: extractDepth,\n        include_images: options.includeImages ?? this.includeImages,\n      };\n\n      if (includeFavicon) {\n        payload.include_favicon = true;\n      }\n      if (format != null) {\n        payload.format = format;\n      }\n\n      const effectiveTimeout =\n        options.timeout ??\n        this.payloadTimeout ??\n        (options.extractDepth != null\n          ? getDefaultTimeout(extractDepth)\n          : this.timeout);\n      const payloadTimeout = options.timeout ?? this.payloadTimeout;\n      if (payloadTimeout != null) {\n        payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);\n      }\n\n      const response = await axios.post<{\n        results?: t.TavilyExtractResult[];\n        failed_results?: t.TavilyExtractResult[];\n      }>(this.apiUrl, payload, {\n        headers: {\n          Authorization: `Bearer ${this.apiKey}`,\n          'Content-Type': 'application/json',\n        },\n        timeout: effectiveTimeout,\n      });\n\n      const data = response.data;\n      const successMap = new Map<string, t.TavilyExtractResult>();\n      const failedMap = new Map<string, t.TavilyExtractResult>();\n\n      for (const result of data.results ?? []) {\n        setUrlResult(successMap, result);\n      }\n      for (const result of data.failed_results ?? []) {\n        setUrlResult(failedMap, result);\n      }\n\n      return urls.map((url): [string, t.TavilyScrapeResponse] => {\n        const success =\n          successMap.get(url) ?? successMap.get(normalizeUrlKey(url));\n        if (success && success.error == null) {\n          return [\n            url,\n            {\n              success: true,\n              data: {\n                rawContent: success.raw_content ?? '',\n                images: success.images ?? [],\n                favicon: success.favicon,\n              },\n            },\n          ];\n        }\n\n        const failed =\n          failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));\n        const error =\n          success?.error ??\n          failed?.error ??\n          'URL not found in Tavily Extract response';\n        return [url, { success: false, error }];\n      });\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return urls.map((url) => [\n        url,\n        {\n          success: false,\n          error: `Tavily Extract API request failed: ${errorMessage}`,\n        },\n      ]);\n    }\n  }\n\n  extractContent(\n    response: t.TavilyScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    const content = response.data.rawContent ?? '';\n    const images = response.data.images ?? [];\n\n    const references: t.References | undefined =\n      images.length > 0\n        ? {\n          links: [],\n          images: images.map((imageUrl) => ({ originalUrl: imageUrl })),\n          videos: [],\n        }\n        : undefined;\n\n    return [content, references];\n  }\n\n  extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {\n    if (!response.success || !response.data) {\n      return {};\n    }\n\n    const metadata: t.GenericScrapeMetadata = {\n      images_count: response.data.images?.length ?? 0,\n    };\n    if (response.data.favicon != null) {\n      metadata.favicon = response.data.favicon;\n    }\n    return metadata;\n  }\n}\n\nexport const createTavilyScraper = (\n  config: t.TavilyScraperConfig = {}\n): TavilyScraper => {\n  return new TavilyScraper(config);\n};\n"],"mappings":";;;AAIA,MAAM,wBAAwB;AAC9B,MAAM,2BAA2B;AACjC,MAAM,iBAAiB;AAEvB,MAAM,qBAAqB,iBACzB,iBAAiB,aACb,2BACA;AAEN,MAAM,mBAAmB,QAAwB;CAC/C,IAAI;EACF,MAAM,YAAY,IAAI,IAAI,GAAG;EAC7B,UAAU,OAAO;EACjB,IAAI,UAAU,SAAS,SAAS,GAC9B,UAAU,WAAW,UAAU,SAAS,QAAQ,QAAQ,EAAE;EAE5D,OAAO,UAAU,SAAS;CAC5B,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,gBACJ,KACA,WACS;CACT,IAAI,IAAI,OAAO,KAAK,MAAM;CAC1B,MAAM,gBAAgB,gBAAgB,OAAO,GAAG;CAChD,IAAI,CAAC,IAAI,IAAI,aAAa,GACxB,IAAI,IAAI,eAAe,MAAM;AAEjC;AAEA,IAAa,gBAAb,MAAoD;CAClD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAgC,CAAC,GAAG;EAC9C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,kBAAkB;EAC7D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,iBAAiB,OAAO;EAC7B,KAAK,eAAe,OAAO,gBAAgB;EAC3C,KAAK,UAAU,OAAO,WAAW,kBAAkB,KAAK,YAAY;EACpE,KAAK,gBAAgB,OAAO,iBAAiB;EAC7C,KAAK,iBAAiB,OAAO,kBAAkB;EAC/C,KAAK,SAAS,OAAO;EACrB,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,oDAAoD;CAEzE;CAEA,MAAM,UACJ,KACA,UAAiC,CAAC,GACS;EAE3C,QAAO,MADe,KAAK,WAAW,CAAC,GAAG,GAAG,OAAO,EAAA,CACrC;CACjB;CAEA,MAAM,WACJ,MACA,UAAiC,CAAC,GACgB;EAClD,IAAI,CAAC,KAAK,QACR,OAAO,KAAK,KAAK,QAAQ,CACvB,KACA;GAAE,SAAS;GAAO,OAAO;EAA4B,CACvD,CAAC;EAGH,MAAM,UAAsB,CAAC;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,gBACpC,QAAQ,KAAK,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC;EAGhD,MAAM,aAAsD,CAAC;EAE7D,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,eAAe,MAAM,KAAK,aAAa,OAAO,OAAO;GAC3D,WAAW,KAAK,GAAG,YAAY;EACjC;EAEA,OAAO;CACT;CAEA,MAAc,aACZ,MACA,UAAiC,CAAC,GACgB;EAClD,IAAI;GACF,MAAM,iBAAiB,QAAQ,kBAAkB,KAAK;GACtD,MAAM,SAAS,QAAQ,UAAU,KAAK;GACtC,MAAM,eAAe,QAAQ,gBAAgB,KAAK;GAClD,MAAM,UAAkC;IACtC;IACA,eAAe;IACf,gBAAgB,QAAQ,iBAAiB,KAAK;GAChD;GAEA,IAAI,gBACF,QAAQ,kBAAkB;GAE5B,IAAI,UAAU,MACZ,QAAQ,SAAS;GAGnB,MAAM,mBACJ,QAAQ,WACR,KAAK,mBACJ,QAAQ,gBAAgB,OACrB,kBAAkB,YAAY,IAC9B,KAAK;GACX,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,IAAI,kBAAkB,MACpB,QAAQ,UAAU,KAAK,IAAI,KAAK,IAAI,iBAAiB,KAAM,CAAC,GAAG,EAAE;GAcnE,MAAM,QAAO,MAXU,MAAM,KAG1B,KAAK,QAAQ,SAAS;IACvB,SAAS;KACP,eAAe,UAAU,KAAK;KAC9B,gBAAgB;IAClB;IACA,SAAS;GACX,CAAC,EAAA,CAEqB;GACtB,MAAM,6BAAa,IAAI,IAAmC;GAC1D,MAAM,4BAAY,IAAI,IAAmC;GAEzD,KAAK,MAAM,UAAU,KAAK,WAAW,CAAC,GACpC,aAAa,YAAY,MAAM;GAEjC,KAAK,MAAM,UAAU,KAAK,kBAAkB,CAAC,GAC3C,aAAa,WAAW,MAAM;GAGhC,OAAO,KAAK,KAAK,QAA0C;IACzD,MAAM,UACJ,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,gBAAgB,GAAG,CAAC;IAC5D,IAAI,WAAW,QAAQ,SAAS,MAC9B,OAAO,CACL,KACA;KACE,SAAS;KACT,MAAM;MACJ,YAAY,QAAQ,eAAe;MACnC,QAAQ,QAAQ,UAAU,CAAC;MAC3B,SAAS,QAAQ;KACnB;IACF,CACF;IAGF,MAAM,SACJ,UAAU,IAAI,GAAG,KAAK,UAAU,IAAI,gBAAgB,GAAG,CAAC;IAK1D,OAAO,CAAC,KAAK;KAAE,SAAS;KAAO,OAH7B,SAAS,SACT,QAAQ,SACR;IACmC,CAAC;GACxC,CAAC;EACH,SAAS,OAAO;GACd,MAAM,eACJ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACvD,OAAO,KAAK,KAAK,QAAQ,CACvB,KACA;IACE,SAAS;IACT,OAAO,sCAAsC;GAC/C,CACF,CAAC;EACH;CACF;CAEA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,UAAU,SAAS,KAAK,cAAc;EAC5C,MAAM,SAAS,SAAS,KAAK,UAAU,CAAC;EAWxC,OAAO,CAAC,SARN,OAAO,SAAS,IACZ;GACA,OAAO,CAAC;GACR,QAAQ,OAAO,KAAK,cAAc,EAAE,aAAa,SAAS,EAAE;GAC5D,QAAQ,CAAC;EACX,IACE,KAAA,CAEqB;CAC7B;CAEA,gBAAgB,UAA2D;EACzE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC;EAGV,MAAM,WAAoC,EACxC,cAAc,SAAS,KAAK,QAAQ,UAAU,EAChD;EACA,IAAI,SAAS,KAAK,WAAW,MAC3B,SAAS,UAAU,SAAS,KAAK;EAEnC,OAAO;CACT;AACF;AAEA,MAAa,uBACX,SAAgC,CAAC,MACf;CAClB,OAAO,IAAI,cAAc,MAAM;AACjC"}
+{"version":3,"file":"tavily-scraper.mjs","names":[],"sources":["../../../../src/tools/search/tavily-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nconst DEFAULT_BASIC_TIMEOUT = 15000;\nconst DEFAULT_ADVANCED_TIMEOUT = 30000;\nconst MAX_BATCH_SIZE = 20;\n\nconst getDefaultTimeout = (extractDepth: 'basic' | 'advanced'): number =>\n  extractDepth === 'advanced'\n    ? DEFAULT_ADVANCED_TIMEOUT\n    : DEFAULT_BASIC_TIMEOUT;\n\nconst normalizeUrlKey = (url: string): string => {\n  try {\n    const parsedUrl = new URL(url);\n    parsedUrl.hash = '';\n    if (parsedUrl.pathname.length > 1) {\n      parsedUrl.pathname = parsedUrl.pathname.replace(/\\/+$/, '');\n    }\n    return parsedUrl.toString();\n  } catch {\n    return url;\n  }\n};\n\nconst setUrlResult = (\n  map: Map<string, t.TavilyExtractResult>,\n  result: t.TavilyExtractResult\n): void => {\n  map.set(result.url, result);\n  const normalizedUrl = normalizeUrlKey(result.url);\n  if (!map.has(normalizedUrl)) {\n    map.set(normalizedUrl, result);\n  }\n};\n\nexport class TavilyScraper implements t.BaseScraper {\n  private apiKey: string;\n  private apiUrl: string;\n  private timeout: number;\n  private payloadTimeout: number | undefined;\n  private logger: t.Logger;\n  private extractDepth: 'basic' | 'advanced';\n  private includeImages: boolean;\n  private includeFavicon: boolean;\n  private format: 'markdown' | 'text' | undefined;\n  private httpAgent?: t.HttpAgent;\n  private httpsAgent?: t.HttpsAgent;\n\n  constructor(config: t.TavilyScraperConfig = {}) {\n    this.apiKey = config.apiKey ?? process.env.TAVILY_API_KEY ?? '';\n    this.apiUrl =\n      config.apiUrl ??\n      process.env.TAVILY_EXTRACT_URL ??\n      'https://api.tavily.com/extract';\n    this.payloadTimeout = config.timeout;\n    this.extractDepth = config.extractDepth ?? 'basic';\n    this.timeout = config.timeout ?? getDefaultTimeout(this.extractDepth);\n    this.includeImages = config.includeImages ?? false;\n    this.includeFavicon = config.includeFavicon ?? false;\n    this.format = config.format;\n    this.httpAgent = config.httpAgent;\n    this.httpsAgent = config.httpsAgent;\n    this.logger = config.logger || createDefaultLogger();\n\n    if (!this.apiKey) {\n      this.logger.warn('TAVILY_API_KEY is not set. Scraping will not work.');\n    }\n  }\n\n  async scrapeUrl(\n    url: string,\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<[string, t.TavilyScrapeResponse]> {\n    const results = await this.scrapeUrls([url], options);\n    return results[0];\n  }\n\n  async scrapeUrls(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    if (!this.apiKey) {\n      return urls.map((url) => [\n        url,\n        { success: false, error: 'TAVILY_API_KEY is not set' },\n      ]);\n    }\n\n    const batches: string[][] = [];\n    for (let i = 0; i < urls.length; i += MAX_BATCH_SIZE) {\n      batches.push(urls.slice(i, i + MAX_BATCH_SIZE));\n    }\n\n    const allResults: Array<[string, t.TavilyScrapeResponse]> = [];\n\n    for (const batch of batches) {\n      const batchResults = await this.extractBatch(batch, options);\n      allResults.push(...batchResults);\n    }\n\n    return allResults;\n  }\n\n  private async extractBatch(\n    urls: string[],\n    options: t.TavilyScrapeOptions = {}\n  ): Promise<Array<[string, t.TavilyScrapeResponse]>> {\n    try {\n      const includeFavicon = options.includeFavicon ?? this.includeFavicon;\n      const format = options.format ?? this.format;\n      const extractDepth = options.extractDepth ?? this.extractDepth;\n      const payload: t.TavilyExtractPayload = {\n        urls,\n        extract_depth: extractDepth,\n        include_images: options.includeImages ?? this.includeImages,\n      };\n\n      if (includeFavicon) {\n        payload.include_favicon = true;\n      }\n      if (format != null) {\n        payload.format = format;\n      }\n\n      const effectiveTimeout =\n        options.timeout ??\n        this.payloadTimeout ??\n        (options.extractDepth != null\n          ? getDefaultTimeout(extractDepth)\n          : this.timeout);\n      const payloadTimeout = options.timeout ?? this.payloadTimeout;\n      if (payloadTimeout != null) {\n        payload.timeout = Math.min(Math.max(payloadTimeout / 1000, 1), 60);\n      }\n\n      const response = await axios.post<{\n        results?: t.TavilyExtractResult[];\n        failed_results?: t.TavilyExtractResult[];\n      }>(this.apiUrl, payload, {\n        headers: {\n          Authorization: `Bearer ${this.apiKey}`,\n          'Content-Type': 'application/json',\n        },\n        timeout: effectiveTimeout,\n        httpAgent: this.httpAgent,\n        httpsAgent: this.httpsAgent,\n      });\n\n      const data = response.data;\n      const successMap = new Map<string, t.TavilyExtractResult>();\n      const failedMap = new Map<string, t.TavilyExtractResult>();\n\n      for (const result of data.results ?? []) {\n        setUrlResult(successMap, result);\n      }\n      for (const result of data.failed_results ?? []) {\n        setUrlResult(failedMap, result);\n      }\n\n      return urls.map((url): [string, t.TavilyScrapeResponse] => {\n        const success =\n          successMap.get(url) ?? successMap.get(normalizeUrlKey(url));\n        if (success && success.error == null) {\n          return [\n            url,\n            {\n              success: true,\n              data: {\n                rawContent: success.raw_content ?? '',\n                images: success.images ?? [],\n                favicon: success.favicon,\n              },\n            },\n          ];\n        }\n\n        const failed =\n          failedMap.get(url) ?? failedMap.get(normalizeUrlKey(url));\n        const error =\n          success?.error ??\n          failed?.error ??\n          'URL not found in Tavily Extract response';\n        return [url, { success: false, error }];\n      });\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      return urls.map((url) => [\n        url,\n        {\n          success: false,\n          error: `Tavily Extract API request failed: ${errorMessage}`,\n        },\n      ]);\n    }\n  }\n\n  extractContent(\n    response: t.TavilyScrapeResponse\n  ): [string, undefined | t.References] {\n    if (!response.success || !response.data) {\n      return ['', undefined];\n    }\n\n    const content = response.data.rawContent ?? '';\n    const images = response.data.images ?? [];\n\n    const references: t.References | undefined =\n      images.length > 0\n        ? {\n          links: [],\n          images: images.map((imageUrl) => ({ originalUrl: imageUrl })),\n          videos: [],\n        }\n        : undefined;\n\n    return [content, references];\n  }\n\n  extractMetadata(response: t.TavilyScrapeResponse): t.GenericScrapeMetadata {\n    if (!response.success || !response.data) {\n      return {};\n    }\n\n    const metadata: t.GenericScrapeMetadata = {\n      images_count: response.data.images?.length ?? 0,\n    };\n    if (response.data.favicon != null) {\n      metadata.favicon = response.data.favicon;\n    }\n    return metadata;\n  }\n}\n\nexport const createTavilyScraper = (\n  config: t.TavilyScraperConfig = {}\n): TavilyScraper => {\n  return new TavilyScraper(config);\n};\n"],"mappings":";;;AAIA,MAAM,wBAAwB;AAC9B,MAAM,2BAA2B;AACjC,MAAM,iBAAiB;AAEvB,MAAM,qBAAqB,iBACzB,iBAAiB,aACb,2BACA;AAEN,MAAM,mBAAmB,QAAwB;CAC/C,IAAI;EACF,MAAM,YAAY,IAAI,IAAI,GAAG;EAC7B,UAAU,OAAO;EACjB,IAAI,UAAU,SAAS,SAAS,GAC9B,UAAU,WAAW,UAAU,SAAS,QAAQ,QAAQ,EAAE;EAE5D,OAAO,UAAU,SAAS;CAC5B,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,gBACJ,KACA,WACS;CACT,IAAI,IAAI,OAAO,KAAK,MAAM;CAC1B,MAAM,gBAAgB,gBAAgB,OAAO,GAAG;CAChD,IAAI,CAAC,IAAI,IAAI,aAAa,GACxB,IAAI,IAAI,eAAe,MAAM;AAEjC;AAEA,IAAa,gBAAb,MAAoD;CAClD;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAAgC,CAAC,GAAG;EAC9C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,kBAAkB;EAC7D,KAAK,SACH,OAAO,UACP,QAAQ,IAAI,sBACZ;EACF,KAAK,iBAAiB,OAAO;EAC7B,KAAK,eAAe,OAAO,gBAAgB;EAC3C,KAAK,UAAU,OAAO,WAAW,kBAAkB,KAAK,YAAY;EACpE,KAAK,gBAAgB,OAAO,iBAAiB;EAC7C,KAAK,iBAAiB,OAAO,kBAAkB;EAC/C,KAAK,SAAS,OAAO;EACrB,KAAK,YAAY,OAAO;EACxB,KAAK,aAAa,OAAO;EACzB,KAAK,SAAS,OAAO,UAAU,oBAAoB;EAEnD,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,oDAAoD;CAEzE;CAEA,MAAM,UACJ,KACA,UAAiC,CAAC,GACS;EAE3C,QAAO,MADe,KAAK,WAAW,CAAC,GAAG,GAAG,OAAO,EAAA,CACrC;CACjB;CAEA,MAAM,WACJ,MACA,UAAiC,CAAC,GACgB;EAClD,IAAI,CAAC,KAAK,QACR,OAAO,KAAK,KAAK,QAAQ,CACvB,KACA;GAAE,SAAS;GAAO,OAAO;EAA4B,CACvD,CAAC;EAGH,MAAM,UAAsB,CAAC;EAC7B,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,gBACpC,QAAQ,KAAK,KAAK,MAAM,GAAG,IAAI,cAAc,CAAC;EAGhD,MAAM,aAAsD,CAAC;EAE7D,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,eAAe,MAAM,KAAK,aAAa,OAAO,OAAO;GAC3D,WAAW,KAAK,GAAG,YAAY;EACjC;EAEA,OAAO;CACT;CAEA,MAAc,aACZ,MACA,UAAiC,CAAC,GACgB;EAClD,IAAI;GACF,MAAM,iBAAiB,QAAQ,kBAAkB,KAAK;GACtD,MAAM,SAAS,QAAQ,UAAU,KAAK;GACtC,MAAM,eAAe,QAAQ,gBAAgB,KAAK;GAClD,MAAM,UAAkC;IACtC;IACA,eAAe;IACf,gBAAgB,QAAQ,iBAAiB,KAAK;GAChD;GAEA,IAAI,gBACF,QAAQ,kBAAkB;GAE5B,IAAI,UAAU,MACZ,QAAQ,SAAS;GAGnB,MAAM,mBACJ,QAAQ,WACR,KAAK,mBACJ,QAAQ,gBAAgB,OACrB,kBAAkB,YAAY,IAC9B,KAAK;GACX,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,IAAI,kBAAkB,MACpB,QAAQ,UAAU,KAAK,IAAI,KAAK,IAAI,iBAAiB,KAAM,CAAC,GAAG,EAAE;GAgBnE,MAAM,QAAO,MAbU,MAAM,KAG1B,KAAK,QAAQ,SAAS;IACvB,SAAS;KACP,eAAe,UAAU,KAAK;KAC9B,gBAAgB;IAClB;IACA,SAAS;IACT,WAAW,KAAK;IAChB,YAAY,KAAK;GACnB,CAAC,EAAA,CAEqB;GACtB,MAAM,6BAAa,IAAI,IAAmC;GAC1D,MAAM,4BAAY,IAAI,IAAmC;GAEzD,KAAK,MAAM,UAAU,KAAK,WAAW,CAAC,GACpC,aAAa,YAAY,MAAM;GAEjC,KAAK,MAAM,UAAU,KAAK,kBAAkB,CAAC,GAC3C,aAAa,WAAW,MAAM;GAGhC,OAAO,KAAK,KAAK,QAA0C;IACzD,MAAM,UACJ,WAAW,IAAI,GAAG,KAAK,WAAW,IAAI,gBAAgB,GAAG,CAAC;IAC5D,IAAI,WAAW,QAAQ,SAAS,MAC9B,OAAO,CACL,KACA;KACE,SAAS;KACT,MAAM;MACJ,YAAY,QAAQ,eAAe;MACnC,QAAQ,QAAQ,UAAU,CAAC;MAC3B,SAAS,QAAQ;KACnB;IACF,CACF;IAGF,MAAM,SACJ,UAAU,IAAI,GAAG,KAAK,UAAU,IAAI,gBAAgB,GAAG,CAAC;IAK1D,OAAO,CAAC,KAAK;KAAE,SAAS;KAAO,OAH7B,SAAS,SACT,QAAQ,SACR;IACmC,CAAC;GACxC,CAAC;EACH,SAAS,OAAO;GACd,MAAM,eACJ,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GACvD,OAAO,KAAK,KAAK,QAAQ,CACvB,KACA;IACE,SAAS;IACT,OAAO,sCAAsC;GAC/C,CACF,CAAC;EACH;CACF;CAEA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,UAAU,SAAS,KAAK,cAAc;EAC5C,MAAM,SAAS,SAAS,KAAK,UAAU,CAAC;EAWxC,OAAO,CAAC,SARN,OAAO,SAAS,IACZ;GACA,OAAO,CAAC;GACR,QAAQ,OAAO,KAAK,cAAc,EAAE,aAAa,SAAS,EAAE;GAC5D,QAAQ,CAAC;EACX,IACE,KAAA,CAEqB;CAC7B;CAEA,gBAAgB,UAA2D;EACzE,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC;EAGV,MAAM,WAAoC,EACxC,cAAc,SAAS,KAAK,QAAQ,UAAU,EAChD;EACA,IAAI,SAAS,KAAK,WAAW,MAC3B,SAAS,UAAU,SAAS,KAAK;EAEnC,OAAO;CACT;AACF;AAEA,MAAa,uBACX,SAAgC,CAAC,MACf;CAClB,OAAO,IAAI,cAAc,MAAM;AACjC"}

package/dist/esm/tools/search/tavily-search.mjs

@@ -256,7 +256,9 @@ const createTavilyAPI = (apiKey, apiUrl, options) => {
 					Authorization: `Bearer ${config.apiKey}`,
 					"Content-Type": "application/json"
 				},
-				timeout: config.timeout
+				timeout: config.timeout,
+				httpAgent: options?.httpAgent,
+				httpsAgent: options?.httpsAgent
 			})).data;
 			const organicResults = (data.results ?? []).map((result) => ({
 				title: result.title ?? "",