@librechat/agents 2.4.320 → 2.4.322

package/dist/cjs/tools/search/rerankers.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"rerankers.cjs","sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\nimport type * as t from './types';\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n\n  constructor() {\n    // Each specific reranker will set its API key\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n\n  protected logDocumentSamples(documents: string[]): void {\n    console.log('Sample documents being sent to API:');\n    for (let i = 0; i < Math.min(3, documents.length); i++) {\n      console.log(`Document ${i}: ${documents[i].substring(0, 100)}...`);\n    }\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  constructor({ apiKey = process.env.JINA_API_KEY }: { apiKey?: string }) {\n    super();\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    console.log(`Reranking ${documents.length} documents with Jina`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        console.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      this.logDocumentSamples(documents);\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        'https://api.jina.ai/v1/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      // Log the response data structure\n      console.log('Jina API response structure:');\n      console.log('Model:', response.data?.model);\n      console.log('Usage:', response.data?.usage);\n      console.log('Results count:', response.data?.results.length);\n\n      // Log a sample of the results\n      if ((response.data?.results.length ?? 0) > 0) {\n        console.log(\n          'Sample result:',\n          JSON.stringify(response.data?.results[0], null, 2)\n        );\n      }\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        console.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      console.error('Error using Jina reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  constructor({ apiKey = process.env.COHERE_API_KEY }: { apiKey?: string }) {\n    super();\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    console.log(`Reranking ${documents.length} documents with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        console.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      this.logDocumentSamples(documents);\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      // Log the response data structure\n      console.log('Cohere API response structure:');\n      console.log('ID:', response.data?.id);\n      console.log('Meta:', response.data?.meta);\n      console.log('Results count:', response.data?.results.length);\n\n      // Log a sample of the results\n      if ((response.data?.results.length ?? 0) > 0) {\n        console.log(\n          'Sample result:',\n          JSON.stringify(response.data?.results[0], null, 2)\n        );\n      }\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        console.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      console.error('Error using Cohere reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor() {\n    super();\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    console.log(\n      `Reranking ${documents.length} documents with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n  rerankerType: t.RerankerType;\n  jinaApiKey?: string;\n  cohereApiKey?: string;\n}): BaseReranker | undefined => {\n  const { rerankerType, jinaApiKey, cohereApiKey } = config;\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({ apiKey: jinaApiKey });\n  case 'cohere':\n    return new CohereReranker({ apiKey: cohereApiKey });\n  case 'infinity':\n    return new InfinityReranker();\n  case 'none':\n    console.log('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    console.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({ apiKey: jinaApiKey });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"names":[],"mappings":";;;;AAAA;MAIsB,YAAY,CAAA;AACtB,IAAA,MAAM;AAEhB,IAAA,WAAA,GAAA;;;IAUU,iBAAiB,CACzB,SAAmB,EACnB,IAAY,EAAA;AAEZ,QAAA,OAAO;AACJ,aAAA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC;AACzC,aAAA,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;;AAGlC,IAAA,kBAAkB,CAAC,SAAmB,EAAA;AAC9C,QAAA,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE;AACtD,YAAA,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA,EAAA,EAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA,GAAA,CAAK,CAAC;;;AAGvE;AAEK,MAAO,YAAa,SAAQ,YAAY,CAAA;IAC5C,WAAY,CAAA,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EAAuB,EAAA;AACpE,QAAA,KAAK,EAAE;AACP,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,OAAO,CAAC,GAAG,CAAC,CAAA,UAAA,EAAa,SAAS,CAAC,MAAM,CAAsB,oBAAA,CAAA,CAAC;AAEhE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,OAAO,CAAC,IAAI,CAAC,iDAAiD,CAAC;gBAC/D,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;AAElC,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,oCAAoC;AAC3C,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;AACpB,gBAAA,gBAAgB,EAAE,IAAI;aACvB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,+BAA+B,EAC/B,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;;AAGD,YAAA,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AAC3C,YAAA,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC;;AAG5D,YAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC5C,OAAO,CAAC,GAAG,CACT,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CACnD;;AAGH,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;oBACpC,IAAI,IAAI,GAAG,EAAE;;AAGb,oBAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,EAAE;AAC3B,wBAAA,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ;wBAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,EAAE;AAC5C,4BAAA,IAAI,GAAG,GAAG,CAAC,IAAI;;AACV,6BAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;4BAClC,IAAI,GAAG,GAAG;;;yBAEP;;AAEL,wBAAA,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;;AAG5B,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,OAAO,CAAC,IAAI,CACV,kEAAkE,CACnE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC;;YAElD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,cAAe,SAAQ,YAAY,CAAA;IAC9C,WAAY,CAAA,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAuB,EAAA;AACtE,QAAA,KAAK,EAAE;AACP,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,OAAO,CAAC,GAAG,CAAC,CAAA,UAAA,EAAa,SAAS,CAAC,MAAM,CAAwB,sBAAA,CAAA,CAAC;AAElE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,OAAO,CAAC,IAAI,CAAC,mDAAmD,CAAC;gBACjE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;AAElC,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,aAAa;AACpB,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,kCAAkC,EAClC,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;;AAGD,YAAA,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;AACzC,YAAA,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC;;AAG5D,YAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC5C,OAAO,CAAC,GAAG,CACT,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CACnD;;AAGH,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;AACpC,oBAAA,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;AAChC,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,OAAO,CAAC,IAAI,CACV,oEAAoE,CACrE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;;YAEpD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,gBAAiB,SAAQ,YAAY,CAAA;AAChD,IAAA,WAAA,GAAA;AACE,QAAA,KAAK,EAAE;;;IAIT,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,OAAO,CAAC,GAAG,CACT,CAAA,UAAA,EAAa,SAAS,CAAC,MAAM,CAAwC,sCAAA,CAAA,CACtE;;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAEjD;AAED;;AAEG;AACU,MAAA,cAAc,GAAG,CAAC,MAI9B,KAA8B;IAC7B,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,MAAM;AAEzD,IAAA,QAAQ,YAAY,CAAC,WAAW,EAAE;AAClC,QAAA,KAAK,MAAM;YACT,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AACjD,QAAA,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AACrD,QAAA,KAAK,UAAU;YACb,OAAO,IAAI,gBAAgB,EAAE;AAC/B,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC;AAC9D,YAAA,OAAO,SAAS;AAClB,QAAA;AACE,YAAA,OAAO,CAAC,IAAI,CACV,0BAA0B,YAAY,CAAA,iCAAA,CAAmC,CAC1E;YACD,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;;AAEnD;AAEA;AACA;AACA;AACA;;;;;;;;"}
+{"version":3,"file":"rerankers.cjs","sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n  protected logger: t.Logger;\n\n  constructor(logger?: t.Logger) {\n    // Each specific reranker will set its API key\n    this.logger = logger || createDefaultLogger();\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n\n  protected logDocumentSamples(documents: string[]): void {\n    this.logger.debug('Sample documents being sent to API:');\n    for (let i = 0; i < Math.min(3, documents.length); i++) {\n      this.logger.debug(`Document ${i}: ${documents[i].substring(0, 100)}...`);\n    }\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  constructor({\n    apiKey = process.env.JINA_API_KEY,\n    logger,\n  }: {\n    apiKey?: string;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} documents with Jina`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      this.logDocumentSamples(documents);\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        'https://api.jina.ai/v1/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      // Log the response data structure\n      this.logger.debug('Jina API response structure:');\n      this.logger.debug('Model:', response.data?.model);\n      this.logger.debug('Usage:', response.data?.usage);\n      this.logger.debug('Results count:', response.data?.results.length);\n\n      // Log a sample of the results\n      if ((response.data?.results.length ?? 0) > 0) {\n        this.logger.debug(\n          'Sample result:',\n          JSON.stringify(response.data?.results[0], null, 2)\n        );\n      }\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Jina reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  constructor({\n    apiKey = process.env.COHERE_API_KEY,\n    logger,\n  }: {\n    apiKey?: string;\n    logger?: t.Logger;\n  }) {\n    super(logger);\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(`Reranking ${documents.length} documents with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        this.logger.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      this.logDocumentSamples(documents);\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      // Log the response data structure\n      this.logger.debug('Cohere API response structure:');\n      this.logger.debug('ID:', response.data?.id);\n      this.logger.debug('Meta:', response.data?.meta);\n      this.logger.debug('Results count:', response.data?.results.length);\n\n      // Log a sample of the results\n      if ((response.data?.results.length ?? 0) > 0) {\n        this.logger.debug(\n          'Sample result:',\n          JSON.stringify(response.data?.results[0], null, 2)\n        );\n      }\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        this.logger.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      this.logger.error('Error using Cohere reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor(logger?: t.Logger) {\n    super(logger);\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    this.logger.debug(\n      `Reranking ${documents.length} documents with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n  rerankerType: t.RerankerType;\n  jinaApiKey?: string;\n  cohereApiKey?: string;\n  logger?: t.Logger;\n}): BaseReranker | undefined => {\n  const { rerankerType, jinaApiKey, cohereApiKey, logger } = config;\n\n  // Create a default logger if none is provided\n  const defaultLogger = logger || createDefaultLogger();\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({ apiKey: jinaApiKey, logger: defaultLogger });\n  case 'cohere':\n    return new CohereReranker({\n      apiKey: cohereApiKey,\n      logger: defaultLogger,\n    });\n  case 'infinity':\n    return new InfinityReranker(defaultLogger);\n  case 'none':\n    defaultLogger.debug('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    defaultLogger.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({ apiKey: jinaApiKey, logger: defaultLogger });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"names":["createDefaultLogger"],"mappings":";;;;;MAIsB,YAAY,CAAA;AACtB,IAAA,MAAM;AACN,IAAA,MAAM;AAEhB,IAAA,WAAA,CAAY,MAAiB,EAAA;;AAE3B,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM,IAAIA,yBAAmB,EAAE;;IASrC,iBAAiB,CACzB,SAAmB,EACnB,IAAY,EAAA;AAEZ,QAAA,OAAO;AACJ,aAAA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC;AACzC,aAAA,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;;AAGlC,IAAA,kBAAkB,CAAC,SAAmB,EAAA;AAC9C,QAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC;QACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE;YACtD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAY,SAAA,EAAA,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAK,GAAA,CAAA,CAAC;;;AAG7E;AAEK,MAAO,YAAa,SAAQ,YAAY,CAAA;IAC5C,WAAY,CAAA,EACV,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EACjC,MAAM,GAIP,EAAA;QACC,KAAK,CAAC,MAAM,CAAC;AACb,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,CAAsB,oBAAA,CAAA,CAAC;AAEtE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC;gBACnE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;AAElC,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,oCAAoC;AAC3C,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;AACpB,gBAAA,gBAAgB,EAAE,IAAI;aACvB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,+BAA+B,EAC/B,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;;AAGD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC;AACjD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AACjD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AACjD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC;;AAGlE,YAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CACnD;;AAGH,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;oBACpC,IAAI,IAAI,GAAG,EAAE;;AAGb,oBAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,EAAE;AAC3B,wBAAA,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ;wBAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,EAAE;AAC5C,4BAAA,IAAI,GAAG,GAAG,CAAC,IAAI;;AACV,6BAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;4BAClC,IAAI,GAAG,GAAG;;;yBAEP;;AAEL,wBAAA,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;;AAG5B,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,kEAAkE,CACnE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC;;YAEtD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,cAAe,SAAQ,YAAY,CAAA;IAC9C,WAAY,CAAA,EACV,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EACnC,MAAM,GAIP,EAAA;QACC,KAAK,CAAC,MAAM,CAAC;AACb,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,CAAwB,sBAAA,CAAA,CAAC;AAExE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mDAAmD,CAAC;gBACrE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;AAElC,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,aAAa;AACpB,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,kCAAkC,EAClC,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;;AAGD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC;AACnD,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;AAC3C,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;AAC/C,YAAA,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC;;AAGlE,YAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CACnD;;AAGH,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;AACpC,oBAAA,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;AAChC,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,oEAAoE,CACrE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;;YAExD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,gBAAiB,SAAQ,YAAY,CAAA;AAChD,IAAA,WAAA,CAAY,MAAiB,EAAA;QAC3B,KAAK,CAAC,MAAM,CAAC;;;IAIf,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,CAAa,UAAA,EAAA,SAAS,CAAC,MAAM,CAAwC,sCAAA,CAAA,CACtE;;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAEjD;AAED;;AAEG;AACU,MAAA,cAAc,GAAG,CAAC,MAK9B,KAA8B;IAC7B,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM;;AAGjE,IAAA,MAAM,aAAa,GAAG,MAAM,IAAIA,yBAAmB,EAAE;AAErD,IAAA,QAAQ,YAAY,CAAC,WAAW,EAAE;AAClC,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;AACxE,QAAA,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC;AACxB,gBAAA,MAAM,EAAE,YAAY;AACpB,gBAAA,MAAM,EAAE,aAAa;AACtB,aAAA,CAAC;AACJ,QAAA,KAAK,UAAU;AACb,YAAA,OAAO,IAAI,gBAAgB,CAAC,aAAa,CAAC;AAC5C,QAAA,KAAK,MAAM;AACT,YAAA,aAAa,CAAC,KAAK,CAAC,iDAAiD,CAAC;AACtE,YAAA,OAAO,SAAS;AAClB,QAAA;AACE,YAAA,aAAa,CAAC,IAAI,CAChB,0BAA0B,YAAY,CAAA,iCAAA,CAAmC,CAC1E;AACD,YAAA,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;;AAE1E;AAEA;AACA;AACA;AACA;;;;;;;;"}

package/dist/cjs/tools/search/schema.cjs

@@ -0,0 +1,70 @@
+'use strict';
+
+var zod = require('zod');
+
+exports.DATE_RANGE = void 0;
+(function (DATE_RANGE) {
+    DATE_RANGE["PAST_HOUR"] = "h";
+    DATE_RANGE["PAST_24_HOURS"] = "d";
+    DATE_RANGE["PAST_WEEK"] = "w";
+    DATE_RANGE["PAST_MONTH"] = "m";
+    DATE_RANGE["PAST_YEAR"] = "y";
+})(exports.DATE_RANGE || (exports.DATE_RANGE = {}));
+const DEFAULT_QUERY_DESCRIPTION = `
+GUIDELINES:
+- Start broad, then narrow: Begin with key concepts, then refine with specifics
+- Think like sources: Use terminology experts would use in the field
+- Consider perspective: Frame queries from different viewpoints for better results
+- Quality over quantity: A precise 3-4 word query often beats lengthy sentences
+
+TECHNIQUES (combine for power searches):
+- EXACT PHRASES: Use quotes ("climate change report")
+- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)
+- SITE-SPECIFIC: Restrict to websites (site:edu research)
+- FILETYPE: Find specific documents (filetype:pdf study)
+- OR OPERATOR: Find alternatives (electric OR hybrid cars)
+- DATE RANGE: Recent information (data after:2020)
+- WILDCARDS: Use * for unknown terms (how to * bread)
+- SPECIFIC QUESTIONS: Use who/what/when/where/why/how
+- DOMAIN TERMS: Include technical terminology for specialized topics
+- CONCISE TERMS: Prioritize keywords over sentences
+`.trim();
+const DEFAULT_COUNTRY_DESCRIPTION = `Country code to localize search results.
+Use standard 2-letter country codes: "us", "uk", "ca", "de", "fr", "jp", "br", etc.
+Provide this when the search should return results specific to a particular country.
+Examples:
+- "us" for United States (default)
+- "de" for Germany
+- "in" for India
+`.trim();
+const querySchema = zod.z.string().describe(DEFAULT_QUERY_DESCRIPTION);
+const dateSchema = zod.z
+    .nativeEnum(exports.DATE_RANGE)
+    .optional()
+    .describe('Date range for search results.');
+const countrySchema = zod.z
+    .string()
+    .optional()
+    .describe(DEFAULT_COUNTRY_DESCRIPTION);
+const imagesSchema = zod.z
+    .boolean()
+    .optional()
+    .describe('Whether to also run an image search.');
+const videosSchema = zod.z
+    .boolean()
+    .optional()
+    .describe('Whether to also run a video search.');
+const newsSchema = zod.z
+    .boolean()
+    .optional()
+    .describe('Whether to also run a news search.');
+
+exports.DEFAULT_COUNTRY_DESCRIPTION = DEFAULT_COUNTRY_DESCRIPTION;
+exports.DEFAULT_QUERY_DESCRIPTION = DEFAULT_QUERY_DESCRIPTION;
+exports.countrySchema = countrySchema;
+exports.dateSchema = dateSchema;
+exports.imagesSchema = imagesSchema;
+exports.newsSchema = newsSchema;
+exports.querySchema = querySchema;
+exports.videosSchema = videosSchema;
+//# sourceMappingURL=schema.cjs.map

package/dist/cjs/tools/search/schema.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.cjs","sources":["../../../../src/tools/search/schema.ts"],"sourcesContent":["import { z } from 'zod';\n\nexport enum DATE_RANGE {\n  PAST_HOUR = 'h',\n  PAST_24_HOURS = 'd',\n  PAST_WEEK = 'w',\n  PAST_MONTH = 'm',\n  PAST_YEAR = 'y',\n}\n\nexport const DEFAULT_QUERY_DESCRIPTION = `\nGUIDELINES:\n- Start broad, then narrow: Begin with key concepts, then refine with specifics\n- Think like sources: Use terminology experts would use in the field\n- Consider perspective: Frame queries from different viewpoints for better results\n- Quality over quantity: A precise 3-4 word query often beats lengthy sentences\n\nTECHNIQUES (combine for power searches):\n- EXACT PHRASES: Use quotes (\"climate change report\")\n- EXCLUDE TERMS: Use minus to remove unwanted results (-wikipedia)\n- SITE-SPECIFIC: Restrict to websites (site:edu research)\n- FILETYPE: Find specific documents (filetype:pdf study)\n- OR OPERATOR: Find alternatives (electric OR hybrid cars)\n- DATE RANGE: Recent information (data after:2020)\n- WILDCARDS: Use * for unknown terms (how to * bread)\n- SPECIFIC QUESTIONS: Use who/what/when/where/why/how\n- DOMAIN TERMS: Include technical terminology for specialized topics\n- CONCISE TERMS: Prioritize keywords over sentences\n`.trim();\n\nexport const DEFAULT_COUNTRY_DESCRIPTION =\n  `Country code to localize search results.\nUse standard 2-letter country codes: \"us\", \"uk\", \"ca\", \"de\", \"fr\", \"jp\", \"br\", etc.\nProvide this when the search should return results specific to a particular country.\nExamples:\n- \"us\" for United States (default)\n- \"de\" for Germany\n- \"in\" for India\n`.trim();\n\nexport const querySchema = z.string().describe(DEFAULT_QUERY_DESCRIPTION);\nexport const dateSchema = z\n  .nativeEnum(DATE_RANGE)\n  .optional()\n  .describe('Date range for search results.');\nexport const countrySchema = z\n  .string()\n  .optional()\n  .describe(DEFAULT_COUNTRY_DESCRIPTION);\nexport const imagesSchema = z\n  .boolean()\n  .optional()\n  .describe('Whether to also run an image search.');\n\nexport const videosSchema = z\n  .boolean()\n  .optional()\n  .describe('Whether to also run a video search.');\n\nexport const newsSchema = z\n  .boolean()\n  .optional()\n  .describe('Whether to also run a news search.');\n"],"names":["DATE_RANGE","z"],"mappings":";;;;AAEYA;AAAZ,CAAA,UAAY,UAAU,EAAA;AACpB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACf,IAAA,UAAA,CAAA,eAAA,CAAA,GAAA,GAAmB;AACnB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACf,IAAA,UAAA,CAAA,YAAA,CAAA,GAAA,GAAgB;AAChB,IAAA,UAAA,CAAA,WAAA,CAAA,GAAA,GAAe;AACjB,CAAC,EANWA,kBAAU,KAAVA,kBAAU,GAMrB,EAAA,CAAA,CAAA;AAEY,MAAA,yBAAyB,GAAG;;;;;;;;;;;;;;;;;;CAkBxC,CAAC,IAAI;AAEO,MAAA,2BAA2B,GACtC,CAAA;;;;;;;CAOD,CAAC,IAAI;AAEC,MAAM,WAAW,GAAGC,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yBAAyB;AACjE,MAAM,UAAU,GAAGA;KACvB,UAAU,CAACD,kBAAU;AACrB,KAAA,QAAQ;KACR,QAAQ,CAAC,gCAAgC;AACrC,MAAM,aAAa,GAAGC;AAC1B,KAAA,MAAM;AACN,KAAA,QAAQ;KACR,QAAQ,CAAC,2BAA2B;AAChC,MAAM,YAAY,GAAGA;AACzB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,sCAAsC;AAE3C,MAAM,YAAY,GAAGA;AACzB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,qCAAqC;AAE1C,MAAM,UAAU,GAAGA;AACvB,KAAA,OAAO;AACP,KAAA,QAAQ;KACR,QAAQ,CAAC,oCAAoC;;;;;;;;;;;"}

package/dist/cjs/tools/search/search.cjs

@@ -4,7 +4,6 @@ var axios = require('axios');
 var textsplitters = require('@langchain/textsplitters');
 var utils = require('./utils.cjs');
 
-/* eslint-disable no-console */
 const chunker = {
     cleanText: (text) => {
         if (!text)
@@ -31,10 +30,11 @@ const chunker = {
         });
         return await splitter.splitText(text);
     },
-    splitTexts: async (texts, options) => {
+    splitTexts: async (texts, options, logger) => {
         // Split multiple texts
+        const logger_ = logger || utils.createDefaultLogger();
         const promises = texts.map((text) => chunker.splitText(text, options).catch((error) => {
-            console.error('Error splitting text:', error);
+            logger_.error('Error splitting text:', error);
             return [text];
         }));
         return Promise.all(promises);
@@ -51,13 +51,14 @@ function createSourceUpdateCallback(sourceMap) {
         }
     };
 }
-const getHighlights = async ({ query, content, reranker, topResults = 5, }) => {
+const getHighlights = async ({ query, content, reranker, topResults = 5, logger, }) => {
+    const logger_ = logger || utils.createDefaultLogger();
     if (!content) {
-        console.warn('No content provided for highlights');
+        logger_.warn('No content provided for highlights');
         return;
     }
     if (!reranker) {
-        console.warn('No reranker provided for highlights');
+        logger_.warn('No reranker provided for highlights');
         return;
     }
     try {
@@ -66,12 +67,12 @@ const getHighlights = async ({ query, content, reranker, topResults = 5, }) => {
             return await reranker.rerank(query, documents, topResults);
         }
         else {
-            console.error('Expected documents to be an array, got:', typeof documents);
+            logger_.error('Expected documents to be an array, got:', typeof documents);
             return;
         }
     }
     catch (error) {
-        console.error('Error in content processing:', error);
+        logger_.error('Error in content processing:', error);
         return;
     }
 };
@@ -84,19 +85,39 @@ const createSerperAPI = (apiKey) => {
     if (config.apiKey == null || config.apiKey === '') {
         throw new Error('SERPER_API_KEY is required for SerperAPI');
     }
-    const getSources = async ({ query, country, numResults = 8, }) => {
+    const getSources = async ({ query, date, country, safeSearch, numResults = 8, type, }) => {
         if (!query.trim()) {
             return { success: false, error: 'Query cannot be empty' };
         }
         try {
+            const safe = ['off', 'moderate', 'active'];
             const payload = {
                 q: query,
+                safe: safe[safeSearch ?? 1],
                 num: Math.min(Math.max(1, numResults), 10),
             };
+            // Set the search type if provided
+            if (type) {
+                payload.type = type;
+            }
+            if (date != null) {
+                payload.tbs = `qdr:${date}`;
+            }
             if (country != null && country !== '') {
                 payload['gl'] = country.toLowerCase();
             }
-            const response = await axios.post(config.apiUrl, payload, {
+            // Determine the API endpoint based on the search type
+            let apiEndpoint = config.apiUrl;
+            if (type === 'images') {
+                apiEndpoint = 'https://google.serper.dev/images';
+            }
+            else if (type === 'videos') {
+                apiEndpoint = 'https://google.serper.dev/videos';
+            }
+            else if (type === 'news') {
+                apiEndpoint = 'https://google.serper.dev/news';
+            }
+            const response = await axios.post(apiEndpoint, payload, {
                 headers: {
                     'X-API-KEY': config.apiKey,
                     'Content-Type': 'application/json',
@@ -112,6 +133,8 @@ const createSerperAPI = (apiKey) => {
                 peopleAlsoAsk: data.peopleAlsoAsk,
                 knowledgeGraph: data.knowledgeGraph,
                 relatedSearches: data.relatedSearches,
+                videos: data.videos ?? [],
+                news: data.news ?? [],
             };
             return { success: true, data: results };
         }
@@ -131,7 +154,7 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
     if (config.instanceUrl == null || config.instanceUrl === '') {
         throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');
     }
-    const getSources = async ({ query, numResults = 8, }) => {
+    const getSources = async ({ query, numResults = 8, type, }) => {
         if (!query.trim()) {
             return { success: false, error: 'Query cannot be empty' };
         }
@@ -144,12 +167,23 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
             if (!searchUrl.endsWith('/search')) {
                 searchUrl = searchUrl.replace(/\/$/, '') + '/search';
             }
+            // Determine the search category based on the type
+            let category = 'general';
+            if (type === 'images') {
+                category = 'images';
+            }
+            else if (type === 'videos') {
+                category = 'videos';
+            }
+            else if (type === 'news') {
+                category = 'news';
+            }
             // Prepare parameters for SearXNG
             const params = {
                 q: query,
                 format: 'json',
                 pageno: 1,
-                categories: 'general',
+                categories: category,
                 language: 'all',
                 safesearch: 0,
                 engines: 'google,bing,duckduckgo',
@@ -190,6 +224,8 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
                 topStories: [],
                 // Use undefined instead of null for optional properties
                 relatedSearches: data.suggestions ?? [],
+                videos: [],
+                news: [],
             };
             return { success: true, data: results };
         }
@@ -222,11 +258,12 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
     const { topResults = 5, 
     // strategies = ['no_extraction'],
     // filterContent = true,
-    reranker, } = config;
+    reranker, logger, } = config;
+    const logger_ = logger || utils.createDefaultLogger();
     const firecrawlScraper = scraperInstance;
     const webScraper = {
         scrapeMany: async ({ query, links, onGetHighlights, }) => {
-            console.log(`Scraping ${links.length} links with Firecrawl`);
+            logger_.debug(`Scraping ${links.length} links with Firecrawl`);
             const promises = [];
             try {
                 for (let i = 0; i < links.length; i++) {
@@ -234,7 +271,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                     const promise = firecrawlScraper
                         .scrapeUrl(currentLink, {})
                         .then(([url, response]) => {
-                        const attribution = utils.getAttribution(url, response.data?.metadata);
+                        const attribution = utils.getAttribution(url, response.data?.metadata, logger_);
                         if (response.success && response.data) {
                             const [content, references] = firecrawlScraper.extractContent(response);
                             return {
@@ -248,13 +285,13 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                             url,
                             attribution,
                             error: true,
-                            content: `Failed to scrape ${url}: ${response.error ?? 'Unknown error'}`,
+                            content: '',
                         };
                     })
                         .then(async (result) => {
                         try {
                             if (result.error != null) {
-                                console.error(`Error scraping ${result.url}: ${result.content}`);
+                                logger_.error(`Error scraping ${result.url}: ${result.content}`, result.error);
                                 return {
                                     ...result,
                                 };
@@ -263,6 +300,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                                 query,
                                 reranker,
                                 content: result.content,
+                                logger: logger_,
                             });
                             if (onGetHighlights) {
                                 onGetHighlights(result.url);
@@ -273,18 +311,18 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                             };
                         }
                         catch (error) {
-                            console.error('Error processing scraped content:', error);
+                            logger_.error('Error processing scraped content:', error);
                             return {
                                 ...result,
                             };
                         }
                     })
                         .catch((error) => {
-                        console.error(`Error scraping ${currentLink}:`, error);
+                        logger_.error(`Error scraping ${currentLink}:`, error);
                         return {
                             url: currentLink,
                             error: true,
-                            content: `Failed to scrape ${currentLink}: ${error.message ?? 'Unknown error'}`,
+                            content: '',
                         };
                     });
                     promises.push(promise);
@@ -292,7 +330,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                 return await Promise.all(promises);
             }
             catch (error) {
-                console.error('Error in scrapeMany:', error);
+                logger_.error('Error in scrapeMany:', error);
                 return [];
             }
         },
@@ -318,7 +356,7 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
             });
         }
     };
-    const processSources = async ({ result, numElements, query, proMode = true, onGetHighlights, }) => {
+    const processSources = async ({ result, numElements, query, news, proMode = true, onGetHighlights, }) => {
         try {
             if (!result.data) {
                 return {
@@ -359,44 +397,48 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
                 return result.data;
             }
             const sourceMap = new Map();
-            const allLinks = [];
-            for (const source of result.data.organic) {
-                if (source.link) {
-                    allLinks.push(source.link);
-                    sourceMap.set(source.link, source);
-                }
-            }
-            if (allLinks.length === 0) {
+            const organicLinksSet = new Set();
+            // Collect organic links
+            const organicLinks = collectLinks(result.data.organic, sourceMap, organicLinksSet);
+            // Collect top story links, excluding any that are already in organic links
+            const topStories = result.data.topStories ?? [];
+            const topStoryLinks = collectLinks(topStories, sourceMap, organicLinksSet);
+            if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {
                 return result.data;
             }
             const onContentScraped = createSourceUpdateCallback(sourceMap);
-            await fetchContents({
-                query,
-                links: allLinks,
-                onGetHighlights,
-                onContentScraped,
-                target: numElements,
-            });
-            for (let i = 0; i < result.data.organic.length; i++) {
-                const source = result.data.organic[i];
-                const updatedSource = sourceMap.get(source.link);
-                if (updatedSource) {
-                    result.data.organic[i] = {
-                        ...source,
-                        ...updatedSource,
-                    };
-                }
+            const promises = [];
+            // Process organic links
+            if (organicLinks.length > 0) {
+                promises.push(fetchContents({
+                    query,
+                    onGetHighlights,
+                    onContentScraped,
+                    links: organicLinks,
+                    target: numElements,
+                }));
             }
-            const successfulSources = result.data.organic
-                .filter((source) => source.content != null && !source.content.startsWith('Failed'))
-                .slice(0, numElements);
-            if (successfulSources.length > 0) {
-                result.data.organic = successfulSources;
+            // Process top story links
+            if (news && topStoryLinks.length > 0) {
+                promises.push(fetchContents({
+                    query,
+                    onGetHighlights,
+                    onContentScraped,
+                    links: topStoryLinks,
+                    target: numElements,
+                }));
+            }
+            await Promise.all(promises);
+            if (result.data.organic.length > 0) {
+                updateSourcesWithContent(result.data.organic, sourceMap);
+            }
+            if (news && topStories.length > 0) {
+                updateSourcesWithContent(topStories, sourceMap);
             }
             return result.data;
         }
         catch (error) {
-            console.error('Error in processSources:', error);
+            logger_.error('Error in processSources:', error);
             return {
                 organic: [],
                 topStories: [],
@@ -412,6 +454,37 @@ const createSourceProcessor = (config = {}, scraperInstance) => {
         topResults,
     };
 };
+/** Helper function to collect links and update sourceMap */
+function collectLinks(sources, sourceMap, existingLinksSet) {
+    const links = [];
+    for (const source of sources) {
+        if (source.link) {
+            // For topStories, only add if not already in organic links
+            if (existingLinksSet && existingLinksSet.has(source.link)) {
+                continue;
+            }
+            links.push(source.link);
+            if (existingLinksSet) {
+                existingLinksSet.add(source.link);
+            }
+            sourceMap.set(source.link, source);
+        }
+    }
+    return links;
+}
+/** Helper function to update sources with scraped content */
+function updateSourcesWithContent(sources, sourceMap) {
+    for (let i = 0; i < sources.length; i++) {
+        const source = sources[i];
+        const updatedSource = sourceMap.get(source.link);
+        if (updatedSource) {
+            sources[i] = {
+                ...source,
+                ...updatedSource,
+            };
+        }
+    }
+}
 
 exports.createSearchAPI = createSearchAPI;
 exports.createSourceProcessor = createSourceProcessor;