@librechat/agents 2.4.30 → 2.4.311

package/dist/cjs/tools/search/rerankers.cjs

@@ -0,0 +1,187 @@
+'use strict';
+
+var axios = require('axios');
+
+/* eslint-disable no-console */
+class BaseReranker {
+    apiKey;
+    constructor() {
+        // Each specific reranker will set its API key
+    }
+    getDefaultRanking(documents, topK) {
+        return documents
+            .slice(0, Math.min(topK, documents.length))
+            .map((doc) => ({ text: doc, score: 0 }));
+    }
+    logDocumentSamples(documents) {
+        console.log('Sample documents being sent to API:');
+        for (let i = 0; i < Math.min(3, documents.length); i++) {
+            console.log(`Document ${i}: ${documents[i].substring(0, 100)}...`);
+        }
+    }
+}
+class JinaReranker extends BaseReranker {
+    constructor({ apiKey = process.env.JINA_API_KEY }) {
+        super();
+        this.apiKey = apiKey;
+    }
+    async rerank(query, documents, topK = 5) {
+        console.log(`Reranking ${documents.length} documents with Jina`);
+        try {
+            if (this.apiKey == null || this.apiKey === '') {
+                console.warn('JINA_API_KEY is not set. Using default ranking.');
+                return this.getDefaultRanking(documents, topK);
+            }
+            this.logDocumentSamples(documents);
+            const requestData = {
+                model: 'jina-reranker-v2-base-multilingual',
+                query: query,
+                top_n: topK,
+                documents: documents,
+                return_documents: true,
+            };
+            const response = await axios.post('https://api.jina.ai/v1/rerank', requestData, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+            });
+            // Log the response data structure
+            console.log('Jina API response structure:');
+            console.log('Model:', response.data?.model);
+            console.log('Usage:', response.data?.usage);
+            console.log('Results count:', response.data?.results.length);
+            // Log a sample of the results
+            if ((response.data?.results.length ?? 0) > 0) {
+                console.log('Sample result:', JSON.stringify(response.data?.results[0], null, 2));
+            }
+            if (response.data && response.data.results.length) {
+                return response.data.results.map((result) => {
+                    const docIndex = result.index;
+                    const score = result.relevance_score;
+                    let text = '';
+                    // If return_documents is true, the document field will be present
+                    if (result.document != null) {
+                        const doc = result.document;
+                        if (typeof doc === 'object' && 'text' in doc) {
+                            text = doc.text;
+                        }
+                        else if (typeof doc === 'string') {
+                            text = doc;
+                        }
+                    }
+                    else {
+                        // Otherwise, use the index to get the document
+                        text = documents[docIndex];
+                    }
+                    return { text, score };
+                });
+            }
+            else {
+                console.warn('Unexpected response format from Jina API. Using default ranking.');
+                return this.getDefaultRanking(documents, topK);
+            }
+        }
+        catch (error) {
+            console.error('Error using Jina reranker:', error);
+            // Fallback to default ranking on error
+            return this.getDefaultRanking(documents, topK);
+        }
+    }
+}
+class CohereReranker extends BaseReranker {
+    constructor({ apiKey = process.env.COHERE_API_KEY }) {
+        super();
+        this.apiKey = apiKey;
+    }
+    async rerank(query, documents, topK = 5) {
+        console.log(`Reranking ${documents.length} documents with Cohere`);
+        try {
+            if (this.apiKey == null || this.apiKey === '') {
+                console.warn('COHERE_API_KEY is not set. Using default ranking.');
+                return this.getDefaultRanking(documents, topK);
+            }
+            this.logDocumentSamples(documents);
+            const requestData = {
+                model: 'rerank-v3.5',
+                query: query,
+                top_n: topK,
+                documents: documents,
+            };
+            const response = await axios.post('https://api.cohere.com/v2/rerank', requestData, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+            });
+            // Log the response data structure
+            console.log('Cohere API response structure:');
+            console.log('ID:', response.data?.id);
+            console.log('Meta:', response.data?.meta);
+            console.log('Results count:', response.data?.results.length);
+            // Log a sample of the results
+            if ((response.data?.results.length ?? 0) > 0) {
+                console.log('Sample result:', JSON.stringify(response.data?.results[0], null, 2));
+            }
+            if (response.data && response.data.results.length) {
+                return response.data.results.map((result) => {
+                    const docIndex = result.index;
+                    const score = result.relevance_score;
+                    const text = documents[docIndex];
+                    return { text, score };
+                });
+            }
+            else {
+                console.warn('Unexpected response format from Cohere API. Using default ranking.');
+                return this.getDefaultRanking(documents, topK);
+            }
+        }
+        catch (error) {
+            console.error('Error using Cohere reranker:', error);
+            // Fallback to default ranking on error
+            return this.getDefaultRanking(documents, topK);
+        }
+    }
+}
+class InfinityReranker extends BaseReranker {
+    constructor() {
+        super();
+        // No API key needed for the placeholder implementation
+    }
+    async rerank(query, documents, topK = 5) {
+        console.log(`Reranking ${documents.length} documents with Infinity (placeholder)`);
+        // This would be replaced with actual Infinity reranker implementation
+        return this.getDefaultRanking(documents, topK);
+    }
+}
+/**
+ * Creates the appropriate reranker based on type and configuration
+ */
+const createReranker = (config) => {
+    const { rerankerType, jinaApiKey, cohereApiKey } = config;
+    switch (rerankerType.toLowerCase()) {
+        case 'jina':
+            return new JinaReranker({ apiKey: jinaApiKey });
+        case 'cohere':
+            return new CohereReranker({ apiKey: cohereApiKey });
+        case 'infinity':
+            return new InfinityReranker();
+        case 'none':
+            console.log('Skipping reranking as reranker is set to "none"');
+            return undefined;
+        default:
+            console.warn(`Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`);
+            return new JinaReranker({ apiKey: jinaApiKey });
+    }
+};
+// Example usage:
+// const jinaReranker = new JinaReranker();
+// const cohereReranker = new CohereReranker();
+// const infinityReranker = new InfinityReranker();
+
+exports.BaseReranker = BaseReranker;
+exports.CohereReranker = CohereReranker;
+exports.InfinityReranker = InfinityReranker;
+exports.JinaReranker = JinaReranker;
+exports.createReranker = createReranker;
+//# sourceMappingURL=rerankers.cjs.map

package/dist/cjs/tools/search/rerankers.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerankers.cjs","sources":["../../../../src/tools/search/rerankers.ts"],"sourcesContent":["/* eslint-disable no-console */\nimport axios from 'axios';\nimport type * as t from './types';\n\nexport abstract class BaseReranker {\n  protected apiKey: string | undefined;\n\n  constructor() {\n    // Each specific reranker will set its API key\n  }\n\n  abstract rerank(\n    query: string,\n    documents: string[],\n    topK?: number\n  ): Promise<t.Highlight[]>;\n\n  protected getDefaultRanking(\n    documents: string[],\n    topK: number\n  ): t.Highlight[] {\n    return documents\n      .slice(0, Math.min(topK, documents.length))\n      .map((doc) => ({ text: doc, score: 0 }));\n  }\n\n  protected logDocumentSamples(documents: string[]): void {\n    console.log('Sample documents being sent to API:');\n    for (let i = 0; i < Math.min(3, documents.length); i++) {\n      console.log(`Document ${i}: ${documents[i].substring(0, 100)}...`);\n    }\n  }\n}\n\nexport class JinaReranker extends BaseReranker {\n  constructor({ apiKey = process.env.JINA_API_KEY }: { apiKey?: string }) {\n    super();\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    console.log(`Reranking ${documents.length} documents with Jina`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        console.warn('JINA_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      this.logDocumentSamples(documents);\n\n      const requestData = {\n        model: 'jina-reranker-v2-base-multilingual',\n        query: query,\n        top_n: topK,\n        documents: documents,\n        return_documents: true,\n      };\n\n      const response = await axios.post<t.JinaRerankerResponse | undefined>(\n        'https://api.jina.ai/v1/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      // Log the response data structure\n      console.log('Jina API response structure:');\n      console.log('Model:', response.data?.model);\n      console.log('Usage:', response.data?.usage);\n      console.log('Results count:', response.data?.results.length);\n\n      // Log a sample of the results\n      if ((response.data?.results.length ?? 0) > 0) {\n        console.log(\n          'Sample result:',\n          JSON.stringify(response.data?.results[0], null, 2)\n        );\n      }\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          let text = '';\n\n          // If return_documents is true, the document field will be present\n          if (result.document != null) {\n            const doc = result.document;\n            if (typeof doc === 'object' && 'text' in doc) {\n              text = doc.text;\n            } else if (typeof doc === 'string') {\n              text = doc;\n            }\n          } else {\n            // Otherwise, use the index to get the document\n            text = documents[docIndex];\n          }\n\n          return { text, score };\n        });\n      } else {\n        console.warn(\n          'Unexpected response format from Jina API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      console.error('Error using Jina reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class CohereReranker extends BaseReranker {\n  constructor({ apiKey = process.env.COHERE_API_KEY }: { apiKey?: string }) {\n    super();\n    this.apiKey = apiKey;\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    console.log(`Reranking ${documents.length} documents with Cohere`);\n\n    try {\n      if (this.apiKey == null || this.apiKey === '') {\n        console.warn('COHERE_API_KEY is not set. Using default ranking.');\n        return this.getDefaultRanking(documents, topK);\n      }\n\n      this.logDocumentSamples(documents);\n\n      const requestData = {\n        model: 'rerank-v3.5',\n        query: query,\n        top_n: topK,\n        documents: documents,\n      };\n\n      const response = await axios.post<t.CohereRerankerResponse | undefined>(\n        'https://api.cohere.com/v2/rerank',\n        requestData,\n        {\n          headers: {\n            'Content-Type': 'application/json',\n            Authorization: `Bearer ${this.apiKey}`,\n          },\n        }\n      );\n\n      // Log the response data structure\n      console.log('Cohere API response structure:');\n      console.log('ID:', response.data?.id);\n      console.log('Meta:', response.data?.meta);\n      console.log('Results count:', response.data?.results.length);\n\n      // Log a sample of the results\n      if ((response.data?.results.length ?? 0) > 0) {\n        console.log(\n          'Sample result:',\n          JSON.stringify(response.data?.results[0], null, 2)\n        );\n      }\n\n      if (response.data && response.data.results.length) {\n        return response.data.results.map((result) => {\n          const docIndex = result.index;\n          const score = result.relevance_score;\n          const text = documents[docIndex];\n          return { text, score };\n        });\n      } else {\n        console.warn(\n          'Unexpected response format from Cohere API. Using default ranking.'\n        );\n        return this.getDefaultRanking(documents, topK);\n      }\n    } catch (error) {\n      console.error('Error using Cohere reranker:', error);\n      // Fallback to default ranking on error\n      return this.getDefaultRanking(documents, topK);\n    }\n  }\n}\n\nexport class InfinityReranker extends BaseReranker {\n  constructor() {\n    super();\n    // No API key needed for the placeholder implementation\n  }\n\n  async rerank(\n    query: string,\n    documents: string[],\n    topK: number = 5\n  ): Promise<t.Highlight[]> {\n    console.log(\n      `Reranking ${documents.length} documents with Infinity (placeholder)`\n    );\n    // This would be replaced with actual Infinity reranker implementation\n    return this.getDefaultRanking(documents, topK);\n  }\n}\n\n/**\n * Creates the appropriate reranker based on type and configuration\n */\nexport const createReranker = (config: {\n  rerankerType: t.RerankerType;\n  jinaApiKey?: string;\n  cohereApiKey?: string;\n}): BaseReranker | undefined => {\n  const { rerankerType, jinaApiKey, cohereApiKey } = config;\n\n  switch (rerankerType.toLowerCase()) {\n  case 'jina':\n    return new JinaReranker({ apiKey: jinaApiKey });\n  case 'cohere':\n    return new CohereReranker({ apiKey: cohereApiKey });\n  case 'infinity':\n    return new InfinityReranker();\n  case 'none':\n    console.log('Skipping reranking as reranker is set to \"none\"');\n    return undefined;\n  default:\n    console.warn(\n      `Unknown reranker type: ${rerankerType}. Defaulting to InfinityReranker.`\n    );\n    return new JinaReranker({ apiKey: jinaApiKey });\n  }\n};\n\n// Example usage:\n// const jinaReranker = new JinaReranker();\n// const cohereReranker = new CohereReranker();\n// const infinityReranker = new InfinityReranker();\n"],"names":[],"mappings":";;;;AAAA;MAIsB,YAAY,CAAA;AACtB,IAAA,MAAM;AAEhB,IAAA,WAAA,GAAA;;;IAUU,iBAAiB,CACzB,SAAmB,EACnB,IAAY,EAAA;AAEZ,QAAA,OAAO;AACJ,aAAA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC;AACzC,aAAA,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;;AAGlC,IAAA,kBAAkB,CAAC,SAAmB,EAAA;AAC9C,QAAA,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE;AACtD,YAAA,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA,EAAA,EAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA,GAAA,CAAK,CAAC;;;AAGvE;AAEK,MAAO,YAAa,SAAQ,YAAY,CAAA;IAC5C,WAAY,CAAA,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,EAAuB,EAAA;AACpE,QAAA,KAAK,EAAE;AACP,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,OAAO,CAAC,GAAG,CAAC,CAAA,UAAA,EAAa,SAAS,CAAC,MAAM,CAAsB,oBAAA,CAAA,CAAC;AAEhE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,OAAO,CAAC,IAAI,CAAC,iDAAiD,CAAC;gBAC/D,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;AAElC,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,oCAAoC;AAC3C,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;AACpB,gBAAA,gBAAgB,EAAE,IAAI;aACvB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,+BAA+B,EAC/B,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;;AAGD,YAAA,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;AAC3C,YAAA,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC;;AAG5D,YAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC5C,OAAO,CAAC,GAAG,CACT,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CACnD;;AAGH,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;oBACpC,IAAI,IAAI,GAAG,EAAE;;AAGb,oBAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,EAAE;AAC3B,wBAAA,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ;wBAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,EAAE;AAC5C,4BAAA,IAAI,GAAG,GAAG,CAAC,IAAI;;AACV,6BAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;4BAClC,IAAI,GAAG,GAAG;;;yBAEP;;AAEL,wBAAA,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;;AAG5B,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,OAAO,CAAC,IAAI,CACV,kEAAkE,CACnE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC;;YAElD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,cAAe,SAAQ,YAAY,CAAA;IAC9C,WAAY,CAAA,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAuB,EAAA;AACtE,QAAA,KAAK,EAAE;AACP,QAAA,IAAI,CAAC,MAAM,GAAG,MAAM;;IAGtB,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,OAAO,CAAC,GAAG,CAAC,CAAA,UAAA,EAAa,SAAS,CAAC,MAAM,CAAwB,sBAAA,CAAA,CAAC;AAElE,QAAA,IAAI;AACF,YAAA,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,EAAE,EAAE;AAC7C,gBAAA,OAAO,CAAC,IAAI,CAAC,mDAAmD,CAAC;gBACjE,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAGhD,YAAA,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC;AAElC,YAAA,MAAM,WAAW,GAAG;AAClB,gBAAA,KAAK,EAAE,aAAa;AACpB,gBAAA,KAAK,EAAE,KAAK;AACZ,gBAAA,KAAK,EAAE,IAAI;AACX,gBAAA,SAAS,EAAE,SAAS;aACrB;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,kCAAkC,EAClC,WAAW,EACX;AACE,gBAAA,OAAO,EAAE;AACP,oBAAA,cAAc,EAAE,kBAAkB;AAClC,oBAAA,aAAa,EAAE,CAAA,OAAA,EAAU,IAAI,CAAC,MAAM,CAAE,CAAA;AACvC,iBAAA;AACF,aAAA,CACF;;AAGD,YAAA,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;AACzC,YAAA,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC;;AAG5D,YAAA,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC5C,OAAO,CAAC,GAAG,CACT,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CACnD;;AAGH,YAAA,IAAI,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;gBACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAI;AAC1C,oBAAA,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK;AAC7B,oBAAA,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe;AACpC,oBAAA,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;AAChC,oBAAA,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE;AACxB,iBAAC,CAAC;;iBACG;AACL,gBAAA,OAAO,CAAC,IAAI,CACV,oEAAoE,CACrE;gBACD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;QAEhD,OAAO,KAAK,EAAE;AACd,YAAA,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC;;YAEpD,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;;AAGnD;AAEK,MAAO,gBAAiB,SAAQ,YAAY,CAAA;AAChD,IAAA,WAAA,GAAA;AACE,QAAA,KAAK,EAAE;;;IAIT,MAAM,MAAM,CACV,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC,EAAA;QAEhB,OAAO,CAAC,GAAG,CACT,CAAA,UAAA,EAAa,SAAS,CAAC,MAAM,CAAwC,sCAAA,CAAA,CACtE;;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC;;AAEjD;AAED;;AAEG;AACU,MAAA,cAAc,GAAG,CAAC,MAI9B,KAA8B;IAC7B,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,MAAM;AAEzD,IAAA,QAAQ,YAAY,CAAC,WAAW,EAAE;AAClC,QAAA,KAAK,MAAM;YACT,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AACjD,QAAA,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AACrD,QAAA,KAAK,UAAU;YACb,OAAO,IAAI,gBAAgB,EAAE;AAC/B,QAAA,KAAK,MAAM;AACT,YAAA,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC;AAC9D,YAAA,OAAO,SAAS;AAClB,QAAA;AACE,YAAA,OAAO,CAAC,IAAI,CACV,0BAA0B,YAAY,CAAA,iCAAA,CAAmC,CAC1E;YACD,OAAO,IAAI,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;;AAEnD;AAEA;AACA;AACA;AACA;;;;;;;;"}

package/dist/cjs/tools/search/search.cjs

@@ -0,0 +1,410 @@
+'use strict';
+
+var axios = require('axios');
+var textsplitters = require('@langchain/textsplitters');
+var firecrawl = require('./firecrawl.cjs');
+
+/* eslint-disable no-console */
+const chunker = {
+    cleanText: (text) => {
+        if (!text)
+            return '';
+        /** Normalized all line endings to '\n' */
+        const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+        /** Handle multiple backslashes followed by newlines
+         * This replaces patterns like '\\\\\\n' with a single newline */
+        const fixedBackslashes = normalizedText.replace(/\\+\n/g, '\n');
+        /** Cleaned up consecutive newlines, tabs, and spaces around newlines */
+        const cleanedNewlines = fixedBackslashes.replace(/[\t ]*\n[\t \n]*/g, '\n');
+        /** Cleaned up excessive spaces and tabs */
+        const cleanedSpaces = cleanedNewlines.replace(/[ \t]+/g, ' ');
+        return cleanedSpaces.trim();
+    },
+    splitText: async (text, options) => {
+        const chunkSize = options?.chunkSize ?? 150;
+        const chunkOverlap = options?.chunkOverlap ?? 50;
+        const separators = options?.separators || ['\n\n', '\n'];
+        const splitter = new textsplitters.RecursiveCharacterTextSplitter({
+            separators,
+            chunkSize,
+            chunkOverlap,
+        });
+        return await splitter.splitText(text);
+    },
+    splitTexts: async (texts, options) => {
+        // Split multiple texts
+        const promises = texts.map((text) => chunker.splitText(text, options).catch((error) => {
+            console.error('Error splitting text:', error);
+            return [text];
+        }));
+        return Promise.all(promises);
+    },
+};
+const createSourceUpdateCallback = (sourceMap) => {
+    return (link, update) => {
+        const source = sourceMap.get(link);
+        if (source) {
+            sourceMap.set(link, {
+                ...source,
+                ...update,
+            });
+        }
+    };
+};
+const getHighlights = async ({ query, content, reranker, topResults = 5, }) => {
+    if (!content) {
+        console.warn('No content provided for highlights');
+        return;
+    }
+    if (!reranker) {
+        console.warn('No reranker provided for highlights');
+        return;
+    }
+    try {
+        const documents = await chunker.splitText(content);
+        if (Array.isArray(documents)) {
+            return await reranker.rerank(query, documents, topResults);
+        }
+        else {
+            console.error('Expected documents to be an array, got:', typeof documents);
+            return;
+        }
+    }
+    catch (error) {
+        console.error('Error in content processing:', error);
+        return;
+    }
+};
+const createSerperAPI = (apiKey) => {
+    const config = {
+        apiKey: apiKey ?? process.env.SERPER_API_KEY,
+        apiUrl: 'https://google.serper.dev/search',
+        defaultLocation: 'us',
+        timeout: 10000,
+    };
+    if (config.apiKey == null || config.apiKey === '') {
+        throw new Error('SERPER_API_KEY is required for SerperAPI');
+    }
+    const getSources = async (query, numResults = 8, storedLocation) => {
+        if (!query.trim()) {
+            return { success: false, error: 'Query cannot be empty' };
+        }
+        try {
+            const searchLocation = (storedLocation ?? config.defaultLocation).toLowerCase();
+            const payload = {
+                q: query,
+                num: Math.min(Math.max(1, numResults), 10),
+                gl: searchLocation,
+            };
+            const response = await axios.post(config.apiUrl, payload, {
+                headers: {
+                    'X-API-KEY': config.apiKey,
+                    'Content-Type': 'application/json',
+                },
+                timeout: config.timeout,
+            });
+            const data = response.data;
+            const results = {
+                organic: data.organic,
+                images: data.images ?? [],
+                topStories: data.topStories ?? [],
+                knowledgeGraph: data.knowledgeGraph,
+                answerBox: data.answerBox,
+                peopleAlsoAsk: data.peopleAlsoAsk,
+                relatedSearches: data.relatedSearches,
+            };
+            return { success: true, data: results };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return { success: false, error: `API request failed: ${errorMessage}` };
+        }
+    };
+    return { getSources };
+};
+const createSearXNGAPI = (instanceUrl, apiKey) => {
+    const config = {
+        instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,
+        apiKey: apiKey ?? process.env.SEARXNG_API_KEY,
+        timeout: 10000,
+    };
+    if (config.instanceUrl == null || config.instanceUrl === '') {
+        throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');
+    }
+    const getSources = async (query, numResults = 8, storedLocation) => {
+        if (!query.trim()) {
+            return { success: false, error: 'Query cannot be empty' };
+        }
+        try {
+            // Ensure the instance URL ends with /search
+            if (config.instanceUrl == null || config.instanceUrl === '') {
+                return { success: false, error: 'Instance URL is not defined' };
+            }
+            let searchUrl = config.instanceUrl;
+            if (!searchUrl.endsWith('/search')) {
+                searchUrl = searchUrl.replace(/\/$/, '') + '/search';
+            }
+            // Prepare parameters for SearXNG
+            const params = {
+                q: query,
+                format: 'json',
+                pageno: 1,
+                categories: 'general',
+                language: 'all',
+                safesearch: 0,
+                engines: 'google,bing,duckduckgo',
+                max_results: Math.min(Math.max(1, numResults), 20),
+            };
+            if (storedLocation != null && storedLocation !== 'all') {
+                params.language = storedLocation;
+            }
+            const headers = {
+                'Content-Type': 'application/json',
+            };
+            if (config.apiKey != null && config.apiKey !== '') {
+                headers['X-API-Key'] = config.apiKey;
+            }
+            const response = await axios.get(searchUrl, {
+                headers,
+                params,
+                timeout: config.timeout,
+            });
+            const data = response.data;
+            // Transform SearXNG results to match SerperAPI format
+            const organicResults = (data.results ?? [])
+                .slice(0, numResults)
+                .map((result) => ({
+                title: result.title ?? '',
+                link: result.url ?? '',
+                snippet: result.content ?? '',
+                date: result.publishedDate ?? '',
+            }));
+            // Extract image results if available
+            const imageResults = (data.results ?? [])
+                .filter((result) => result.img_src)
+                .slice(0, 6)
+                .map((result) => ({
+                title: result.title ?? '',
+                imageUrl: result.img_src ?? '',
+            }));
+            // Format results to match SerperAPI structure
+            const results = {
+                organic: organicResults,
+                images: imageResults,
+                topStories: [],
+                // Use undefined instead of null for optional properties
+                relatedSearches: data.suggestions ?? [],
+            };
+            return { success: true, data: results };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                success: false,
+                error: `SearXNG API request failed: ${errorMessage}`,
+            };
+        }
+    };
+    return { getSources };
+};
+const createSearchAPI = (config) => {
+    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, } = config;
+    if (searchProvider.toLowerCase() === 'serper') {
+        return createSerperAPI(serperApiKey);
+    }
+    else if (searchProvider.toLowerCase() === 'searxng') {
+        return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
+    }
+    else {
+        throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper' or 'searxng'`);
+    }
+};
+const createSourceProcessor = (config = {}, scraperInstance) => {
+    if (!scraperInstance) {
+        throw new Error('Firecrawl scraper instance is required');
+    }
+    const { topResults = 5, 
+    // strategies = ['no_extraction'],
+    // filterContent = true,
+    reranker, } = config;
+    const firecrawlScraper = scraperInstance;
+    const webScraper = {
+        scrapeMany: async ({ query, links, }) => {
+            console.log(`Scraping ${links.length} links with Firecrawl`);
+            const promises = [];
+            try {
+                for (const currentLink of links) {
+                    const promise = firecrawlScraper
+                        .scrapeUrl(currentLink, {})
+                        .then(([url, response]) => {
+                        const attribution = firecrawl.getAttribution(url, response.data?.metadata);
+                        if (response.success && response.data) {
+                            const content = firecrawlScraper.extractContent(response);
+                            return {
+                                url,
+                                attribution,
+                                content: chunker.cleanText(content),
+                            };
+                        }
+                        return {
+                            url,
+                            attribution,
+                            error: true,
+                            content: `Failed to scrape ${url}: ${response.error ?? 'Unknown error'}`,
+                        };
+                    })
+                        .then(async (result) => {
+                        try {
+                            if (result.error != null) {
+                                console.error(`Error scraping ${result.url}: ${result.content}`);
+                                return {
+                                    ...result,
+                                };
+                            }
+                            const highlights = await getHighlights({
+                                query,
+                                reranker,
+                                content: result.content,
+                            });
+                            return {
+                                ...result,
+                                highlights,
+                            };
+                        }
+                        catch (error) {
+                            console.error('Error processing scraped content:', error);
+                            return {
+                                ...result,
+                            };
+                        }
+                    })
+                        .catch((error) => {
+                        console.error(`Error scraping ${currentLink}:`, error);
+                        return {
+                            url: currentLink,
+                            error: true,
+                            content: `Failed to scrape ${currentLink}: ${error.message ?? 'Unknown error'}`,
+                        };
+                    });
+                    promises.push(promise);
+                }
+                return await Promise.all(promises);
+            }
+            catch (error) {
+                console.error('Error in scrapeMany:', error);
+                return [];
+            }
+        },
+    };
+    const fetchContents = async ({ links, query, target, onContentScraped, }) => {
+        const initialLinks = links.slice(0, target);
+        // const remainingLinks = links.slice(target).reverse();
+        const results = await webScraper.scrapeMany({ query, links: initialLinks });
+        for (const result of results) {
+            if (result.error === true) {
+                continue;
+            }
+            const { url, content, attribution, highlights } = result;
+            onContentScraped?.(url, {
+                content,
+                attribution,
+                highlights,
+            });
+        }
+    };
+    const processSources = async (result, numElements, query, proMode = false) => {
+        try {
+            if (!result.data) {
+                return {
+                    organic: [],
+                    topStories: [],
+                    images: [],
+                    relatedSearches: [],
+                };
+            }
+            else if (!result.data.organic) {
+                return result.data;
+            }
+            if (!proMode) {
+                const wikiSources = result.data.organic.filter((source) => source.link.includes('wikipedia.org'));
+                if (!wikiSources.length) {
+                    return result.data;
+                }
+                const wikiSourceMap = new Map();
+                wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);
+                const onContentScraped = createSourceUpdateCallback(wikiSourceMap);
+                await fetchContents({
+                    query,
+                    target: 1,
+                    onContentScraped,
+                    links: [wikiSources[0].link],
+                });
+                for (let i = 0; i < result.data.organic.length; i++) {
+                    const source = result.data.organic[i];
+                    const updatedSource = wikiSourceMap.get(source.link);
+                    if (updatedSource) {
+                        result.data.organic[i] = {
+                            ...source,
+                            ...updatedSource,
+                        };
+                    }
+                }
+                return result.data;
+            }
+            const sourceMap = new Map();
+            const allLinks = [];
+            for (const source of result.data.organic) {
+                if (source.link) {
+                    allLinks.push(source.link);
+                    sourceMap.set(source.link, source);
+                }
+            }
+            if (allLinks.length === 0) {
+                return result.data;
+            }
+            const onContentScraped = createSourceUpdateCallback(sourceMap);
+            await fetchContents({
+                links: allLinks,
+                query,
+                onContentScraped,
+                target: numElements,
+            });
+            for (let i = 0; i < result.data.organic.length; i++) {
+                const source = result.data.organic[i];
+                const updatedSource = sourceMap.get(source.link);
+                if (updatedSource) {
+                    result.data.organic[i] = {
+                        ...source,
+                        ...updatedSource,
+                    };
+                }
+            }
+            const successfulSources = result.data.organic
+                .filter((source) => source.content != null && !source.content.startsWith('Failed'))
+                .slice(0, numElements);
+            if (successfulSources.length > 0) {
+                result.data.organic = successfulSources;
+            }
+            return result.data;
+        }
+        catch (error) {
+            console.error('Error in processSources:', error);
+            return {
+                organic: [],
+                topStories: [],
+                images: [],
+                relatedSearches: [],
+                ...result.data,
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    };
+    return {
+        processSources,
+        topResults,
+    };
+};
+
+exports.createSearchAPI = createSearchAPI;
+exports.createSourceProcessor = createSourceProcessor;
+//# sourceMappingURL=search.cjs.map