@tiledesk/tiledesk-server 2.10.94 → 2.10.96

package/CHANGELOG.md

@@ -5,6 +5,17 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.10.96
+- Added: support for hybrid search
+- Added: support for chunks_only option
+- Added: support for native logs
+- Bug fix: unexpceted indexing run of all urls in the project
+
+# 2.10.95 (Aborted)
+- Added: support for hybrid search
+- Added: support for chunks_only option
+- Added: support for native logs
+
 # 2.10.94
 - Update: fix uncaughtException in tagSchedule method
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.10.94",
+  "version": "2.10.96",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -47,13 +47,13 @@
     "@tiledesk/tiledesk-messenger-connector": "^0.1.23",
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
-    "@tiledesk/tiledesk-tybot-connector": "^2.0.15",
+    "@tiledesk/tiledesk-tybot-connector": "^2.0.19",
     "@tiledesk/tiledesk-whatsapp-connector": "^0.1.83",
     "@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.12",
     "@tiledesk/tiledesk-sms-connector": "^0.1.11",
     "@tiledesk/tiledesk-vxml-connector": "^0.1.76",
     "@tiledesk/tiledesk-voice-twilio-connector": "^0.1.22",
-    "@tiledesk/tiledesk-multi-worker": "^0.3.1",
+    "@tiledesk/tiledesk-multi-worker": "^0.3.2",
     "passport-oauth2": "^1.8.0",
     "amqplib": "^0.5.5",
     "app-root-path": "^3.0.0",

package/routes/kb.js

@@ -57,8 +57,9 @@ let default_preview_settings = {
   max_tokens: 256,
   temperature: 0.7,
   top_k: 4,
-  //context: "You are an awesome AI Assistant."
+  alpha: 0.5,
   context: null
+  //context: "You are an awesome AI Assistant."
 }
 let default_engine = {
   name: "pinecone",
@@ -93,10 +94,6 @@ router.post('/scrape/single', async (req, res) => {
   let data = req.body;
   winston.debug("/scrape/single data: ", data);
 
-  // if (!data.scrape_type) {
-  //   data.scrape_type = 1;
-  // }
-
   let namespaces = await Namespace.find({ id_project: project_id }).catch((err) => {
     winston.error("find namespaces error: ", err)
     res.status(500).send({ success: false, error: err })
@@ -137,11 +134,6 @@ router.post('/scrape/single', async (req, res) => {
         json.content = kb.content;
       }
 
-      // json.scrape_type = 1;
-      // if (data.scrape_type) {
-      //   json.scrape_type = data.scrape_type;
-      // }
-
       if (kb.scrape_type) {
         json.scrape_type = kb.scrape_type
       }
@@ -157,6 +149,10 @@ router.post('/scrape/single', async (req, res) => {
       let ns = namespaces.find(n => n.id === kb.namespace);
       json.engine = ns.engine || default_engine;
 
+      if (json.engine.type === 'serverless') {
+        json.hybrid = true;
+      }
+
       winston.verbose("/scrape/single json: ", json);
 
       startScrape(json).then((response) => {
@@ -302,12 +298,17 @@ router.post('/qa', async (req, res) => {
 
   let ns = namespaces.find(n => n.id === data.namespace);
   data.engine = ns.engine || default_engine;
+
+  if (data.engine.type === 'serverless') {
+    data.search_type = 'hybrid';
+  }
+
   
   delete data.advancedPrompt;
   winston.verbose("ask data: ", data);
   
   if (process.env.NODE_ENV === 'test') {
-    return res.status(200).send({ success: true, message: "Question skipped in test environment"});
+    return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
   }
 
   data.debug = true;
@@ -1217,6 +1218,10 @@ router.post('/', async (req, res) => {
       let ns = namespaces.find(n => n.id === body.namespace);
       json.engine = ns.engine || default_engine;
 
+      if (json.engine.type === 'serverless') {
+        json.hybrid = true;
+      }
+
       let resources = [];
 
       resources.push(json);
@@ -1336,14 +1341,19 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
     }
   })
 
-  saveBulk(operations, kbs, project_id).then((result) => {
+  saveBulk(operations, kbs, project_id, namespace_id).then((result) => {
 
     let ns = namespaces.find(n => n.id === namespace_id);
     let engine = ns.engine || default_engine;
 
+    let hybrid;
+    if (engine.type === 'serverless') {
+      hybrid = true;
+    }
+
     let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
     resources = resources.map(({ _id, scrape_options, ...rest }) => {
-      return { id: _id, webhook: webhook, parameters_scrape_type_4: scrape_options, engine: engine, ...rest}
+      return { id: _id, webhook: webhook, parameters_scrape_type_4: scrape_options, engine: engine, hybrid: hybrid, ...rest}
     });
     winston.verbose("resources to be sent to worker: ", resources);
 
@@ -1446,7 +1456,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
         }
       })
 
-      saveBulk(operations, kbs, project_id).then((result) => {
+      saveBulk(operations, kbs, project_id, namespace_id).then((result) => {
 
         let ns = namespaces.find(n => n.id === namespace_id);
         let engine = ns.engine || default_engine;
@@ -1613,13 +1623,13 @@ router.delete('/:kb_id', async (req, res) => {
 * ****************************************
 */
 
-async function saveBulk(operations, kbs, project_id) {
+async function saveBulk(operations, kbs, project_id, namespace) {
 
   return new Promise((resolve, reject) => {
     KB.bulkWrite(operations, { ordered: false }).then((result) => {
       winston.verbose("bulkWrite operations result: ", result);
 
-      KB.find({ id_project: project_id, source: { $in: kbs.map(kb => kb.source) } }).lean().then((documents) => {
+      KB.find({ id_project: project_id, namespace: namespace, source: { $in: kbs.map(kb => kb.source) } }).lean().then((documents) => {
         winston.debug("documents: ", documents);
         resolve(documents)
       }).catch((err) => {

package/services/logsService.js

@@ -3,7 +3,7 @@
 const { FlowLogs } = require("../models/flowLogs");
 const default_log_level = 'info';
 
-const levels = { error: 0, warn: 1, info: 2, debug: 3 };
+const levels = { error: 0, warn: 1, info: 2, debug: 3, native: 4 };
 
 class LogsService {
 
@@ -16,7 +16,7 @@ class LogsService {
         return FlowLogs.aggregate([
             { $match: { request_id: request_id } },
             { $unwind: "$rows" },
-            { $match: { "rows.nlevel": { $lt: nlevel } } },
+            { $match: { "rows.nlevel": { $lte: nlevel } } },
             { $sort: { "rows.timestamp": -1, "rows._id": -1 } },
             { $limit: limit }
         ]).then(rows => rows.reverse())
@@ -31,7 +31,7 @@ class LogsService {
         return FlowLogs.aggregate([
             { $match: { request_id: request_id } },
             { $unwind: "$rows" },
-            { $match: { "rows.nlevel": { $lt: nlevel }, "rows.timestamp": { $lt: timestamp } } },
+            { $match: { "rows.nlevel": { $lte: nlevel }, "rows.timestamp": { $lt: timestamp } } },
             { $sort: { "rows.timestamp": -1, "rows._id": -1 } },
             { $limit: limit }
         ]).then(rows => rows.reverse())
@@ -46,7 +46,7 @@ class LogsService {
         return FlowLogs.aggregate([
             { $match: { request_id: request_id } },
             { $unwind: "$rows" },
-            { $match: { "rows.nlevel": { $lt: nlevel }, "rows.timestamp": { $gt: timestamp } } },
+            { $match: { "rows.nlevel": { $lte: nlevel }, "rows.timestamp": { $gt: timestamp } } },
             { $sort: { "rows.timestamp": 1, "rows._id": 1 } },
             { $limit: limit }
         ])

package/services/webhookService.js

@@ -44,6 +44,7 @@ class WebhookService {
                 }
                 let json_value = JSON.parse(value);
                 payload.preloaded_request_id = json_value.request_id;
+                payload.draft = true;
             }
 
             let token = await this.generateChatbotToken(chatbot);

package/test/kbRoute.js

@@ -1,10 +1,15 @@
 //During the test the env variable is set to test
 process.env.NODE_ENV = 'test';
 process.env.GPTKEY = "fakegptkey";
-process.env.KB_WEBHOOK_TOKEN = "testtoken"
-process.env.PINECONE_INDEX = "test_index"
-process.env.PINECONE_TYPE = "pod"
 process.env.LOG_LEVEL = 'critical'
+process.env.KB_WEBHOOK_TOKEN = "testtoken"
+// Similarity
+// process.env.PINECONE_INDEX = "test_index"
+// process.env.PINECONE_TYPE = "pod"
+// Hybrid
+process.env.PINECONE_INDEX = "test_hybrid_index"
+process.env.PINECONE_TYPE = "serverless"
+
 
 var userService = require('../services/userService');
 var projectService = require('../services/projectService');
@@ -58,7 +63,7 @@ describe('KbRoute', () => {
 
                             res.should.have.status(200);
                             expect(res.body.length).to.equal(1);
-                            expect(res.body[0].engine.index_name).to.equal('test_index')
+                            expect(res.body[0].engine.index_name).to.equal('test_hybrid_index')
 
                             let namespace_id = res.body[0].id;
 
@@ -813,6 +818,49 @@ describe('KbRoute', () => {
             })
         }).timeout(10000)
 
+        it('askkb-with-hybrid-search', (done) => {
+
+            var email = "test-signup-" + Date.now() + "@email.com";
+            var pwd = "pwd";
+
+            userService.signup(email, pwd, "Test Firstname", "Test Lastname").then((savedUser) => {
+                projectService.create("test-kb-qa", savedUser._id).then((savedProject) => {
+
+                    chai.request(server)
+                        .get('/' + savedProject._id + '/kb/namespace/all')
+                        .auth(email, pwd)
+                        .end((err, res) => {
+
+                            if (err) { console.error("err: ", err); }
+                            if (log) { console.log("get all namespaces res.body: ", res.body); }
+
+                            res.should.have.status(200)
+                            expect(res.body.length).to.equal(1);
+                            expect(res.body[0].type === "serverless");
+
+
+                            chai.request(server)
+                                .post('/' + savedProject._id + "/kb/qa")
+                                .auth(email, pwd)
+                                .send({ model: "gpt-4o", namespace: savedProject._id, question: "sample question", advancedPrompt: true, system_context: "You are a robot coming from future" })
+                                .end((err, res) => {
+
+                                    if (err) { console.error("err: ", err) };
+                                    if (log) { console.log("res.body: ", res.body) };
+
+                                    res.should.have.status(200);
+                                    expect(res.body.data);
+                                    expect(res.body.data.search_type === "hybrid");
+
+                                    done();
+                                })
+
+
+                        })
+                })
+            })
+        }).timeout(10000)
+
         it('webhook', (done) => {
 
             var email = "test-signup-" + Date.now() + "@email.com";
@@ -1096,7 +1144,7 @@ describe('KbRoute', () => {
                         expect(res.body.name).to.equal('MyCustomNamespace');
                         should.exist(res.body.engine)
                         expect(res.body.engine.name).to.equal('pinecone');
-                        expect(res.body.engine.type).to.equal('pod');
+                        expect(res.body.engine.type).to.equal('serverless');
 
                         // Get again all namespace. A new default namespace should not be created.
                         chai.request(server)

package/test/openaiRoute.js

@@ -1,6 +1,5 @@
 //During the test the env variable is set to test
 process.env.NODE_ENV = 'test';
-process.env.GPTKEY = '';
 //process.env.AI_MODELS = 'gpt-3.5-turbo:0.6;gpt-4:25;gpt-4-turbo-preview:12;gpt-4o:6'
 //process.env.AI_MODELS = 'gpt-3.5-turbo;gpt-4-turbo-preview;gpt-4o'
 process.env.AI_MODELS = '    gpt-3.5-turbo:0   .6;gpt -4:2  5; g  pt-4-tur   bo-preview:12;gp   t-4o:6'
@@ -110,6 +109,8 @@ describe('OpenaiRoute', () => {
 
         it('newCompletionsMissingGptkey', (done) => {
 
+            let original_value = process.env.GPTKEY;
+            process.env.GPTKEY = "";
             var email = "test-signup-" + Date.now() + "@email.com";
             var pwd = "pwd";
 
@@ -130,6 +131,8 @@ describe('OpenaiRoute', () => {
                             expect(res.body.success).to.equal(false);
                             expect(res.body.message).to.equal("Missing gptkey parameter");
 
+                            process.env.GPTKEY = original_value;
+
                             done();
                         })