@tiledesk/tiledesk-server 2.18.3 → 2.18.4

package/CHANGELOG.md

@@ -5,6 +5,12 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.18.4
+- Added HyDE support for Knowledge Base Q&A
+- Introduced Cache (cRag) functionality in Knowledge Base Q&A
+- Enabled Situated Context for enhanced Knowledge Base content indexing
+- Bumped tybot-connector version to 2.0.48
+
 # 2.18.3
 - Added permissions logic
 - Added custom roles support

package/config/kb/situatedContext.js

@@ -1,6 +1,6 @@
 module.exports = {
     enable: process.env.SITUATED_CONTEXT_ENABLE === "true",
     provider: process.env.SITUATED_CONTEXT_PROVIDER || "openai",
-    model: process.env.SITUATED_CONTEXT_MODEL || "gpt-4o",
+    model: process.env.SITUATED_CONTEXT_MODEL || "gpt-5.4-nano",
     api_key: ""
 }

package/models/kb_setting.js

@@ -198,6 +198,11 @@ var KBSchema = new Schema({
     type: Array,
     default: undefined,
     required: false
+  },
+  situated_context: {
+    type: Boolean,
+    default: false,
+    required: false
   }
 }, {
   timestamps: true

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.18.3",
+  "version": "2.18.4",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -49,7 +49,7 @@
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-sms-connector": "^0.1.13",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
-    "@tiledesk/tiledesk-tybot-connector": "^2.0.45",
+    "@tiledesk/tiledesk-tybot-connector": "^2.0.48",
     "@tiledesk/tiledesk-voice-twilio-connector": "^0.3.2",
     "@tiledesk/tiledesk-vxml-connector": "^0.1.91",
     "@tiledesk/tiledesk-whatsapp-connector": "1.0.26",

package/routes/kb.js

@@ -121,7 +121,8 @@ function normalizeEmbedding(embedding) {
   return { ...normalizedEmbedding };
 }
 
-function normalizeSituatedContext() {
+function normalizeSituatedContext(enable = false) {
+  situatedContext.enable = enable;
   return situatedContext.enable
     ? {
       ...situatedContext,
@@ -199,6 +200,11 @@ router.post('/scrape/single', async (req, res) => {
       })
     }
 
+    let situated_context;
+    if (sitemapKb.situated_context && sitemapKb.situated_context === true && sitemapKb.scrape_type === 0) {
+      situated_context = normalizeSituatedContext(true);
+    }
+
     if (addedUrls.length > 0) {
       const options = {
         sitemap_origin_id: sitemapKb._id,
@@ -206,7 +212,8 @@ router.post('/scrape/single', async (req, res) => {
         scrape_type: sitemapKb.scrape_type,
         scrape_options: sitemapKb.scrape_options,
         refresh_rate: sitemapKb.refresh_rate,
-        tags: sitemapKb.tags
+        tags: sitemapKb.tags,
+        ...(situated_context && { situated_context: situated_context }),
       }
       aiManager.addMultipleUrls(namespace, addedUrls, options).catch((err) => {
         winston.error("(webhook) error adding multiple urls contents: ", err);
@@ -263,9 +270,8 @@ router.post('/scrape/single', async (req, res) => {
         json.hybrid = true;
       }
 
-      const situated_context = normalizeSituatedContext();
-      if (situated_context) {
-        json.situated_context = situated_context;
+      if (json.situated_context && json.situated_context === true && json.scrape_type === 0) {
+        json.situated_context = normalizeSituatedContext(true);
       }
 
       winston.verbose("/scrape/single json: ", json);
@@ -447,12 +453,11 @@ router.post('/qa', async (req, res) => {
   }
 
   data.stream = data.stream === true;
+  data.use_hyde = data.use_hyde === true;
+  data.use_cache = data.use_cache === true;
   data.debug = true;
   delete data.advancedPrompt;
   winston.verbose("ask data: ", data);
-
-  console.log("data: ", data);
-
   if (process.env.NODE_ENV === 'test') {
     return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
   }
@@ -1591,7 +1596,7 @@ router.post('/', async (req, res) => {
   const id_project = req.projectid;
   const project = req.project
 
-  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
+  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context } = req.body;
   const namespace_id = req.body?.namespace;
   
   if (!namespace_id) {
@@ -1643,6 +1648,10 @@ router.post('/', async (req, res) => {
     new_kb.tags = tags;
   }
 
+  if (situated_context && situated_context === true && (type !== "url" || scrape_type === 0)) {
+    new_kb.situated_context = situated_context;
+  }
+
   winston.debug("adding kb: ", new_kb);
 
   KB.findOneAndUpdate({ id_project, type, source }, new_kb, { upsert: true, new: true, rawResult: true }, async (err, raw_content) => {
@@ -1661,7 +1670,7 @@ router.post('/', async (req, res) => {
       const embedding = normalizeEmbedding(namespace.embedding);
       embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
 
-      const situated_context = normalizeSituatedContext();
+      const situated_context_obj = normalizeSituatedContext(saved_kb.situated_context);
 
       const json = {
         id: saved_kb._id,
@@ -1673,7 +1682,7 @@ router.post('/', async (req, res) => {
         hybrid: namespace.hybrid,
         engine: namespace.engine || default_engine,
         embedding: embedding,
-        ...(situated_context && { situated_context: situated_context }),
+        ...(situated_context_obj && { situated_context: situated_context_obj }),
         ...(saved_kb.scrape_type && { scrape_type: saved_kb.scrape_type }),
         ...(saved_kb.scrape_options && { parameters_scrape_type_4: saved_kb.scrape_options }),
         ...(saved_kb.tags && { tags: saved_kb.tags }),
@@ -1705,7 +1714,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
 
   const id_project = req.projectid;
   const project = req.project;
-  let { refresh_rate = 'never', scrape_type = 2, scrape_options } = req.body;
+  let { refresh_rate = 'never', scrape_type = 2, scrape_options, situated_context } = req.body;
   let tags = parseStringArrayField(req.body.tags);
 
 
@@ -1748,6 +1757,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
     scrape_type,
     scrape_options,
     refresh_rate,
+    ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
     ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
   }
 
@@ -1772,6 +1782,11 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
   const { delimiter = ';' } = req.body;
   let tags = parseStringArrayField(req.body.tags);
 
+  let situated_context = false;
+  if (req.body.situated_context) {
+    situated_context = req.body.situated_context === true || req.body.situated_context === "true";
+  }
+
 
   let namespace;
   try {
@@ -1799,6 +1814,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
         content: question + "\n" + answer,
         namespace: namespace_id,
         status: -1,
+        ...(situated_context && { situated_context: situated_context }),
         ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
       })
     })
@@ -1830,16 +1846,20 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
         let embedding = normalizeEmbedding(namespace.embedding);
         embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
         let hybrid = namespace.hybrid;
-        const situated_context = normalizeSituatedContext();
 
-        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project,  ...keepAttrs }) => keepAttrs)
+        let situated_context_obj;
+        if (situated_context) {
+          situated_context_obj = normalizeSituatedContext(situated_context);
+        }
+
+        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context,  ...keepAttrs }) => keepAttrs)
         resources = resources.map(({ _id, ...rest}) => {
           return { 
             id: _id, 
             webhook: webhook, 
             embedding: embedding, 
             engine: engine, 
-            ...(situated_context && { situated_context: situated_context }), 
+            ...(situated_context_obj && { situated_context: situated_context_obj }), 
             ...rest 
           };
         })
@@ -1891,7 +1911,7 @@ router.post('/sitemap/import', async (req, res) => {
   const id_project = req.projectid;
   const namespace_id = req.query.namespace;
 
-  let { type, source, refresh_rate = 'never', scrape_type = 2, scrape_options, tags } = req.body;
+  let { type, source, refresh_rate = 'never', scrape_type = 2, scrape_options, tags, situated_context } = req.body;
   if (scrape_type === 2 && !scrape_options) {
     scrape_options = aiManager.setDefaultScrapeOptions();
   }
@@ -1957,6 +1977,7 @@ router.post('/sitemap/import', async (req, res) => {
     scrape_type,
     scrape_options,
     refresh_rate,
+    ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
     ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
   }
 
@@ -1974,7 +1995,8 @@ router.post('/sitemap/import', async (req, res) => {
     scrape_type,
     scrape_options,
     refresh_rate,
-    ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
+    ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {}),
+    ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
   }
   
   try {
@@ -1998,7 +2020,7 @@ router.put('/:kb_id', async (req, res) => {
   const project = req.project;
   const kb_id = req.params.kb_id;
   
-  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
+  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context } = req.body;
   const namespace_id = req.body.namespace;
 
   if (!namespace_id) {
@@ -2065,6 +2087,10 @@ router.put('/:kb_id', async (req, res) => {
     return res.status(500).send({ success: false, error: err });
   }
 
+  if (situated_context && situated_context === true && scrape_type === 0) {
+    situated_context = true;
+  }
+
   let new_content = {
     id_project,
     name,
@@ -2096,6 +2122,10 @@ router.put('/:kb_id', async (req, res) => {
     new_content.tags = tags;
   }
 
+  if (situated_context && situated_context === true && scrape_type === 0) {
+    new_content.situated_context = situated_context;
+  }
+
   winston.debug("Update content. New content: ", new_content);
 
   let updated_content;
@@ -2109,7 +2139,7 @@ router.put('/:kb_id', async (req, res) => {
   const embedding = normalizeEmbedding(namespace.embedding);
   embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
   let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
-  const situated_context = normalizeSituatedContext();
+  const situated_context_obj = normalizeSituatedContext(updated_content.situated_context);
 
   const json = {
     id: updated_content._id,
@@ -2121,7 +2151,7 @@ router.put('/:kb_id', async (req, res) => {
     hybrid: namespace.hybrid,
     engine: namespace.engine || default_engine,
     embedding: embedding,
-    ...(situated_context && { situated_context: situated_context }),
+    ...(situated_context_obj && { situated_context: situated_context_obj }),
     ...(updated_content.scrape_type && { scrape_type: updated_content.scrape_type }),
     ...(updated_content.scrape_options && { parameters_scrape_type_4: updated_content.scrape_options }),
     ...(updated_content.tags && { tags: updated_content.tags }),

package/routes/webhook.js

@@ -148,7 +148,11 @@ router.post('/kb/reindex', async (req, res) => {
         sitemap_origin: kb.source,
         scrape_type: kb.scrape_type,
         scrape_options: kb.scrape_options,
-        refresh_rate: kb.refresh_rate
+        refresh_rate: kb.refresh_rate,
+        ...(kb.tags ? { tags: kb.tags } : {}),
+        ...(kb.situated_context === true && (kb.scrape_type === 0 || kb.scrape_type === '0')
+          ? { situated_context: true }
+          : {}),
       }
       aiManager.addMultipleUrls(namespace, addedUrls, options).catch((err) => {
         winston.error("(webhook) error adding multiple urls contents: ", err);
@@ -198,9 +202,9 @@ router.post('/kb/reindex', async (req, res) => {
     embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
     json.embedding = embedding;
 
-    const situated_context = aiManager.normalizeSituatedContext();
-    if (situated_context) {
-      json.situated_context = situated_context;
+    const situated_context_obj = aiManager.normalizeSituatedContext(kb.situated_context);
+    if (situated_context_obj) {
+      json.situated_context = situated_context_obj;
     }
 
     let resources = [];

package/services/aiManager.js

@@ -52,6 +52,7 @@ jobManagerHybrid.connectAndStartPublisher((status, error) => {
   }
 });
 
+
 class AiManager {
 
   constructor() { }
@@ -74,6 +75,7 @@ class AiManager {
           ...(options.sitemap_origin_id && { sitemap_origin_id: options.sitemap_origin_id }),
           ...(options.sitemap_origin && { sitemap_origin: options.sitemap_origin }),
           ...(options.tags && { tags: options.tags }),
+          ...(options.situated_context && { situated_context: options.situated_context }),
         }
         return kb;
       })
@@ -95,11 +97,14 @@ class AiManager {
         let embedding = namespace.embedding || default_embedding;
         embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
 
-        let situated_context = this.normalizeSituatedContext();
+        let situated_context;
+        if (options.situated_context) {
+          situated_context = this.normalizeSituatedContext(options.situated_context);
+        }
 
         let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
 
-        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
+        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
         resources = resources.map(({ _id, scrape_options, ...rest }) => {
           return { 
             id: _id, 
@@ -132,8 +137,6 @@ class AiManager {
   async scheduleSitemap(namespace, sitemap_content, options) {
     return new Promise((resolve, reject) => {
 
-      const situated_context = this.normalizeSituatedContext();
-
       let kb = {
         id: sitemap_content._id,
         source: sitemap_content.source,
@@ -144,7 +147,7 @@ class AiManager {
         engine: namespace.engine,
         embedding: namespace.embedding,
         hybrid: namespace.hybrid,
-        ...(situated_context && { situated_context }),
+        ...(options.situated_context && { situated_context: options.situated_context }),
       }
 
       if (process.env.NODE_ENV === 'test') {
@@ -190,6 +193,8 @@ class AiManager {
       throw err;
     }
 
+
+
     // Recreate all url contents with sitemap_origin_id
     let result;
     try {
@@ -567,7 +572,8 @@ class AiManager {
     })
   }
 
-  normalizeSituatedContext() {
+  normalizeSituatedContext(enable = false) {
+    situatedContext.enable = enable;
     return situatedContext.enable
       ? {
         ...situatedContext,

package/test/kbRoute.js

@@ -1011,6 +1011,7 @@ describe('KbRoute', () => {
                                 .auth(email, pwd)
                                 //.set('Content-Type', 'text/csv')
                                 .field('delimiter', ';')
+                                .field('situated_context', true)
                                 .field('tags', JSON.stringify(['tag1', 'tag2']))
                                 .attach('uploadFile', fs.readFileSync(path.resolve(__dirname, './fixtures/example-kb-faqs.csv')), 'example-kb-faqs.csv')
                                 .end((err, res) => {