@tiledesk/tiledesk-server 2.18.1 → 2.18.4

package/CHANGELOG.md

@@ -5,9 +5,19 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
-# 2.18.1
+# 2.18.4
+- Added HyDE support for Knowledge Base Q&A
+- Introduced Cache (cRag) functionality in Knowledge Base Q&A
+- Enabled Situated Context for enhanced Knowledge Base content indexing
+- Bumped tybot-connector version to 2.0.48
+
+# 2.18.3
 - Added permissions logic
 - Added custom roles support
+- Add answered questions functionality
+- Added AnsweredQuestion schema with TTL index for automatic deletion.
+- Updated UnansweredQuestion schema to include additional fields and improved query handling for searching and sorting.
+- Enhanced the deletion process for unanswered questions.
 
 # 2.17.4
 - Refactor error handling and code structure in webhook.js

package/app.js

@@ -130,6 +130,7 @@ var integration = require('./routes/integration')
 var kbsettings = require('./routes/kbsettings');
 var kb = require('./routes/kb');
 var unanswered = require('./routes/unanswered');
+var answered = require('./routes/answered');
 
 // var admin = require('./routes/admin');
 var faqpub = require('./routes/faqpub');
@@ -643,6 +644,7 @@ app.use('/:projectid/mcp', [passport.authenticate(['basic', 'jwt'], { session: f
 
 app.use('/:projectid/kbsettings', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('agent', ['bot','subscription'])], kbsettings);
 app.use('/:projectid/kb/unanswered', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], unanswered);
+app.use('/:projectid/kb/answered', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], answered);
 app.use('/:projectid/kb', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])], kb);
 
 app.use('/:projectid/logs', [passport.authenticate(['basic', 'jwt'], { session: false }), validtoken, roleChecker.hasRole('agent')], logs);

package/config/kb/situatedContext.js

@@ -1,6 +1,6 @@
 module.exports = {
     enable: process.env.SITUATED_CONTEXT_ENABLE === "true",
     provider: process.env.SITUATED_CONTEXT_PROVIDER || "openai",
-    model: process.env.SITUATED_CONTEXT_MODEL || "gpt-4o",
+    model: process.env.SITUATED_CONTEXT_MODEL || "gpt-5.4-nano",
     api_key: ""
 }

package/docs/routes-answered.md

@@ -0,0 +1,153 @@
+# Route `kb/answered` — domande con risposta (Knowledge Base)
+
+Documentazione per [`routes/answered.js`](../routes/answered.js).
+
+## Scopo
+
+API REST per gestire le **domande già risposte** associate a un **namespace** della Knowledge Base del progetto. I documenti sono persistiti nel modello Mongoose `AnsweredQuestion` (vedi [`models/kb_setting.js`](../models/kb_setting.js)).
+
+Ogni operazione è vincolata al **progetto corrente** (`req.projectid`, derivato dal segmento `:projectid` nell’URL) e alla validità del **namespace** (deve esistere un `Namespace` con `id` uguale al `namespace` richiesto e `id_project` uguale al progetto).
+
+## Montaggio e autenticazione
+
+In [`app.js`](../app.js) il router è montato così:
+
+- **Path base:** `/:projectid/kb/answered`
+- **Middleware:** `passport.authenticate(['basic', 'jwt'])`, `validtoken`, `roleChecker.hasRoleOrTypes('admin', ['bot','subscription'])`
+
+Sono quindi richiesti autenticazione (Basic o JWT), token valido e ruolo **admin** oppure tipi **bot** / **subscription**.
+
+Esempio di URL completo (sviluppo locale):
+
+```http
+GET http://localhost:3000/{projectId}/kb/answered/{namespace}
+```
+
+Sostituire `{projectId}` con l’ID MongoDB del progetto.
+
+## Modello dati (`AnsweredQuestion`)
+
+| Campo         | Tipo     | Note                                      |
+|---------------|----------|-------------------------------------------|
+| `id_project`  | string   | Impostato dal server dal contesto progetto |
+| `namespace`   | string   | ID del namespace (validato contro `Namespace`) |
+| `question`    | string   | Obbligatorio                              |
+| `answer`      | string   | Obbligatorio                              |
+| `tokens`      | number   | Opzionale                                 |
+| `request_id`  | string   | Opzionale                                 |
+| `created_at` / `updated_at` | date | Da `timestamps: true`              |
+
+Indici rilevanti: TTL su `created_at`, indice composto `(id_project, namespace, created_at)`, indice **testo** su `question` e `answer` (pesi question 5, answer 1) per la ricerca full-text.
+
+---
+
+## Endpoint
+
+Tutti i path qui sotto sono relativi a `/:projectid/kb/answered`.
+
+### `POST /`
+
+Aggiunge una nuova domanda risposta.
+
+**Body JSON**
+
+| Campo        | Obbligatorio | Descrizione        |
+|-------------|--------------|--------------------|
+| `namespace` | sì           | ID namespace       |
+| `question`  | sì           | Testo domanda      |
+| `answer`    | sì           | Testo risposta     |
+| `tokens`    | no           | Numero opzionale   |
+| `request_id`| no           | Riferimento richiesta |
+
+**Risposte**
+
+| HTTP | Significato |
+|------|-------------|
+| 200  | Documento salvato (corpo = documento MongoDB creato) |
+| 400  | Parametri mancanti (`namespace`, `question`, `answer`) |
+| 403  | Namespace non appartenente al progetto |
+| 500  | Errore server |
+
+---
+
+### `GET /:namespace`
+
+Elenco paginato delle domande risposte per il namespace.
+
+**Query**
+
+| Parametro    | Default | Descrizione |
+|--------------|---------|-------------|
+| `page`       | `0`     | Pagina (0-based) |
+| `limit`      | `20`    | Elementi per pagina |
+| `sortField`  | `created_at` | Campo ordinamento |
+| `direction`  | `-1`    | `1` crescente, `-1` decrescente |
+| `search`     | —       | Se presente, attiva ricerca **full-text** (`$text`); l’ordinamento usa lo score di testo |
+
+**Risposta 200**
+
+```json
+{
+  "count": 0,
+  "questions": [],
+  "query": {
+    "page": 0,
+    "limit": 20,
+    "sortField": "created_at",
+    "direction": -1,
+    "search": "..."
+  }
+}
+```
+
+**Altri codici:** `400` (namespace mancante), `403` (namespace non valido per il progetto), `500`.
+
+---
+
+### `DELETE /:id`
+
+Elimina una singola domanda per `_id` MongoDB.
+
+**Risposte:** `200` con `{ success, message }`, `404` se non trovata (o non del progetto), `500`.
+
+---
+
+### `DELETE /namespace/:namespace`
+
+Elimina **tutte** le domande risposte per `namespace` nel progetto corrente.
+
+**Risposta 200:** `{ success, count: <deletedCount>, message }`.
+
+**403** se il namespace non appartiene al progetto. **500** in errore.
+
+---
+
+### `GET /count/:namespace`
+
+Restituisce il conteggio documenti per `id_project` + `namespace`.
+
+**Risposta 200:** `{ count: <number> }`.
+
+**400** / **403** / **500** come negli altri endpoint con validazione namespace.
+
+---
+
+## Funzione interna `validateNamespace(id_project, namespace_id)`
+
+Verifica che esista un documento `Namespace` con `id_project` e `id: namespace_id`. Usata in creazione, lettura elenco, cancellazione bulk e conteggio.
+
+---
+
+## Nota sull’ordine delle route Express
+
+Nel file è definito `GET /:namespace` **prima** di `GET /count/:namespace`. In Express la prima route che corrisponde vince: una richiesta del tipo `GET .../kb/answered/count/<namespace>` può essere interpretata come `GET /:namespace` con `namespace` uguale alla stringa letterale `count`, invece che come endpoint di conteggio.
+
+Per far funzionare l’URL `GET /count/:namespace` come previsto dal codice, in genere occorre registrare **`GET /count/:namespace` prima di `GET /:namespace`**, oppure usare un prefisso path diverso per uno dei due. Verificare il comportamento in ambiente reale o adeguare l’ordine delle route se il conteggio non risponde come atteso.
+
+---
+
+## Riferimenti
+
+- Router: [`routes/answered.js`](../routes/answered.js)
+- Modello e indici: [`models/kb_setting.js`](../models/kb_setting.js) (`AnsweredQuestionSchema`)
+- Montaggio app: [`app.js`](../app.js) (`/:projectid/kb/answered`)

package/migrations/1757601159298-project_user_role_type.js

@@ -1,45 +1,104 @@
 var Project_user = require("../models/project_user");
 var winston = require('../config/winston');
 
+const BATCH_SIZE = 100;
+/** Log a progress line every this many bulkWrite rounds (plus always after the 1st). */
+const PROGRESS_LOG_EVERY_BATCHES = 10;
+
+/** Only documents that still need roleType (idempotent re-runs). */
+const WITHOUT_ROLE_TYPE = {
+  $or: [{ roleType: { $exists: false } }, { roleType: null }]
+};
+
+const AGENT_ROLES_FILTER = {
+  $and: [
+    {
+      $or: [
+        { role: 'agent' },
+        { role: 'supervisor' },
+        { role: 'admin' },
+        { role: 'owner' }
+      ]
+    },
+    WITHOUT_ROLE_TYPE
+  ]
+};
+
+const USER_ROLES_FILTER = {
+  $and: [{ $or: [{ role: 'user' }, { role: 'guest' }] }, WITHOUT_ROLE_TYPE]
+};
+
+async function batchSetRoleType(filter, roleType, phaseLabel) {
+  const totalMatching = await Project_user.countDocuments(filter);
+  const started = Date.now();
+  winston.info(
+    `[project_user_role_type] ${phaseLabel}: ${totalMatching} documents match filter; streaming _id + bulkWrite in chunks of ${BATCH_SIZE}`
+  );
+
+  let modified = 0;
+  let scanned = 0;
+  let bulkRounds = 0;
+  const cursor = Project_user.find(filter).select('_id').lean().cursor();
+  let batch = [];
+
+  function logProgress(reason) {
+    const elapsedSec = ((Date.now() - started) / 1000).toFixed(1);
+    winston.info(
+      `[project_user_role_type] ${phaseLabel} ${reason}: scanned=${scanned}/${totalMatching}, modifiedThisPhase=${modified}, ${elapsedSec}s elapsed`
+    );
+  }
+
+  for await (const doc of cursor) {
+    scanned++;
+    batch.push({
+      updateOne: {
+        filter: { _id: doc._id, ...WITHOUT_ROLE_TYPE },
+        update: { $set: { roleType } }
+      }
+    });
+    if (batch.length >= BATCH_SIZE) {
+      const result = await Project_user.bulkWrite(batch);
+      modified += result.modifiedCount;
+      batch = [];
+      bulkRounds++;
+      if (bulkRounds === 1 || bulkRounds % PROGRESS_LOG_EVERY_BATCHES === 0) {
+        logProgress(`progress (bulk round ${bulkRounds})`);
+      }
+    }
+  }
+  if (batch.length > 0) {
+    const result = await Project_user.bulkWrite(batch);
+    modified += result.modifiedCount;
+    bulkRounds++;
+    logProgress(`final partial batch (bulk round ${bulkRounds})`);
+  }
+
+  logProgress('phase complete');
+  return modified;
+}
+
 /**
  * Make any changes you need to make to the database here
  */
-async function up () {
-  let p = await Project_user.find({}).exec()
-  // Write migration here
-  await new Promise((resolve, reject) => {
-     Project_user.updateMany({"$or":[{ "role": "agent" }, { "role": "supervisor" }, { "role":  "admin" }, { "role": "owner" }]}, {"$set": {"roleType": 1 }}, function (err, updates) {
-      if (err) {
-        winston.error("Error appling the migration script", err);
-      }
-      winston.info("Schema updated for " + updates.nModified + " project_user of type agents")
-      winston.debug("updates",updates)      
-      //  return resolve('ok'); 
-    });  
-
-     return Project_user.updateMany({"$or":[{ "role": "user" }, { "role": "guest" }]}, {"$set": {"roleType": 2 }}, function (err, updates) {
-          if (err) {
-            winston.error("Error appling the migration script", err);
-          }
-          winston.info("Schema updated for " + updates.nModified + " project_user of type user"),
-          winston.debug("updates",updates)
-           return resolve('ok'); 
-        });  
-    
-  
-  });
-     
- 
-} 
-  
-/** 
+async function up() {
+  try {
+    const agentsModified = await batchSetRoleType(AGENT_ROLES_FILTER, 1, 'agents roleType=1');
+    winston.info(`[project_user_role_type] Agents phase done: ${agentsModified} documents modified`);
+
+    const usersModified = await batchSetRoleType(USER_ROLES_FILTER, 2, 'users/guest roleType=2');
+    winston.info(`[project_user_role_type] Users/guest phase done: ${usersModified} documents modified`);
+  } catch (err) {
+    winston.error('[project_user_role_type] Error applying migration:', err);
+    throw err;
+  }
+}
+
+/**
  * Make any changes that UNDO the up function side effects here (if possible)
  */
-async function down () {
-  
+async function down() {
   // Write migration here
   // console.log("down*********");
 }
 
-module.exports = { up, down }; 
-  
+module.exports = { up, down };

package/models/kb_setting.js

@@ -2,10 +2,24 @@ let mongoose = require('mongoose');
 let Schema = mongoose.Schema;
 let winston = require('../config/winston');
 
-const expireAfterSeconds = (() => {
-  const n = Number(process.env.UNANSWERED_QUESTION_EXPIRATION_TIME);
-  return !isNaN(n) && n >= 0 ? n : 7 * 24 * 60 * 60; // default 7 days
-})();
+const DEFAULT_UNANSWERED_TTL_SEC = 7 * 24 * 60 * 60; // 7 days
+const DEFAULT_ANSWERED_TTL_SEC = 7 * 24 * 60 * 60; // 7 days
+
+function ttlSecondsFromEnv(raw, fallbackSec) {
+  if (raw == null || String(raw).trim() === '') return fallbackSec;
+  const n = Number(raw);
+  return Number.isFinite(n) && n >= 0 ? n : fallbackSec;
+}
+
+let expireAfterSeconds = ttlSecondsFromEnv(
+  process.env.UNANSWERED_QUESTION_EXPIRATION_TIME,
+  DEFAULT_UNANSWERED_TTL_SEC
+);
+let expireAnsweredAfterSeconds = ttlSecondsFromEnv(
+  process.env.ANSWERED_QUESTION_EXPIRATION_TIME,
+  DEFAULT_ANSWERED_TTL_SEC
+);
+
 
 const EngineSchema = new Schema({
   name: {
@@ -184,6 +198,11 @@ var KBSchema = new Schema({
     type: Array,
     default: undefined,
     required: false
+  },
+  situated_context: {
+    type: Boolean,
+    default: false,
+    required: false
   }
 }, {
   timestamps: true
@@ -203,13 +222,63 @@ const UnansweredQuestionSchema = new Schema({
   question: {
     type: String,
     required: true
+  },
+  request_id: {
+    type: String,
+    required: false,
+  },
+  sender: {
+    type: String,
+    required: false,
   }
 },{
   timestamps: true
 });
 
-// Add TTL index to automatically delete documents after 30 days
-UnansweredQuestionSchema.index({ createdAt: 1 }, { expireAfterSeconds: expireAfterSeconds }); // 30 days
+const AnsweredQuestionSchema = new Schema({
+  id_project: {
+    type: String,
+    required: true,
+    index: true
+  },
+  namespace: {
+    type: String,
+    required: true,
+    index: true
+  },
+  question: {
+    type: String,
+    required: true
+  },
+  answer: {
+    type: String,
+    required: true
+  },
+  tokens: {
+    type: Number,
+    required: false
+  },
+  request_id: {
+    type: String,
+    required: false
+  }
+}, {
+  timestamps: true
+});
+
+// Add TTL index to automatically delete documents
+UnansweredQuestionSchema.index({ createdAt: 1 }, { expireAfterSeconds: expireAfterSeconds }); 
+UnansweredQuestionSchema.index({ id_project: 1, namespace: 1, createdAt: -1 });
+UnansweredQuestionSchema.index({ question: "text" });
+
+AnsweredQuestionSchema.index({ createdAt: 1 }, { expireAfterSeconds: expireAnsweredAfterSeconds });
+AnsweredQuestionSchema.index({ id_project: 1, namespace: 1, createdAt: -1 });
+AnsweredQuestionSchema.index(
+  { question: "text", answer: "text" },
+  { weights: { question: 3, answer: 1 } }
+);
+
+
 
 // DEPRECATED !! - Start
 const KBSettingSchema = new Schema({
@@ -246,6 +315,7 @@ const Engine = mongoose.model('Engine', EngineSchema)
 const Namespace = mongoose.model('Namespace', NamespaceSchema)
 const KB = mongoose.model('KB', KBSchema)
 const UnansweredQuestion = mongoose.model('UnansweredQuestion', UnansweredQuestionSchema)
+const AnsweredQuestion = mongoose.model('AnsweredQuestion', AnsweredQuestionSchema)
 
 
 module.exports = {
@@ -253,5 +323,6 @@ module.exports = {
   Namespace: Namespace,
   Engine: Engine,
   KB: KB,
-  UnansweredQuestion: UnansweredQuestion
+  UnansweredQuestion: UnansweredQuestion,
+  AnsweredQuestion: AnsweredQuestion
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.18.1",
+  "version": "2.18.4",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -49,7 +49,7 @@
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-sms-connector": "^0.1.13",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
-    "@tiledesk/tiledesk-tybot-connector": "^2.0.45",
+    "@tiledesk/tiledesk-tybot-connector": "^2.0.48",
     "@tiledesk/tiledesk-voice-twilio-connector": "^0.3.2",
     "@tiledesk/tiledesk-vxml-connector": "^0.1.91",
     "@tiledesk/tiledesk-whatsapp-connector": "1.0.26",

package/routes/answered.js

@@ -0,0 +1,227 @@
+const express = require('express');
+const router = express.Router();
+const { Namespace, AnsweredQuestion } = require('../models/kb_setting');
+const winston = require('../config/winston');
+
+// Add a new unanswerd question
+router.post('/', async (req, res) => {
+    try {
+        const { namespace, question, answer, tokens, request_id } = req.body;
+        const id_project = req.projectid;
+
+        if (!namespace || !question || !answer) {
+            return res.status(400).json({
+                success: false,
+                error: "Missing required parameters: namespace, question and answer"
+            })
+        }
+
+        // Check if namespae belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            })
+        }
+
+        const answeredQuestion = new AnsweredQuestion({
+            id_project,
+            namespace,
+            question,
+            answer,
+            tokens,
+            request_id,
+        });
+
+        const savedQuestion = await answeredQuestion.save();
+        res.status(200).json(savedQuestion);
+
+    } catch (error) {
+        winston.error('Error adding answered question:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error adding answered question"
+        });
+    }
+})
+
+// Get all answered questions for a namespace
+router.get('/:namespace', async (req, res) => {
+
+    try {
+        const { namespace } = req.params;
+        const id_project = req.projectid;
+
+        if (!namespace) {
+            return res.status(400).json({
+                success: false,
+                error: "Missing required parameter: namespace"
+            })
+        }
+
+        // Check if namespace belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            })
+        }
+        
+        const page = parseInt(req.query.page) || 0;
+        const limit = parseInt(req.query.limit) || 20;
+        const sortField = req.query.sortField || 'created_at';
+        const direction = parseInt(req.query.direction) || -1;
+
+        const filter = { id_project, namespace };
+
+        let projection = undefined;
+
+        if (req.query.search) {
+            filter.$text = { $search: req.query.search };
+            // Add score to projection if it's a text search
+            projection = { score: { $meta: "textScore" } };
+        }
+
+        let sortObj;
+        if (projection && projection.score) {
+            sortObj = { score: { $meta: "textScore" } };
+        } else {
+            sortObj = { [sortField]: direction };
+        }
+
+        const questions = await AnsweredQuestion.find(filter, projection)
+            .sort(sortObj)
+            .skip(page * limit)
+            .limit(limit);
+
+        const count = await AnsweredQuestion.countDocuments(filter);
+
+        res.status(200).json({
+            count,
+            questions,
+            query: {
+                page,
+                limit,
+                sortField,
+                direction,
+                search: req.query.search || undefined
+            }
+        });
+        
+    } catch (error) {
+        winston.error('Error getting answered questions:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error getting answered questions"
+        });
+    }
+
+})
+
+router.delete('/:id', async (req, res) => {
+    try {
+        const { id } = req.params;
+        const id_project = req.projectid;
+
+        const deleted = await AnsweredQuestion.findOneAndDelete({ _id: id, id_project });
+        if (!deleted) {
+            return res.status(404).json({
+                success: false,
+                error: "Question not found"
+            });
+        }
+
+        res.status(200).json({
+            success: true,
+            message: "Question deleted successfully"
+        });
+
+    } catch (error) {
+        winston.error('Error deleting answered question:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error deleting answered question"
+        });
+    }
+})
+
+router.delete('/namespace/:namespace', async (req, res) => {
+    try {
+        const { namespace } = req.params;
+        const id_project = req.projectid;
+
+        // Check if namespace belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            });
+        }
+        
+        const result = await AnsweredQuestion.deleteMany({ id_project, namespace });
+        res.status(200).json({
+            success: true,
+            count: result.deletedCount,
+            message: "All questions deleted successfully"
+        });
+
+    } catch (error) {
+        winston.error('Error deleting answered questions:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error deleting answered questions"
+        });
+    }
+})
+
+router.get('/count/:namespace', async (req, res) => {
+    try {
+        const { namespace } = req.params;
+        const id_project = req.projectid;
+
+        if (!namespace) {
+            return res.status(400).json({
+                success: false,
+                error: "Missing required parameter: namespace"
+            });
+        }
+
+        // Check if namespace belongs to project
+        const isValidNamespace = await validateNamespace(id_project, namespace);
+        if (!isValidNamespace) {
+            return res.status(403).json({
+                success: false,
+                error: "Not allowed. The namespace does not belong to the current project."
+            });
+        }
+
+        const count = await AnsweredQuestion.countDocuments({ id_project, namespace });
+        res.status(200).json({ count });
+
+    } catch (error) {
+        winston.error('Error counting answered questions:', error);
+        res.status(500).json({
+            success: false,
+            error: "Error counting answered questions"
+        });
+    }
+})
+
+// Helper function to validate namespace
+async function validateNamespace(id_project, namespace_id) {
+    try {
+        const namespace = await Namespace.findOne({
+            id_project: id_project,
+            id: namespace_id
+        });
+        return !!namespace;
+    } catch (err) {
+        winston.error('validate namespace error: ', err);
+        throw err;
+    }
+}
+
+module.exports = router;

package/routes/kb.js

@@ -121,7 +121,8 @@ function normalizeEmbedding(embedding) {
   return { ...normalizedEmbedding };
 }
 
-function normalizeSituatedContext() {
+function normalizeSituatedContext(enable = false) {
+  situatedContext.enable = enable;
   return situatedContext.enable
     ? {
       ...situatedContext,
@@ -199,6 +200,11 @@ router.post('/scrape/single', async (req, res) => {
       })
     }
 
+    let situated_context;
+    if (sitemapKb.situated_context && sitemapKb.situated_context === true && sitemapKb.scrape_type === 0) {
+      situated_context = normalizeSituatedContext(true);
+    }
+
     if (addedUrls.length > 0) {
       const options = {
         sitemap_origin_id: sitemapKb._id,
@@ -206,7 +212,8 @@ router.post('/scrape/single', async (req, res) => {
         scrape_type: sitemapKb.scrape_type,
         scrape_options: sitemapKb.scrape_options,
         refresh_rate: sitemapKb.refresh_rate,
-        tags: sitemapKb.tags
+        tags: sitemapKb.tags,
+        ...(situated_context && { situated_context: situated_context }),
       }
       aiManager.addMultipleUrls(namespace, addedUrls, options).catch((err) => {
         winston.error("(webhook) error adding multiple urls contents: ", err);
@@ -263,9 +270,8 @@ router.post('/scrape/single', async (req, res) => {
         json.hybrid = true;
       }
 
-      const situated_context = normalizeSituatedContext();
-      if (situated_context) {
-        json.situated_context = situated_context;
+      if (json.situated_context && json.situated_context === true && json.scrape_type === 0) {
+        json.situated_context = normalizeSituatedContext(true);
       }
 
       winston.verbose("/scrape/single json: ", json);
@@ -447,12 +453,11 @@ router.post('/qa', async (req, res) => {
   }
 
   data.stream = data.stream === true;
+  data.use_hyde = data.use_hyde === true;
+  data.use_cache = data.use_cache === true;
   data.debug = true;
   delete data.advancedPrompt;
   winston.verbose("ask data: ", data);
-
-  console.log("data: ", data);
-
   if (process.env.NODE_ENV === 'test') {
     return res.status(200).send({ success: true, message: "Question skipped in test environment", data: data });
   }
@@ -1591,7 +1596,7 @@ router.post('/', async (req, res) => {
   const id_project = req.projectid;
   const project = req.project
 
-  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
+  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context } = req.body;
   const namespace_id = req.body?.namespace;
   
   if (!namespace_id) {
@@ -1643,6 +1648,10 @@ router.post('/', async (req, res) => {
     new_kb.tags = tags;
   }
 
+  if (situated_context && situated_context === true && (type !== "url" || scrape_type === 0)) {
+    new_kb.situated_context = situated_context;
+  }
+
   winston.debug("adding kb: ", new_kb);
 
   KB.findOneAndUpdate({ id_project, type, source }, new_kb, { upsert: true, new: true, rawResult: true }, async (err, raw_content) => {
@@ -1661,7 +1670,7 @@ router.post('/', async (req, res) => {
       const embedding = normalizeEmbedding(namespace.embedding);
       embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
 
-      const situated_context = normalizeSituatedContext();
+      const situated_context_obj = normalizeSituatedContext(saved_kb.situated_context);
 
       const json = {
         id: saved_kb._id,
@@ -1673,7 +1682,7 @@ router.post('/', async (req, res) => {
         hybrid: namespace.hybrid,
         engine: namespace.engine || default_engine,
         embedding: embedding,
-        ...(situated_context && { situated_context: situated_context }),
+        ...(situated_context_obj && { situated_context: situated_context_obj }),
         ...(saved_kb.scrape_type && { scrape_type: saved_kb.scrape_type }),
         ...(saved_kb.scrape_options && { parameters_scrape_type_4: saved_kb.scrape_options }),
         ...(saved_kb.tags && { tags: saved_kb.tags }),
@@ -1705,7 +1714,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
 
   const id_project = req.projectid;
   const project = req.project;
-  let { refresh_rate = 'never', scrape_type = 2, scrape_options } = req.body;
+  let { refresh_rate = 'never', scrape_type = 2, scrape_options, situated_context } = req.body;
   let tags = parseStringArrayField(req.body.tags);
 
 
@@ -1748,6 +1757,7 @@ router.post('/multi', upload.single('uploadFile'), async (req, res) => {
     scrape_type,
     scrape_options,
     refresh_rate,
+    ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
     ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
   }
 
@@ -1772,6 +1782,11 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
   const { delimiter = ';' } = req.body;
   let tags = parseStringArrayField(req.body.tags);
 
+  let situated_context = false;
+  if (req.body.situated_context) {
+    situated_context = req.body.situated_context === true || req.body.situated_context === "true";
+  }
+
 
   let namespace;
   try {
@@ -1799,6 +1814,7 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
         content: question + "\n" + answer,
         namespace: namespace_id,
         status: -1,
+        ...(situated_context && { situated_context: situated_context }),
         ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
       })
     })
@@ -1830,16 +1846,20 @@ router.post('/csv', upload.single('uploadFile'), async (req, res) => {
         let embedding = normalizeEmbedding(namespace.embedding);
         embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
         let hybrid = namespace.hybrid;
-        const situated_context = normalizeSituatedContext();
 
-        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project,  ...keepAttrs }) => keepAttrs)
+        let situated_context_obj;
+        if (situated_context) {
+          situated_context_obj = normalizeSituatedContext(situated_context);
+        }
+
+        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context,  ...keepAttrs }) => keepAttrs)
         resources = resources.map(({ _id, ...rest}) => {
           return { 
             id: _id, 
             webhook: webhook, 
             embedding: embedding, 
             engine: engine, 
-            ...(situated_context && { situated_context: situated_context }), 
+            ...(situated_context_obj && { situated_context: situated_context_obj }), 
             ...rest 
           };
         })
@@ -1891,7 +1911,7 @@ router.post('/sitemap/import', async (req, res) => {
   const id_project = req.projectid;
   const namespace_id = req.query.namespace;
 
-  let { type, source, refresh_rate = 'never', scrape_type = 2, scrape_options, tags } = req.body;
+  let { type, source, refresh_rate = 'never', scrape_type = 2, scrape_options, tags, situated_context } = req.body;
   if (scrape_type === 2 && !scrape_options) {
     scrape_options = aiManager.setDefaultScrapeOptions();
   }
@@ -1957,6 +1977,7 @@ router.post('/sitemap/import', async (req, res) => {
     scrape_type,
     scrape_options,
     refresh_rate,
+    ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
     ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
   }
 
@@ -1974,7 +1995,8 @@ router.post('/sitemap/import', async (req, res) => {
     scrape_type,
     scrape_options,
     refresh_rate,
-    ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {})
+    ...(Array.isArray(tags) && tags.length > 0 ? { tags } : {}),
+    ...(situated_context && situated_context === true && scrape_type === 0 ? { situated_context: true } : {}),
   }
   
   try {
@@ -1998,7 +2020,7 @@ router.put('/:kb_id', async (req, res) => {
   const project = req.project;
   const kb_id = req.params.kb_id;
   
-  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags } = req.body;
+  const { name, type, source, content, refresh_rate, scrape_type, scrape_options, tags, situated_context } = req.body;
   const namespace_id = req.body.namespace;
 
   if (!namespace_id) {
@@ -2065,6 +2087,10 @@ router.put('/:kb_id', async (req, res) => {
     return res.status(500).send({ success: false, error: err });
   }
 
+  if (situated_context && situated_context === true && scrape_type === 0) {
+    situated_context = true;
+  }
+
   let new_content = {
     id_project,
     name,
@@ -2096,6 +2122,10 @@ router.put('/:kb_id', async (req, res) => {
     new_content.tags = tags;
   }
 
+  if (situated_context && situated_context === true && scrape_type === 0) {
+    new_content.situated_context = situated_context;
+  }
+
   winston.debug("Update content. New content: ", new_content);
 
   let updated_content;
@@ -2109,7 +2139,7 @@ router.put('/:kb_id', async (req, res) => {
   const embedding = normalizeEmbedding(namespace.embedding);
   embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
   let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
-  const situated_context = normalizeSituatedContext();
+  const situated_context_obj = normalizeSituatedContext(updated_content.situated_context);
 
   const json = {
     id: updated_content._id,
@@ -2121,7 +2151,7 @@ router.put('/:kb_id', async (req, res) => {
     hybrid: namespace.hybrid,
     engine: namespace.engine || default_engine,
     embedding: embedding,
-    ...(situated_context && { situated_context: situated_context }),
+    ...(situated_context_obj && { situated_context: situated_context_obj }),
     ...(updated_content.scrape_type && { scrape_type: updated_content.scrape_type }),
     ...(updated_content.scrape_options && { parameters_scrape_type_4: updated_content.scrape_options }),
     ...(updated_content.tags && { tags: updated_content.tags }),

package/routes/unanswered.js

@@ -6,7 +6,7 @@ var winston = require('../config/winston');
 // Add a new unanswered question
 router.post('/', async (req, res) => {
     try {
-        const { namespace, question } = req.body;
+        const { namespace, question, request_id, sender } = req.body;
         const id_project = req.projectid;
 
         if (!namespace || !question) {
@@ -28,7 +28,9 @@ router.post('/', async (req, res) => {
         const unansweredQuestion = new UnansweredQuestion({
             id_project,
             namespace,
-            question
+            question,
+            request_id,
+            sender
         });
 
         const savedQuestion = await unansweredQuestion.save();
@@ -70,18 +72,29 @@ router.get('/:namespace', async (req, res) => {
         const sortField = req.query.sortField || 'createdAt';
         const direction = parseInt(req.query.direction) || -1;
 
-        const questions = await UnansweredQuestion.find({
-            id_project,
-            namespace
-        })
-        .sort({ [sortField]: direction })
-        .skip(page * limit)
-        .limit(limit);
+        const filter = { id_project, namespace };
 
-        const count = await UnansweredQuestion.countDocuments({
-            id_project,
-            namespace
-        });
+        let projection = undefined;
+
+        if (req.query.search) {
+            filter.$text = { $search: req.query.search };
+            // Add score to projection if it's a text search
+            projection = { score: { $meta: "textScore" } };
+        }
+
+        let sortObj;
+        if (projection && projection.score) {
+            sortObj = { score: { $meta: "textScore" } };
+        } else {
+            sortObj = { [sortField]: direction };
+        }
+
+        const questions = await UnansweredQuestion.find(filter, projection)
+            .sort(sortObj)
+            .skip(page * limit)
+            .limit(limit);
+
+        const count = await UnansweredQuestion.countDocuments(filter);
 
         res.status(200).json({
             count,
@@ -90,7 +103,8 @@ router.get('/:namespace', async (req, res) => {
                 page,
                 limit,
                 sortField,
-                direction
+                direction,
+                search: req.query.search || undefined
             }
         });
 
@@ -109,20 +123,19 @@ router.delete('/:id', async (req, res) => {
         const { id } = req.params;
         const id_project = req.projectid;
 
-        const question = await UnansweredQuestion.findOne({ _id: id, id_project });
-        if (!question) {
+        const deleted = await UnansweredQuestion.findOneAndDelete({ _id: id, id_project });
+        if (!deleted) {
             return res.status(404).json({
                 success: false,
                 error: "Question not found"
             });
         }
 
-        await UnansweredQuestion.deleteOne({ _id: id });
         res.status(200).json({
             success: true,
             message: "Question deleted successfully"
         });
-
+        
     } catch (error) {
         winston.error('Error deleting unanswered question:', error);
         res.status(500).json({

package/routes/webhook.js

@@ -148,7 +148,11 @@ router.post('/kb/reindex', async (req, res) => {
         sitemap_origin: kb.source,
         scrape_type: kb.scrape_type,
         scrape_options: kb.scrape_options,
-        refresh_rate: kb.refresh_rate
+        refresh_rate: kb.refresh_rate,
+        ...(kb.tags ? { tags: kb.tags } : {}),
+        ...(kb.situated_context === true && (kb.scrape_type === 0 || kb.scrape_type === '0')
+          ? { situated_context: true }
+          : {}),
       }
       aiManager.addMultipleUrls(namespace, addedUrls, options).catch((err) => {
         winston.error("(webhook) error adding multiple urls contents: ", err);
@@ -198,9 +202,9 @@ router.post('/kb/reindex', async (req, res) => {
     embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
     json.embedding = embedding;
 
-    const situated_context = aiManager.normalizeSituatedContext();
-    if (situated_context) {
-      json.situated_context = situated_context;
+    const situated_context_obj = aiManager.normalizeSituatedContext(kb.situated_context);
+    if (situated_context_obj) {
+      json.situated_context = situated_context_obj;
     }
 
     let resources = [];

package/services/aiManager.js

@@ -52,6 +52,7 @@ jobManagerHybrid.connectAndStartPublisher((status, error) => {
   }
 });
 
+
 class AiManager {
 
   constructor() { }
@@ -74,6 +75,7 @@ class AiManager {
           ...(options.sitemap_origin_id && { sitemap_origin_id: options.sitemap_origin_id }),
           ...(options.sitemap_origin && { sitemap_origin: options.sitemap_origin }),
           ...(options.tags && { tags: options.tags }),
+          ...(options.situated_context && { situated_context: options.situated_context }),
         }
         return kb;
       })
@@ -95,11 +97,14 @@ class AiManager {
         let embedding = namespace.embedding || default_embedding;
         embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
 
-        let situated_context = this.normalizeSituatedContext();
+        let situated_context;
+        if (options.situated_context) {
+          situated_context = this.normalizeSituatedContext(options.situated_context);
+        }
 
         let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
 
-        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
+        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, situated_context, ...keepAttrs }) => keepAttrs)
         resources = resources.map(({ _id, scrape_options, ...rest }) => {
           return { 
             id: _id, 
@@ -132,8 +137,6 @@ class AiManager {
   async scheduleSitemap(namespace, sitemap_content, options) {
     return new Promise((resolve, reject) => {
 
-      const situated_context = this.normalizeSituatedContext();
-
       let kb = {
         id: sitemap_content._id,
         source: sitemap_content.source,
@@ -144,7 +147,7 @@ class AiManager {
         engine: namespace.engine,
         embedding: namespace.embedding,
         hybrid: namespace.hybrid,
-        ...(situated_context && { situated_context }),
+        ...(options.situated_context && { situated_context: options.situated_context }),
       }
 
       if (process.env.NODE_ENV === 'test') {
@@ -190,6 +193,8 @@ class AiManager {
       throw err;
     }
 
+
+
     // Recreate all url contents with sitemap_origin_id
     let result;
     try {
@@ -567,7 +572,8 @@ class AiManager {
     })
   }
 
-  normalizeSituatedContext() {
+  normalizeSituatedContext(enable = false) {
+    situatedContext.enable = enable;
     return situatedContext.enable
       ? {
         ...situatedContext,

package/test/kbRoute.js

@@ -1011,6 +1011,7 @@ describe('KbRoute', () => {
                                 .auth(email, pwd)
                                 //.set('Content-Type', 'text/csv')
                                 .field('delimiter', ';')
+                                .field('situated_context', true)
                                 .field('tags', JSON.stringify(['tag1', 'tag2']))
                                 .attach('uploadFile', fs.readFileSync(path.resolve(__dirname, './fixtures/example-kb-faqs.csv')), 'example-kb-faqs.csv')
                                 .end((err, res) => {