@tiledesk/tiledesk-server 2.19.3 → 2.19.5

package/CHANGELOG.md

@@ -5,6 +5,16 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.19.5
+- Improved management of answered and unanswered questions, duration and token consumption
+
+# 2.19.4
+- Added missing id_project in data sent for qa and single scrape
+
+# 2.19.3
+- Fixed bug on AiPrompt action with vllm models
+- Updated tybot-connector to 2.1.2
+
 # 2.19.2
 - Improved vLLM integration in order to support multiple servers
 - Added custom_regex option for .txt and .md file uploaded in a Knowledge Base

package/models/kb_setting.js

@@ -20,6 +20,67 @@ let expireAnsweredAfterSeconds = ttlSecondsFromEnv(
   DEFAULT_ANSWERED_TTL_SEC
 );
 
+const EMPTY_EMBEDDING_API_KEY = '';
+
+function logEmbeddingApiKeyWriteAttempt(source, attemptedValue, meta = {}) {
+  if (attemptedValue === EMPTY_EMBEDDING_API_KEY || attemptedValue == null) {
+    return;
+  }
+  winston.warn('[Namespace.embedding.api_key] non-empty value blocked on ' + source, {
+    ...meta,
+    attemptedValue,
+    stack: new Error().stack
+  });
+}
+
+function sanitizeEmbeddingApiKeyObject(embedding, source, meta = {}) {
+  if (!embedding || typeof embedding !== 'object') {
+    return false;
+  }
+  if (!Object.prototype.hasOwnProperty.call(embedding, 'api_key')) {
+    return false;
+  }
+  if (embedding.api_key === EMPTY_EMBEDDING_API_KEY || embedding.api_key == null) {
+    embedding.api_key = EMPTY_EMBEDDING_API_KEY;
+    return false;
+  }
+  logEmbeddingApiKeyWriteAttempt(source, embedding.api_key, meta);
+  embedding.api_key = EMPTY_EMBEDDING_API_KEY;
+  return true;
+}
+
+function sanitizeEmbeddingApiKeyUpdate(update, source, meta = {}) {
+  if (!update || typeof update !== 'object') {
+    return false;
+  }
+
+  let changed = false;
+
+  const sanitizeContainer = (container) => {
+    if (!container || typeof container !== 'object') {
+      return;
+    }
+    if (sanitizeEmbeddingApiKeyObject(container.embedding, source, meta)) {
+      changed = true;
+    }
+    if (Object.prototype.hasOwnProperty.call(container, 'embedding.api_key')
+      && container['embedding.api_key'] !== EMPTY_EMBEDDING_API_KEY) {
+      logEmbeddingApiKeyWriteAttempt(source, container['embedding.api_key'], meta);
+      container['embedding.api_key'] = EMPTY_EMBEDDING_API_KEY;
+      changed = true;
+    }
+  };
+
+  sanitizeContainer(update);
+  ['$set', '$setOnInsert'].forEach((key) => {
+    if (update[key]) {
+      sanitizeContainer(update[key]);
+    }
+  });
+
+  return changed;
+}
+
 
 const EngineSchema = new Schema({
   name: {
@@ -77,7 +138,12 @@ const EmbeddingSchema = new Schema({
   },
   api_key: {
     type: String,
-    required: false
+    required: false,
+    default: EMPTY_EMBEDDING_API_KEY,
+    set: function (value) {
+      logEmbeddingApiKeyWriteAttempt('embedding.api_key setter', value);
+      return EMPTY_EMBEDDING_API_KEY;
+    }
   }
 }, {
   _id: false  // This is schema is always used as an embedded object inside NamespaceSchema
@@ -120,6 +186,40 @@ const NamespaceSchema = new Schema({
   timestamps: true
 })
 
+NamespaceSchema.pre('save', function (next) {
+  if (this.isModified('embedding.api_key')) {
+    const attemptedValue = this.get('embedding.api_key');
+    logEmbeddingApiKeyWriteAttempt('save', attemptedValue, {
+      namespaceId: this.id,
+      id_project: this.id_project
+    });
+    this.set('embedding.api_key', EMPTY_EMBEDDING_API_KEY);
+  }
+  next();
+});
+
+['updateOne', 'updateMany', 'findOneAndUpdate'].forEach((hook) => {
+  NamespaceSchema.pre(hook, function (next) {
+    const update = this.getUpdate();
+    if (sanitizeEmbeddingApiKeyUpdate(update, hook, { query: this.getQuery() })) {
+      this.setUpdate(update);
+    }
+    next();
+  });
+});
+
+NamespaceSchema.pre('insertMany', function (next, docs) {
+  if (Array.isArray(docs)) {
+    docs.forEach((doc) => {
+      sanitizeEmbeddingApiKeyObject(doc.embedding, 'insertMany', {
+        namespaceId: doc.id,
+        id_project: doc.id_project
+      });
+    });
+  }
+  next();
+});
+
 var KBSchema = new Schema({
   id_project: {
     type: String,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.19.3",
+  "version": "2.19.5",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",

package/routes/answered.js

@@ -6,295 +6,274 @@ var fastCsv = require('fast-csv');
 
 // Add a new unanswerd question
 router.post('/', async (req, res) => {
-    try {
-        const { namespace, question, answer, tokens, request_id } = req.body;
-        const id_project = req.projectid;
-
-        if (!namespace || !question || !answer) {
-            return res.status(400).json({
-                success: false,
-                error: "Missing required parameters: namespace, question and answer"
-            })
-        }
-
-        // Check if namespae belongs to project
-        const isValidNamespace = await validateNamespace(id_project, namespace);
-        if (!isValidNamespace) {
-            return res.status(403).json({
-                success: false,
-                error: "Not allowed. The namespace does not belong to the current project."
-            })
-        }
-
-        const answeredQuestion = new AnsweredQuestion({
-            id_project,
-            namespace,
-            question,
-            answer,
-            tokens,
-            request_id,
-        });
-
-        const savedQuestion = await answeredQuestion.save();
-        res.status(200).json(savedQuestion);
-
-    } catch (error) {
-        winston.error('Error adding answered question:', error);
-        res.status(500).json({
-            success: false,
-            error: "Error adding answered question"
-        });
-    }
+  try {
+
+    const data = req.body;
+    const id_project = req.projectid;
+
+    const savedQuestion = await KbQuestionService.createAnsweredQuestion(
+      id_project, 
+      data
+    );
+
+    res.status(200).json(savedQuestion);
+
+  } catch (error) {
+    winston.error('Error adding answered question:', error);
+    res.status(error.status || 500).json({
+      success: false,
+      error: error.error || "Error adding answered question"
+    });
+  }
 })
 
 // Get all answered questions for a namespace
 router.get('/:namespace', async (req, res) => {
 
-    try {
-        const { namespace } = req.params;
-        const id_project = req.projectid;
-
-        if (!namespace) {
-            return res.status(400).json({
-                success: false,
-                error: "Missing required parameter: namespace"
-            })
-        }
-
-        // Check if namespace belongs to project
-        const isValidNamespace = await validateNamespace(id_project, namespace);
-        if (!isValidNamespace) {
-            return res.status(403).json({
-                success: false,
-                error: "Not allowed. The namespace does not belong to the current project."
-            })
-        }
-        
-        const page = parseInt(req.query.page) || 0;
-        const limit = parseInt(req.query.limit) || 20;
-        const sortField = req.query.sortField || 'created_at';
-        const direction = parseInt(req.query.direction) || -1;
-
-        const filter = { id_project, namespace };
-
-        let projection = undefined;
-
-        if (req.query.search) {
-            filter.$text = { $search: req.query.search };
-            // Add score to projection if it's a text search
-            projection = { score: { $meta: "textScore" } };
-        }
-
-        let sortObj;
-        if (projection && projection.score) {
-            sortObj = { score: { $meta: "textScore" } };
-        } else {
-            sortObj = { [sortField]: direction };
-        }
-
-        const questions = await AnsweredQuestion.find(filter, projection)
-            .sort(sortObj)
-            .skip(page * limit)
-            .limit(limit);
-
-        const count = await AnsweredQuestion.countDocuments(filter);
-
-        res.status(200).json({
-            count,
-            questions,
-            query: {
-                page,
-                limit,
-                sortField,
-                direction,
-                search: req.query.search || undefined
-            }
-        });
-        
-    } catch (error) {
-        winston.error('Error getting answered questions:', error);
-        res.status(500).json({
-            success: false,
-            error: "Error getting answered questions"
-        });
+  try {
+    const { namespace } = req.params;
+    const id_project = req.projectid;
+
+    if (!namespace) {
+      return res.status(400).json({
+        success: false,
+        error: "Missing required parameter: namespace"
+      })
+    }
+
+    // Check if namespace belongs to project
+    const isValidNamespace = await validateNamespace(id_project, namespace);
+    if (!isValidNamespace) {
+      return res.status(403).json({
+        success: false,
+        error: "Not allowed. The namespace does not belong to the current project."
+      })
     }
 
+    const page = parseInt(req.query.page) || 0;
+    const limit = parseInt(req.query.limit) || 20;
+    const sortField = req.query.sortField || 'createdAt';
+    const direction = parseInt(req.query.direction) || -1;
+
+    const filter = { id_project, namespace };
+
+    let projection = undefined;
+
+    if (req.query.search) {
+      filter.$text = { $search: req.query.search };
+      // Add score to projection if it's a text search
+      projection = { score: { $meta: "textScore" } };
+    }
+
+    let sortObj;
+    if (projection && projection.score) {
+      sortObj = { score: { $meta: "textScore" } };
+    } else {
+      sortObj = { [sortField]: direction };
+    }
+
+    const questions = await AnsweredQuestion.find(filter, projection)
+      .sort(sortObj)
+      .skip(page * limit)
+      .limit(limit);
+
+    const count = await AnsweredQuestion.countDocuments(filter);
+
+    res.status(200).json({
+      count,
+      questions,
+      query: {
+        page,
+        limit,
+        sortField,
+        direction,
+        search: req.query.search || undefined
+      }
+    });
+
+  } catch (error) {
+    winston.error('Error getting answered questions:', error);
+    res.status(500).json({
+      success: false,
+      error: "Error getting answered questions"
+    });
+  }
+
 })
 
 router.delete('/:id', async (req, res) => {
-    try {
-        const { id } = req.params;
-        const id_project = req.projectid;
-
-        const deleted = await AnsweredQuestion.findOneAndDelete({ _id: id, id_project });
-        if (!deleted) {
-            return res.status(404).json({
-                success: false,
-                error: "Question not found"
-            });
-        }
-
-        res.status(200).json({
-            success: true,
-            message: "Question deleted successfully"
-        });
-
-    } catch (error) {
-        winston.error('Error deleting answered question:', error);
-        res.status(500).json({
-            success: false,
-            error: "Error deleting answered question"
-        });
+  try {
+    const { id } = req.params;
+    const id_project = req.projectid;
+
+    const deleted = await AnsweredQuestion.findOneAndDelete({ _id: id, id_project });
+    if (!deleted) {
+      return res.status(404).json({
+        success: false,
+        error: "Question not found"
+      });
     }
+
+    res.status(200).json({
+      success: true,
+      message: "Question deleted successfully"
+    });
+
+  } catch (error) {
+    winston.error('Error deleting answered question:', error);
+    res.status(500).json({
+      success: false,
+      error: "Error deleting answered question"
+    });
+  }
 })
 
 router.delete('/namespace/:namespace', async (req, res) => {
-    try {
-        const { namespace } = req.params;
-        const id_project = req.projectid;
-
-        // Check if namespace belongs to project
-        const isValidNamespace = await validateNamespace(id_project, namespace);
-        if (!isValidNamespace) {
-            return res.status(403).json({
-                success: false,
-                error: "Not allowed. The namespace does not belong to the current project."
-            });
-        }
-        
-        const result = await AnsweredQuestion.deleteMany({ id_project, namespace });
-        res.status(200).json({
-            success: true,
-            count: result.deletedCount,
-            message: "All questions deleted successfully"
-        });
-
-    } catch (error) {
-        winston.error('Error deleting answered questions:', error);
-        res.status(500).json({
-            success: false,
-            error: "Error deleting answered questions"
-        });
+  try {
+    const { namespace } = req.params;
+    const id_project = req.projectid;
+
+    // Check if namespace belongs to project
+    const isValidNamespace = await validateNamespace(id_project, namespace);
+    if (!isValidNamespace) {
+      return res.status(403).json({
+        success: false,
+        error: "Not allowed. The namespace does not belong to the current project."
+      });
     }
+
+    const result = await AnsweredQuestion.deleteMany({ id_project, namespace });
+    res.status(200).json({
+      success: true,
+      count: result.deletedCount,
+      message: "All questions deleted successfully"
+    });
+
+  } catch (error) {
+    winston.error('Error deleting answered questions:', error);
+    res.status(500).json({
+      success: false,
+      error: "Error deleting answered questions"
+    });
+  }
 })
 
 router.get('/count/:namespace', async (req, res) => {
-    try {
-        const { namespace } = req.params;
-        const id_project = req.projectid;
-
-        if (!namespace) {
-            return res.status(400).json({
-                success: false,
-                error: "Missing required parameter: namespace"
-            });
-        }
-
-        // Check if namespace belongs to project
-        const isValidNamespace = await validateNamespace(id_project, namespace);
-        if (!isValidNamespace) {
-            return res.status(403).json({
-                success: false,
-                error: "Not allowed. The namespace does not belong to the current project."
-            });
-        }
-
-        const count = await AnsweredQuestion.countDocuments({ id_project, namespace });
-        res.status(200).json({ count });
-
-    } catch (error) {
-        winston.error('Error counting answered questions:', error);
-        res.status(500).json({
-            success: false,
-            error: "Error counting answered questions"
-        });
+  try {
+    const { namespace } = req.params;
+    const id_project = req.projectid;
+
+    if (!namespace) {
+      return res.status(400).json({
+        success: false,
+        error: "Missing required parameter: namespace"
+      });
+    }
+
+    // Check if namespace belongs to project
+    const isValidNamespace = await validateNamespace(id_project, namespace);
+    if (!isValidNamespace) {
+      return res.status(403).json({
+        success: false,
+        error: "Not allowed. The namespace does not belong to the current project."
+      });
     }
+
+    const count = await AnsweredQuestion.countDocuments({ id_project, namespace });
+    res.status(200).json({ count });
+
+  } catch (error) {
+    winston.error('Error counting answered questions:', error);
+    res.status(500).json({
+      success: false,
+      error: "Error counting answered questions"
+    });
+  }
 })
 
 router.get('/:namespace/export', async (req, res) => {
-    try {
-        const { namespace } = req.params;
-        const id_project = req.projectid;
-        const mode = String(req.query.mode || 'csv').toLowerCase();
-
-        if (mode !== 'csv' && mode !== 'json') {
-            return res.status(400).json({
-                success: false,
-                error: 'Invalid format. Use mode=json or mode=csv'
-            });
-        }
-
-        const isValidNamespace = await validateNamespace(id_project, namespace);
-        if (!isValidNamespace) {
-            return res.status(403).json({
-                success: false,
-                error: "Not allowed. The namespace does not belong to the current project."
-            });
-        }
-
-        const questions = await AnsweredQuestion.find({ id_project, namespace })
-            .sort({ createdAt: -1 })
-            .lean();
-
-        const safeFilename = String(namespace).replace(/[^\w.-]+/g, '_') || 'export';
-
-        if (mode === 'json') {
-            const questionsJson = questions.map((q) => {
-                const { __v, updatedAt, ...doc } = q;
-                return doc;
-            });
-            const payload = {
-                namespace,
-                exportedAt: new Date().toISOString(),
-                count: questionsJson.length,
-                questions: questionsJson
-            };
-            res.setHeader('Content-Type', 'application/json; charset=utf-8');
-            res.setHeader('Content-Disposition', `attachment; filename="answered-${safeFilename}.json"`);
-            return res.send(JSON.stringify(payload, null, 2));
-        }
-
-        res.setHeader('Content-Type', 'text/csv; charset=utf-8');
-        res.setHeader('Content-Disposition', `attachment; filename="answered-${safeFilename}.csv"`);
-
-        const csvStream = fastCsv.format({ headers: true });
-        csvStream.pipe(res);
-
-        for (const q of questions) {
-            csvStream.write({
-                id: String(q._id),
-                namespace: q.namespace,
-                question: q.question,
-                answer: q.answer,
-                tokens: q.tokens != null ? q.tokens : '',
-                request_id: q.request_id || '',
-                createdAt: q.createdAt ? new Date(q.createdAt).toISOString() : ''
-            });
-        }
-        csvStream.end();
-    } catch (error) {
-        winston.error('Error exporting answered questions:', error);
-        if (!res.headersSent) {
-            res.status(500).json({
-                success: false,
-                error: "Error exporting answered questions"
-            });
-        }
+  try {
+    const { namespace } = req.params;
+    const id_project = req.projectid;
+    const mode = String(req.query.mode || 'csv').toLowerCase();
+
+    if (mode !== 'csv' && mode !== 'json') {
+      return res.status(400).json({
+        success: false,
+        error: 'Invalid format. Use mode=json or mode=csv'
+      });
+    }
+
+    const isValidNamespace = await validateNamespace(id_project, namespace);
+    if (!isValidNamespace) {
+      return res.status(403).json({
+        success: false,
+        error: "Not allowed. The namespace does not belong to the current project."
+      });
     }
+
+    const questions = await AnsweredQuestion.find({ id_project, namespace })
+      .sort({ createdAt: -1 })
+      .lean();
+
+    const safeFilename = String(namespace).replace(/[^\w.-]+/g, '_') || 'export';
+
+    if (mode === 'json') {
+      const questionsJson = questions.map((q) => {
+        const { __v, updatedAt, ...doc } = q;
+        return doc;
+      });
+      const payload = {
+        namespace,
+        exportedAt: new Date().toISOString(),
+        count: questionsJson.length,
+        questions: questionsJson
+      };
+      res.setHeader('Content-Type', 'application/json; charset=utf-8');
+      res.setHeader('Content-Disposition', `attachment; filename="answered-${safeFilename}.json"`);
+      return res.send(JSON.stringify(payload, null, 2));
+    }
+
+    res.setHeader('Content-Type', 'text/csv; charset=utf-8');
+    res.setHeader('Content-Disposition', `attachment; filename="answered-${safeFilename}.csv"`);
+
+    const csvStream = fastCsv.format({ headers: true });
+    csvStream.pipe(res);
+
+    for (const q of questions) {
+      csvStream.write({
+        id: String(q._id),
+        namespace: q.namespace,
+        question: q.question,
+        answer: q.answer,
+        tokens: q.tokens != null ? q.tokens : '',
+        request_id: q.request_id || '',
+        createdAt: q.createdAt ? new Date(q.createdAt).toISOString() : ''
+      });
+    }
+    csvStream.end();
+  } catch (error) {
+    winston.error('Error exporting answered questions:', error);
+    if (!res.headersSent) {
+      res.status(500).json({
+        success: false,
+        error: "Error exporting answered questions"
+      });
+    }
+  }
 });
 
 // Helper function to validate namespace
 async function validateNamespace(id_project, namespace_id) {
-    try {
-        const namespace = await Namespace.findOne({
-            id_project: id_project,
-            id: namespace_id
-        });
-        return !!namespace;
-    } catch (err) {
-        winston.error('validate namespace error: ', err);
-        throw err;
-    }
+  try {
+    const namespace = await Namespace.findOne({
+      id_project: id_project,
+      id: namespace_id
+    });
+    return !!namespace;
+  } catch (err) {
+    winston.error('validate namespace error: ', err);
+    throw err;
+  }
 }
-
-module.exports = router;
+module.exports = router;