@tiledesk/tiledesk-server 2.13.49 → 2.13.50

package/routes/webhook.js

@@ -1,17 +1,13 @@
 var express = require('express');
 var router = express.Router();
-const uuidv4 = require('uuid/v4');
 var { KB, Namespace } = require('../models/kb_setting');
 var winston = require('../config/winston');
 const JobManager = require('../utils/jobs-worker-queue-manager/JobManagerV2');
-const { Scheduler } = require('../services/Scheduler');
 const { AiReindexService } = require('../services/aiReindexService');
 const { Webhook } = require('../models/webhook');
-const httpUtil = require('../utils/httpUtil');
-var jwt = require('jsonwebtoken');
-const Faq_kb = require('../models/faq_kb');
 const webhookService = require('../services/webhookService');
 const errorCodes = require('../errorCodes');
+const aiManager = require('../services/aiManager');
 var ObjectId = require('mongoose').Types.ObjectId;
 
 const port = process.env.PORT || '3000';
@@ -41,12 +37,20 @@ jobManager.connectAndStartPublisher((status, error) => {
 
 let default_engine = {
   name: "pinecone",
-  type: process.env.PINECONE_TYPE,
+  type: process.env.PINECONE_TYPE || "pod",
   apikey: "",
   vector_size: 1536,
   index_name: process.env.PINECONE_INDEX
 }
 
+let default_engine_hybrid = {
+  name: "pinecone",
+  type: process.env.PINECONE_TYPE_HYBRID || "serverless",
+  apikey: "",
+  vector_size: 1536,
+  index_name: process.env.PINECONE_INDEX_HYBRID
+}
+
 router.post('/kb/reindex', async (req, res) => {
 
   winston.verbose("/kb/reindex webhook called")
@@ -69,6 +73,15 @@ router.post('/kb/reindex', async (req, res) => {
     return res.status(500).send({ success: false, error: "Error getting content with id " + content_id });
   })
 
+  const namespace_id = kb.namespace;
+  let namespace;
+  try {
+    namespace = await aiManager.checkNamespace(kb.id_project, namespace_id);
+  } catch (err) {
+    let errorCode = err?.errorCode ?? 500;
+    return res.status(errorCode).send({ success: false, error: err.error });
+  }
+
   if (!kb) {
     winston.warn("(webhook) Kb content not found with id " + content_id + ". Deleting scheduler...");
 
@@ -86,6 +99,58 @@ router.post('/kb/reindex', async (req, res) => {
       return;
     }, 10000);
     
+  } else if (kb.type === 'sitemap') {
+
+    const urls = await aiManager.fetchSitemap(kb.source).catch((err) => {
+      winston.error("(webhook) Error fetching sitemap: ", err);
+      return res.status(500).send({ success: false, error: err });
+    })
+
+    if (urls.length === 0) {
+      return res.status(400).send({ success: false, error: "No url found on sitemap" });
+    }
+
+    let existingKbs;
+    try {
+      existingKbs = await KB.find({ id_project: kb.id_project, namespace: namespace_id, sitemap_origin_id: content_id}).lean().exec();
+    } catch(err) {
+      winston.error("(webhook) Error finding existing contents: ", err);
+      return res.status(500).send({ success: false, error: "Error finding existing sitemap contents" });
+    }
+
+    const result = await aiManager.foundSitemapChanges(existingKbs, urls).catch((err) => {
+      winston.error("(webhook) error finding sitemap differecens ", err);
+      return res.status(400).send({ success: false, error: "Error finding sitemap differecens" });
+    })
+
+    if (!result) return; // esco qui
+
+    const { addedUrls, removedIds } = result;
+
+    if (removedIds.length > 0) {
+      const idsSet = new Set(removedIds);
+      const kbsToDelete = existingKbs.filter(obj => idsSet.has(obj._id));
+
+      aiManager.removeMultipleContents(namespace, kbsToDelete).catch((err) => {
+        winston.error("(webhook) error deleting multiple contents: ", err);
+      })
+    }
+
+    if (addedUrls.length > 0) {
+      const options = {
+        sitemap_origin_id: kb.id,
+        sitemap_origin: kb.source,
+        scrape_type: kb.scrape_type,
+        scrape_options: kb.scrape_options,
+        refresh_rate: kb.refresh_rate
+      }
+      aiManager.addMultipleUrls(namespace, addedUrls, options).catch((err) => {
+        winston.error("(webhook) error adding multiple urls contents: ", err);
+      })
+    }
+
+    res.status(200).send({ success: true, message: "Content queued for reindexing" });
+
   } else {
 
     let json = {
@@ -120,13 +185,13 @@ router.post('/kb/reindex', async (req, res) => {
       return res.status(500).send({ success: false, error: err })
     }
   
-    json.engine = namespace.engine || default_engine;
-  
+    json.engine = namespace.engine || (namespace.hybrid ? default_engine_hybrid : default_engine);
+
     let resources = [];
     resources.push(json);
   
     if (process.env.NODE_ENV !== 'test') {
-      scheduleScrape(resources);
+      aiManager.scheduleScrape(resources, namespace.hybrid);
     }
   
     res.status(200).send({ success: true, message: "Content queued for reindexing" });
@@ -273,37 +338,20 @@ router.all('/:webhook_id/dev', async (req, res) => {
   
 })
 
-async function scheduleScrape(resources) {
 
-  let scheduler = new Scheduler({ jobManager: jobManager });
+// async function generateChatbotToken(chatbot) {
+//   let signOptions = {
+//     issuer: 'https://tiledesk.com',
+//     subject: 'bot',
+//     audience: 'https://tiledesk.com/bots/' + chatbot._id,
+//     jwtid: uuidv4()
+//   };
 
-  resources.forEach(r => {
-    winston.debug("(Webhook) Schedule job with following data: ", r);
-    scheduler.trainSchedule(r, async (err, result) => {
-      if (err) {
-        winston.error("Scheduling error: ", err);
-      } else {
-        winston.verbose("Scheduling result: ", result);
-      }
-    });
-  })
+//   let botPayload = chatbot.toObject();
+//   let botSecret = botPayload.secret;
 
-  return true;
-}
-
-async function generateChatbotToken(chatbot) {
-  let signOptions = {
-    issuer: 'https://tiledesk.com',
-    subject: 'bot',
-    audience: 'https://tiledesk.com/bots/' + chatbot._id,
-    jwtid: uuidv4()
-  };
-
-  let botPayload = chatbot.toObject();
-  let botSecret = botPayload.secret;
-
-  var bot_token = jwt.sign(botPayload, botSecret, signOptions);
-  return bot_token;
-}
+//   var bot_token = jwt.sign(botPayload, botSecret, signOptions);
+//   return bot_token;
+// }
 
 module.exports = router;

package/services/aiManager.js

@@ -0,0 +1,464 @@
+const { Namespace, KB, Engine } = require('../models/kb_setting');
+const Integrations = require("../models/integrations");
+const aiService = require("./aiService");
+const { Scheduler } = require("./Scheduler");
+const { default: Sitemapper } = require('sitemapper');
+const winston = require('../config/winston');
+const configGlobal = require('../config/global');
+const JobManager = require('../utils/jobs-worker-queue-manager/JobManagerV2');
+
+// Constants
+const apiUrl = process.env.API_URL || configGlobal.apiUrl;
+const KB_WEBHOOK_TOKEN = process.env.KB_WEBHOOK_TOKEN || 'kbcustomtoken';
+const AMQP_MANAGER_URL = process.env.AMQP_MANAGER_URL || 'amqp://localhost';
+const JOB_TOPIC_EXCHANGE = process.env.JOB_TOPIC_EXCHANGE_TRAIN || 'tiledesk-trainer';
+const JOB_TOPIC_EXCHANGE_HYBRID = process.env.JOB_TOPIC_EXCHANGE_TRAIN_HYBRID || 'tiledesk-trainer-hybrid';
+
+// Default engine configuration
+const default_engine = require('../config/kb/engine');
+const default_engine_hybrid = require('../config/kb/engine.hybrid');
+const default_embedding = require('../config/kb/embedding');
+const integrationService = require('./integrationService');
+
+// Job managers
+let jobManager = new JobManager(AMQP_MANAGER_URL, {
+  debug: false,
+  topic: JOB_TOPIC_EXCHANGE,
+  exchange: JOB_TOPIC_EXCHANGE
+})
+
+let jobManagerHybrid = new JobManager(AMQP_MANAGER_URL, {
+  debug: false,
+  topic: JOB_TOPIC_EXCHANGE_HYBRID,
+  exchange: JOB_TOPIC_EXCHANGE_HYBRID
+})
+
+// Connect job managers
+jobManager.connectAndStartPublisher((status, error) => {
+  if (error) {
+    winston.error("aiManager jobManager connectAndStartPublisher error: ", error);
+  } else {
+    winston.info("aiManager jobManager - ConnectPublisher done with status: ", status);
+  }
+});
+
+jobManagerHybrid.connectAndStartPublisher((status, error) => {
+  if (error) {
+    winston.error("aiManager jobManagerHybrid connectAndStartPublisher error: ", error);
+  } else {
+    winston.info("aiManager jobManagerHybrid - ConnectPublisher done with status: ", status);
+  }
+});
+
+class AiManager {
+
+  constructor() { }
+
+  async addMultipleUrls(namespace, urls, options) {
+    return new Promise(async (resolve, reject) => {
+
+      let kbs = urls.map((url) => {
+        let kb = {
+          id_project: namespace.id_project,
+          name: url,
+          source: url,
+          type: 'url',
+          content: "",
+          namespace: namespace.id,
+          status: -1,
+          sitemap_origin_id: options.sitemap_origin_id,
+          sitemap_origin: options.sitemap_origin,
+          scrape_type: options.scrape_type,
+          scrape_options: options.scrape_options,
+          refresh_rate: options.refresh_rate
+        }
+        return kb;
+      })
+
+      let operations = kbs.map(doc => {
+        return {
+          updateOne: {
+            filter: { id_project: doc.id_project, type: 'url', source: doc.source, namespace: namespace.id },
+            update: doc,
+            upsert: true,
+            returnOriginal: false
+          }
+        }
+      })
+
+      this.saveBulk(operations, kbs, namespace.id_project, namespace.id).then( async (result) => {
+        let hybrid = namespace.hybrid;
+        let engine = namespace.engine || default_engine;
+        let embedding = namespace.embedding || default_embedding;
+        embedding.api_key = process.env.EMBEDDING_API_KEY || process.env.GPTKEY;
+        let webhook = apiUrl + '/webhook/kb/status?token=' + KB_WEBHOOK_TOKEN;
+
+        let resources = result.map(({ name, status, __v, createdAt, updatedAt, id_project, ...keepAttrs }) => keepAttrs)
+        resources = resources.map(({ _id, scrape_options, ...rest }) => {
+          return { id: _id, webhook: webhook, parameters_scrape_type_4: scrape_options, embedding: embedding, engine: engine, hybrid: hybrid, ...rest}
+        });
+
+        winston.verbose("resources to be sent to worker: ", resources);
+
+        if (process.env.NODE_ENV === 'test') {
+          resolve({ result, schedule_json: resources });
+          return;
+        }
+  
+        this.scheduleScrape(resources, hybrid);
+        resolve(result);
+
+      }).catch((err) => {
+        winston.error("Error save contents in bulk: ", err);
+        reject(err);
+      })
+    })
+  }
+
+  async checkNamespace(id_project, namespace_id) {
+    return new Promise( async (resolve, reject) => {
+
+      let namespace = await Namespace.findOne({ id: namespace_id }).catch((err) => {
+        winston.error("Error getting namespace ", err);
+        reject(err);
+      })
+      if (!namespace) {
+        winston.warn("Namespace not found with id " + namespace_id);
+        reject({ errorCode: 404, error: "Namespace not found with id " + namespace_id });
+      }
+      if (namespace.id_project !== id_project) {
+        winston.warn("Namespace not belonging to project " + id_project);
+        reject({ errorCode: 403,  error: "Namespace not belonging to project " + id_project });
+      }
+
+      resolve(namespace);
+    })
+  }
+
+  async resolveLLMConfig(id_project, provider = 'openai', model) {
+
+    if (provider === 'ollama' || provider === 'vllm') {
+      try {
+        const integration = await integrationService.getIntegration(id_project, provider);
+        
+        if (!integration?.value?.url) {
+          throw { code: 422, error: `Server url for ${provider} is empty or invalid`}
+        }
+
+        return {
+          provider,
+          name: model,
+          url: integration.value.url,
+          api_key: integration.value.apikey || ""
+        }
+
+      } catch (err) {
+        throw { code: err.code, error: err.error }
+      }
+    }
+
+    try {
+      let key = await integrationService.getKeyFromIntegration(id_project, provider)
+
+      return {
+        provider,
+        name: model,
+        api_key: key
+      }
+
+    } catch (err) {
+      throw { code: err.code, error: err.error }
+    }
+
+  }
+
+  async checkQuotaAvailability(quoteManager, project, ncontents) {
+
+    return new Promise( async (resolve, reject) => {
+
+      let limits = await quoteManager.getPlanLimits(project);
+      let kbs_limit = limits.kbs;
+      winston.verbose("Limit of kbs for current plan: " + kbs_limit);
+  
+      let kbs_count = await KB.countDocuments({ id_project: project._id }).exec();
+      winston.verbose("Kbs count: " + kbs_count);
+  
+      if (kbs_count >= kbs_limit) {
+        reject({ errorCode: 403, error: "Maximum number of resources reached for the current plan", plan_limit: kbs_limit });
+      }
+
+      let total_count = kbs_count + ncontents;
+      if (total_count > kbs_limit) {
+        reject({ errorCode: 403, error: "Cannot exceed the number of resources in the current plan", plan_limit: kbs_limit });
+      }
+
+      resolve(true);
+    })
+  }
+
+  async fetchSitemap(sitemapUrl) {
+
+    return new Promise(async (resolve, reject) => {
+      const sitemap = new Sitemapper({
+        url: sitemapUrl,
+        timeout: 15000,
+        debug: false
+      });
+
+      const data = await sitemap.fetch().catch((err) => {
+        reject(err);
+      })
+
+      if (data.errors && data.errors.length > 0) {
+        reject(data.errors[0]);
+      }
+
+      const urls = Array.isArray(data.sites) ? data.sites : [];
+      resolve(urls);
+    })
+  }
+
+  async foundSitemapChanges(existingKbs, urls) {
+  
+    return new Promise( async (resolve, reject) => {
+      let existingIdsBySource = {};
+      existingKbs.forEach(doc => {
+        existingIdsBySource[doc.source] = doc._id;
+      });
+  
+      let addedUrls = urls.filter(url => !existingIdsBySource.hasOwnProperty(url));
+      let removedIds = existingKbs
+        .filter(doc => !urls.includes(doc.source))
+        .map(doc => doc._id);
+  
+      resolve({ addedUrls, removedIds });
+    })
+  }
+
+  async generateFilename(name) {
+    return name
+      .toLowerCase()
+      .trim()
+      .normalize("NFD") // Normalize characters with accents
+      .replace(/[\u0300-\u036f]/g, "") // Removes diacritics (e.g. à becomes a)
+      .replace(/[^a-z0-9\s-_]/g, "") // Remove special characters
+      .replace(/\s+/g, "-") // Replaces spaces with dashes
+      .replace(/_/g, "-")
+      .replace(/-+/g, "-"); // Removes consecutive hyphens
+  }
+
+  async getKeyFromIntegrations(project_id) {
+
+    return new Promise( async (resolve) => {
+
+      let integration = await Integrations.findOne({ id_project: project_id, name: 'openai' }).catch((err) => {
+        winston.error("Unable to find openai integration for the current project " + project_id);
+        resolve(null);
+      })
+      if (integration && integration.value && integration.value.apikey) {
+        resolve(integration.value.apikey);
+      } else {
+        resolve(null);
+      }
+    })
+  }
+
+  async removeMultipleContents(namespace, kbs) {
+  
+    return new Promise( async (resolve, reject) => {
+      
+      kbs.forEach((kb) => {
+        let data = {
+          id: kb._id,
+          namespace: kb.namespace,
+          engine: namespace.engine || default_engine
+        }
+  
+        aiService.deleteIndex(data).then((resp) => {
+          winston.debug("delete content response: ", resp);
+          if (resp.data.success === true) {
+            KB.findByIdAndDelete(kb._id, (err, deletedKb) => {
+              if (err) {
+                winston.error("Delete kb error: ", err);
+                reject(err);
+              }
+            })
+          } else {
+            KB.findOneAndDelete({ _id: kb._id, status: { $in: [-1, 400 ]}}, (err, deletedKb) => {
+              if (err) {
+                winston.error("Delete kb error: ", err);
+                reject(err);
+              }
+            })
+          }
+        })
+      })
+      resolve(true);
+    })
+  }
+
+  async saveBulk(operations, kbs, project_id, namespace) {
+
+    return new Promise((resolve, reject) => {
+      KB.bulkWrite(operations, { ordered: false }).then((result) => {
+        winston.verbose("bulkWrite operations result: ", result);
+
+        KB.find({ id_project: project_id, namespace: namespace, source: { $in: kbs.map(kb => kb.source) } }).lean().then((documents) => {
+          winston.debug("documents: ", documents);
+          resolve(documents)
+        }).catch((err) => {
+          winston.error("Error finding documents ", err)
+          reject(err);
+        })
+
+      }).catch((err) => {
+        reject(err);
+      })
+    })
+  }
+
+  setDefaultScrapeOptions() {
+    return {
+      tags_to_extract: ["body"],
+      unwanted_tags: [],
+      unwanted_classnames: []
+    }
+  }
+
+  async scheduleScrape(resources, hybrid) {
+
+    let scheduler;
+    if (hybrid) {
+      scheduler = new Scheduler({ jobManager: jobManagerHybrid });
+    } else {
+      scheduler = new Scheduler({ jobManager: jobManager });
+    }
+
+    if (!scheduler) {
+      winston.error("ScheduleScrape JobManager is not defined");
+      return false;
+    }
+
+    resources.forEach(r => {
+      winston.debug("Schedule job with following data: ", r);
+      scheduler.trainSchedule(r, async (err, result) => {
+        let error_code = 100;
+        if (err) {
+          winston.error("Scheduling error: ", err);
+          error_code = 400;
+        } else {
+          winston.verbose("Scheduling result: ", result);
+        }
+        await this.updateStatus(r.id, error_code);
+      });
+    })
+
+    return true;
+  }
+
+  // from webhook
+  // async scheduleScrape(resources) {
+
+  //   let scheduler = new Scheduler({ jobManager: jobManager });
+  
+  //   resources.forEach(r => {
+  //     winston.debug("(Webhook) Schedule job with following data: ", r);
+  //     scheduler.trainSchedule(r, async (err, result) => {
+  //       if (err) {
+  //         winston.error("Scheduling error: ", err);
+  //       } else {
+  //         winston.info("Scheduling result: ", result);
+  //       }
+  //     });
+  //   })
+  
+  //   return true;
+  // }
+
+  async startScrape(data) {
+
+    if (!data.gptkey) {
+      let gptkey = process.env.GPTKEY;
+      if (!gptkey) {
+        return { error: "GPT apikey undefined" }
+      }
+      data.gptkey = gptkey;
+    }
+  
+    let status_updated = await this.updateStatus(data.id, 200);
+    winston.verbose("status of kb " + data.id + " updated: " + status_updated);
+  
+    return new Promise((resolve, reject) => {
+      aiService.singleScrape(data).then(async (resp) => {
+        winston.debug("singleScrape resp: ", resp.data);
+        let status_updated = await this.updateStatus(data.id, 300);
+        winston.verbose("status of kb " + data.id + " updated: " + status_updated);
+        resolve(resp.data);
+      }).catch( async (err) => {
+        winston.error("singleScrape err: ", err);
+        let error_message = err.response?.data?.error || "An unexpected error occurred";
+        let status_updated = await this.updateStatus(data.id, 400, error_message);
+        winston.verbose("status of kb " + data.id + " updated: " + status_updated);
+        reject(err);
+      })
+    })
+  }
+
+  async statusConverter(status) {
+    return new Promise((resolve) => {
+
+      let td_status;
+      switch (status) {
+        case 0:
+          td_status = -1;
+          break;
+        case 2:
+          td_status = 200;
+          break;
+        case 3:
+          td_status = 300;
+          break;
+        case 4:
+          td_status = 400;
+          break;
+        default:
+          td_status = -1
+      }
+      resolve(td_status);
+    })
+  }
+
+  async updateStatus(id, status, error) {
+    return new Promise((resolve) => {
+
+      let update = {
+        status: status,
+        last_refresh: new Date()
+      }
+
+      if (error) {
+        update.last_error = {
+          timestamp: Date.now(),
+          message: error
+        }
+      }
+
+      KB.findByIdAndUpdate(id, update, { new: true }, (err, updatedKb) => {
+        if (err) {
+          resolve(false)
+        } else if (!updatedKb) {
+          winston.verbose("Unable to update status. Data source not found.")
+          resolve(false)
+        } else {
+          winston.debug("updatedKb: ", updatedKb)
+          resolve(true);
+        }
+      })
+    })
+  }
+
+}
+
+const aiManager = new AiManager();
+
+module.exports = aiManager;

package/services/aiService.js

@@ -205,7 +205,7 @@ class AiService {
       base_url = kb_endpoint_qa_gpu;
     }
     winston.debug("[OPENAI SERVICE] kb endpoint: " + base_url);
-    
+    console.log("aa1")
     return new Promise((resolve, reject) => {
 
       axios({
@@ -216,8 +216,10 @@ class AiService {
         data: data,
         method: 'POST'
       }).then((resbody) => {
+        console.log("aa2")
         resolve(resbody);
       }).catch((err) => {
+        console.log("aa3")
         reject(err);
       })
 
@@ -291,6 +293,6 @@ class AiService {
 
 }
 
-var aiService = new AiService();
+const aiService = new AiService();
 
 module.exports = aiService;