@tiledesk/tiledesk-server 2.13.49 → 2.13.51

package/CHANGELOG.md

@@ -5,6 +5,15 @@
 🚀        IN PRODUCTION                        🚀
 (https://www.npmjs.com/package/@tiledesk/tiledesk-server/v/2.3.77) 
 
+# 2.13.51
+- Refactoring of route() and create() method on RequestService in order to improve performance for firt message
+- Updated tests
+
+# 2.13.50
+- Updated kb route to support embeddings
+- Updated tybot-connector to 2.0.41
+- Updated vxml-connector to 0.1.89
+
 # 2.13.49
 - Updated tybot-connector to 2.0.41
 

package/app.js

@@ -64,10 +64,12 @@ var autoIndex = true;
 if (process.env.MONGOOSE_AUTOINDEX) {
   autoIndex = process.env.MONGOOSE_AUTOINDEX;
 }
-
 winston.info("DB AutoIndex: " + autoIndex);
 
-var connection = mongoose.connect(databaseUri, { "useNewUrlParser": true, "autoIndex": autoIndex }, function(err) {
+let useUnifiedTopology = process.env.MONGOOSE_UNIFIED_TOPOLOGY === 'true';
+winston.info("DB useUnifiedTopology: ", useUnifiedTopology, typeof useUnifiedTopology);
+
+var connection = mongoose.connect(databaseUri, { "useNewUrlParser": true, "autoIndex": autoIndex, "useUnifiedTopology": useUnifiedTopology }, function(err) {
   if (err) { 
     winston.error('Failed to connect to MongoDB on ' + databaseUri + " ", err);
     process.exit(1);
@@ -79,7 +81,7 @@ if (process.env.MONGOOSE_DEBUG==="true") {
 }
 mongoose.set('useFindAndModify', false); // https://mongoosejs.com/docs/deprecations.html#-findandmodify-
 mongoose.set('useCreateIndex', true);
-mongoose.set('useUnifiedTopology', false); 
+//mongoose.set('useUnifiedTopology', false); 
 
 // CONNECT REDIS - CHECK IT
 const { TdCache } = require('./utils/TdCache');
@@ -180,6 +182,7 @@ var geoService = require('./services/geoService');
 // geoService.listen(); //queued
 
 var updateLeadQueued = require('./services/updateLeadQueued');
+var updateRequestSnapshotQueued = require('./services/updateRequestSnapshotQueued');
 
 let JobsManager = require('./jobsManager');
 
@@ -189,7 +192,7 @@ if (process.env.JOB_WORKER_ENABLED=="true" || process.env.JOB_WORKER_ENABLED ==
 }
 winston.info("JobsManager jobWorkerEnabled: "+ jobWorkerEnabled);  
 
-let jobsManager = new JobsManager(jobWorkerEnabled, geoService, botEvent, subscriptionNotifierQueued, botSubscriptionNotifier, updateLeadQueued);
+let jobsManager = new JobsManager(jobWorkerEnabled, geoService, botEvent, subscriptionNotifierQueued, botSubscriptionNotifier, updateLeadQueued, updateRequestSnapshotQueued);
 
 var faqBotHandler = require('./services/faqBotHandler');
 faqBotHandler.listen();

package/config/kb/embedding.js

@@ -0,0 +1,7 @@
+module.exports = {
+    provider: process.env.EMBEDDINGS_PROVIDER || "openai",
+    name: process.env.EMBEDDINGS_NAME || "text-embedding-ada-002",
+    api_key: "",
+    dimension: Number(process.env.EMBEDDINGS_DIMENSION) || 1536,
+    url: process.env.EMBEDDINGS_URL
+}

package/config/kb/engine.hybrid.js

@@ -0,0 +1,10 @@
+module.exports = {
+    name: process.env.VECTOR_STORE_NAME || 'pinecone',
+    type: process.env.INDEX_TYPE_HYBRID || process.env.PINECONE_TYPE_HYBRID || 'serverless',
+    apikey: process.env.VECTOR_STORE_APIKEY || '',
+    vector_size: Number(process.env.VECTOR_SIZE_HYBRID) || 1536,
+    index_name: process.env.INDEX_NAME_HYBRID || process.env.PINECONE_INDEX_HYBRID || 'llm-sample-hybrid-index',
+    host: process.env.VECTOR_STORE_HOST,
+    port: process.env.VECTOR_STORE_PORT ? Number(process.env.VECTOR_STORE_PORT) : undefined,
+    deployment: process.env.VECTOR_STORE_DEPLOYMENT
+  }

package/config/kb/engine.js

@@ -0,0 +1,10 @@
+module.exports = {
+    name: process.env.VECTOR_STORE_NAME || 'pinecone',
+    type: process.env.INDEX_TYPE || process.env.PINECONE_TYPE || 'serverless',
+    apikey: process.env.VECTOR_STORE_APIKEY || '',
+    vector_size: Number(process.env.VECTOR_SIZE) || 1536,
+    index_name: process.env.INDEX_NAME || process.env.PINECONE_INDEX || 'llm-sample-index',
+    host: process.env.VECTOR_STORE_HOST,
+    port: process.env.VECTOR_STORE_PORT ? Number(process.env.VECTOR_STORE_PORT) : undefined,
+    deployment: process.env.VECTOR_STORE_DEPLOYMENT
+  }

package/jobs.js

@@ -29,6 +29,7 @@ const botEvent = require('./event/botEvent');
 var channelManager = require('./channels/channelManager');
 
 var updateLeadQueued = require('./services/updateLeadQueued');
+var updateRequestSnapshotQueued = require('./services/updateRequestSnapshotQueued');
 
 
 require('./services/mongoose-cache-fn')(mongoose);
@@ -47,7 +48,10 @@ if (!databaseUri) { //TODO??
   winston.warn('DATABASE_URI not specified, falling back to localhost.');
 }
 
-var connection = mongoose.connect(databaseUri, { "useNewUrlParser": true, "autoIndex": autoIndex }, function(err) {
+let useUnifiedTopology = process.env.MONGOOSE_UNIFIED_TOPOLOGY === 'true';
+winston.info("DB useUnifiedTopology: ", useUnifiedTopology, typeof useUnifiedTopology);
+
+var connection = mongoose.connect(databaseUri, { "useNewUrlParser": true, "autoIndex": autoIndex, "useUnifiedTopology": useUnifiedTopology }, function(err) {
   if (err) { 
     winston.error('Failed to connect to MongoDB on ' + databaseUri + " ", err);
     process.exit(1);
@@ -83,7 +87,7 @@ async function main()
     
 
 
-    let jobsManager = new JobsManager(undefined, geoService, botEvent, subscriptionNotifierQueued, botSubscriptionNotifier, updateLeadQueued);
+    let jobsManager = new JobsManager(undefined, geoService, botEvent, subscriptionNotifierQueued, botSubscriptionNotifier, updateLeadQueued, updateRequestSnapshotQueued);
 
     jobsManager.listen();
 

package/jobsManager.js

@@ -2,7 +2,7 @@
 var winston = require('./config/winston');
 
 class JobsManager {
-    constructor(jobWorkerEnabled, geoService, botEvent, subscriptionNotifierQueued, botSubscriptionNotifier, updateLeadQueued) {
+    constructor(jobWorkerEnabled, geoService, botEvent, subscriptionNotifierQueued, botSubscriptionNotifier, updateLeadQueued, updateRequestSnapshotQueued) {
         this.geoService = geoService;
         this.botEvent = botEvent;
         // this.subscriptionNotifier = subscriptionNotifier;
@@ -22,6 +22,7 @@ class JobsManager {
         // winston.info("JobsManager jobWorkerEnabled: "+ this.jobWorkerEnabled);  
 
         this.updateLeadQueued = updateLeadQueued;
+        this.updateRequestSnapshotQueued = updateRequestSnapshotQueued;
     }
 
 
@@ -39,6 +40,10 @@ class JobsManager {
 
         this.updateLeadQueued.listen();
 
+        if (this.updateRequestSnapshotQueued) {
+            this.updateRequestSnapshotQueued.listen();
+        }
+
         // this.botSubscriptionNotifier.start(); // disabled
     }
 

package/models/kb_setting.js

@@ -1,9 +1,9 @@
-var mongoose = require('mongoose');
-var Schema = mongoose.Schema;
-var winston = require('../config/winston');
+let mongoose = require('mongoose');
+let Schema = mongoose.Schema;
+let winston = require('../config/winston');
 let expireAfterSeconds = process.env.UNANSWERED_QUESTION_EXPIRATION_TIME || 7 * 24 * 60 * 60; // 7 days
 
-var EngineSchema = new Schema({
+const EngineSchema = new Schema({
   name: {
     type: String,
     required: true
@@ -23,12 +23,49 @@ var EngineSchema = new Schema({
   index_name: {
     type: String,
     required: true
+  },
+  host: {
+    type: String,
+    required: false
+  },
+  port: {
+    type: String,
+    required: false
+  },
+  deployment: {
+    type: String,
+    required: false
+  },
+}, {
+  _id: false  // This is schema is always used as an embedded object inside NamespaceSchema
+})
+
+const EmbeddingSchema = new Schema({
+  provider: {
+    type: String,
+    required: true
+  },
+  name: {
+    type: String,
+    required: true
+  },
+  dimension: {
+    type: Number,
+    reuired: true
+  },
+  url: {
+    type: String,
+    required: false
+  },
+  api_key: {
+    type: String,
+    required: false
   }
 }, {
   _id: false  // This is schema is always used as an embedded object inside NamespaceSchema
 })
 
-var NamespaceSchema = new Schema({
+const NamespaceSchema = new Schema({
   id_project: {
     type: String,
     required: true
@@ -55,7 +92,11 @@ var NamespaceSchema = new Schema({
   },
   engine: {
     type: EngineSchema,
-    required: false
+    required: true
+  },
+  embedding: {
+    type: EmbeddingSchema,
+    required: true
   }
 }, {
   timestamps: true
@@ -86,6 +127,14 @@ var KBSchema = new Schema({
     type: String,
     required: false
   },
+  sitemap_origin_id: {
+    type: String,
+    required: false
+  },
+  sitemap_origin: {
+    type: String,
+    required: false
+  },
   namespace: {
     type: String,
     required: false
@@ -122,6 +171,10 @@ var KBSchema = new Schema({
   last_refresh: {
     type: Date,
     required: false
+  },
+  last_error: {
+    type: Object,
+    required: false
   }
 }, {
   timestamps: true
@@ -141,14 +194,6 @@ const UnansweredQuestionSchema = new Schema({
   question: {
     type: String,
     required: true
-  },
-  created_at: {
-    type: Date,
-    default: Date.now
-  },
-  updated_at: {
-    type: Date,
-    default: Date.now
   }
 },{
   timestamps: true
@@ -158,7 +203,7 @@ const UnansweredQuestionSchema = new Schema({
 UnansweredQuestionSchema.index({ created_at: 1 }, { expireAfterSeconds: expireAfterSeconds }); // 30 days
 
 // DEPRECATED !! - Start
-var KBSettingSchema = new Schema({
+const KBSettingSchema = new Schema({
   id_project: {
     type: String,
     required: true,

package/models/profile.js

@@ -1,11 +1,17 @@
 var mongoose = require('mongoose');
 var Schema = mongoose.Schema;
 var winston = require('../config/winston');
+let isCommunity = false;
+if (process.env.COMMUNITY_VERSION === true || process.env.COMMUNITY_VERSION === 'true') {
+  isCommunity = true;
+}
 
 var ProfileSchema = new Schema({
   name: {
     type: String,
-    default: 'Sandbox',
+    default: function() {
+      return (isCommunity) ? 'Custom' : 'Sandbox';
+    },
     index: true
   },
   trialDays: {
@@ -14,23 +20,66 @@ var ProfileSchema = new Schema({
   },
   agents: {
     type: Number,
-    default: 0 //??
+    default: function() {
+      return (isCommunity) ? 1000 : 0;
+    }
   },
   type: {
     type: String,
-    default: 'free',
+    default: function() {
+      return (isCommunity) ? 'payment' : 'free';
+    }
   },
   quotes: {
-    type: Object
+    type: Object,
+    default: function() {
+      if (isCommunity) {
+        return {
+          chatbots: 10000,
+          kbs: 10000,
+          namespace: 10000,
+        }
+      }
+      return undefined;
+    }
   },
   customization: {
-    type: Object
+    type: Object,
+    default: function() {
+      if (isCommunity) {
+        return {
+          copilot: true,
+          webhook: true,
+          widgetUnbranding: true,
+          smtpSettings: true,
+          knowledgeBases: true,
+          reindex: true,
+          messanger: true,
+          telegram: true,
+          chatbot: true
+        };
+      }
+      return undefined;
+    }
   },
   subStart: {
     type: Date,
+    default: function() {
+      if (isCommunity) {
+        return new Date();
+      }
+      return undefined;
+    }
   },
   subEnd: {
     type: Date,
+    default: function() {
+      if (isCommunity) {
+        // Set date to 31 December 2099
+        return new Date('2099-12-31T23:59:59.999Z');
+      }
+      return undefined;
+    }
   },
   subscriptionId:  {
     type: String,

package/models/request.js

@@ -460,7 +460,7 @@ RequestSchema.index({ id_project: 1, request_id: 1 }
 //TODO cambiare dummy con language? attento che il codice deve essere compatibile
 
 
-RequestSchema.index({transcript: 'text', rating_message: 'text', subject: 'text', "tags.tag": 'text', "notes.text": 'text', "snapshot.lead.email": 'text', "snapshot.lead.fullname": 'text' },  
+RequestSchema.index({id_project: 1, transcript: 'text', rating_message: 'text', subject: 'text', "tags.tag": 'text', "notes.text": 'text', "snapshot.lead.email": 'text', "snapshot.lead.fullname": 'text' },  
  {"name":"request_fulltext","default_language": defaultFullTextLanguage,"language_override": "dummy"}); // schema level
 
 //  let query = {id_project: operatorSelectedEvent.id_project, participants: { $exists: true, $ne: [] }};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tiledesk/tiledesk-server",
   "description": "The Tiledesk server module",
-  "version": "2.13.49",
+  "version": "2.13.51",
   "scripts": {
     "start": "node ./bin/www",
     "pretest": "mongodb-runner start",
@@ -49,9 +49,9 @@
     "@tiledesk/tiledesk-rasa-connector": "^1.0.10",
     "@tiledesk/tiledesk-sms-connector": "^0.1.11",
     "@tiledesk/tiledesk-telegram-connector": "^0.1.14",
-    "@tiledesk/tiledesk-tybot-connector": "^2.0.41",
+    "@tiledesk/tiledesk-tybot-connector": "^2.0.42",
     "@tiledesk/tiledesk-voice-twilio-connector": "^0.1.26",
-    "@tiledesk/tiledesk-vxml-connector": "^0.1.87",
+    "@tiledesk/tiledesk-vxml-connector": "^0.1.89",
     "@tiledesk/tiledesk-whatsapp-connector": "1.0.18",
     "@tiledesk/tiledesk-whatsapp-jobworker": "^0.0.13",
     "amqplib": "^0.5.5",
@@ -113,7 +113,7 @@
     "retry-request": "^4.2.2",
     "serve-favicon": "~2.5.0",
     "sharp": "^0.27.2",
-    "sitemapper": "^3.2.8",
+    "sitemapper": "^3.2.24",
     "stripe": "^7.2.0",
     "uniqid": "^5.4.0",
     "uuid": "^3.3.3",

package/pubmodules/apps/listener.js

@@ -29,7 +29,8 @@ class Listener {
             ACCESS_TOKEN_SECRET: process.env.APPS_ACCESS_TOKEN_SECRET || configSecretOrPubicKay,
             MONGODB_URI: process.env.APPS_MONGODB_URI || config.databaseUri,
             KALEYRA_ENABLED: process.env.KALEYRA_ENABLED || config.kaleyra_enabled,
-            VOICE_ENABLED: process.env.VOICE_ENABLED || false
+            VOICE_ENABLED: process.env.VOICE_ENABLED || false,
+            isCommunity: process.env.COMMUNITY_VERSION || false
         }, () => {
             winston.info("Tiledesk Apps proxy server succesfully started.")
         })

package/pubmodules/queue/reconnect.js

@@ -204,9 +204,10 @@ function startWorker() {
           winston.info("Data queue", oka)
         });
 
-        
-
-
+        ch.bindQueue(_ok.queue, exchange, "request_snapshot_update", {}, function(err3, oka) {
+          winston.info("Queue bind: "+_ok.queue+ " err: "+err3+ " key: request_snapshot_update");
+          winston.info("Data queue", oka)
+        });
 
 
         ch.consume(queueName, processMsg, { noAck: false });
@@ -320,6 +321,11 @@ function work(msg, cb) {
     leadEvent.emit('lead.fullname.email.update.queue',  JSON.parse(message_string));
   }
 
+  if (topic === 'request_snapshot_update') {
+    winston.debug("reconnect here topic request_snapshot_update:" + topic); 
+    requestEvent.emit('request.snapshot.update.queue',  JSON.parse(message_string));
+  }
+
 
 
   cb(true);
@@ -404,6 +410,13 @@ function listen() {
       });
     });
 
+    requestEvent.on('request.snapshot.update', function(data) {
+      setImmediate(() => {
+        winston.debug("reconnect request.snapshot.update")
+        console.log("reconnect request.snapshot.update")
+        publish(exchange, "request_snapshot_update", Buffer.from(JSON.stringify(data)));
+      });
+    });
 
 
     messageEvent.on('message.create', function(message) {

package/pubmodules/queue/reconnectFanout.js

@@ -184,6 +184,10 @@ function work(msg, cb) {
     winston.debug("reconnectfanout here topic faqbot_update:" + topic);
     botEvent.emit('faqbot.update.queue.pubsub', JSON.parse(message_string));
   }
+  if (topic === 'request_snapshot_add') {
+    winston.debug("reconnectfanout here topic request_snapshot_add:" + topic); 
+    requestEvent.emit('request.snapshot.add.queue.pubsub', JSON.parse(message_string));
+  }
   cb(true);
 //   WebSocket.cb(true);
 //   requestEvent.on(msg.KEYYYYYYY+'.ws', msg.content);