@tiledesk/tiledesk-voice-twilio-connector 0.1.22 → 0.1.23

package/index.js

@@ -58,6 +58,7 @@ let API_URL = null;
 let BASE_URL = null;
 let BASE_FILE_URL = null;
 let OPENAI_ENDPOINT = null;
+let ELEVENLABS_ENDPOINT = null;
 let REDIS_HOST = null;
 let REDIS_PORT = null;
 let REDIS_PASSWORD = null;
@@ -164,7 +165,7 @@ router.post('/webhook/:id_project', async (req, res) => {
   winston.debug("(voice) conversation returned:"+ conversation_id);
   
   
-   //GET AND SAVE GPT-KET IF 
+  //GET AND SAVE GPT-KET IF 
   let integrations = [], publicKey = false;
   let key = await integrationService.getKeyFromIntegrations(project_id, 'openai', settings.token)
   if (!key) {
@@ -179,6 +180,12 @@ router.post('/webhook/:id_project', async (req, res) => {
   }
   integrations.push({type: 'openai', key: key, publicKey: publicKey})
   
+  let eleven_labs = await integrationService.getKeyFromIntegrations(project_id, 'elevenlabs', settings.token)
+  if (eleven_labs) {
+      winston.debug("(voice) - Key found in Integrations: "+ eleven_labs);
+      integrations.push({type: 'elevenlabs', key: eleven_labs, publicKey: false})
+  }
+  
   //save data to redis
   let session_data = {
     from: from,
@@ -207,7 +214,7 @@ router.post('/webhook/:id_project', async (req, res) => {
     attributes: {
       subtype: 'info',
       payload: {
-        ... req.query //send all attributes back to chatbot
+        ... req.body //send all attributes back to chatbot
       }
     },
     channel: { name: CHANNEL_NAME },
@@ -363,6 +370,8 @@ async function getMessage(callSid, from, project_id, conversation_id){
           await tdChannel.removeMessageFromQueue(conversation_id, message._id)
           //reset delayIndex for wait command message time
           await voiceChannel.clearDelayTimeForCallId(callSid)
+          //manage attributes for current callId
+          await voiceChannel.saveSettingsForCallId(message.attributes, callSid)
 
           resolve(message);
           return;
@@ -918,23 +927,16 @@ router.post('/record/:callSid/',async (req, res) => {
     return res.status(404).send({error: "VOICE Channel not already connected"})
   }
   
-  //SPEECH TO TEXT
-  let key = sessionInfo.integrations.find((el => el.type === VOICE_PROVIDER.OPEN_AI))?.key
-  let publicKey = sessionInfo.integrations.find((el => el.type === VOICE_PROVIDER.OPEN_AI))?.publicKey
-  //check quotes if user is using public GPT_KEY
-  if(publicKey){
-    let keep_going = await aiService.checkQuoteAvailability(project_id, user.token);
-    if(!keep_going){
-      //no toke is available
-    }
-  }
   
-  let textMessage = await aiService.speechToText(audioFileUrl, vxmlAttributes.STT_MODEL, key).catch((err)=>{
-    console.log('errr while transcript', err.response?.data)
-  })
-  console.log('(voice) Message captured after STT -->', textMessage)
+  let attributes = await voiceChannel.getSettingsForCallId(callSid);
+  console.log('attributessss', attributes)
   
-  if(!textMessage){
+  //SPEECH TO TEXT
+  console.log('getting text message  . . . ', audioFileUrl, attributes.STT_MODEL)
+  let tiledeskMessage = await generateSTT(audioFileUrl, attributes, sessionInfo, settings)
+  console.log('(voice) Message captured after STT -->', tiledeskMessage)
+  
+  if(!tiledeskMessage){
     //case NO_INPUT
     const queryString = utils.buildQueryString(req.query);
     winston.debug('case no input.. redirect '+ queryString)
@@ -953,13 +955,7 @@ router.post('/record/:callSid/',async (req, res) => {
     })
   }
   
-  
-  let tiledeskMessage= {
-      text:textMessage,
-      senderFullname: from,
-      type: 'text',
-      channel: { name: CHANNEL_NAME }
-  };
+
   let tdMessage = await tdChannel.send(tiledeskMessage, user.token, conversation_id);
   winston.debug("message sent : ", tdMessage);
   
@@ -979,6 +975,77 @@ router.post('/record/:callSid/',async (req, res) => {
 
 })
 
+async function generateSTT(audioFileUrl, attributes, sessionInfo, settings){
+    
+  winston.debug("(voice) generateSTT: "+ attributes.VOICE_PROVIDER);
+    
+  let tiledeskMessage = {}, text = null;
+  switch(attributes.VOICE_PROVIDER){
+    case VOICE_PROVIDER.OPENAI:
+      let GPT_KEY = sessionInfo.integrations.find((el => el.type === VOICE_PROVIDER.OPENAI))?.key
+      let publicKey = sessionInfo.integrations.find((el => el.type === VOICE_PROVIDER.OPENAI))?.publicKey
+      if(publicKey){
+        let keep_going = await aiService.checkQuoteAvailability(sessionInfo.project_id, settings.token).catch((err)=>{
+            winston.error('errr while checkQuoteAvailability for project:', sessionInfo.project_id, err.response?.data)
+        })
+        winston.verbose('(voice) checkQuoteAvailability return: '+ keep_going);
+        if(!keep_going){
+          //no token is available --> close conversation
+          return tiledeskMessage= {
+            //text:'\\close',
+            text:'/close',
+            senderFullname: sessionInfo.from,
+            type: 'text',
+            channel: { name: CHANNEL_NAME },
+            attributes: {
+              subtype: "info",
+              action: 'close'+JSON.stringify({event: 'quota_exceeded'}),
+              payload: {
+                catchEvent: 'quota_exceeded'
+              },
+              timestamp: 'xxxxxx'
+            }
+          };
+
+        }
+      }
+      
+      text = await aiService.speechToText(audioFileUrl, attributes.STT_MODEL, GPT_KEY).catch((err)=>{
+          winston.error('errr while transcript', err.response?.data)
+      })
+      tiledeskMessage= {
+        text: text,
+        senderFullname: sessionInfo.from,
+        type: 'text',
+        channel: { name: CHANNEL_NAME }
+      };
+      break;
+    case VOICE_PROVIDER.ELEVENLABS:
+      let ELEVENLABS_APIKEY = sessionInfo.integrations.find((el => el.type === VOICE_PROVIDER.ELEVENLABS))?.key
+      // La condizione negli input del metodo è corretta, ma può essere scritta in modo più leggibile:
+      const ttsLanguage = attributes.TTS_LANGUAGE || 'en';
+      text = await this.aiService.speechToTextElevenLabs(
+        audioFileUrl,
+        attributes.STT_MODEL,
+        ttsLanguage,
+        ELEVENLABS_APIKEY
+      ).catch((err) => {
+        winston.error('errr while creating elevenlabs audio message', err?.response?.data);
+      });
+      tiledeskMessage= {
+        text: text,
+        senderFullname: sessionInfo.from,
+        type: 'text',
+        channel: { name: CHANNEL_NAME }
+      };
+      break;
+
+  }
+
+  return tiledeskMessage
+}
+
+
 
 router.get('/addon/transcript', async (req, res) => {
   winston.verbose("(vxml) called GET /transcript query-->" , req.query);
@@ -1225,6 +1292,9 @@ async function startApp(settings, callback) {
   if(settings.OPENAI_ENDPOINT){
     OPENAI_ENDPOINT = settings.OPENAI_ENDPOINT
   }
+  if(settings.ELEVENLABS_ENDPOINT){
+    ELEVENLABS_ENDPOINT = settings.ELEVENLABS_ENDPOINT
+  }
   
   if(settings.MAX_POLLING_TIME){
     MAX_POLLING_TIME = settings.MAX_POLLING_TIME;
@@ -1249,6 +1319,7 @@ async function startApp(settings, callback) {
   //init Services
   aiService = new AiService({
     OPENAI_ENDPOINT: OPENAI_ENDPOINT,
+    ELEVENLABS_ENDPOINT: ELEVENLABS_ENDPOINT,
     API_URL: API_URL
   })
   integrationService = new IntegrationService({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiledesk/tiledesk-voice-twilio-connector",
-  "version": "0.1.22",
+  "version": "0.1.23",
   "description": "Tiledesk VOICE Twilio connector",
   "license": "MIT",
   "author": "Gabriele Panico",

package/tiledesk/TiledeskTwilioTranslator.js

@@ -102,7 +102,7 @@ class TiledeskTwilioTranslator {
     
     
     this.user = sessionInfo.user
-    this.voiceSettings = sessionInfo.integrations.find((el => el.type === VOICE_PROVIDER.OPENAI))
+    this.integrations = sessionInfo.integrations
     
     
     const xml = xmlbuilder.create("Response", {});
@@ -310,7 +310,7 @@ class TiledeskTwilioTranslator {
       gather.att("action", this.BASE_URL + '/speechresult/' + xmlAttributes.callSid + queryUrl)
             .att("method", "POST")
             .att("language", xmlAttributes.TTS_VOICE_LANGUAGE)
-            .att('speechTimeout', "0")
+            .att('speechTimeout', "auto")
 
       //if(xmlAttributes && xmlAttributes.noInputTimeout){
       //  gather.att("timeout",  xmlAttributes.noInputTimeout/1000 ).up();
@@ -534,7 +534,7 @@ class TiledeskTwilioTranslator {
           if (command.type === "message") {
             //case type: TEXT
             if(command.message.type === 'text'){
-              if(that.voiceProvider === VOICE_PROVIDER.OPENAI){
+              if(that.voiceProvider !== VOICE_PROVIDER.TWILIO){
                 let voiceMessageUrl = await that.generateTTS(command.message.text, attributes)
                 rootEle.ele('Play', {}, voiceMessageUrl )
               }else{
@@ -622,11 +622,24 @@ class TiledeskTwilioTranslator {
   }
   
   async generateTTS(text, attributes){
-    let GPT_KEY = this.voiceSettings?.key
     
-    let audioData = await this.aiService.textToSpeech(text, attributes.TTS_VOICE_NAME, attributes.TTS_MODEL, GPT_KEY).catch((err)=>{
-      console.log('errr while creating audio message', err.response?.data)
-    })
+    let audioData = null;
+    switch(this.voiceProvider){
+      case VOICE_PROVIDER.OPENAI:
+        let GPT_KEY = this.integrations.find((el => el.type === VOICE_PROVIDER.OPENAI))?.key
+        audioData = await this.aiService.textToSpeech(text, attributes.TTS_VOICE_NAME, attributes.TTS_MODEL, GPT_KEY).catch((err)=>{
+          console.log('errr while creating audio message', err.response?.data)
+        })
+        break;
+      case VOICE_PROVIDER.ELEVENLABS:
+        let ELEVENLABS_APIKEY = this.integrations.find((el => el.type === VOICE_PROVIDER.ELEVENLABS))?.key
+        audioData = await this.aiService.textToSpeechElevenLabs(text, attributes.TTS_VOICE_NAME, attributes.TTS_MODEL, ELEVENLABS_APIKEY).catch((err)=>{
+          console.log('errr while creating elevenlabs audio message', err.response?.data)
+        })
+        break;
+
+    }
+    
     let fileUrl = await this.uploadService.upload(attributes.callSid, audioData, this.user).catch((err)=>{
       console.log('errr while uploading audioData', err.response)
     })

package/tiledesk/VoiceChannel.js

@@ -10,7 +10,8 @@ const TYPE_MESSAGE = require('./constants').TYPE_MESSAGE
 const MESSAGE_TYPE_MINE = require('./constants').MESSAGE_TYPE_MINE
 const MESSAGE_TYPE_OTHERS = require('./constants').MESSAGE_TYPE_OTHERS
 const CHANNEL_NAME = require('./constants').CHANNEL_NAME
-
+const VOICE_PROVIDER = require('./constants').VOICE_PROVIDER;
+const OPENAI_SETTINGS = require('./constants').OPENAI_SETTINGS;
 
 const winston = require("../winston");
 
@@ -89,6 +90,56 @@ class VoiceChannel {
     //if index is not present: set to default (0)
     await this.redis_client.set('tiledesk:vxml:'+callId + ':delayIndex', 0, {'EX': 86400});
   }
+  
+  
+  async saveSettingsForCallId(attributes, callId){
+    
+    winston.debug('saveSettingsForCallId: attributes -->', attributes)
+    let flowAttributes = {}
+    if(attributes && attributes.flowAttributes){
+      
+      flowAttributes = attributes.flowAttributes;
+      
+      //MANAGE VOICE SETTINGS from globals attributes
+      let voiceProvider = VOICE_PROVIDER.TWILIO
+      if(flowAttributes.VOICE_PROVIDER){
+        voiceProvider = flowAttributes.VOICE_PROVIDER
+        
+      }
+      
+      // IF VOICE_PROVIDER is TWILIO --> default values is on user account twilio settings
+      // IF VOICE_PROVIDER is OPENAI --> set default values from constants
+      if(voiceProvider === VOICE_PROVIDER.OPENAI){
+        flowAttributes.TTS_VOICE_NAME = flowAttributes.TTS_VOICE_NAME? flowAttributes.TTS_VOICE_NAME : OPENAI_SETTINGS.TTS_VOICE_NAME;
+        flowAttributes.TTS_MODEL = flowAttributes.TTS_MODEL? flowAttributes.TTS_MODEL : OPENAI_SETTINGS.TTS_MODEL;
+        flowAttributes.STT_MODEL = flowAttributes.STT_MODEL? flowAttributes.STT_MODEL : OPENAI_SETTINGS.STT_MODEL;
+      }
+            
+      
+    }
+    
+    const index = await this.redis_client.get('tiledesk:vxml:'+callId + ':attributes');
+    winston.debug('saveSettingsForCallId: attributes found -->'+index)
+    if(index){
+      //set index to default (0)
+      await this.redis_client.set('tiledesk:vxml:'+callId + ':attributes', JSON.stringify(flowAttributes), {'EX': 86400});
+      return;
+    }
+    //if index is not present: set to default (0)
+    await this.redis_client.set('tiledesk:vxml:'+callId + ':attributes', JSON.stringify(flowAttributes), {'EX': 86400});
+  
+  }
+  
+  
+  async getSettingsForCallId(callId){
+    const attributes = await this.redis_client.get('tiledesk:vxml:'+callId + ':attributes');
+    if(attributes){
+      return JSON.parse(attributes)
+    }
+    return {};
+  }
+  
+  
 
   
   

package/tiledesk/constants.js

@@ -38,8 +38,8 @@ module.exports = {
     SPEECH_FORM:     'speech_form',
     
   },
-  BASE_POOLING_DELAY: 250,
-  MAX_POLLING_TIME: 30000,
+  BASE_POOLING_DELAY: 500,
+  MAX_POLLING_TIME: 50000,
   VOICE_NAME: 'Polly.Danielle',
   VOICE_LANGUAGE: 'en-US',
   CALL_STATUS: {
@@ -51,7 +51,8 @@ module.exports = {
   },
   VOICE_PROVIDER: {
     OPENAI: 'openai',
-    TWILIO: 'twilio'
+    TWILIO: 'twilio',
+    ELEVENLABS: 'elevenlabs'
   },
   OPENAI_SETTINGS:{
     TTS_VOICE_NAME: 'alloy',

package/tiledesk/fileUtils.js

@@ -39,8 +39,9 @@ class FileUtils {
               responseType: 'arraybuffer',
               method: 'GET'
             }).then((resbody) => {
-              console.log('okkkkkkk')
-              resolve(resbody.data);
+              const buffer = Buffer.from(resbody.data, 'binary');
+              resolve(buffer);
+              //resolve(resbody.data);
             }).catch((err) => {
               reject(err);
             })       

package/tiledesk/services/AiService.js

@@ -15,11 +15,16 @@ class AiService {
     if (!config.OPENAI_ENDPOINT) {
       throw new Error("[AiService] config.OPENAI_ENDPOINT is mandatory");
     }
+    if(!config.ELEVENLABS_ENDPOINT){
+      throw new Error("[AiService] config.ELEVENLABS_ENDPOINT is mandatory");
+    }
     if (!config.API_URL) {
       throw new Error("[AiService] config.API_URL is mandatory");
     }
+    
 
     this.OPENAI_ENDPOINT = config.OPENAI_ENDPOINT;
+    this.ELEVENLABS_ENDPOINT = config.ELEVENLABS_ENDPOINT;
     this.API_URL = config.API_URL;
         
   }
@@ -28,15 +33,20 @@ class AiService {
 
     winston.debug("[AiService] speechToText url: "+ fileUrl);   
     let file = await fileUtils.downloadFromUrl(fileUrl).catch((err) => {
-        winston.error("[AiService] err: ", err)
+        winston.error("[AiService] err while downloadFromUrl: ", err)
         return null; // fallback per evitare undefined
     })
     
+    if (!file) {
+      winston.error('file non esisteeeeeeee')
+      return;
+    }
     
     return new Promise((resolve, reject) => {
-
+      
+      
       const formData = new FormData();
-      formData.append('file', file, { filename: 'audiofile', contentType: 'audio/mpeg' });
+      formData.append('file', file, { filename: 'audiofile.wav', contentType: 'audio/wav' });
       formData.append('model', model);
 
       axios({
@@ -91,10 +101,87 @@ class AiService {
     
   }
   
+  
+  
+  async speechToTextElevenLabs(fileUrl, model, language, API_KEY) {
+
+    winston.debug("[AiService] ELEVEN Labs speechToText url: "+ fileUrl);   
+    let file = await fileUtils.downloadFromUrl(fileUrl).catch((err) => {
+        winston.error("[AiService] err: ", err)
+        return null; // fallback per evitare undefined
+    })
+    
+    if (!file) {
+      winston.debug('[AiService] ELEVEN Labs speechToText file NOT EXIST: . . . return')
+      return;
+    }
+    
+    return new Promise((resolve, reject) => {
+      
+      
+      const formData = new FormData();
+      formData.append('file', file, { filename: 'audiofile.wav', contentType: 'audio/wav' });
+      formData.append('model_id', "scribe_v1");
+      formData.append('language_code', language)
+
+      axios({
+        url: this.ELEVENLABS_ENDPOINT + "/v1/speech-to-text",
+        headers: {
+          ...formData.getHeaders(),
+          "xi-api-key": API_KEY
+        },
+        data: formData,
+        method: 'POST'
+      }).then((resbody) => {
+        console.log('dataaaaaa', resbody)
+        resolve(resbody.data.text);
+      }).catch((err) => {
+        console.log('errrrrrr', err?.response)
+        reject(err);
+      })
+
+    })
+  }
+  
+  async textToSpeechElevenLabs(text, voice_id, model, API_KEY){
+    
+    
+    const data = {
+      model_id: model,
+      text: text,
+      output_format: "mp3_44100_128",
+    };
+    
+    
+    winston.debug('[AiService] ELEVEN Labs textToSpeech config:', data)
+    
+    return new Promise((resolve, reject) => {
+      axios({
+        url: this.ELEVENLABS_ENDPOINT + "/v1/text-to-speech/"+ voice_id,
+        headers: {
+          "Content-Type": "application/json",
+          "xi-api-key": API_KEY
+        },
+        responseType: 'arraybuffer',
+        data: data,
+        method: "POST",
+      }).then( async (response) => {
+        resolve(response?.data)            
+      })
+      .catch((err) => {
+        winston.error("[AiService] ELEVEN Labs textToSpeech error: ", err);
+        reject(err)
+      });
+    });
+    
+  }
+  
+  
+  
   async checkQuoteAvailability(projectId, token) {
 
     winston.debug("[AiService] checkQuoteAvailability for project: "+ projectId);   
-        
+          
     return new Promise((resolve, reject) => {
 
       axios({
@@ -105,12 +192,13 @@ class AiService {
         },
         method: 'GET'
       }).then((resbody) => {
-        if (resbody && resbody.isAvailable === true) {
+        if (resbody && resbody.data?.isAvailable === true) {
           resolve(true)
         } else {
           resolve(false)
         }
       }).catch((err) => {
+        winston.error("[AiService] checkQuoteAvailability error: ", err.response?.data);
         reject(err);
       })
 

package/tiledesk/services/IntegrationService.js

@@ -24,7 +24,7 @@ class IntegrationService {
   
   async getKeyFromIntegrations(id_project, integration_name, token){
     
-    winston.debug('[IntegrationService] getKeyFromIntegrations id_project:', id_project)
+    winston.debug('[IntegrationService] getKeyFromIntegrations id_project:'+ id_project + ' ' + integration_name)
         
     return await axios({
       url: this.API_URL + "/"+ id_project + "/integration/name/" + integration_name,
@@ -35,10 +35,10 @@ class IntegrationService {
       data: {},
       method: "GET",
     }).then( async (response) => {
-      if (!response.data || response.data?.value) {
+      if (!response.data || !response.data?.value) {
         return null;
       }
-
+      
       return response.data?.value?.apikey             
     })
     .catch((err) => {
@@ -69,7 +69,6 @@ class IntegrationService {
       return response.data?.gptkey            
     })
     .catch((err) => {
-      winston.error("[IntegrationService] getKeyFromKbSettings error: ", err.response?.data);
       return null;
     });
     

package/tiledesk/services/UploadService.js

@@ -47,7 +47,8 @@ class UploadService {
 
       //const formData = new FormData();
       //formData.append('file', file, { filename: 'audiofile_'+user._id+'_'+id+'.mp3', contentType: 'audio/mpeg' });
-
+      user.token = 'JWT eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJfaWQiOiI2NWM1ZjExNjlmYWYyZDA0Y2Q3ZGE1MjciLCJlbWFpbCI6ImdhYnJpZWxlQHRpbGVkZXNrLmNvbSIsImZpcnN0bmFtZSI6IkdhYnJpZWxlIiwibGFzdG5hbWUiOiJQYW5pY28iLCJlbWFpbHZlcmlmaWVkIjp0cnVlLCJpYXQiOjE3NDgyNTY2MTUsImF1ZCI6Imh0dHBzOi8vdGlsZWRlc2suY29tIiwiaXNzIjoiaHR0cHM6Ly90aWxlZGVzay5jb20iLCJzdWIiOiJ1c2VyIiwianRpIjoiNWUyZDhhYmUtYzQ0YS00MjJiLWE3MjUtYWYwMjcxNDgyZTczIn0.AcT1tNbE3AcfctJXfOsfUbytRNUQlhBqPUctxzXMjehZOS2ORJThWaPqPxrvqTTIyeOU2l6eoTw8_tqfRJGlp6X4m9KLio87axGl1z3WYBgh8bSMIkAw2zSIUuJmpjBuT8EZdjXZClXRUAliAvAoFRgCmhWJ1tODVvBynLiSb37sB_zscqWH5L5eF1vdt6HHizEO4HbGABQS00I2hEPn99ssC9Y3W4_UhDcitZG80ACwS_Bpl6uk8OxAFybZ1DHHkBS1AK-lCO2P2JJCFRyM33mcvTgb9B6pADETzgJT2qfgOU4-1Pm0l55Mij1LS-h7QTj95DTFQMM7DD6elP0WcA'
+      
       axios({
         url: this.API_URL + "/files/users",
         headers: {