@peopl-health/nexus 2.2.10 → 2.3.0

package/lib/helpers/messageHelper.js

@@ -0,0 +1,136 @@
+const moment = require('moment-timezone');
+const { Message } = require('../models/messageModel.js');
+
+const addMessageToThread = async (reply, messagesChat, provider, thread) => {
+  const threadId = thread.getConversationId();
+  
+  if (reply.origin === 'whatsapp_platform') {
+    await provider.addMessage({
+      threadId,
+      role: 'assistant',
+      content: messagesChat
+    });
+  } else if (reply.origin === 'patient') {
+    await provider.addMessage({
+      threadId,
+      role: 'user',
+      content: messagesChat
+    });
+  }
+  
+  console.log(`[addMessageToThread] Message added - ID: ${reply.message_id}, Thread: ${threadId}, Origin: ${reply.origin}`);
+};
+
+const updateMessageRecord = async (reply, thread) => {
+  const threadId = thread.getConversationId();
+  
+  await Message.updateOne(
+    { message_id: reply.message_id, timestamp: reply.timestamp },
+    { $set: { 
+      assistant_id: thread.getAssistantId(), 
+      thread_id: threadId,
+      processed: true 
+    } }
+  );
+  
+  console.log(`[updateMessageRecord] Record updated - ID: ${reply.message_id}, Thread: ${threadId}, Processed: true`);
+};
+
+
+async function getLastMessages(code) {
+  try {
+    let query = { processed: false };
+    if (code.endsWith('@g.us')) {
+      query = { ...query, numero: code, $or: [{ origin: 'patient' }, { origin: 'whatsapp_platform' }] };
+    } else {
+      query = { ...query, numero: code, $or: [{ origin: 'patient' }] };
+    }
+
+    const lastMessages = await Message.find(query)
+      .sort({ createdAt: 1 });
+
+    if (!lastMessages || lastMessages.length === 0) {
+      console.log(`[getLastMessages] No messages found for code: ${code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown'}`);
+      return null;
+    }
+
+    const unprocessedMessages = lastMessages.filter(msg => !msg.processed);
+    if (unprocessedMessages.length === 0) {
+      console.log(`[getLastMessages] No unprocessed messages for code: ${code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown'}`);
+      return null;
+    }
+
+    console.log(`[getLastMessages] Found ${unprocessedMessages.length} unprocessed messages for code: ${code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown'}`);
+    return unprocessedMessages;
+
+  } catch (error) {
+    console.error(`[getLastMessages] Error for code: ${code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown'}:`, error?.message || String(error));
+    return null;
+  }
+}
+
+async function getLastNMessages(code, n) {
+  try {
+    const lastMessages = await Message.find({ numero: code })
+      .sort({ createdAt: -1 })
+      .limit(n);
+
+    if (!lastMessages || lastMessages.length === 0) {
+      console.log(`[getLastNMessages] No messages found for code: ${code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown'}, limit: ${n}`);
+      return [];
+    }
+
+    console.log(`[getLastNMessages] Found ${lastMessages.length} messages for code: ${code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown'}, limit: ${n}`);
+    return lastMessages;
+
+  } catch (error) {
+    console.error(`[getLastNMessages] Error for code: ${code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown'}, limit: ${n}:`, error?.message || String(error));
+    return [];
+  }
+}
+
+function formatMessage(reply) {
+  try {
+    if (!reply.timestamp) {
+      return null;
+    }
+    
+    const timestamp = parseInt(reply.timestamp) * 1000;
+    const msgDate = new Date(timestamp);
+    
+    if (isNaN(msgDate.getTime())) {
+      console.warn(`[formatMessage] Invalid timestamp for message ID: ${reply.message_id}`);
+      return reply.body;
+    }
+    
+    const mexicoCityTime = moment(msgDate)
+      .tz('America/Mexico_City')
+      .locale('es')
+      .format('dddd, D [de] MMMM [de] YYYY [a las] h:mm A');
+    
+    return `[${mexicoCityTime}] ${reply.body}`;
+  } catch (error) {
+    console.error(`[formatMessage] Error for message ID: ${reply.message_id}:`, error?.message || String(error));
+    return null;
+  } 
+}
+
+async function isRecentMessage(chatId) {
+  const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000);
+
+  const recentMessage = await Message.find({
+    $or: [{ group_id: chatId }, { numero: chatId }],
+    createdAt: { $gte: fiveMinutesAgo }
+  }).sort({ createdAt: -1 }).limit(1);
+
+  return !!recentMessage;
+}
+
+module.exports = {
+  addMessageToThread,
+  updateMessageRecord,
+  getLastMessages,
+  getLastNMessages,
+  formatMessage,
+  isRecentMessage
+};

package/lib/helpers/processHelper.js

@@ -0,0 +1,238 @@
+const fs = require('fs');
+const { generatePresignedUrl } = require('../config/awsConfig.js');
+const { analyzeImage } = require('./llmsHelper.js');
+const { cleanupFiles, downloadMediaAndCreateFile } = require('./filesHelper.js');
+const { formatMessage, addMessageToThread, updateMessageRecord } = require('./messageHelper.js');
+const { sanitizeLogMetadata } = require('../utils/sanitizer.js');
+
+/**
+ * Structured logging with PHI protection
+ */
+const DEBUG_ENABLED = process.env.DEBUG_MESSAGE_CONTENT === 'true';
+const logger = {
+  info: (context, metadata = {}) => {
+    const safeMetadata = sanitizeLogMetadata(metadata);
+    console.log(`[${context}]`, JSON.stringify(safeMetadata));
+  },
+  warn: (context, metadata = {}) => {
+    const safeMetadata = sanitizeLogMetadata(metadata);
+    console.warn(`[${context}]`, JSON.stringify(safeMetadata));
+  },
+  error: (context, error, metadata = {}) => {
+    const safeMetadata = sanitizeLogMetadata(metadata);
+    console.error(`[${context}]`, {
+      error: error?.message || String(error),
+      ...safeMetadata
+    });
+  },
+  debug: (context, sensitiveData = {}) => {
+    if (DEBUG_ENABLED) {
+      console.log(`[DEBUG:${context}]`, sensitiveData);
+    }
+  }
+};
+
+/**
+ * Dedicated message processing utilities
+ * Handles text messages, media files, audio transcription, and thread operations
+ */
+const processTextMessage = (reply) => {
+  const formattedMessage = formatMessage(reply);
+  logger.info('processTextMessage', { 
+    message_id: reply.message_id,
+    timestamp: reply.timestamp,
+    from_me: reply.from_me,
+    body: reply.body,
+    hasContent: !!formattedMessage 
+  });
+  logger.debug('processTextMessage_content', { formattedMessage });
+  
+  const messagesChat = [];
+  if (formattedMessage) {
+    messagesChat.push({ type: 'text', text: formattedMessage });
+  }
+  
+  return messagesChat;
+};
+
+const processImageFile = async (fileName, reply) => {
+  let imageAnalysis = null;
+  let url = null;
+  const messagesChat = [];
+
+  try {
+    imageAnalysis = await analyzeImage(fileName);
+    
+    logger.info('processImageFile', {
+      message_id: reply.message_id,
+      bucketName: reply.media?.bucketName,
+      key: reply.media?.key,
+      medical_relevance: imageAnalysis?.medical_relevance,
+      has_table: imageAnalysis?.has_table,
+      analysis_type: imageAnalysis?.medical_analysis ? 'medical' : 'general'
+    });
+    
+    logger.debug('processImageFile_analysis', { imageAnalysis });
+    
+    const invalidAnalysis = ['NOT_MEDICAL', 'QUALITY_INSUFFICIENT'];
+    
+    // Generate presigned URL if medically relevant
+    if (imageAnalysis?.medical_relevance) {
+      url = await generatePresignedUrl(reply.media.bucketName, reply.media.key);            
+    }
+    
+    // Add appropriate text based on analysis
+    if (imageAnalysis?.has_table && imageAnalysis.table_data) {
+      messagesChat.push({
+        type: 'text',
+        text: imageAnalysis.table_data,
+      });
+    } else if (imageAnalysis?.medical_analysis && !invalidAnalysis.some(tag => imageAnalysis.medical_analysis.includes(tag))) {
+      messagesChat.push({
+        type: 'text',
+        text: imageAnalysis.medical_analysis,
+      });
+    } else {
+      messagesChat.push({
+        type: 'text',
+        text: imageAnalysis?.description || 'Image processed',
+      });
+    }
+  } catch (error) {
+    logger.error('processImageFile', error, {
+      message_id: reply.message_id,
+      fileName: fileName ? fileName.split('/').pop() : 'unknown'
+    });
+    messagesChat.push({
+      type: 'text',
+      text: 'Image received but could not be analyzed',
+    });
+  }
+
+  return { messagesChat, url };
+};
+
+const processAudioFile = async (fileName, provider) => {
+  const messagesChat = [];
+  
+  try {
+    const audioTranscript = await provider.transcribeAudio({
+      file: fs.createReadStream(fileName),
+      responseFormat: 'text',
+      language: 'es'
+    });
+    
+    const transcriptText = audioTranscript?.text || audioTranscript;
+    messagesChat.push({
+      type: 'text',
+      text: transcriptText,
+    });
+    
+    logger.info('processAudioFile', {
+      fileName: fileName ? fileName.split('/').pop().replace(/^[^-]+-[^-]+-/, 'xxx-xxx-') : 'unknown',
+      transcription_success: true,
+      transcript_length: transcriptText?.length || 0
+    });
+    
+    logger.debug('processAudioFile_transcript', { transcriptText });
+    
+  } catch (error) {
+    logger.error('processAudioFile', error, {
+      fileName: fileName ? fileName.split('/').pop().replace(/^[^-]+-[^-]+-/, 'xxx-xxx-') : 'unknown'
+    });
+    messagesChat.push({
+      type: 'text',
+      text: 'Audio received but could not be transcribed',
+    });
+  }
+
+  return messagesChat;
+};
+
+const processMediaFiles = async (code, reply, provider) => {
+  let url = null;
+  const messagesChat = [];
+  const tempFiles = [];
+
+  if (!reply.is_media) {
+    return { messagesChat, url, tempFiles };
+  }
+
+  logger.info('processMediaFiles', {
+    message_id: reply.message_id,
+    processing_media: true
+  });
+  
+  const fileNames = await downloadMediaAndCreateFile(code, reply);
+  tempFiles.push(...fileNames);
+
+  for (const fileName of fileNames) {
+    const safeFileName = fileName ? fileName.split('/').pop().replace(/^[^-]+-[^-]+-/, 'xxx-xxx-') : 'unknown';
+    
+    logger.info('processMediaFiles_file', {
+      message_id: reply.message_id,
+      fileName: safeFileName
+    });
+    
+    // Skip unsupported file types
+    if (fileName.toLowerCase().includes('.wbmp') || fileName.toLowerCase().includes('sticker')) {
+      logger.info('processMediaFiles_skip', {
+        message_id: reply.message_id,
+        fileName: safeFileName,
+        reason: 'unsupported_format'
+      });
+      continue;
+    }
+
+    if (fileName.includes('image') || fileName.includes('document') || fileName.includes('application')) {
+      const { messagesChat: imageMessages, url: imageUrl } = await processImageFile(fileName, reply);
+      messagesChat.push(...imageMessages);
+      if (imageUrl) url = imageUrl;
+    } else if (fileName.includes('audio')) {
+      const audioMessages = await processAudioFile(fileName, provider);
+      messagesChat.push(...audioMessages);
+    }
+  }
+
+  return { messagesChat, url, tempFiles };
+};
+
+const processIndividualMessage = async (code, reply, provider, thread) => {  
+  let tempFiles = [];
+  try {
+    const isPatient = reply.origin === 'patient';
+    const textMessages = processTextMessage(reply);
+    
+    const { messagesChat: mediaMessages, url, tempFiles: mediaFiles } = await processMediaFiles(code, reply, provider);
+    tempFiles = mediaFiles;
+    
+    const allMessages = [...textMessages, ...mediaMessages];
+    
+    if (allMessages.length > 0) {
+      await addMessageToThread(reply, allMessages, provider, thread);
+      await updateMessageRecord(reply, thread);
+    }
+    
+    return { isPatient, url };
+  } catch (error) {
+    logger.error('processIndividualMessage', error, {
+      message_id: reply.message_id,
+      code: code ? `${code.substring(0, 3)}***${code.slice(-4)}` : 'unknown',
+      origin: reply.origin
+    });
+    await cleanupFiles(tempFiles);
+    return { isPatient: false, url: null };
+  } finally {
+    if (tempFiles.length > 0) {
+      await cleanupFiles(tempFiles);
+    }
+  }
+};
+
+module.exports = {
+  processTextMessage,
+  processImageFile,
+  processAudioFile,
+  processMediaFiles,
+  processIndividualMessage
+};

package/lib/helpers/threadHelper.js

@@ -0,0 +1,73 @@
+const { Thread } = require('../models/threadModel.js');
+const { createProvider } = require('../providers/createProvider.js');
+const llmConfig = require('../config/llmConfig.js');
+
+const log = (level, context, data, error = null) => {
+  const safeData = { ...data };
+  if (safeData.code) safeData.code = `${safeData.code.substring(0, 3)}***${safeData.code.slice(-4)}`;
+  console[level](`[${context}]`, error ? { error: error?.message || String(error), ...safeData } : safeData);
+};
+
+const TERMINAL_STATUSES = ['cancelled', 'expired', 'completed', 'failed', 'incomplete'];
+const MAX_ATTEMPTS = 30;
+const MAX_WAIT_MS = 30000;
+
+const getThread = async (code, message = null) => {
+  try {
+    let thread = await Thread.findOne({ code: code });
+    log('log', 'getThread', { code, hasThread: !!thread, hasRunId: !!(thread?.run_id) });
+
+    if (!thread && message) {
+      thread = new Thread({ code, active: true });
+      await thread.save();
+    }
+    if (thread?.run_id) {
+      const provider = createProvider({ variant: process.env.VARIANT || 'assistants' }) || llmConfig.requireOpenAIProvider();
+      let attempts = 0;
+      const start = Date.now();
+
+      while (thread.run_id && attempts < MAX_ATTEMPTS && (Date.now() - start) < MAX_WAIT_MS) {
+        attempts++;
+        try {
+          const run = await provider.getRun({ threadId: thread.getConversationId(), runId: thread.run_id });
+          
+          if (TERMINAL_STATUSES.includes(run.status)) {
+            log('log', 'getThread_terminal', { code, status: run.status, attempts });
+            break;
+          }
+        } catch (error) {
+          log('error', 'getThread_getRun_failed', { code, attempt: attempts }, error);
+          break;
+        }
+        await new Promise(r => setTimeout(r, 1000));
+      }
+
+      if (attempts >= MAX_ATTEMPTS || (Date.now() - start) >= MAX_WAIT_MS) {
+        log('warn', 'getThread_timeout', { code, attempts, elapsed: Date.now() - start });
+      }
+      
+      await Thread.updateOne({ code, active: true }, { $set: { run_id: null } });
+      thread = await Thread.findOne({ code });
+    }
+
+    return thread;
+  } catch (error) {
+    log('error', 'getThread', { code }, error);
+    return null;
+  }
+};
+
+const getThreadInfo = async (code) => {
+  try {
+    let thread = await Thread.findOne({ code: code });
+    return thread;
+  } catch (error) {
+    log('error', 'getThreadInfo', { code }, error);
+    return null;
+  }
+};
+
+module.exports = {
+  getThread,
+  getThreadInfo
+};

package/lib/helpers/twilioHelper.js

@@ -1,5 +1,5 @@
-const { Message } = require('../models/messageModel');
-
+const { Message } = require('../models/messageModel.js');
+const { isRecentMessage } = require('./messageHelper.js');
 const axios = require('axios');
 const { v4: uuidv4 } = require('uuid');
 
@@ -65,18 +65,6 @@ function extractTitle(message, mediaType) {
 }
 
 
-async function isRecentMessage(chatId) {
-  const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000);
-
-  const recentMessage = await Message.find({
-    $or: [{ group_id: chatId }, { numero: chatId }],
-    createdAt: { $gte: fiveMinutesAgo }
-  }).sort({ createdAt: -1 }).limit(1);
-
-  return !!recentMessage;
-}
-
-
 async function getLastMessages(chatId, n) {
   const messages = await Message.find({ numero: chatId })
     .sort({ createdAt: -1 })

package/lib/models/messageModel.js

@@ -106,7 +106,6 @@ async function insertMessage(values) {
       origin: values.origin,
       raw: values.raw || null
     };
-    console.log('[MongoStorage] Inserting message', messageData);
 
     await Message.findOneAndUpdate(
       { message_id: values.message_id, body: values.body },

package/lib/observability/index.js

@@ -0,0 +1,184 @@
+const { initTelemetry, shutdownTelemetry } = require('./telemetry');
+const { trace, metrics } = require('@opentelemetry/api');
+
+const tracer = trace.getTracer('nexus-assistant');
+const meter = metrics.getMeter('nexus-assistant');
+
+const responseTimeHistogram = meter.createHistogram('nexus_response_time', {
+  description: 'Response time in milliseconds',
+  unit: 'ms',
+});
+
+const operationCounter = meter.createCounter('nexus_operations_total', {
+  description: 'Total number of operations',
+});
+
+const activeOperationsGauge = meter.createUpDownCounter('nexus_active_operations', {
+  description: 'Currently active operations',
+});
+
+const assistantReplyDuration = meter.createHistogram('nexus_assistant_reply_duration', {
+  description: 'Assistant reply duration in milliseconds',
+  unit: 'ms',
+});
+
+const assistantInstructionDuration = meter.createHistogram('nexus_assistant_instruction_duration', {
+  description: 'Assistant instruction execution duration in milliseconds',
+  unit: 'ms',
+});
+
+const assistantRetryCounter = meter.createCounter('nexus_assistant_retries_total', {
+  description: 'Total number of assistant operation retries',
+});
+
+const threadOperationsCounter = meter.createCounter('nexus_thread_operations_total', {
+  description: 'Total number of thread operations',
+});
+
+const messageProcessingDuration = meter.createHistogram('nexus_message_processing_duration', {
+  description: 'Message processing duration in milliseconds',
+  unit: 'ms',
+});
+
+const fileOperationsCounter = meter.createCounter('nexus_file_operations_total', {
+  description: 'Total number of file operations',
+});
+
+const s3UploadDuration = meter.createHistogram('nexus_s3_upload_duration', {
+  description: 'S3 upload duration in milliseconds',
+  unit: 'ms',
+});
+
+/**
+ * Wrapper function to trace any async operation
+ */
+async function traceOperation(name, operation, attributes = {}) {
+  const span = tracer.startSpan(name, { attributes });
+  const startTime = Date.now();
+  
+  try {
+    activeOperationsGauge.add(1, attributes);
+    
+    const result = await operation(span);
+    
+    const duration = Date.now() - startTime;
+    
+    responseTimeHistogram.record(duration, attributes);
+    operationCounter.add(1, { ...attributes, success: true });
+    
+    span.setAttributes({
+      'operation.duration_ms': duration,
+      'operation.success': true,
+    });
+    
+    span.setStatus({ code: 1 });
+    return result;
+    
+  } catch (error) {
+    const duration = Date.now() - startTime;
+    
+    operationCounter.add(1, { ...attributes, success: false });
+    responseTimeHistogram.record(duration, { ...attributes, error: true });
+    
+    span.recordException(error);
+    span.setStatus({ code: 2, message: error.message });
+    span.setAttributes({
+      'operation.duration_ms': duration,
+      'operation.success': false,
+      'error.message': error.message,
+    });
+    
+    throw error;
+  } finally {
+    activeOperationsGauge.add(-1, attributes);
+    span.end();
+  }
+}
+
+/**
+ * Helper to add custom span with automatic cleanup
+ */
+function createSpan(name, attributes = {}) {
+  return tracer.startSpan(name, { attributes });
+}
+
+/**
+ * Initialize telemetry with config
+ */
+function init(config = {}) {
+  initTelemetry(config);
+  console.log('🎯 Custom tracing and metrics ready');
+}
+
+/**
+ * Enhanced tracing functions for specific operations
+ */
+function traceAssistantReply(operation, attributes = {}) {
+  return traceOperation('assistant_reply', async (span) => {
+    const startTime = Date.now();
+    try {
+      const result = await operation(span);
+      const duration = Date.now() - startTime;
+      assistantReplyDuration.record(duration, attributes);
+      return result;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      assistantReplyDuration.record(duration, { ...attributes, error: true });
+      throw error;
+    }
+  }, attributes);
+}
+
+function traceAssistantInstruction(operation, attributes = {}) {
+  return traceOperation('assistant_instruction', async (span) => {
+    const startTime = Date.now();
+    try {
+      const result = await operation(span);
+      const duration = Date.now() - startTime;
+      assistantInstructionDuration.record(duration, attributes);
+      return result;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      assistantInstructionDuration.record(duration, { ...attributes, error: true });
+      throw error;
+    }
+  }, attributes);
+}
+
+function recordAssistantRetry(attributes = {}) {
+  assistantRetryCounter.add(1, attributes);
+}
+
+function recordThreadOperation(operationType, attributes = {}) {
+  threadOperationsCounter.add(1, { ...attributes, operation_type: operationType });
+}
+
+function recordFileOperation(operationType, attributes = {}) {
+  fileOperationsCounter.add(1, { ...attributes, operation_type: operationType });
+}
+
+module.exports = {
+  init,
+  shutdown: shutdownTelemetry,
+  traceOperation,
+  createSpan,
+  tracer,
+  meter,
+  // Enhanced tracing functions
+  traceAssistantReply,
+  traceAssistantInstruction,
+  recordAssistantRetry,
+  recordThreadOperation,
+  recordFileOperation,
+  // Metrics
+  responseTimeHistogram,
+  operationCounter,
+  activeOperationsGauge,
+  assistantReplyDuration,
+  assistantInstructionDuration,
+  assistantRetryCounter,
+  threadOperationsCounter,
+  messageProcessingDuration,
+  fileOperationsCounter,
+  s3UploadDuration,
+};