@peopl-health/nexus 3.3.19 → 3.4.0

package/lib/eval/EvalProvider.js

@@ -0,0 +1,309 @@
+const { OpenAI } = require('openai');
+
+const { Config_ID } = require('../config/airtableConfig');
+const { getCurrentMexicoDateTime } = require('../utils/dateUtils');
+const { retryWithBackoff } = require('../utils/retryUtils');
+const { logger } = require('../utils/logger');
+const { Thread } = require('../models/threadModel');
+const { DefaultMemoryManager } = require('../memory/DefaultMemoryManager');
+const { OpenAIResponsesProvider } = require('../providers/OpenAIResponsesProvider');
+const { handleFunctionCalls } = require('../providers/OpenAIResponsesProviderTools');
+const { getRecordByFilter } = require('../services/airtableService');
+const { getAssistantById } = require('../services/assistantResolver');
+
+const MAX_FUNCTION_ROUNDS = parseInt(process.env.MAX_FUNCTION_ROUNDS || '5', 10);
+
+/**
+ * Promptfoo-compatible eval provider that wraps the real Nexus pipeline.
+ *
+ * Modes:
+ *  - 'context-only':   Builds context, calls LLM, no tool schemas sent.
+ *  - 'dry-run':        Sends tool schemas so LLM can decide to call tools,
+ *                       but captures calls without executing them. (default)
+ *  - 'full-pipeline':  Executes tools for real. Use only with safe/mocked tools.
+ */
+class EvalProvider {
+  constructor(options = {}) {
+    const config = options.config || {};
+    this.model = config.model || process.env.OPENAI_MODEL || 'gpt-5';
+    this.temperature = config.temperature ?? 0.7;
+    this.maxOutputTokens = config.maxOutputTokens ?? 400;
+    this.assistantId = config.assistantId || null;
+    this.mode = config.mode || 'dry-run';
+    this.promptSource = config.promptSource || null;
+    // tool_choice: 'auto' (default, matches production), 'required', 'none',
+    // or { type: 'function', name: 'toolName' } to force a specific tool
+    this.toolChoice = config.toolChoice || 'auto';
+    this.promptVersions = config.promptVersions || {};
+    this.label = options.label || `nexus:${this.model}`;
+
+    this.client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+    this.memoryManager = new DefaultMemoryManager();
+    this.provider = new OpenAIResponsesProvider({
+      client: this.client,
+      defaultModels: { responseModel: this.model },
+    });
+  }
+
+  id() {
+    return this.label;
+  }
+
+  async callApi(prompt, context) {
+    const vars = context.vars || {};
+    const numero = vars.numero;
+    if (!numero) return { error: 'Missing required var: numero' };
+
+    try {
+      const beforeCheckpoint = vars.beforeCheckpoint ? new Date(vars.beforeCheckpoint) : null;
+      let assistantId = vars.assistantId || this.assistantId;
+
+      const thread = await Thread.findOne({ code: numero });
+      if (!assistantId && thread) {
+        assistantId = thread.prompt_id || thread.assistant_id || null;
+      }
+
+      const { messages, promptVariables, lastUserMessage } = await this._buildContext(numero, beforeCheckpoint);
+
+      const { devContent, assistant, toolSchemas } = await this._resolvePrompt(prompt, assistantId, thread, promptVariables);
+
+      const apiConfig = this._buildApiConfig(devContent, messages, assistantId, promptVariables, toolSchemas);
+
+      const startTime = Date.now();
+      const { finalResponse, toolCallsRequested, allToolsExecuted, accumulatedUsage } =
+        await this._executeWithToolLoop(apiConfig, assistant);
+
+      return this._formatResult({
+        finalResponse, toolCallsRequested, allToolsExecuted,
+        accumulatedUsage, assistantId, numero, lastUserMessage,
+        messages, promptVariables, startTime,
+      });
+    } catch (error) {
+      logger.error('[NexusEvalProvider] callApi failed', { error: error.message, numero });
+      return { error: error.message };
+    }
+  }
+
+  async _buildContext(numero, beforeCheckpoint) {
+    const messages = await this.memoryManager.buildContext({
+      thread: { code: numero },
+      config: { beforeCheckpoint },
+    });
+
+    const lastUserMsg = [...messages].reverse().find(m => m.role === 'user');
+    const lastUserMessage = (typeof lastUserMsg?.content === 'string'
+      ? lastUserMsg.content : ''
+    ).substring(0, 200);
+
+    const clinicalData = await this.memoryManager.getClinicalData(numero);
+    const promptVariables = {
+      clinical_context: clinicalData?.clinicalContext ?? '',
+      last_symptoms: clinicalData?.lastSymptoms ?? '',
+      current_date: getCurrentMexicoDateTime(),
+    };
+
+    return { messages, promptVariables, lastUserMessage };
+  }
+
+  async _resolvePrompt(fallbackPrompt, assistantId, thread, promptVariables) {
+    let devContent;
+    if (this.promptSource === 'airtable' && assistantId) {
+      const devRecord = await getRecordByFilter(Config_ID, 'responses', `{prompt_id} = "${assistantId}"`);
+      devContent = devRecord?.[0]?.content || '';
+    } else {
+      devContent = fallbackPrompt;
+    }
+    devContent = devContent.replace(/\{\{(\w+)\}\}/g, (_, key) => promptVariables[key] ?? '');
+
+    let assistant = null;
+    let toolSchemas = [];
+    if (this.mode !== 'context-only' && assistantId) {
+      try {
+        assistant = getAssistantById(assistantId, thread);
+        toolSchemas = assistant.getToolSchemas?.() || [];
+        if (assistant.tools?.size) {
+          const toolNames = Array.from(assistant.tools.keys()).join(', ');
+          devContent += `\n\nYou only have access to these tools: ${toolNames}. Do not call or reference any tools not listed here.`;
+        }
+      } catch {
+        logger.warn('[NexusEvalProvider] Failed to resolve assistant', { assistantId });
+      }
+    }
+
+    return { devContent, assistant, toolSchemas };
+  }
+
+  _buildApiConfig(devContent, messages, assistantId, promptVariables, toolSchemas) {
+    const convertedMessages = this.provider._convertItemsToApiFormat(messages);
+    const input = [{ role: 'developer', content: devContent }, ...convertedMessages];
+    const apiConfig = { input, instructions: '' };
+
+    if (assistantId) {
+      apiConfig.prompt = { id: assistantId, variables: promptVariables };
+      const version = this.promptVersions[assistantId];
+      if (version) apiConfig.prompt.version = String(version);
+    } else {
+      apiConfig.model = this.model;
+      apiConfig.temperature = this.temperature;
+      apiConfig.max_output_tokens = this.maxOutputTokens;
+    }
+
+    if (this.mode !== 'context-only') {
+      apiConfig.tool_choice = this.toolChoice;
+    }
+
+    if (toolSchemas.length > 0 && !assistantId) {
+      apiConfig.tools = toolSchemas.map(schema => {
+        if (schema.type === 'function' && schema.function) {
+          const { name, description, parameters, strict } = schema.function;
+          return { type: 'function', name, description, parameters, strict };
+        }
+        return schema;
+      });
+    }
+
+    return apiConfig;
+  }
+
+  async _executeWithToolLoop(apiConfig, assistant) {
+    const { result: response } = await retryWithBackoff(
+      () => this.client.responses.create(apiConfig),
+      { providerName: 'NexusEvalProvider' }
+    );
+
+    let finalResponse = response;
+    const toolCallsRequested = [];
+    const allToolsExecuted = [];
+    const accumulatedUsage = {
+      input_tokens: response.usage?.input_tokens || 0,
+      output_tokens: response.usage?.output_tokens || 0,
+      total_tokens: response.usage?.total_tokens || 0,
+    };
+
+    const functionCalls = (response.output || []).filter(item => item.type === 'function_call');
+    if (!functionCalls.length) {
+      return { finalResponse, toolCallsRequested, allToolsExecuted, accumulatedUsage };
+    }
+
+    // Record initial tool calls
+    for (const call of functionCalls) {
+      toolCallsRequested.push({
+        name: call.name,
+        arguments: call.arguments ? JSON.parse(call.arguments) : {},
+        call_id: call.call_id,
+      });
+    }
+
+    if (this.mode === 'full-pipeline' && assistant) {
+      finalResponse = await this._executeFullPipeline(
+        apiConfig, finalResponse, assistant, toolCallsRequested, allToolsExecuted, accumulatedUsage
+      );
+    } else if (this.mode === 'dry-run') {
+      finalResponse = await this._executeDryRun(apiConfig, finalResponse, functionCalls, accumulatedUsage);
+    }
+
+    return { finalResponse, toolCallsRequested, allToolsExecuted, accumulatedUsage };
+  }
+
+  async _executeFullPipeline(apiConfig, initialResponse, assistant, toolCallsRequested, allToolsExecuted, usage) {
+    let finalResponse = initialResponse;
+    let currentInput = [...apiConfig.input];
+
+    for (let round = 1; round <= MAX_FUNCTION_ROUNDS; round++) {
+      const calls = finalResponse.output.filter(item => item.type === 'function_call');
+      if (!calls.length) break;
+
+      const { outputs, toolsExecuted } = await handleFunctionCalls(calls, assistant);
+      currentInput.push(...finalResponse.output, ...outputs);
+      allToolsExecuted.push(...toolsExecuted);
+
+      for (const call of calls) {
+        if (!toolCallsRequested.find(t => t.call_id === call.call_id)) {
+          toolCallsRequested.push({
+            name: call.name,
+            arguments: call.arguments ? JSON.parse(call.arguments) : {},
+            call_id: call.call_id,
+          });
+        }
+      }
+
+      const { result: followUp } = await retryWithBackoff(
+        () => this.client.responses.create({ ...apiConfig, input: currentInput, tool_choice: 'auto' }),
+        { providerName: 'NexusEvalProvider' }
+      );
+      this._addUsage(usage, followUp.usage);
+      finalResponse = followUp;
+    }
+
+    return finalResponse;
+  }
+
+  async _executeDryRun(apiConfig, initialResponse, functionCalls, usage) {
+    const currentInput = [...apiConfig.input, ...initialResponse.output];
+    for (const call of functionCalls) {
+      currentInput.push({
+        type: 'function_call_output',
+        call_id: call.call_id,
+        output: JSON.stringify({ success: true }),
+      });
+    }
+
+    logger.info('[NexusEvalProvider] Dry-run: sending stub outputs', {
+      tools: functionCalls.map(c => c.name),
+    });
+
+    const { result: followUp } = await retryWithBackoff(
+      () => this.client.responses.create({ ...apiConfig, input: currentInput, tool_choice: 'auto' }),
+      { providerName: 'NexusEvalProvider' }
+    );
+    this._addUsage(usage, followUp.usage);
+    return followUp;
+  }
+
+  _addUsage(accumulated, newUsage) {
+    accumulated.input_tokens += newUsage?.input_tokens || 0;
+    accumulated.output_tokens += newUsage?.output_tokens || 0;
+    accumulated.total_tokens += newUsage?.total_tokens || 0;
+  }
+
+  _formatResult({ finalResponse, toolCallsRequested, allToolsExecuted, accumulatedUsage, assistantId, numero, lastUserMessage, messages, promptVariables, startTime }) {
+    const output = this.provider._extractMessageOutput(finalResponse);
+    const durationMs = Date.now() - startTime;
+    const toolNames = toolCallsRequested.map(t => t.name);
+
+    logger.info('[NexusEvalProvider] Result', {
+      assistantId, model: finalResponse.model || this.model,
+      lastUserMessage, toolCallsCaptured: toolNames,
+      outputLength: (output || '').length, durationMs,
+    });
+
+    const toolsHeader = toolNames.length ? `[tools: ${toolNames.join(', ')}]\n---\n` : '';
+
+    return {
+      output: `${toolsHeader}${output || ''}`,
+      tokenUsage: {
+        total: accumulatedUsage.total_tokens,
+        prompt: accumulatedUsage.input_tokens,
+        completion: accumulatedUsage.output_tokens,
+      },
+      metadata: {
+        model: finalResponse.model || this.model,
+        numero, assistantId, lastUserMessage,
+        mode: this.mode,
+        contextMessages: messages.length,
+        clinicalContext: promptVariables.clinical_context ? 'present' : 'absent',
+        durationMs,
+        toolCalls: toolCallsRequested,
+        toolsExecuted: allToolsExecuted.map(t => ({
+          name: t.tool_name,
+          arguments: t.tool_arguments,
+          output: t.tool_output,
+          success: t.success,
+          duration_ms: t.execution_time_ms,
+        })),
+      },
+    };
+  }
+}
+
+module.exports = { EvalProvider };

package/lib/index.js

@@ -10,7 +10,7 @@ const { OpenAIAssistantsProvider } = require('./providers/OpenAIAssistantsProvid
 const { OpenAIResponsesProvider } = require('./providers/OpenAIResponsesProvider');
 const runtimeConfig = require('./config/runtimeConfig');
 const llmConfigModule = require('./config/llmConfig');
-const { setModelDatabases, setModelDatabase, getModelDatabase } = require('./config/mongoConfig');
+const { setModelDatabases, setModelDatabase, getModelDatabase, connect: mongoConnect, disconnect: mongoDisconnect, getConnection: mongoGetConnection } = require('./config/mongoConfig');
 const { TwilioProvider } = require('./adapters/TwilioProvider');
 const { BaileysProvider } = require('./adapters/BaileysProvider');
 const { BaseAssistant } = require('./assistants/BaseAssistant');
@@ -219,6 +219,8 @@ class Nexus {
 
 const routes = require('./routes');
 const { resetAll } = require('./config/lifecycle');
+const { EvalProvider } = require('./eval/EvalProvider');
+const airtableService = require('./services/airtableService');
 
 module.exports = {
   Nexus,
@@ -254,5 +256,11 @@ module.exports = {
   createQueueAdapter,
   registerQueueAdapter,
 
-  resetAll
+  resetAll,
+
+  mongoConnect,
+  mongoDisconnect,
+  mongoGetConnection,
+  EvalProvider,
+  airtableService,
 };

package/lib/services/airtableService.js

@@ -2,6 +2,13 @@ const { airtable } = require('../config/airtableConfig');
 
 const { logger } = require('../utils/logger');
 
+let evalMode = false;
+
+function setEvalMode(enabled) {
+  evalMode = !!enabled;
+  logger.info(`[airtableService] Eval mode ${evalMode ? 'ON' : 'OFF'} — writes will be ${evalMode ? 'muted' : 'live'}`);
+}
+
 function getBase(baseID) {
   if (!airtable) throw new Error('Airtable not configured. Set AIRTABLE_API_KEY');
   return airtable.base(baseID);
@@ -17,6 +24,10 @@ async function collectRecords(query, mapper = r => r.fields) {
 }
 
 async function addRecord(baseID, tableName, fields) {
+  if (evalMode) {
+    logger.info('[addRecord:eval] Muted', { tableName });
+    return { id: 'eval_mock_record', fields: Array.isArray(fields) ? fields[0]?.fields || {} : fields };
+  }
   try {
     const record = await getBase(baseID)(tableName).create(fields);
     logger.info('[addRecord] Created', { tableName });
@@ -48,6 +59,10 @@ async function getRecordByFilter(baseID, tableName, filter, view = 'Grid view')
 }
 
 async function updateRecordByFilter(baseID, tableName, filter, updateFields) {
+  if (evalMode) {
+    logger.info('[updateRecordByFilter:eval] Muted', { tableName, filter });
+    return [{ id: 'eval_mock_record', fields: updateFields }];
+  }
   try {
     const base = getBase(baseID);
     const updatedRecords = [];
@@ -88,6 +103,7 @@ async function addLinkedRecord(baseID, targetTable, fields, linkConfig) {
 }
 
 module.exports = {
+  setEvalMode,
   addRecord,
   getRecords,
   getRecordByFilter,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peopl-health/nexus",
-  "version": "3.3.19",
+  "version": "3.4.0",
   "description": "Core messaging and assistant library for WhatsApp communication platforms",
   "keywords": [
     "whatsapp",