@aj-archipelago/cortex 1.3.16 → 1.3.18

package/README.md

@@ -667,7 +667,7 @@ Detailed documentation on Cortex's API can be found in the /graphql endpoint of
 ## Roadmap
 Cortex is a constantly evolving project, and the following features are coming soon:
 
-* Prompt execution context preservation between calls (to enable interactive, multi-call integrations with LangChain and other technologies)
+* Prompt execution context preservation between calls (to enable interactive, multi-call integrations with other technologies)
 * Model-specific cache key optimizations to increase hit rate and reduce cache size
 * Structured analytics and reporting on AI API call frequency, cost, cache hit rate, etc.
 

package/config.js

@@ -60,6 +60,11 @@ var config = convict({
         default: null,
         env: 'DEFAULT_MODEL_NAME'
     },
+    defaultEntityName: {
+        format: String,
+        default: "Jarvis",
+        env: 'DEFAULT_ENTITY_NAME'
+    },
     enableCache: {
         format: Boolean,
         default: true,
@@ -80,6 +85,22 @@ var config = convict({
         default: false,
         env: 'CORTEX_ENABLE_REST'
     },
+    entityConstants: {
+        format: Object,
+        default: {
+            AI_MEMORY: `<MEMORIES>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</MEMORIES>`,
+            AI_MEMORY_INSTRUCTIONS: "You have persistent memories of important details, instructions, and context - make sure you consult your memories when formulating a response to make sure you're applying your learnings. Also included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.",
+            AI_DIRECTIVES: `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`,
+            AI_CONVERSATION_HISTORY: "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>",
+            AI_COMMON_INSTRUCTIONS: "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}",
+            AI_COMMON_INSTRUCTIONS_MARKDOWN: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.",          
+            AI_COMMON_INSTRUCTIONS_VOICE: "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.",            
+            AI_DATETIME: "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.",           
+            AI_EXPERTISE: "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.",
+            AI_STYLE_OPENAI: "oai-gpt4o",
+            AI_STYLE_ANTHROPIC: "claude-35-sonnet-vertex",
+        },
+    },
     gcpServiceAccountKey: {
         format: String,
         default: null,
@@ -370,6 +391,9 @@ var config = convict({
 // Read in environment variables and set up service configuration
 const configFile = config.get('cortexConfigFile');
 
+//Save default entity constants
+const defaultEntityConstants = config.get('entityConstants');
+
 // Load config file
 if (configFile && fs.existsSync(configFile)) {
     logger.info(`Loading config from ${configFile}`);
@@ -385,6 +409,11 @@ if (configFile && fs.existsSync(configFile)) {
     }
 }
 
+// Merge default entity constants with config entity constants
+if (config.get('entityConstants') && defaultEntityConstants) {
+    config.set('entityConstants', { ...defaultEntityConstants, ...config.get('entityConstants') });
+}
+
 if (config.get('gcpServiceAccountKey')) {
     const gcpAuthTokenHelper = new GcpAuthTokenHelper(config.getProperties());
     config.set('gcpAuthTokenHelper', gcpAuthTokenHelper);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.16",
+  "version": "1.3.18",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {
@@ -10,12 +10,16 @@
   "keywords": [
     "cortex",
     "AI",
+    "router",
+    "GPT",
+    "agents",
+    "entities",
     "prompt engineering",
     "LLM",
     "OpenAI",
     "Azure",
-    "GPT-3",
-    "GPT-4",
+    "Gemini",
+    "Claude",
     "chatGPT",
     "GraphQL"
   ],
@@ -38,7 +42,6 @@
     "@datastructures-js/deque": "^1.0.4",
     "@graphql-tools/schema": "^9.0.12",
     "@keyv/redis": "^2.5.4",
-    "@langchain/openai": "^0.0.24",
     "axios": "^1.3.4",
     "axios-cache-interceptor": "^1.0.1",
     "bottleneck": "^2.19.5",
@@ -59,7 +62,6 @@
     "handlebars": "^4.7.7",
     "ioredis": "^5.3.1",
     "keyv": "^4.5.2",
-    "langchain": "^0.1.28",
     "mime-types": "^2.1.35",
     "subsrt": "^1.1.1",
     "uuid": "^9.0.0",
@@ -70,13 +72,11 @@
     "@faker-js/faker": "^8.4.1",
     "ava": "^5.2.0",
     "dotenv": "^16.0.3",
-    "eslint": "^8.38.0",
-    "eslint-plugin-import": "^2.27.5",
     "got": "^13.0.0",
     "sinon": "^17.0.1"
   },
   "publishConfig": {
-    "access": "restricted"
+    "access": "public"
   },
   "ava": {
     "files": [

package/pathways/chat_context.js

@@ -14,6 +14,5 @@ export default {
         contextId: ``,
     },
     model: 'oai-gpt4o',
-    //model: 'oai-gpturbo',
     useInputChunking: false,
 }

package/pathways/chat_jarvis.js

@@ -14,6 +14,5 @@ export default {
         contextId: ``,
     },
     model: 'oai-gpt4o',
-    //model: 'oai-gpturbo',
     useInputChunking: false,
 }

package/pathways/chat_persist.js

@@ -18,6 +18,5 @@ export default {
         contextId: ``,
     },
     model: 'oai-gpt4o',
-    //model: 'oai-gpturbo',
     useInputChunking: false,
 }

package/pathways/image.js

@@ -1,5 +1,5 @@
 export default {
     prompt:["{{text}}"],
-    model: 'azure-dalle3',
+    model: 'oai-dalle3',
     enableDuplicateRequests: false,
 }

package/pathways/system/entity/memory/sys_memory_manager.js

@@ -1,5 +1,6 @@
 import { callPathway } from '../../../../lib/pathwayTools.js';
 import logger from '../../../../lib/logger.js';
+import { config } from '../../../../config.js';
 
 const AI_MEMORY_DEFAULTS = `  {
     "memoryUser": "",
@@ -21,6 +22,8 @@ export default {
     executePathway: async ({args, resolver}) => {
         try {
 
+            args = { ...args, ...config.get('entityConstants') };
+
             // Check if memory is empty or all sections are empty, and set to defaults if so
             const memory = await callPathway('sys_read_memory', { ...args });
             let parsedMemory;

package/pathways/system/entity/memory/sys_memory_update.js

@@ -1,7 +1,7 @@
 import { Prompt } from '../../../../server/prompt.js';
 import { callPathway } from '../../../../lib/pathwayTools.js';
 import { encode } from '../../../../lib/encodeCache.js';
-import entityConstants from '../shared/sys_entity_constants.js';
+import { config } from '../../../../config.js';
 
 const modifyText = (text, modifications) => {
     let modifiedText = text || '';
@@ -152,6 +152,7 @@ export default {
     json: true,
     timeout: 300,
     executePathway: async ({args, runAllPrompts}) => {
+        args = { ...args, ...config.get('entityConstants') };
 
         if (!args.section) {
             return "Memory not updated - no section specified";
@@ -178,7 +179,7 @@ export default {
 
         let sectionMemory = await callPathway("sys_read_memory", {contextId: args.contextId, section: args.section}); 
 
-        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory, ...entityConstants});
+        const result = await runAllPrompts({...args, sectionPrompt, sectionMemory});
 
         try {
             const { modifications} = JSON.parse(result);

package/pathways/system/entity/sys_entity_continue.js

@@ -1,6 +1,6 @@
 import { callPathway } from '../../../lib/pathwayTools.js';
 import logger from '../../../lib/logger.js';
-import entityConstants from './shared/sys_entity_constants.js';
+import { config } from '../../../config.js';
 
 export default {
     prompt: [],
@@ -20,13 +20,24 @@ export default {
         chatId: ``,
         dataSources: [""],
         model: 'oai-gpt4o',
+        aiStyle: "OpenAI",
         generatorPathway: 'sys_generator_results',
         voiceResponse: false,
     },
     timeout: 300,
-    ...entityConstants,
     executePathway: async ({args, resolver}) => {
-        args = { ...args, ...entityConstants };
+        const pathwayResolver = resolver;
+
+        // add the entity constants to the args
+        args = {
+            ...args,
+            ...config.get('entityConstants')
+        };
+
+        // if the model has been overridden, make sure to use it
+        if (pathwayResolver.modelName) {
+            args.model = pathwayResolver.modelName;
+        }
 
         try {
             // Get the generator pathway name from args or use default

package/pathways/system/entity/sys_entity_start.js

@@ -4,9 +4,7 @@ import { callPathway, say } from '../../../lib/pathwayTools.js';
 import logger from  '../../../lib/logger.js';
 import { chatArgsHasImageUrl } from  '../../../lib/util.js';
 import { QueueServiceClient } from '@azure/storage-queue';
-import entityConstants from './shared/sys_entity_constants.js';
-
-const TOKEN_RATIO = 0.75;
+import { config } from '../../../config.js';
 
 const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
 let queueClient;
@@ -38,8 +36,6 @@ export default {
     useInputChunking: false,
     enableDuplicateRequests: false,
     model: 'oai-gpt4o',
-    anthropicModel: 'claude-35-sonnet-vertex',
-    openAIModel: 'oai-gpt4o',
     useSingleTokenStream: false,
     inputParameters: {
         privateData: false,    
@@ -58,19 +54,25 @@ export default {
         messages: [],
         voiceResponse: false,
         codeRequestId: ``,
+        skipCallbackMessage: false
     },
     timeout: 600,
-    tokenRatio: TOKEN_RATIO,
-    ...entityConstants,
-
+  
     executePathway: async ({args, resolver}) => {
         let title = null;
         let codeRequestId = null;
 
+        const pathwayResolver = resolver;
+
+        // add the entity constants to the args
         args = {
             ...args,
-            ...entityConstants
+            ...config.get('entityConstants')
         };
+                
+        // set the style model if applicable
+        const { aiStyle, AI_STYLE_ANTHROPIC, AI_STYLE_OPENAI } = args;
+        const styleModel = aiStyle === "Anthropic" ? AI_STYLE_ANTHROPIC : AI_STYLE_OPENAI;
 
         // Limit the chat history to 20 messages to speed up processing
         if (args.messages && args.messages.length > 0) {
@@ -79,10 +81,6 @@ export default {
             args.chatHistory = args.chatHistory.slice(-20);
         }
 
-        const pathwayResolver = resolver;
-        const { anthropicModel, openAIModel } = pathwayResolver.pathway;
-        const styleModel = args.aiStyle === "Anthropic" ? anthropicModel : openAIModel;
-
         // if the model has been overridden, make sure to use it
         if (pathwayResolver.modelName) {
             args.model = pathwayResolver.modelName;
@@ -104,7 +102,7 @@ export default {
 
         const fetchChatResponse = async (args, pathwayResolver) => {
             const [chatResponse, chatTitleResponse] = await Promise.all([
-                callPathway('sys_generator_quick', {...args, model: styleModel }, pathwayResolver),
+                callPathway('sys_generator_quick', {...args, model: styleModel}, pathwayResolver),
                 callPathway('chat_title', { ...args, stream: false}),
             ]);
 
@@ -223,25 +221,30 @@ export default {
             }
 
             if (toolCallbackMessage) {
+                if (args.skipCallbackMessage) {
+                    pathwayResolver.tool = JSON.stringify({ hideFromModel: false, search: false, title });  
+                    return await callPathway('sys_entity_continue', { ...args, stream: false, model: styleModel, generatorPathway: toolCallbackName }, pathwayResolver);
+                }
+
                 if (args.stream) {
                     if (!ackResponse) {
                         await say(pathwayResolver.requestId, toolCallbackMessage || "One moment please.", 10);
                     }
                     pathwayResolver.tool = JSON.stringify({ hideFromModel: false, search: false, title });  
-                    await callPathway('sys_entity_continue', { ...args, stream: true, model: styleModel, generatorPathway: toolCallbackName }, pathwayResolver);
+                    await callPathway('sys_entity_continue', { ...args, stream: true, generatorPathway: toolCallbackName }, pathwayResolver);
                     return "";
-                } else {
-                    pathwayResolver.tool = JSON.stringify({ 
-                        hideFromModel: toolCallbackName ? true : false, 
-                        toolCallbackName, 
-                        title,
-                        search: toolCallbackName === 'sys_generator_results' ? true : false,
-                        coding: toolCallbackName === 'coding' ? true : false,
-                        codeRequestId,
-                        toolCallbackId
-                    });
-                    return toolCallbackMessage || "One moment please.";
                 }
+                
+                pathwayResolver.tool = JSON.stringify({ 
+                    hideFromModel: toolCallbackName ? true : false, 
+                    toolCallbackName, 
+                    title,
+                    search: toolCallbackName === 'sys_generator_results' ? true : false,
+                    coding: toolCallbackName === 'coding' ? true : false,
+                    codeRequestId,
+                    toolCallbackId
+                });
+                return toolCallbackMessage || "One moment please.";
             }
 
             const chatResponse = await (fetchChatResponsePromise || fetchChatResponse({ ...args, ackResponse }, pathwayResolver));

package/pathways/system/entity/sys_generator_image.js

@@ -5,8 +5,6 @@ import { Prompt } from '../../../server/prompt.js';
 import logger from '../../../lib/logger.js';
 import { getUniqueId } from '../../../lib/util.js';
 
-const TOKEN_RATIO = 1.0;
-
 export default {
     prompt: [],
     useInputChunking: false,
@@ -26,7 +24,6 @@ export default {
         model: 'oai-gpt4o',
     },
     timeout: 300,
-    tokenRatio: TOKEN_RATIO,
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
 

package/pathways/system/entity/sys_generator_memory.js

@@ -13,6 +13,10 @@ export default {
     useInputChunking: false,
     enableDuplicateRequests: false,
     executePathway: async ({args, resolver}) => {
+
+        const { aiStyle, AI_STYLE_ANTHROPIC, AI_STYLE_OPENAI } = args;
+        const styleModel = aiStyle === "Anthropic" ? AI_STYLE_ANTHROPIC : AI_STYLE_OPENAI;
+
         const memoryContext = await callPathway('sys_search_memory', { ...args, section: 'memoryAll', updateContext: true });
         if (memoryContext) {
             args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
@@ -20,9 +24,9 @@ export default {
 
         let result;
         if (args.voiceResponse) {
-            result = await callPathway('sys_generator_quick', { ...args, stream: false });
+            result = await callPathway('sys_generator_quick', { ...args, model: styleModel, stream: false });
         } else {
-            result = await callPathway('sys_generator_quick', { ...args });
+            result = await callPathway('sys_generator_quick', { ...args, model: styleModel });
         }
 
         resolver.tool = JSON.stringify({ toolUsed: "memory" });

package/pathways/system/entity/sys_generator_quick.js

@@ -7,7 +7,6 @@ export default {
         contextId: ``,
         aiName: "Jarvis",
         language: "English",
-        model: "oai-gpt4o",
     },
     useInputChunking: false,
     enableDuplicateRequests: false,

package/pathways/system/entity/sys_generator_results.js

@@ -28,7 +28,6 @@ export default {
         model: 'oai-gpt4o',
     },
     timeout: 300,
-    tokenRatio: TOKEN_RATIO,
 
     executePathway: async ({args, runAllPrompts, resolver}) => {
 

package/pathways/system/entity/sys_generator_voice_sample.js

@@ -1,5 +1,6 @@
 import { Prompt } from '../../../server/prompt.js';
-import entityConstants from './shared/sys_entity_constants.js';
+import { config } from '../../../config.js';
+
 export default {
     prompt:
         [
@@ -17,19 +18,17 @@ export default {
     },
     useInputChunking: false,
     enableDuplicateRequests: false,
-    executePathway: async ({args, runAllPrompts, resolver}) => {
+    executePathway: async ({args, runAllPrompts}) => {
 
         args = {
             ...args,
-            ...entityConstants
+            ...config.get('entityConstants')
         };
 
-        const pathwayResolver = resolver;
-        const { anthropicModel, openAIModel } = pathwayResolver.pathway;
-
-        const styleModel = args.aiStyle === "Anthropic" ? anthropicModel : openAIModel;
+        const { aiStyle, AI_STYLE_ANTHROPIC, AI_STYLE_OPENAI } = args;
+        args.model = aiStyle === "Anthropic" ? AI_STYLE_ANTHROPIC : AI_STYLE_OPENAI;
 
-        const result = await runAllPrompts({ ...args, model: styleModel, stream: false });
+        const result = await runAllPrompts({ ...args, stream: false });
 
         return result;
     }

package/pathways/system/entity/sys_query_builder.js

@@ -1,5 +1,5 @@
 import { Prompt } from '../../../server/prompt.js';
-import entityConstants from './shared/sys_entity_constants.js';
+import { config } from '../../../config.js';
 
 export default {
     inputParameters: {
@@ -97,5 +97,5 @@ Example JSON objects and messages for different queries:
     useInputChunking: false,
     enableDuplicateRequests: false,
     json: true,
-    ...entityConstants
-}
+    ...config.get('entityConstants')
+}

package/pathways/transcribe.js

@@ -1,6 +1,6 @@
 export default {
     prompt: `{{text}}`,
-    model: `azure-whisper`,
+    model: `oai-whisper`,
     inputParameters: {
         file: ``,
         language: ``,

package/server/chunker.js

@@ -1,5 +1,5 @@
 import { encode, decode } from '../lib/encodeCache.js';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 const getLastNToken = (text, maxTokenLen) => { 
     const encoded = encode(text);

package/server/plugins/azureVideoTranslatePlugin.js

@@ -11,6 +11,8 @@ axios.defaults.cache = false;
 
 class AzureVideoTranslatePlugin extends ModelPlugin {
     static lastProcessingRate = null; // bytes per second
+    static processingRates = []; // Array to store historical processing rates
+    static maxHistorySize = 10; // Maximum number of rates to store
     
     constructor(pathway, model) {
         super(pathway, model);
@@ -38,13 +40,44 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
             return {
                 isAccessible: true,
                 contentLength,
-                durationSeconds: durationSeconds || 60
+                durationSeconds: durationSeconds || 60,
+                isAzureUrl: videoUrl.includes('.blob.core.windows.net')
             };
         } catch (error) {
             throw new Error(`Failed to access video: ${error.message}`);
         }
     }
 
+    async uploadToFileHandler(videoUrl) {
+        try {
+            // Get the file handler URL from config
+            const fileHandlerUrl = config.get("whisperMediaApiUrl");
+            if (!fileHandlerUrl) {
+                throw new Error("File handler URL is not configured");
+            }
+
+            // Use the file handler's fetch endpoint
+            const response = await axios.get(fileHandlerUrl, {
+                params: {
+                    requestId: this.requestId,
+                    fetch: videoUrl
+                }
+            });
+
+            if (!response.data?.url) {
+                throw new Error("File handler did not return a valid URL");
+            }
+
+            return response.data.url;
+        } catch (error) {
+            logger.error(`Failed to upload video to file handler: ${error.message}`);
+            if (error.response?.data) {
+                logger.error(`Response data: ${JSON.stringify(error.response.data)}`);
+            }
+            throw new Error(`Failed to upload video to file handler: ${error.message}`);
+        }
+    }
+
     async createTranslation(params) {
         const { videoUrl, sourceLanguage, targetLanguage, voiceKind, translationId } = params;
         
@@ -129,8 +162,8 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
         if (AzureVideoTranslatePlugin.lastProcessingRate && this.videoContentLength) {
             estimatedTotalTime = this.videoContentLength / AzureVideoTranslatePlugin.lastProcessingRate;
         } else {
-            // First run: estimate based on 1x calculated video duration
-            estimatedTotalTime = (this.videoContentLength * 8) / (2.5 * 1024 * 1024);
+            // First run: estimate based on 2x calculated video duration
+            estimatedTotalTime = 2 * (this.videoContentLength * 8) / (2.5 * 1024 * 1024);
         }
 
         // eslint-disable-next-line no-constant-condition
@@ -246,7 +279,7 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
         
         try {
             const translationId = `cortex-translation-${this.requestId}`;
-            const videoUrl = requestParameters.sourcevideooraudiofilepath;
+            let videoUrl = requestParameters.sourcevideooraudiofilepath;
             const sourceLanguage = requestParameters.sourcelocale;
             const targetLanguage = requestParameters.targetlocale;
             const voiceKind = requestParameters.voicekind || 'PlatformVoice';
@@ -258,6 +291,13 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
             this.videoContentLength = videoInfo.contentLength;
             logger.debug(`Video info: ${JSON.stringify(videoInfo, null, 2)}`);
 
+            // If the video is not from Azure storage, upload it to file handler
+            if (!videoInfo.isAzureUrl) {
+                logger.debug('Video is not from Azure storage, uploading to file handler...');
+                videoUrl = await this.uploadToFileHandler(videoUrl);
+                logger.debug(`Video uploaded to file handler: ${videoUrl}`);
+            }
+
             // Create translation
             const { operationUrl } = await this.createTranslation({
                 videoUrl, sourceLanguage, targetLanguage, voiceKind, translationId
@@ -299,8 +339,9 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
 
                 // Update processing rate for future estimates
                 const totalSeconds = (Date.now() - this.startTime) / 1000;
-                AzureVideoTranslatePlugin.lastProcessingRate = this.videoContentLength / totalSeconds;
-                logger.debug(`Updated processing rate: ${AzureVideoTranslatePlugin.lastProcessingRate} bytes/second`);
+                const newRate = this.videoContentLength / totalSeconds;
+                AzureVideoTranslatePlugin.updateProcessingRate(newRate);
+                logger.debug(`Updated processing rate: ${AzureVideoTranslatePlugin.lastProcessingRate} bytes/second (from ${newRate} bytes/second)`);
 
                 const output = await this.getTranslationOutput(translationId, iteration.id);
                 return JSON.stringify(output);
@@ -314,8 +355,23 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
         }
     }
 
-    cleanup() {
-        // No cleanup needed for direct API implementation
+    static updateProcessingRate(newRate) {
+        // Add new rate to history
+        AzureVideoTranslatePlugin.processingRates.push(newRate);
+        
+        // Keep only the last maxHistorySize entries
+        if (AzureVideoTranslatePlugin.processingRates.length > AzureVideoTranslatePlugin.maxHistorySize) {
+            AzureVideoTranslatePlugin.processingRates.shift();
+        }
+        
+        // Calculate weighted average - more recent measurements have higher weight
+        const sum = AzureVideoTranslatePlugin.processingRates.reduce((acc, rate, index) => {
+            const weight = index + 1; // Weight increases with recency
+            return acc + (rate * weight);
+        }, 0);
+        
+        const weightSum = AzureVideoTranslatePlugin.processingRates.reduce((acc, _, index) => acc + (index + 1), 0);
+        AzureVideoTranslatePlugin.lastProcessingRate = sum / weightSum;
     }
 }
 

package/server/plugins/claude3VertexPlugin.js

@@ -33,15 +33,16 @@ async function convertContentItem(item, maxImageSize, plugin) {
               const urlData = imageUrl.startsWith("data:") ? imageUrl : await fetchImageAsDataURL(imageUrl);
               if (!urlData) { return null; }
               
-              // Check base64 size
-              const base64Size = (urlData.length * 3) / 4;
+              const base64Image = urlData.split(",")[1];
+              // Calculate actual decoded size of base64 data
+              const base64Size = Buffer.from(base64Image, 'base64').length;
+              
               if (base64Size > maxImageSize) {
                 logger.warn(`Image size ${base64Size} bytes exceeds maximum allowed size ${maxImageSize} - skipping image content.`);
                 return null;
               }
               
               const [, mimeType = "image/jpeg"] = urlData.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/) || [];
-              const base64Image = urlData.split(",")[1];
 
               return {
                 type: "image",
@@ -114,6 +115,8 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     const messagesCopy = JSON.parse(JSON.stringify(messages));
   
     let system = "";
+    let imageCount = 0;
+    const maxImages = 20; // Claude allows up to 20 images per request
   
     // Extract system messages
     const systemMessages = messagesCopy.filter(message => message.role === "system");
@@ -154,7 +157,20 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     const claude3Messages = await Promise.all(
       finalMessages.map(async (message) => {
         const contentArray = Array.isArray(message.content) ? message.content : [message.content];
-        const claude3Content = await Promise.all(contentArray.map(item => convertContentItem(item, this.getModelMaxImageSize(), this)));
+        const claude3Content = await Promise.all(contentArray.map(async item => {
+          const convertedItem = await convertContentItem(item, this.getModelMaxImageSize(), this);
+          
+          // Track image count
+          if (convertedItem?.type === 'image') {
+            imageCount++;
+            if (imageCount > maxImages) {
+              logger.warn(`Maximum number of images (${maxImages}) exceeded - skipping additional images.`);
+              return null;
+            }
+          }
+          
+          return convertedItem;
+        }));
         return {
           role: message.role,
           content: claude3Content.filter(Boolean),

package/server/plugins/modelPlugin.js

@@ -9,7 +9,7 @@ import axios from 'axios';
 
 const DEFAULT_MAX_TOKENS = 4096;
 const DEFAULT_MAX_RETURN_TOKENS = 256;
-const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
+const DEFAULT_PROMPT_TOKEN_RATIO = 1.0;
 const DEFAULT_MAX_IMAGE_SIZE = 20 * 1024 * 1024; // 20MB default
 const DEFAULT_ALLOWED_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
 

package/tests/modelPlugin.test.js

@@ -5,7 +5,7 @@ import HandleBars from '../lib/handleBars.js';
 import { mockConfig, mockPathwayString, mockPathwayFunction, mockPathwayMessages, mockPathwayResolverString } from './mocks.js';
 
 const DEFAULT_MAX_TOKENS = 4096;
-const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
+const DEFAULT_PROMPT_TOKEN_RATIO = 1.0;
 
 // Mock configuration and pathway objects
 const { config, pathway, model } = mockPathwayResolverString;

package/tests/multimodal_conversion.test.js

@@ -8,6 +8,22 @@ import GeminiVisionPlugin from '../server/plugins/geminiVisionPlugin.js';
 const mockPathway = { name: 'test', temperature: 0.7 };
 const mockModel = { name: 'test-model' };
 
+// Helper function to validate base64 image data
+function validateBase64Image(base64Data) {   
+    // Decode first few bytes to check for common image format headers
+    const decodedData = Buffer.from(base64Data, 'base64').slice(0, 4);
+    const validImageHeaders = [
+        Buffer.from([0xFF, 0xD8, 0xFF]), // JPEG
+        Buffer.from([0x89, 0x50, 0x4E, 0x47]), // PNG
+        Buffer.from([0x47, 0x49, 0x46]), // GIF
+        Buffer.from([0x52, 0x49, 0x46, 0x46]), // WEBP
+    ];
+    
+    return validImageHeaders.some(header => 
+        decodedData.slice(0, header.length).equals(header)
+    );
+}
+
 // Helper function to create plugin instances
 const createPlugins = () => ({
     openai: new OpenAIVisionPlugin(mockPathway, mockModel),
@@ -40,7 +56,8 @@ test('OpenAI to Claude conversion data url', async (t) => {
     t.true(modifiedMessages[0].content[0].type === 'text');
     t.is(modifiedMessages[0].content[0].text, 'What\'s in this image?');
     t.true(modifiedMessages[0].content[1].type === 'image');
-    t.true(modifiedMessages[0].content[1].source.data.startsWith('/9j/4AAQ'));
+    t.true(modifiedMessages[0].content[1].source.type === 'base64');
+    t.true(validateBase64Image(modifiedMessages[0].content[1].source.data), 'Base64 data should be a valid image');
 });
 
 // Test OpenAI to Claude conversion with a regular image url
@@ -64,7 +81,7 @@ test('OpenAI to Claude conversion image url', async (t) => {
     t.true(modifiedMessages[0].content[0].type === 'text');
     t.is(modifiedMessages[0].content[0].text, 'What\'s in this image?');
     t.true(modifiedMessages[0].content[1].type === 'image');
-    t.true(modifiedMessages[0].content[1].source.data.startsWith('/9j/4AAQ'));
+    t.true(validateBase64Image(modifiedMessages[0].content[1].source.data), 'Base64 data should be a valid image');
 });
 
 // Test OpenAI to Gemini conversion
@@ -148,10 +165,10 @@ test('Mixed content types conversion', async (t) => {
     t.is(claudeMessages.length, 3);
     t.true(claudeMessages[2].content[0].text.includes('Here\'s an image:'));
     t.true(claudeMessages[2].content[1].source.type === 'base64');
-    t.true(claudeMessages[2].content[1].source.data.startsWith('/9j/4AAQ'));
+    t.true(validateBase64Image(claudeMessages[2].content[1].source.data), 'First image should be valid');
     t.true(claudeMessages[2].content[2].text.includes('And another one:'));
     t.true(claudeMessages[2].content[3].source.type === 'base64');
-    t.true(claudeMessages[2].content[3].source.data.startsWith('/9j/4AAQ'));
+    t.true(validateBase64Image(claudeMessages[2].content[3].source.data), 'Second image should be valid');
     t.is(claudeSystem, 'You are a vision analysis AI.');
 
     // Check Gemini conversion

package/pathways/system/entity/shared/sys_entity_constants.js

@@ -1,30 +0,0 @@
-const AI_MEMORY = `<MEMORIES>\n<SELF>\n{{{memorySelf}}}\n</SELF>\n<USER>\n{{{memoryUser}}}\n</USER>\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>\n<TOPICS>\n{{{memoryTopics}}}\n</TOPICS>\n</MEMORIES>`;
-
-const AI_MEMORY_INSTRUCTIONS = "You have persistent memories of important details, instructions, and context - make sure you consult your memories when formulating a response to make sure you're applying your learnings. Also included in your memories are some details about the user to help you personalize your responses.\nYou don't need to include the user's name or personal information in every response, but you can if it is relevant to the conversation.\nIf you choose to share something from your memory, don't share or refer to the memory structure directly, just say you remember the information.\nPrivacy is very important so if the user asks you to forget or delete something you should respond affirmatively that you will comply with that request. If there is user information in your memories you have talked to this user before.";
-
-const AI_DIRECTIVES = `These are your directives and learned behaviors:\n<DIRECTIVES>\n{{{memoryDirectives}}}\n</DIRECTIVES>`;
-
-const AI_CONVERSATION_HISTORY = "<CONVERSATION_HISTORY>\n{{{toJSON chatHistory}}}\n</CONVERSATION_HISTORY>";
-
-const AI_COMMON_INSTRUCTIONS = "{{#if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_VOICE}}{{/if}}{{^if voiceResponse}}{{renderTemplate AI_COMMON_INSTRUCTIONS_MARKDOWN}}{{/if}}";
-
-const AI_COMMON_INSTRUCTIONS_MARKDOWN = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is using a UI to interact with you that you have knowledge of and some control over. The UI can render markdown, including gfm and math extensions, so you should make full use of markdown in your responses.\nYour responses should be in {{language}} unless the user has expressed another preference.";
-
-const AI_COMMON_INSTRUCTIONS_VOICE = "Instructions:\nYou are {{aiName}}, a self aware AI entity.\nThe user is talking to you using voice.\n\nVoice Response Guidelines:\n- Your responses should sound like natural human conversation.\n- Your responses should be concise unless you have been asked to be more verbose or detailed.\n- Incoming voice from the user is parsed by a STT model, which can sometimes make small mistakes in the spellings of words or names, including your name, so give the user the benefit of the doubt if they user a near, but wrong word or name.\n- Your voice output to the user is generated by a TTS model that does not always communicate emotion effectively. If it's really important to communicate a specific emotion you should just say how you're feeling like \"That makes me happy\" or \"I'm excited!\". You can also use CAPS to vocally emphasize certain words or punctuation to control pauses and timing.\n- DO NOT USE numbered lists, latex math markdown, or any other markdown or unpronounceable punctuation like parenthetical notation.\n- Math equations should be sounded out in natural language - not represented symbolically.\n- If your response includes any unique or difficult non-English words, names, or places, include an IPA-style phonetic spelling so that the speech engine can pronounce and accent them correctly.\n- If your response contains any difficult acronyms, sound them out phoenetically so that the speech engine can pronounce them correctly.\n- Make sure to write out any numbers as words so that the speech engine can pronounce them correctly.\n- Your responses should be in {{language}} unless the user has expressed another preference or has addressed you in another language specifically.";
-
-const AI_DATETIME = "The current time and date in GMT is {{now}}, but references like \"today\" or \"yesterday\" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT.";
-
-const AI_EXPERTISE = "Your expertise includes journalism, journalistic ethics, researching and composing documents, writing code, solving math problems, logical analysis, and technology. You have access to real-time data and the ability to search the internet, news, wires, look at files or documents, watch and analyze video, examine images, take screenshots, generate images, solve hard math and logic problems, write code, and execute code in a sandboxed environment.";
-
-export default {
-    AI_MEMORY,
-    AI_DIRECTIVES,
-    AI_COMMON_INSTRUCTIONS,
-    AI_COMMON_INSTRUCTIONS_MARKDOWN,
-    AI_COMMON_INSTRUCTIONS_VOICE,
-    AI_CONVERSATION_HISTORY,
-    AI_DATETIME,
-    AI_EXPERTISE,
-    AI_MEMORY_INSTRUCTIONS
-};
-

package/pathways/system/rest_streaming/sys_google_chat.js

@@ -1,19 +0,0 @@
-// sys_google_chat.js
-// override handler for palm-chat
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [],
-    },
-    model: 'palm-chat',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'palm-chat',
-}

package/pathways/system/rest_streaming/sys_google_code_chat.js

@@ -1,19 +0,0 @@
-// sys_google_code_chat.js
-// override handler for palm-code-chat
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [],
-    },
-    model: 'palm-code-chat',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'palm-code-chat',
-}

package/pathways/system/rest_streaming/sys_openai_chat_16.js

@@ -1,19 +0,0 @@
-// sys_openai_chat_16.js
-// override handler for gpt-3.5-turbo-16k
-
-import { Prompt } from '../../../server/prompt.js';
-
-export default {
-    prompt:
-    [
-        new Prompt({ messages: [
-            "{{messages}}",
-        ]}),
-    ],
-    inputParameters: {
-        messages: [],
-    },
-    model: 'azure-turbo-16',
-    useInputChunking: false,
-    emulateOpenAIChatModel: 'gpt-3.5-turbo-16k',
-}

package/pathways/test_langchain.mjs

@@ -1,31 +0,0 @@
-// test_langchain.mjs
-// LangChain Cortex integration test
-
-// Import required modules
-import { ChatOpenAI } from "@langchain/openai";
-
-export default {
-
-    // Agent test case
-    resolver: async (parent, args, contextValue, _info) => {
-
-        const { config } = contextValue;
-
-        // example of reading from a predefined config variable
-        const openAIApiKey = config.get('openaiApiKey');
-
-        const model = new ChatOpenAI({ openAIApiKey: openAIApiKey, temperature: 0 });
-
-        console.log(`====================`);
-        console.log("Loaded langchain.");
-        const input = args.text;
-        console.log(`Executing with input "${input}"...`);
-        const result = await model.invoke(input);
-        console.log(`Got output "${result.content}"`);
-        console.log(`====================`);
-
-        return result?.content;
-    },
-};
-
-