@aj-archipelago/cortex 1.0.3 → 1.0.5

package/README.md

@@ -17,7 +17,7 @@ Just about anything! It's kind of an LLM swiss army knife.  Here are some ideas:
 ## Features
 
 * Simple architecture to build custom functional endpoints (called `pathways`), that implement common NL AI tasks. Default pathways include chat, summarization, translation, paraphrasing, completion, spelling and grammar correction, entity extraction, sentiment analysis, and bias analysis.
-* Allows for building multi-model, multi-tool, multi-vendor, and model-agnostic pathways (choose the right model or combination of models and tools for the job, implement redundancy) with built-in support for OpenAI GPT-3, GPT-3.5 (chatGPT), and GPT-4 models - both from OpenAI directly and through Azure OpenAI, OpenAI Whisper, Azure Translator, LangChain.js and more.
+* Allows for building multi-model, multi-tool, multi-vendor, and model-agnostic pathways (choose the right model or combination of models and tools for the job, implement redundancy) with built-in support for OpenAI GPT-3, GPT-3.5 (chatGPT), and GPT-4 models - both from OpenAI directly and through Azure OpenAI, PaLM Text and PaLM Chat from Google, OpenAI Whisper, Azure Translator, LangChain.js and more.
 * Easy, templatized prompt definition with flexible support for most prompt engineering techniques and strategies ranging from simple single prompts to complex custom prompt chains with context continuity.
 * Built in support for long-running, asynchronous operations with progress updates or streaming responses
 * Integrated context persistence: have your pathways "remember" whatever you want and use it on the next request to the model

package/config/default.example.json

@@ -51,6 +51,24 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 8192
         },
+        "palm-text": {
+            "type": "PALM-COMPLETION",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/text-bison@001:predict",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 2048
+        },
+        "palm-chat": {
+            "type": "PALM-CHAT",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/chat-bison@001:predict",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 2048
+        },
         "local-llama13B": {
             "type": "LOCAL-CPP-MODEL",
             "executablePath": "../llm/llama.cpp/main",

package/config.js

@@ -1,8 +1,11 @@
 import path from 'path';
-const __dirname = path.dirname(new URL(import.meta.url).pathname);
 import convict from 'convict';
 import HandleBars from './lib/handleBars.js';
 import fs from 'fs';
+import { fileURLToPath, pathToFileURL } from 'url';
+import GcpAuthTokenHelper from './lib/gcpAuthTokenHelper.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 // Schema for config
 var config = convict({
@@ -55,7 +58,8 @@ var config = convict({
     cortexApiKey: {
         format: String,
         default: null,
-        env: 'CORTEX_API_KEY'
+        env: 'CORTEX_API_KEY',
+        sensitive: true
     },
     defaultModelName: {
         format: String,
@@ -75,6 +79,7 @@ var config = convict({
                 "params": {
                     "model": "{{openaiDefaultModel}}"
                 },
+                "requestsPerSecond": 2,
             },
             "oai-whisper": {
                 "type": "OPENAI_WHISPER",
@@ -115,6 +120,12 @@ var config = convict({
         default: 'null',
         env: 'WHISPER_MEDIA_API_URL'
     },
+    gcpServiceAccountKey: {
+        format: String,
+        default: null,
+        env: 'GCP_SERVICE_ACCOUNT_KEY',
+        sensitive: true
+    },
 });
 
 // Read in environment variables and set up service configuration
@@ -133,21 +144,30 @@ if (configFile && fs.existsSync(configFile)) {
     }
 }
 
+if (config.get('gcpServiceAccountKey')) {
+    const gcpAuthTokenHelper = new GcpAuthTokenHelper(config.getProperties());
+    config.set('gcpAuthTokenHelper', gcpAuthTokenHelper);
+}
+
 // Build and load pathways to config
 const buildPathways = async (config) => {
     const { pathwaysPath, corePathwaysPath, basePathwayPath } = config.getProperties();
 
+    const pathwaysURL = pathToFileURL(pathwaysPath).toString();
+    const corePathwaysURL = pathToFileURL(corePathwaysPath).toString();
+    const basePathwayURL = pathToFileURL(basePathwayPath).toString();
+
     // Load cortex base pathway 
-    const basePathway = await import(basePathwayPath).then(module => module.default);
+    const basePathway = await import(basePathwayURL).then(module => module.default);
 
     // Load core pathways, default from the Cortex package
     console.log('Loading core pathways from', corePathwaysPath)
-    let loadedPathways = await import(`${corePathwaysPath}/index.js`).then(module => module);
+    let loadedPathways = await import(`${corePathwaysURL}/index.js`).then(module => module);
 
     // Load custom pathways and override core pathways if same
     if (pathwaysPath && fs.existsSync(pathwaysPath)) {
         console.log('Loading custom pathways from', pathwaysPath)
-        const customPathways = await import(`${pathwaysPath}/index.js`).then(module => module);
+        const customPathways = await import(`${pathwaysURL}/index.js`).then(module => module);
         loadedPathways = { ...loadedPathways, ...customPathways };
     }
 

package/graphql/pathwayPrompter.js

@@ -4,6 +4,8 @@ import OpenAICompletionPlugin from './plugins/openAiCompletionPlugin.js';
 import AzureTranslatePlugin from './plugins/azureTranslatePlugin.js';
 import OpenAIWhisperPlugin from './plugins/openAiWhisperPlugin.js';
 import LocalModelPlugin from './plugins/localModelPlugin.js';
+import PalmChatPlugin from './plugins/palmChatPlugin.js';
+import PalmCompletionPlugin from './plugins/palmCompletionPlugin.js';
 
 class PathwayPrompter {
     constructor({ config, pathway }) {
@@ -33,6 +35,12 @@ class PathwayPrompter {
             case 'LOCAL-CPP-MODEL':
                 plugin = new LocalModelPlugin(config, pathway);
                 break;
+            case 'PALM-CHAT':
+                plugin = new PalmChatPlugin(config, pathway);
+                break;
+            case 'PALM-COMPLETION':
+                plugin = new PalmCompletionPlugin(config, pathway);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }

package/graphql/pathwayResolver.js

@@ -289,7 +289,15 @@ class PathwayResolver {
         if (requestState[this.requestId].canceled) {
             return;
         }
-        const result = await this.pathwayPrompter.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
+        let result = '';
+
+        // If this text is empty, skip applying the prompt as it will likely be a nonsensical result
+        if (!/^\s*$/.test(text)) {
+            result = await this.pathwayPrompter.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
+        } else {
+            result = text;
+        }
+        
         requestState[this.requestId].completedCount++;
 
         const { completedCount, totalCount } = requestState[this.requestId];

package/graphql/plugins/azureTranslatePlugin.js

@@ -35,6 +35,28 @@ class AzureTranslatePlugin extends ModelPlugin {
 
         return this.executeRequest(url, data, params, headers, prompt);
     }
+    
+    // Parse the response from the Azure Translate API
+    parseResponse(data) {
+        if (Array.isArray(data) && data.length > 0 && data[0].translations) {
+            return data[0].translations[0].text.trim();
+        } else {
+            return data;
+        }
+    }
+    
+    // Override the logging function to display the request and response
+    logRequestData(data, responseData, prompt) {
+        const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
+        console.log(separator);
+    
+        const modelInput = data[0].Text;
+    
+        console.log(`\x1b[36m${modelInput}\x1b[0m`);
+        console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
+    
+        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+    }
 }
 
 export default AzureTranslatePlugin;

package/graphql/plugins/modelPlugin.js

@@ -62,7 +62,7 @@ class ModelPlugin {
             const message = tokenLengths[index].message;
 
             // Skip system messages
-            if (message.role === 'system') {
+            if (message?.role === 'system') {
                 index++;
                 continue;
             }
@@ -113,7 +113,7 @@ class ModelPlugin {
         let output = "";
         if (messages && messages.length) {
             for (let message of messages) {
-                output += (message.role && (message.content || message.content === '')) ? `<|im_start|>${message.role}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
+                output += ((message.author || message.role) && (message.content || message.content === '')) ? `<|im_start|>${(message.author || message.role)}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
             }
             // you always want the assistant to respond next so add a
             // directive for that
@@ -124,6 +124,7 @@ class ModelPlugin {
         return output;
     }
 
+    // compile the Prompt    
     getCompiledPrompt(text, parameters, prompt) {
         const combinedParameters = { ...this.promptParameters, ...parameters };
         const modelPrompt = this.getModelPrompt(prompt, parameters);
@@ -132,9 +133,9 @@ class ModelPlugin {
         const modelPromptMessagesML = this.messagesToChatML(modelPromptMessages);
 
         if (modelPromptMessagesML) {
-            return { modelPromptMessages, tokenLength: encode(modelPromptMessagesML).length };
+            return { modelPromptMessages, tokenLength: encode(modelPromptMessagesML).length, modelPrompt };
         } else {
-            return { modelPromptText, tokenLength: encode(modelPromptText).length };
+            return { modelPromptText, tokenLength: encode(modelPromptText).length, modelPrompt };
         }
     }
 
@@ -147,12 +148,11 @@ class ModelPlugin {
         return this.promptParameters.inputParameters?.tokenRatio ?? this.promptParameters.tokenRatio ?? DEFAULT_PROMPT_TOKEN_RATIO;
     }
 
-
     getModelPrompt(prompt, parameters) {
         if (typeof(prompt) === 'function') {
-        return prompt(parameters);
+            return prompt(parameters);
         } else {
-        return prompt;
+            return prompt;
         }
     }
 
@@ -160,20 +160,20 @@ class ModelPlugin {
         if (!modelPrompt.messages) {
             return null;
         }
-
+    
         // First run handlebars compile on the pathway messages
         const compiledMessages = modelPrompt.messages.map((message) => {
             if (message.content) {
                 const compileText = HandleBars.compile(message.content);
                 return {
-                    role: message.role,
+                    ...message,
                     content: compileText({ ...combinedParameters, text }),
                 };
             } else {
                 return message;
             }
         });
-
+    
         // Next add in any parameters that are referenced by name in the array
         const expandedMessages = compiledMessages.flatMap((message) => {
             if (typeof message === 'string') {
@@ -188,7 +188,7 @@ class ModelPlugin {
                 return [message];
             }
         });
-
+     
         return expandedMessages;
     }
 
@@ -197,44 +197,17 @@ class ModelPlugin {
         return generateUrl({ ...this.model, ...this.environmentVariables, ...this.config });
     }
 
-    //simples form string single or list return
-    parseResponse(data) {
-        const { choices } = data;
-        if (!choices || !choices.length) {
-            if (Array.isArray(data) && data.length > 0 && data[0].translations) {
-                return data[0].translations[0].text.trim();
-            } else {
-                return data;
-            }
-        }
-
-        // if we got a choices array back with more than one choice, return the whole array
-        if (choices.length > 1) {
-            return choices;
-        }
-
-        // otherwise, return the first choice
-        const textResult = choices[0].text && choices[0].text.trim();
-        const messageResult = choices[0].message && choices[0].message.content && choices[0].message.content.trim();
-
-        return messageResult ?? textResult ?? null;
-    }
+    // Default response parsing
+    parseResponse(data) { return data; };
 
+    // Default simple logging
     logRequestData(data, responseData, prompt) {
         const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
         console.log(separator);
     
         const modelInput = data.prompt || (data.messages && data.messages[0].content) || (data.length > 0 && data[0].Text) || null;
     
-        if (data && data.messages && data.messages.length > 1) {
-            data.messages.forEach((message, index) => {
-                const words = message.content.split(" ");
-                const tokenCount = encode(message.content).length;
-                const preview = words.length < 41 ? message.content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
-    
-                console.log(`\x1b[36mMessage ${index + 1}: Role: ${message.role}, Tokens: ${tokenCount}, Content: "${preview}"\x1b[0m`);
-            });
-        } else {
+        if (modelInput) {
             console.log(`\x1b[36m${modelInput}\x1b[0m`);
         }
     

package/graphql/plugins/openAiChatPlugin.js

@@ -1,20 +1,64 @@
 // OpenAIChatPlugin.js
 import ModelPlugin from './modelPlugin.js';
+import { encode } from 'gpt-3-encoder';
 
 class OpenAIChatPlugin extends ModelPlugin {
     constructor(config, pathway) {
         super(config, pathway);
     }
 
+    // convert to OpenAI messages array format if necessary
+    convertPalmToOpenAIMessages(context, examples, messages) {
+        let openAIMessages = [];
+        
+        // Add context as a system message
+        if (context) {
+            openAIMessages.push({
+            role: 'system',
+            content: context,
+            });
+        }
+        
+        // Add examples to the messages array
+        examples.forEach(example => {
+            openAIMessages.push({
+            role: example.input.author || 'user',
+            content: example.input.content,
+            });
+            openAIMessages.push({
+            role: example.output.author || 'assistant',
+            content: example.output.content,
+            });
+        });
+        
+        // Add remaining messages to the messages array
+        messages.forEach(message => {
+            openAIMessages.push({
+            role: message.author,
+            content: message.content,
+            });
+        });
+        
+        return openAIMessages;
+    }
+
     // Set up parameters specific to the OpenAI Chat API
     getRequestParameters(text, parameters, prompt) {
-        const { modelPromptText, modelPromptMessages, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
+        const { modelPromptText, modelPromptMessages, tokenLength, modelPrompt } = this.getCompiledPrompt(text, parameters, prompt);
         const { stream } = parameters;
     
         // Define the model's max token length
         const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();
     
         let requestMessages = modelPromptMessages || [{ "role": "user", "content": modelPromptText }];
+        
+        // Check if the messages are in Palm format and convert them to OpenAI format if necessary
+        const isPalmFormat = requestMessages.some(message => 'author' in message);
+        if (isPalmFormat) {
+            const context = modelPrompt.context || '';
+            const examples = modelPrompt.examples || [];
+            requestMessages = this.convertPalmToOpenAIMessages(context, examples, expandedMessages);
+        }
     
         // Check if the token length exceeds the model's max token length
         if (tokenLength > modelTargetTokenLength) {
@@ -25,7 +69,7 @@ class OpenAIChatPlugin extends ModelPlugin {
         const requestParameters = {
         messages: requestMessages,
         temperature: this.temperature ?? 0.7,
-        stream
+        ...(stream !== undefined ? { stream } : {}),
         };
     
         return requestParameters;
@@ -41,6 +85,45 @@ class OpenAIChatPlugin extends ModelPlugin {
         const headers = this.model.headers || {};
         return this.executeRequest(url, data, params, headers, prompt);
     }
+
+    // Parse the response from the OpenAI Chat API
+    parseResponse(data) {
+        const { choices } = data;
+        if (!choices || !choices.length) {
+            return null;
+        }
+
+        // if we got a choices array back with more than one choice, return the whole array
+        if (choices.length > 1) {
+            return choices;
+        }
+
+        // otherwise, return the first choice
+        const messageResult = choices[0].message && choices[0].message.content && choices[0].message.content.trim();
+        return messageResult ?? null;
+    }
+
+    // Override the logging function to display the messages and responses
+    logRequestData(data, responseData, prompt) {
+        const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
+        console.log(separator);
+    
+        if (data && data.messages && data.messages.length > 1) {
+            data.messages.forEach((message, index) => {
+                const words = message.content.split(" ");
+                const tokenCount = encode(message.content).length;
+                const preview = words.length < 41 ? message.content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
+    
+                console.log(`\x1b[36mMessage ${index + 1}: Role: ${message.role}, Tokens: ${tokenCount}, Content: "${preview}"\x1b[0m`);
+            });
+        } else {
+            console.log(`\x1b[36m${data.messages[0].content}\x1b[0m`);
+        }
+    
+        console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
+    
+        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+    }
 }
 
 export default OpenAIChatPlugin;

package/graphql/plugins/openAiCompletionPlugin.js

@@ -1,7 +1,6 @@
 // OpenAICompletionPlugin.js
 
 import ModelPlugin from './modelPlugin.js';
-
 import { encode } from 'gpt-3-encoder';
 
 // Helper function to truncate the prompt if it is too long
@@ -52,7 +51,7 @@ class OpenAICompletionPlugin extends ModelPlugin {
                 frequency_penalty: 0,
                 presence_penalty: 0,
                 stop: ["<|im_end|>"],
-                stream
+                ...(stream !== undefined ? { stream } : {}),
             };
         } else {
 
@@ -83,8 +82,39 @@ class OpenAICompletionPlugin extends ModelPlugin {
         const data = { ...(this.model.params || {}), ...requestParameters };
         const params = {};
         const headers = this.model.headers || {};
+        
         return this.executeRequest(url, data, params, headers, prompt);
     }
+
+    // Parse the response from the OpenAI Completion API
+    parseResponse(data) {
+        const { choices } = data;
+        if (!choices || !choices.length) {
+            return data;
+        }
+
+        // if we got a choices array back with more than one choice, return the whole array
+        if (choices.length > 1) {
+            return choices;
+        }
+
+        // otherwise, return the first choice
+        const textResult = choices[0].text && choices[0].text.trim();
+        return textResult ?? null;
+    }
+
+    // Override the logging function to log the prompt and response
+    logRequestData(data, responseData, prompt) {
+        const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
+        console.log(separator);
+    
+        const modelInput = data.prompt;
+    
+        console.log(`\x1b[36m${modelInput}\x1b[0m`);
+        console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
+    
+        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+    }
 }
 
 export default OpenAICompletionPlugin;

package/graphql/plugins/openAiWhisperPlugin.js

@@ -14,11 +14,33 @@ import http from 'http';
 import https from 'https';
 import url from 'url';
 import { promisify } from 'util';
+import subsrt from 'subsrt';
 const pipeline = promisify(stream.pipeline);
 
 
 const API_URL = config.get('whisperMediaApiUrl');
 
+function alignSubtitles(subtitles) {
+    const result = [];
+    const offset = 1000 * 60 * 10; // 10 minutes for each chunk
+
+    function preprocessStr(str) {
+        return str.trim().replace(/(\n\n)(?!\n)/g, '\n\n\n');
+    }
+
+    function shiftSubtitles(subtitle, shiftOffset) {
+        const captions = subsrt.parse(preprocessStr(subtitle));
+        const resynced = subsrt.resync(captions, { offset: shiftOffset });
+        return resynced;
+    }
+
+    for (let i = 0; i < subtitles.length; i++) {
+        const subtitle = subtitles[i];
+        result.push(...shiftSubtitles(subtitle, i * offset));
+    }
+    return subsrt.build(result);
+}
+
 function generateUniqueFilename(extension) {
     return `${uuidv4()}.${extension}`;
 }
@@ -52,6 +74,7 @@ const downloadFile = async (fileUrl) => {
             fs.unlink(localFilePath, () => {
                 reject(error);
             });
+            throw error;
         }
     });
 };
@@ -73,41 +96,50 @@ class OpenAIWhisperPlugin extends ModelPlugin {
             }
         } catch (err) {
             console.log(`Error getting media chunks list from api:`, err);
+            throw err;
         }
     }
 
     async markCompletedForCleanUp(requestId) {
-        if (API_URL) {
-            //call helper api to mark processing as completed
-            const res = await axios.delete(API_URL, { params: { requestId } });
-            console.log(`Marked request ${requestId} as completed:`, res.data);
-            return res.data;
+        try {
+            if (API_URL) {
+                //call helper api to mark processing as completed
+                const res = await axios.delete(API_URL, { params: { requestId } });
+                console.log(`Marked request ${requestId} as completed:`, res.data);
+                return res.data;
+            }
+        } catch (err) {
+            console.log(`Error marking request ${requestId} as completed:`, err);
         }
     }
 
     // Execute the request to the OpenAI Whisper API
     async execute(text, parameters, prompt, pathwayResolver) {
+        const { responseFormat } = parameters;
         const url = this.requestUrl(text);
         const params = {};
         const { modelPromptText } = this.getCompiledPrompt(text, parameters, prompt);
 
         const processChunk = async (chunk) => {
             try {
-                const { language } = parameters;
+                const { language, responseFormat } = parameters;
+                const response_format = responseFormat || 'text';
+
                 const formData = new FormData();
                 formData.append('file', fs.createReadStream(chunk));
                 formData.append('model', this.model.params.model);
-                formData.append('response_format', 'text');
+                formData.append('response_format', response_format);
                 language && formData.append('language', language);
                 modelPromptText && formData.append('prompt', modelPromptText);
 
                 return this.executeRequest(url, formData, params, { ...this.model.headers, ...formData.getHeaders() });
             } catch (err) {
                 console.log(err);
+                throw err;
             }
         }
 
-        let result = ``;
+        let result = [];
         let { file } = parameters;
         let totalCount = 0;
         let completedCount = 0;
@@ -144,7 +176,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
 
             // sequential processing of chunks
             for (const chunk of chunks) {
-                result += await processChunk(chunk);
+                result.push(await processChunk(chunk));
                 sendProgress();
             }
 
@@ -152,11 +184,11 @@ class OpenAIWhisperPlugin extends ModelPlugin {
             // result = await Promise.all(mediaSplit.chunks.map(processChunk));
 
         } catch (error) {
-            console.error("An error occurred:", error);
+            const errMsg = `Transcribe error: ${error?.message || JSON.stringify(error)}`;
+            console.error(errMsg);
+            return errMsg;
         }
         finally {
-            // isYoutubeUrl && (await deleteTempPath(file));
-            // folder && (await deleteTempPath(folder));
             try {
                 for (const chunk of chunks) {
                     await deleteTempPath(chunk);
@@ -177,7 +209,11 @@ class OpenAIWhisperPlugin extends ModelPlugin {
                 console.error("An error occurred while deleting:", error);
             }
         }
-        return result;
+
+        if (['srt','vtt'].includes(responseFormat)) { // align subtitles for formats
+            return alignSubtitles(result);
+        }
+        return result.join(` `);
     }
 }