@aj-archipelago/cortex 1.1.18 → 1.1.19

package/helper-apps/cortex-file-handler/package.json

@@ -9,6 +9,7 @@
   },
   "dependencies": {
     "@azure/storage-blob": "^12.13.0",
+    "@google-cloud/storage": "^7.10.0",
     "axios": "^1.3.6",
     "busboy": "^1.6.0",
     "cors": "^2.8.5",

package/helper-apps/cortex-file-handler/redis.js

@@ -27,6 +27,106 @@ const publishRequestProgress = async (data) => {
     }
 };
 
+// Function to get all key value pairs in "FileStoreMap" hash map
+const getAllFileStoreMap = async () => {
+    try {
+        const allKeyValuePairs = await client.hgetall("FileStoreMap");
+        // Parse each JSON value in the returned object
+        for (const key in allKeyValuePairs) {
+            try {
+                // Modify the value directly in the returned object
+                allKeyValuePairs[key] = JSON.parse(allKeyValuePairs[key]);
+            } catch (error) {
+                console.error(`Error parsing JSON for key ${key}: ${error}`);
+                // keep original value if parsing failed
+            }
+        }
+        return allKeyValuePairs;
+    } catch (error) {
+        console.error(`Error getting all key-value pairs from FileStoreMap: ${error}`);
+        return {}; // Return null or any default value indicating an error occurred
+    }
+};
+
+// Function to set key value in "FileStoreMap" hash map
+const setFileStoreMap = async (key, value) => {
+    try {
+        value.timestamp = new Date().toISOString();
+        await client.hset("FileStoreMap", key, JSON.stringify(value));
+    } catch (error) {
+        console.error(`Error setting key in FileStoreMap: ${error}`);
+    }
+};
+
+const getFileStoreMap = async (key) => {
+    try {
+        const value = await client.hget("FileStoreMap", key);
+        if (value) {
+            try {
+                // parse the value back to an object before returning
+                return JSON.parse(value);
+            } catch (error) {
+                console.error(`Error parsing JSON: ${error}`);
+                return value; // return original value if parsing failed
+            }
+        }
+        return value;
+    } catch (error) {
+        console.error(`Error getting key from FileStoreMap: ${error}`);
+        return null; // Return null or any default value indicating an error occurred
+    }
+};
+
+// Function to remove key from "FileStoreMap" hash map
+const removeFromFileStoreMap = async (key) => {
+    try {
+        // hdel returns the number of keys that were removed.
+        // If the key does not exist, 0 is returned.
+        const result = await client.hdel("FileStoreMap", key);
+        if (result === 0) {
+            console.log(`The key ${key} does not exist`);
+        } else {
+            console.log(`The key ${key} was removed successfully`);
+        }
+    } catch (error) {
+        console.error(`Error removing key from FileStoreMap: ${error}`);
+    }
+};
+
+const cleanupRedisFileStoreMap = async (nDays=1) => {
+    let cleaned = [];
+    try {
+        // Get all key-value pairs from "FileStoreMap"
+        const fileStoreMap = await getAllFileStoreMap();
+
+        if(!fileStoreMap){
+            console.log("FileStoreMap is empty");
+            return;
+        }
+        
+        // Iterate over each key-value pair in the fileStoreMap
+        for (const [key, value] of Object.entries(fileStoreMap)) {
+            //check timestamp of each value compare to nDays and remove if older
+            const timestamp = new Date(value.timestamp);
+            const now = new Date();
+            const diffTime = Math.abs(now - timestamp);
+            const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
+            if (diffDays > nDays) {
+                // Remove the key from the "FileStoreMap" hash map
+                await removeFromFileStoreMap(key);
+                console.log(`Removed key ${key} from FileStoreMap`);
+                cleaned.push(Object.assign({hash:key}, value));
+            }
+
+        }
+    } catch (error) {
+        console.error(`Error cleaning FileStoreMap: ${error}`);
+    }finally{
+        return cleaned;
+    }
+};
+
+
 export {
-    publishRequestProgress, connectClient
-}
+    publishRequestProgress, connectClient, setFileStoreMap, getFileStoreMap, removeFromFileStoreMap, cleanupRedisFileStoreMap
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.18",
+  "version": "1.1.19",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/summary.js

@@ -21,7 +21,7 @@ export default {
         const originalTargetLength = args.targetLength;
 
         // If targetLength is not provided, execute the prompt once and return the result.
-        if (originalTargetLength === 0 || originalTargetLength === null) {
+        if (!originalTargetLength) {
             let pathwayResolver = new PathwayResolver({ config, pathway, args });
             return await pathwayResolver.resolve(args);
         }

package/server/modelExecutor.js

@@ -19,6 +19,8 @@ import OpenAIDallE3Plugin from './plugins/openAiDallE3Plugin.js';
 import OpenAIVisionPlugin from './plugins/openAiVisionPlugin.js';
 import GeminiChatPlugin from './plugins/geminiChatPlugin.js';
 import GeminiVisionPlugin from './plugins/geminiVisionPlugin.js';
+import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
+import Gemini15VisionPlugin from './plugins/gemini15VisionPlugin.js';
 import AzureBingPlugin from './plugins/azureBingPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
 
@@ -82,6 +84,12 @@ class ModelExecutor {
             case 'GEMINI-VISION':
                 plugin = new GeminiVisionPlugin(pathway, model);
                 break;
+            case 'GEMINI-1.5-CHAT':
+                plugin = new Gemini15ChatPlugin(pathway, model);
+                break;
+            case 'GEMINI-1.5-VISION':
+                plugin = new Gemini15VisionPlugin(pathway, model);
+                break;
             case 'AZURE-BING':
                 plugin = new AzureBingPlugin(pathway, model);
                 break;

package/server/pathwayResolver.js

@@ -226,6 +226,7 @@ class PathwayResolver {
             }
 
             logger.warn(`Bad pathway result - retrying pathway. Attempt ${retries + 1} of ${MAX_RETRIES}`);
+            this.savedContext = JSON.parse(savedContextStr);
         }
 
         // Update saved context if it has changed, generating a new contextId if necessary

package/server/plugins/azureCognitivePlugin.js

@@ -34,7 +34,7 @@ class AzureCognitivePlugin extends ModelPlugin {
     async getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId, cortexRequest) {
         const combinedParameters = { ...this.promptParameters, ...parameters };
         const { modelPromptText } = this.getCompiledPrompt(text, combinedParameters, prompt);
-        const { inputVector, calculateInputVector, privateData, filter, docId } = combinedParameters;
+        const { inputVector, calculateInputVector, privateData, filter, docId, title, chunkNo } = combinedParameters;
         const data = {};
 
         if (mode == 'delete') {
@@ -82,6 +82,15 @@ class AzureCognitivePlugin extends ModelPlugin {
                 doc.contentVector = inputVector ? inputVector : await this.getInputVector(text);
             }
 
+
+            if(title){
+                doc.title = title;
+            }
+
+            if(chunkNo!=null){
+                doc.chunkNo = chunkNo;
+            }
+
             if(!privateData){ //if public, remove owner
                 delete doc.owner;
             }
@@ -183,7 +192,22 @@ class AzureCognitivePlugin extends ModelPlugin {
             const chunkTokenLength = this.promptParameters.inputChunkSize || 1000;
             const chunks = getSemanticChunks(data, chunkTokenLength);
 
-            for (const text of chunks) {
+
+            //extract filename as the title from file
+            try {
+                // Extract filename from file
+                let filename = file.split("/").pop();
+                // Remove everything before and including first underscore
+                let title = filename.replace(/^.*?_/, "");
+            
+                parameters.title = title;
+            } catch (error) {
+                logger.error(`Error extracting title from file ${file}: ${error}`);
+            }
+
+            for (let i = 0; i < chunks.length; i++) {
+                const text = chunks[i];
+                parameters.chunkNo = i;
                 const { data: singleData } = await this.getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId, cortexRequest) 
                 fileData.value.push(singleData.value[0]);
             }

package/server/plugins/gemini15ChatPlugin.js

@@ -0,0 +1,215 @@
+// gemini15ChatPlugin.js
+import ModelPlugin from './modelPlugin.js';
+import logger from '../../lib/logger.js';
+
+const mergeResults = (data) => {
+    let output = '';
+    let safetyRatings = [];
+    const RESPONSE_BLOCKED = 'The response was blocked because the input or response potentially violates policies. Try rephrasing the prompt or adjusting the parameter settings.';
+
+    for (let chunk of data) {
+        const { promptfeedback } = chunk;
+        if (promptfeedback) {
+            const { blockReason } = promptfeedback;
+            if (blockReason) {
+                logger.warn(`Response blocked due to prompt feedback: ${blockReason}`);
+                return {mergedResult: RESPONSE_BLOCKED, safetyRatings: safetyRatings};
+            }
+        }
+
+        const { candidates } = chunk;
+        if (!candidates || !candidates.length) {
+            continue;
+        }
+
+        // If it was blocked, return the blocked message
+        if (candidates[0].safetyRatings?.some(rating => rating.blocked)) {
+            safetyRatings = candidates[0].safetyRatings;
+            logger.warn(`Response blocked due to safety ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
+            return {mergedResult: RESPONSE_BLOCKED, safetyRatings: safetyRatings};
+        }
+
+        // Append the content of the first part of the first candidate to the output
+        const message = candidates[0].content.parts[0].text;
+        output += message;
+    }
+
+    return {mergedResult: output || null, safetyRatings: safetyRatings};
+};
+
+class Gemini15ChatPlugin extends ModelPlugin {
+    constructor(pathway, model) {
+        super(pathway, model);
+    }
+
+    // This code converts either OpenAI or PaLM messages to the Gemini messages format
+    convertMessagesToGemini(messages) {
+        let modifiedMessages = [];
+        let systemParts = [];
+        let lastAuthor = '';
+
+        // Check if the messages are already in the Gemini format
+        if (messages[0] && Object.prototype.hasOwnProperty.call(messages[0], 'parts')) {
+            modifiedMessages = messages;
+        } else {
+            messages.forEach(message => {
+                const { role, author, content } = message;
+        
+                if (role === 'system') {
+                    systemParts.push({ text: content });
+                    return;
+                }
+        
+                // Aggregate consecutive author messages, appending the content
+                if ((role === lastAuthor || author === lastAuthor) && modifiedMessages.length > 0) {
+                    modifiedMessages[modifiedMessages.length - 1].parts.push({ text: content });
+                }
+
+                // Push messages that are role: 'user' or 'assistant', changing 'assistant' to 'model'
+                else if (role === 'user' || role === 'assistant' || author) {
+                    modifiedMessages.push({
+                        role: author || role,
+                        parts: [{ text: content }],
+                    });
+                    lastAuthor = author || role;
+                }
+            });
+        }
+    
+        // Gemini requires an even number of messages
+        if (modifiedMessages.length % 2 === 0) {
+            modifiedMessages = modifiedMessages.slice(1);
+        }
+
+        const system = { role: 'user', parts: systemParts };
+
+        return {
+            modifiedMessages,
+            system,
+        };
+    }
+
+    // Set up parameters specific to the Gemini API
+    getRequestParameters(text, parameters, prompt, cortexRequest) {
+        const { modelPromptText, modelPromptMessages, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
+        const { geminiSafetySettings, geminiTools, max_tokens } = cortexRequest ? cortexRequest.pathway : {};
+
+        // Define the model's max token length
+        const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();
+    
+        const geminiMessages = this.convertMessagesToGemini(modelPromptMessages || [{ "role": "user", "parts": [{ "text": modelPromptText }]}]);
+        
+        let requestMessages = geminiMessages.modifiedMessages;
+        let system = geminiMessages.system;
+
+        // Check if the token length exceeds the model's max token length
+        if (tokenLength > modelTargetTokenLength) {
+            // Remove older messages until the token length is within the model's limit
+            requestMessages = this.truncateMessagesToTargetLength(requestMessages, modelTargetTokenLength);
+        }
+        
+        if (max_tokens < 0) {
+            throw new Error(`Prompt is too long to successfully call the model at ${tokenLength} tokens.  The model will not be called.`);
+        }
+    
+        const requestParameters = {
+        contents: requestMessages,
+        generationConfig: {
+            temperature: this.temperature || 0.7,
+            maxOutputTokens: max_tokens || this.getModelMaxReturnTokens(),
+            topP: parameters.topP || 0.95,
+            topK: parameters.topK || 40,
+        },
+        safety_settings: geminiSafetySettings || undefined,
+        systemInstruction: system,
+        tools: geminiTools || undefined
+        };
+    
+        return requestParameters;
+    }
+
+    // Parse the response from the new Chat API
+    parseResponse(data) {
+        // If data is not an array, return it directly
+        let dataToMerge = [];
+        if (data && data.contents && Array.isArray(data.contents)) {
+            dataToMerge = data.contents;
+        } else if (data && data.candidates && Array.isArray(data.candidates)) {
+            return data.candidates[0].content.parts[0].text;
+        } else if (Array.isArray(data)) {
+            dataToMerge = data;
+        } else {
+            return data;
+        }
+
+        return mergeResults(dataToMerge).mergedResult || null;
+    
+    }
+
+    // Execute the request to the new Chat API
+    async execute(text, parameters, prompt, cortexRequest) {
+        const requestParameters = this.getRequestParameters(text, parameters, prompt, cortexRequest);
+        const { stream } = parameters;
+
+        cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
+        cortexRequest.params = {}; // query params
+        cortexRequest.stream = stream;
+        cortexRequest.stream = stream;
+        cortexRequest.urlSuffix = cortexRequest.stream ? ':streamGenerateContent?alt=sse' : ':generateContent';
+
+        const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
+        const authToken = await gcpAuthTokenHelper.getAccessToken();
+        cortexRequest.headers.Authorization = `Bearer ${authToken}`;
+
+        return this.executeRequest(cortexRequest);
+    }
+
+    // Override the logging function to display the messages and responses
+    logRequestData(data, responseData, prompt) {
+        const messages = data && data.contents;
+        
+        if (messages && messages.length > 1) {
+            logger.info(`[chat request contains ${messages.length} messages]`);
+            messages.forEach((message, index) => {
+                const messageContent = message.parts.reduce((acc, part) => {
+                    if (part.text) {
+                        return acc + part.text;
+                    }
+                    return acc;
+                } , '');
+                const words = messageContent.split(" ");
+                const { length, units } = this.getLength(messageContent);
+                const preview = words.length < 41 ? messageContent : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
+    
+                logger.debug(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${preview}"`);
+            });
+        } else if (messages && messages.length === 1) {
+            logger.debug(`${messages[0].parts[0].text}`);
+        }
+
+        // check if responseData is an array or string
+        if (typeof responseData === 'string') {
+            const { length, units } = this.getLength(responseData);
+            logger.info(`[response received containing ${length} ${units}]`);
+            logger.debug(`${responseData}`);
+        } else if (Array.isArray(responseData)) {
+            const { mergedResult, safetyRatings } = mergeResults(responseData);
+            if (safetyRatings?.length) {
+                logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
+                logger.debug(`Safety Ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
+            }
+            const { length, units } = this.getLength(mergedResult);
+            logger.info(`[response received containing ${length} ${units}]`);
+            logger.debug(`${mergedResult}`);
+        } else {
+            logger.info(`[response received as an SSE stream]`);
+        }
+
+        if (prompt && prompt.debugInfo) {
+            prompt.debugInfo += `\n${JSON.stringify(data)}`;
+        }
+    }
+
+}
+
+export default Gemini15ChatPlugin;

package/server/plugins/gemini15VisionPlugin.js

@@ -0,0 +1,100 @@
+import Gemini15ChatPlugin from './gemini15ChatPlugin.js';
+import mime from 'mime-types';
+import logger from '../../lib/logger.js';
+
+class Gemini15VisionPlugin extends Gemini15ChatPlugin {
+
+    // Override the convertMessagesToGemini method to handle multimodal vision messages
+    // This function can operate on messages in Gemini native format or in OpenAI's format
+    // It will convert the messages to the Gemini format
+    convertMessagesToGemini(messages) {
+        let modifiedMessages = [];
+        let lastAuthor = '';
+        let systemParts = [];
+    
+        // Check if the messages are already in the Gemini format
+        if (messages[0] && Object.prototype.hasOwnProperty.call(messages[0], 'parts')) {
+            modifiedMessages = messages;
+        } else {
+            messages.forEach(message => {
+                const { role, author, content } = message;
+    
+                if (role === 'system') {
+                    systemParts.push({ text: content });
+                    return;
+                }
+    
+                // Convert content to Gemini format, trying to maintain compatibility
+                const convertPartToGemini = (partString) => {
+                    try {
+                        const part = JSON.parse(partString);
+                        if (typeof part === 'string') {
+                            return { text: part };
+                        } else if (part.type === 'text') {
+                            return { text: part.text };
+                        } else if (part.type === 'image_url') {
+                            if (part.image_url.url.startsWith('gs://')) {
+                                return {
+                                    fileData: {
+                                        mimeType: mime.lookup(part.image_url.url),
+                                        fileUri: part.image_url.url
+                                    }
+                                };
+                            } else {
+                                return {
+                                    inlineData: {
+                                        mimeType: 'image/jpeg', // fixed for now as there's no MIME type in the request
+                                        data: part.image_url.url.split('base64,')[1]
+                                    }
+                                };
+                            }
+                        }
+                    } catch (e) {
+                        logger.warn(`Unable to parse part - including as string: ${partString}`);
+                    }
+                    return { text: partString };
+                };
+    
+                const addPartToMessages = (geminiPart) => {
+                    // Gemini requires alternating user: and model: messages
+                    if ((role === lastAuthor || author === lastAuthor) && modifiedMessages.length > 0) {
+                        modifiedMessages[modifiedMessages.length - 1].parts.push(geminiPart);
+                    }
+                    // Gemini only supports user: and model: roles
+                    else if (role === 'user' || role === 'assistant' || author) {
+                        modifiedMessages.push({
+                            role: author || role,
+                            parts: [geminiPart],
+                        });
+                        lastAuthor = author || role;
+                    }
+                };
+
+                // Content can either be in the "vision" format (array) or in the "chat" format (string)
+                if (Array.isArray(content)) {
+                    content.forEach(part => {
+                        addPartToMessages(convertPartToGemini(part));
+                    });
+                } 
+                else {
+                    addPartToMessages(convertPartToGemini(content));
+                }
+            });
+        }
+    
+        // Gemini requires an even number of messages
+        if (modifiedMessages.length % 2 === 0) {
+            modifiedMessages = modifiedMessages.slice(1);
+        }
+    
+        const system = { role: 'user', parts: systemParts };
+
+        return {
+            modifiedMessages,
+            system,
+        };
+    }
+
+}
+
+export default Gemini15VisionPlugin;

package/server/plugins/geminiChatPlugin.js

@@ -131,11 +131,18 @@ class GeminiChatPlugin extends ModelPlugin {
     // Parse the response from the new Chat API
     parseResponse(data) {
         // If data is not an array, return it directly
-        if (!Array.isArray(data)) {
+        let dataToMerge = [];
+        if (data && data.contents && Array.isArray(data.contents)) {
+            dataToMerge = data.contents;
+        } else if (data && data.candidates && Array.isArray(data.candidates)) {
+            return data.candidates[0].content.parts[0].text;
+        } else if (Array.isArray(data)) {
+            dataToMerge = data;
+        } else {
             return data;
         }
 
-        return mergeResults(data).mergedResult || null;
+        return mergeResults(dataToMerge).mergedResult || null;
     
     }
 
@@ -147,7 +154,8 @@ class GeminiChatPlugin extends ModelPlugin {
         cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
         cortexRequest.params = {}; // query params
         cortexRequest.stream = stream;
-        cortexRequest.url = cortexRequest.stream ? `${cortexRequest.url}?alt=sse` : cortexRequest.url;
+        cortexRequest.stream = stream;
+        cortexRequest.urlSuffix = cortexRequest.stream ? ':streamGenerateContent?alt=sse' : ':generateContent';
 
         const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
         const authToken = await gcpAuthTokenHelper.getAccessToken();
@@ -179,10 +187,12 @@ class GeminiChatPlugin extends ModelPlugin {
             logger.debug(`${messages[0].parts[0].text}`);
         }
 
-        // check if responseData is an array
-        if (!Array.isArray(responseData)) {
-            logger.info(`[response received as an SSE stream]`);    
-        } else {
+        // check if responseData is an array or string
+        if (typeof responseData === 'string') {
+            const { length, units } = this.getLength(responseData);
+            logger.info(`[response received containing ${length} ${units}]`);
+            logger.debug(`${responseData}`);
+        } else if (Array.isArray(responseData)) {
             const { mergedResult, safetyRatings } = mergeResults(responseData);
             if (safetyRatings?.length) {
                 logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
@@ -191,6 +201,8 @@ class GeminiChatPlugin extends ModelPlugin {
             const { length, units } = this.getLength(mergedResult);
             logger.info(`[response received containing ${length} ${units}]`);
             logger.debug(`${mergedResult}`);
+        } else {
+            logger.info(`[response received as an SSE stream]`);
         }
 
         if (prompt && prompt.debugInfo) {

package/server/typeDef.js

@@ -1,30 +1,30 @@
 const getGraphQlType = (value) => {
   switch (typeof value) {
     case 'boolean':
-      return {type: 'Boolean', defaultValue: 'false'};
+      return {type: 'Boolean'};
     case 'string':
-      return {type: 'String', defaultValue: `""`};
+      return {type: 'String'};
     case 'number':
-      return {type: 'Int', defaultValue: 'null'};
+      return {type: 'Int'};
     case 'object':
       if (Array.isArray(value)) {
         if (value.length > 0 && typeof(value[0]) === 'string') {
-          return {type: '[String]', defaultValue: '[]'};
+          return {type: '[String]'};
         }
         else {
           // New case for MultiMessage type
           if (Array.isArray(value[0]?.content)) {
-            return {type: '[MultiMessage]', defaultValue: '[]'};
+            return {type: '[MultiMessage]'};
           }
           else {
-            return {type: '[Message]', defaultValue: '[]'};
+            return {type: '[Message]'};
           }
         }
       } else {
-        return {type: `[${value.objName}]`, defaultValue: 'null'};
+        return {type: `[${value.objName}]`};
       }
     default:
-      return {type: 'String', defaultValue: `""`};
+      return {type: 'String'};
   }
 };