@aj-archipelago/cortex 1.1.33 → 1.1.35

package/server/plugins/geminiVisionPlugin.js

@@ -3,6 +3,11 @@ import mime from 'mime-types';
 
 class GeminiVisionPlugin extends GeminiChatPlugin {
 
+    constructor(pathway, model) {
+        super(pathway, model);
+        this.isMultiModal = true;
+    }
+    
     // Override the convertMessagesToGemini method to handle multimodal vision messages
     // This function can operate on messages in Gemini native format or in OpenAI's format
     // It will convert the messages to the Gemini format
@@ -15,40 +20,33 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
             modifiedMessages = messages;
         } else {
             messages.forEach(message => {
-                const { role, author, content } = message;
-    
-                // Right now Gemini API has no direct translation for system messages,
-                // so we insert them as parts of the first user: role message
-                if (role === 'system') {
-                    modifiedMessages.push({
-                        role: 'user',
-                        parts: [{ text: content }],
-                    });
-                    lastAuthor = 'user';
-                    return;
-                }
-    
+                let role = message.role;
+                const { author, content } = message;
+
                 // Convert content to Gemini format, trying to maintain compatibility
-                const convertPartToGemini = (partString) => {
+                const convertPartToGemini = (inputPart) => {
                     try {
-                        const part = JSON.parse(partString);
+                        const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart;
+                        const {type, text, image_url, gcs} = part;
+                        let fileUrl = gcs || image_url?.url;
+
                         if (typeof part === 'string') {
-                            return { text: part };
-                        } else if (part.type === 'text') {
-                            return { text: part.text };
-                        } else if (part.type === 'image_url') {
-                            if (part.image_url.url.startsWith('gs://')) {
+                            return { text: text };
+                        } else if (type === 'text') {
+                            return { text: text };
+                        } else if (type === 'image_url') {
+                            if (fileUrl.startsWith('gs://')) {
                                 return {
                                     fileData: {
-                                        mimeType: mime.lookup(part.image_url.url),
-                                        fileUri: part.image_url.url
+                                        mimeType: mime.lookup(fileUrl) || 'image/jpeg',
+                                        fileUri: fileUrl
                                     }
                                 };
                             } else {
                                 return {
                                     inlineData: {
                                         mimeType: 'image/jpeg', // fixed for now as there's no MIME type in the request
-                                        data: part.image_url.url.split('base64,')[1]
+                                        data: fileUrl.split('base64,')[1]
                                     }
                                 };
                             }
@@ -56,10 +54,11 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
                     } catch (e) {
                         // this space intentionally left blank
                     }
-                    return { text: partString };
+                    return inputPart ? { text: inputPart } : null;
                 };
-    
+
                 const addPartToMessages = (geminiPart) => {
+                    if (!geminiPart) { return; }
                     // Gemini requires alternating user: and model: messages
                     if ((role === lastAuthor || author === lastAuthor) && modifiedMessages.length > 0) {
                         modifiedMessages[modifiedMessages.length - 1].parts.push(geminiPart);
@@ -74,6 +73,14 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
                     }
                 };
 
+                // Right now Gemini API has no direct translation for system messages,
+                // so we insert them as parts of the first user: role message
+                if (role === 'system') {
+                    role = 'user';
+                    addPartToMessages(convertPartToGemini(content));
+                    return;
+                }
+
                 // Content can either be in the "vision" format (array) or in the "chat" format (string)
                 if (Array.isArray(content)) {
                     content.forEach(part => {

package/server/plugins/modelPlugin.js

@@ -20,6 +20,7 @@ class ModelPlugin {
         this.pathwayPrompt = pathway.prompt;
         this.pathwayName = pathway.name;
         this.promptParameters = {};
+        this.isMultiModal = false;
 
         // Make all of the parameters defined on the pathway itself available to the prompt
         for (const [k, v] of Object.entries(pathway)) {
@@ -205,6 +206,15 @@ class ModelPlugin {
                 message.content = '';
             }
         });
+
+        // Flatten content arrays for non-multimodal models
+        if (!this.isMultiModal) {
+            expandedMessages.forEach(message => {
+                if (Array.isArray(message?.content)) {
+                    message.content = message.content.join("\n");
+                }
+            });
+        }
         
         return expandedMessages;
     }
@@ -288,8 +298,11 @@ class ModelPlugin {
             return parsedData;
         } catch (error) {
             // Log the error and continue
-            logger.error(error.message || error);
-            throw error;
+            logger.error(`Error in executeRequest for ${this.pathwayName}: ${error.message || error}`);
+            if (error.data) {
+                logger.error(`Additional error data: ${JSON.stringify(error.data)}`);
+            }
+            throw new Error(`Execution failed for ${this.pathwayName}: ${error.message || error}`);
         }
     }
 

package/server/plugins/openAiChatPlugin.js

@@ -112,7 +112,7 @@ class OpenAIChatPlugin extends ModelPlugin {
             let totalUnits;
             messages.forEach((message, index) => {
                 //message.content string or array
-                const content = Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content;
+                const content = message.content === undefined ? JSON.stringify(message) : (Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content);
                 const words = content.split(" ");
                 const { length, units } = this.getLength(content);
                 const preview = words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");

package/server/plugins/openAiVisionPlugin.js

@@ -10,22 +10,35 @@ function safeJsonParse(content) {
 }
 
 class OpenAIVisionPlugin extends OpenAIChatPlugin {
+
+    constructor(pathway, model) {
+        super(pathway, model);
+        this.isMultiModal = true;
+    }
     
     tryParseMessages(messages) {
         return messages.map(message => {
             try {
+                if (message.role === "tool") {
+                    return message;
+                }
                 if (typeof message.content === 'string') {
                     message.content = safeJsonParse(message.content);
                 }
                 if (Array.isArray(message.content)) {
                     message.content = message.content.map(item => {
-                        if (typeof item === 'string') {
-                            return { type: 'text', text: item };
-                        } else {
-                            const parsedItem = safeJsonParse(item);
-                            const { type, text, image_url, url } = parsedItem;
-                            return { type, text, image_url: url || image_url };
+                        const parsedItem = safeJsonParse(item);
+
+                        if (typeof parsedItem === 'string') {
+                            return { type: 'text', text: parsedItem };
+                        }
+
+                        if (typeof parsedItem === 'object' && parsedItem !== null && parsedItem.type === 'image_url') {
+                            parsedItem.image_url.url = parsedItem.url || parsedItem.image_url.url;
+                            return parsedItem;
                         }
+                        
+                        return parsedItem;
                     });
                 }
             } catch (e) {

package/server/plugins/runwareAIPlugin.js

@@ -0,0 +1,81 @@
+// RunwareAiPlugin.js
+import ModelPlugin from "./modelPlugin.js";
+import logger from "../../lib/logger.js";
+import { config } from "../../config.js";
+import { v4 as uuidv4 } from "uuid";
+
+class RunwareAiPlugin extends ModelPlugin {
+  constructor(pathway, model) {
+    super(pathway, model);
+  }
+
+  // Set up parameters specific to the Runware REST API
+  getRequestParameters(text, parameters, prompt) {
+    const combinedParameters = { ...this.promptParameters, ...parameters };
+    const { modelPromptText } = this.getCompiledPrompt(
+      text,
+      parameters,
+      prompt,
+    );
+
+    const requestParameters = {
+      data: [
+        {
+          taskType: "authentication",
+          apiKey: config.get("runwareAiApiKey"),
+        },
+        {
+          taskType: "imageInference",
+          taskUUID: uuidv4(),
+          positivePrompt: modelPromptText,
+          width: combinedParameters.width,
+          height: combinedParameters.height,
+          modelId: "runware:100@1",
+          CFGScale: 4.0,
+          negative_prompt: combinedParameters.negativePrompt,
+          numberResults: combinedParameters.numberResults,
+          steps: combinedParameters.steps,
+          checkNSFW: false,
+        },
+      ],
+    };
+
+    return requestParameters;
+  }
+
+  // Execute the request to the Runware REST API
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = this.getRequestParameters(
+      text,
+      parameters,
+      prompt,
+    );
+
+    cortexRequest.data = requestParameters.data;
+    cortexRequest.params = requestParameters.params;
+
+    return this.executeRequest(cortexRequest);
+  }
+
+  // Parse the response from the Azure Translate API
+  parseResponse(data) {
+    if (data.data) {
+      return JSON.stringify(data.data);
+    }
+    return JSON.stringify(data);
+  }
+
+  // Override the logging function to display the request and response
+  logRequestData(data, responseData, prompt) {
+    const modelInput = data[1].positivePrompt;
+
+    logger.verbose(`${modelInput}`);
+    logger.verbose(`${this.parseResponse(responseData)}`);
+
+    prompt &&
+      prompt.debugInfo &&
+      (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+  }
+}
+
+export default RunwareAiPlugin;

package/server/rest.js

@@ -137,13 +137,35 @@ const processIncomingStream = (requestId, res, jsonResponse, pathway) => {
             if (subscription) {
                 try {
                     const subPromiseResult = await subscription;
-                    subPromiseResult && pubsub.unsubscribe(subPromiseResult);
+                    if (subPromiseResult && pubsub.subscriptions?.[subPromiseResult]) {
+                        pubsub.unsubscribe(subPromiseResult);
+                    }
                 } catch (error) {
-                    logger.error(`Error unsubscribing from pubsub: ${error}`);
+                    logger.warn(`Pubsub unsubscribe threw error: ${error}`);
                 }
             }
         }
 
+        const processStringData = (stringData) => {
+            if (progress === 1 && stringData.trim() === "[DONE]") {
+                fillJsonResponse(jsonResponse, stringData, "stop");
+                safeUnsubscribe();
+                finishStream(res, jsonResponse);
+                return;
+            }
+
+            chunkTextIntoTokens(stringData, false, useSingleTokenStream).forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
+
+            if (progress === 1) {
+                safeUnsubscribe();
+                finishStream(res, jsonResponse);
+            }
+
+        }
+
         if (data.requestProgress.requestId !== requestId) return;
 
         logger.debug(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
@@ -152,6 +174,12 @@ const processIncomingStream = (requestId, res, jsonResponse, pathway) => {
 
         try {
             const messageJson = JSON.parse(progressData);
+
+            if (typeof messageJson === 'string') {
+                processStringData(messageJson);
+                return;
+            }
+
             if (messageJson.error) {
                 logger.error(`Stream error REST: ${messageJson?.error?.message || 'unknown error'}`);
                 safeUnsubscribe();
@@ -178,17 +206,7 @@ const processIncomingStream = (requestId, res, jsonResponse, pathway) => {
         } catch (error) {
             logger.debug(`progressData not JSON: ${progressData}`);
             if (typeof progressData === 'string') {
-                if (progress === 1 && progressData.trim() === "[DONE]") {
-                    fillJsonResponse(jsonResponse, progressData, "stop");
-                    safeUnsubscribe();
-                    finishStream(res, jsonResponse);
-                    return;
-                }
-
-                chunkTextIntoTokens(progressData, false, useSingleTokenStream).forEach(token => {
-                    fillJsonResponse(jsonResponse, token, null);
-                    sendStreamData(jsonResponse);
-                });
+                processStringData(progressData);
             } else {
                 fillJsonResponse(jsonResponse, progressData, "stop");
                 sendStreamData(jsonResponse);

package/server/typeDef.js

@@ -28,7 +28,26 @@ const getGraphQlType = (value) => {
   }
 };
 
-const typeDef = (pathway) => {
+const getMessageTypeDefs = () => {
+  const messageType = `input Message { role: String, content: String, name: String }`;
+  const multiMessageType = `input MultiMessage { role: String, content: [String], name: String }`;
+  
+  return `${messageType}\n\n${multiMessageType}`;
+};
+
+const getPathwayTypeDef = (name, returnType) => {
+  return `type ${name} {
+    debug: String
+    result: ${returnType}
+    previousResult: String
+    warnings: [String]
+    errors: [String]
+    contextId: String
+    tool: String
+  }`
+};
+
+const getPathwayTypeDefAndExtendQuery = (pathway) => {
   const { name, objName, defaultInputParameters, inputParameters, format } = pathway;
 
   const fields = format ? format.match(/\b(\w+)\b/g) : null;
@@ -36,24 +55,13 @@ const typeDef = (pathway) => {
 
   const typeName = fields ? `${objName}Result` : `String`;
 
-  const messageType = `input Message { role: String, content: String, name: String }`;
-  const multiMessageType = `input MultiMessage { role: String, content: [String], name: String }`;
-
   const type = fields ? `type ${typeName} {
     ${fieldsStr}
     }` : ``;
 
-  const resultStr = pathway.list ? `[${typeName}]` : typeName;
+  const returnType = pathway.list ? `[${typeName}]` : typeName;
 
-  const responseType = `type ${objName} {
-        debug: String
-        result: ${resultStr}
-        previousResult: String
-        warnings: [String]
-        errors: [String]
-        contextId: String
-        tool: String
-}`;
+  const responseType = getPathwayTypeDef(objName, returnType);
 
   const params = { ...defaultInputParameters, ...inputParameters };
 
@@ -71,7 +79,7 @@ const typeDef = (pathway) => {
     };
   });
 
-  const gqlDefinition = `${messageType}\n\n${multiMessageType}\n\n${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
+  const gqlDefinition = `${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
 
   return {
     gqlDefinition,
@@ -79,6 +87,16 @@ const typeDef = (pathway) => {
   };
 };
 
+const typeDef = (pathway) => {
+  return getPathwayTypeDefAndExtendQuery(pathway);
+};
+
+const userPathwayInputParameters = `text: String`;
+
+
 export {
   typeDef,
+  getMessageTypeDefs,
+  getPathwayTypeDef,
+  userPathwayInputParameters,
 };

package/tests/claude3VertexPlugin.test.js

@@ -199,7 +199,7 @@ test('convertMessagesToClaudeVertex system message with user message', async (t)
 test('convertMessagesToClaudeVertex user message with unsupported image type', async (t) => {
   const plugin = new Claude3VertexPlugin(pathway, model);
   // Test with unsupported image type
-  const messages = [{ role: 'user', content: { type: 'image_url', image_url: 'http://example.com/image.svg' } }];
+  const messages = [{ role: 'user', content: { type: 'image_url', image_url: 'https://unec.edu.az/application/uploads/2014/12/pdf-sample.pdf' } }];
   const output = await plugin.convertMessagesToClaudeVertex(messages);
   t.deepEqual(output, { system: '', modifiedMessages: [{role: 'user', content: [] }] });
 });