@aj-archipelago/cortex 1.1.32 → 1.1.34

package/server/plugins/geminiVisionPlugin.js

@@ -3,6 +3,11 @@ import mime from 'mime-types';
 
 class GeminiVisionPlugin extends GeminiChatPlugin {
 
+    constructor(pathway, model) {
+        super(pathway, model);
+        this.isMultiModal = true;
+    }
+    
     // Override the convertMessagesToGemini method to handle multimodal vision messages
     // This function can operate on messages in Gemini native format or in OpenAI's format
     // It will convert the messages to the Gemini format
@@ -15,40 +20,33 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
             modifiedMessages = messages;
         } else {
             messages.forEach(message => {
-                const { role, author, content } = message;
-    
-                // Right now Gemini API has no direct translation for system messages,
-                // so we insert them as parts of the first user: role message
-                if (role === 'system') {
-                    modifiedMessages.push({
-                        role: 'user',
-                        parts: [{ text: content }],
-                    });
-                    lastAuthor = 'user';
-                    return;
-                }
-    
+                let role = message.role;
+                const { author, content } = message;
+
                 // Convert content to Gemini format, trying to maintain compatibility
-                const convertPartToGemini = (partString) => {
+                const convertPartToGemini = (inputPart) => {
                     try {
-                        const part = JSON.parse(partString);
+                        const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart;
+                        const {type, text, image_url, gcs} = part;
+                        let fileUrl = gcs || image_url?.url;
+
                         if (typeof part === 'string') {
-                            return { text: part };
-                        } else if (part.type === 'text') {
-                            return { text: part.text };
-                        } else if (part.type === 'image_url') {
-                            if (part.image_url.url.startsWith('gs://')) {
+                            return { text: text };
+                        } else if (type === 'text') {
+                            return { text: text };
+                        } else if (type === 'image_url') {
+                            if (fileUrl.startsWith('gs://')) {
                                 return {
                                     fileData: {
-                                        mimeType: mime.lookup(part.image_url.url),
-                                        fileUri: part.image_url.url
+                                        mimeType: mime.lookup(fileUrl) || 'image/jpeg',
+                                        fileUri: fileUrl
                                     }
                                 };
                             } else {
                                 return {
                                     inlineData: {
                                         mimeType: 'image/jpeg', // fixed for now as there's no MIME type in the request
-                                        data: part.image_url.url.split('base64,')[1]
+                                        data: fileUrl.split('base64,')[1]
                                     }
                                 };
                             }
@@ -56,10 +54,11 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
                     } catch (e) {
                         // this space intentionally left blank
                     }
-                    return { text: partString };
+                    return inputPart ? { text: inputPart } : null;
                 };
-    
+
                 const addPartToMessages = (geminiPart) => {
+                    if (!geminiPart) { return; }
                     // Gemini requires alternating user: and model: messages
                     if ((role === lastAuthor || author === lastAuthor) && modifiedMessages.length > 0) {
                         modifiedMessages[modifiedMessages.length - 1].parts.push(geminiPart);
@@ -74,6 +73,14 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
                     }
                 };
 
+                // Right now Gemini API has no direct translation for system messages,
+                // so we insert them as parts of the first user: role message
+                if (role === 'system') {
+                    role = 'user';
+                    addPartToMessages(convertPartToGemini(content));
+                    return;
+                }
+
                 // Content can either be in the "vision" format (array) or in the "chat" format (string)
                 if (Array.isArray(content)) {
                     content.forEach(part => {

package/server/plugins/modelPlugin.js

@@ -20,6 +20,7 @@ class ModelPlugin {
         this.pathwayPrompt = pathway.prompt;
         this.pathwayName = pathway.name;
         this.promptParameters = {};
+        this.isMultiModal = false;
 
         // Make all of the parameters defined on the pathway itself available to the prompt
         for (const [k, v] of Object.entries(pathway)) {
@@ -205,6 +206,15 @@ class ModelPlugin {
                 message.content = '';
             }
         });
+
+        // Flatten content arrays for non-multimodal models
+        if (!this.isMultiModal) {
+            expandedMessages.forEach(message => {
+                if (Array.isArray(message?.content)) {
+                    message.content = message.content.join("\n");
+                }
+            });
+        }
         
         return expandedMessages;
     }
@@ -288,8 +298,11 @@ class ModelPlugin {
             return parsedData;
         } catch (error) {
             // Log the error and continue
-            logger.error(error.message || error);
-            throw error;
+            logger.error(`Error in executeRequest for ${this.pathwayName}: ${error.message || error}`);
+            if (error.data) {
+                logger.error(`Additional error data: ${JSON.stringify(error.data)}`);
+            }
+            throw new Error(`Execution failed for ${this.pathwayName}: ${error.message || error}`);
         }
     }
 

package/server/plugins/openAiChatPlugin.js

@@ -112,7 +112,7 @@ class OpenAIChatPlugin extends ModelPlugin {
             let totalUnits;
             messages.forEach((message, index) => {
                 //message.content string or array
-                const content = Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content;
+                const content = message.content === undefined ? JSON.stringify(message) : (Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content);
                 const words = content.split(" ");
                 const { length, units } = this.getLength(content);
                 const preview = words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");

package/server/plugins/openAiVisionPlugin.js

@@ -10,22 +10,34 @@ function safeJsonParse(content) {
 }
 
 class OpenAIVisionPlugin extends OpenAIChatPlugin {
+
+    constructor(pathway, model) {
+        super(pathway, model);
+        this.isMultiModal = true;
+    }
     
     tryParseMessages(messages) {
         return messages.map(message => {
             try {
+                if (message.role === "tool") {
+                    return message;
+                }
                 if (typeof message.content === 'string') {
                     message.content = safeJsonParse(message.content);
                 }
                 if (Array.isArray(message.content)) {
                     message.content = message.content.map(item => {
                         if (typeof item === 'string') {
-                            return { type: 'text', text: item };
-                        } else {
                             const parsedItem = safeJsonParse(item);
-                            const { type, text, image_url, url } = parsedItem;
-                            return { type, text, image_url: url || image_url };
+                            return parsedItem.type ? parsedItem : { type: 'text', text: item };
+                        } else if (typeof item === 'object') {
+                            const { type, image_url, url } = item;
+                            if (type === 'image_url') {
+                                image_url.url = url || image_url.url;
+                                return { type, image_url };
+                            }
                         }
+                        return item;
                     });
                 }
             } catch (e) {

package/server/plugins/runwareAIPlugin.js

@@ -0,0 +1,81 @@
+// RunwareAiPlugin.js
+import ModelPlugin from "./modelPlugin.js";
+import logger from "../../lib/logger.js";
+import { config } from "../../config.js";
+import { v4 as uuidv4 } from "uuid";
+
+class RunwareAiPlugin extends ModelPlugin {
+  constructor(pathway, model) {
+    super(pathway, model);
+  }
+
+  // Set up parameters specific to the Runware REST API
+  getRequestParameters(text, parameters, prompt) {
+    const combinedParameters = { ...this.promptParameters, ...parameters };
+    const { modelPromptText } = this.getCompiledPrompt(
+      text,
+      parameters,
+      prompt,
+    );
+
+    const requestParameters = {
+      data: [
+        {
+          taskType: "authentication",
+          apiKey: config.get("runwareAiApiKey"),
+        },
+        {
+          taskType: "imageInference",
+          taskUUID: uuidv4(),
+          positivePrompt: modelPromptText,
+          width: combinedParameters.width,
+          height: combinedParameters.height,
+          modelId: "runware:100@1",
+          CFGScale: 4.0,
+          negative_prompt: combinedParameters.negativePrompt,
+          numberResults: combinedParameters.numberResults,
+          steps: combinedParameters.steps,
+          checkNSFW: false,
+        },
+      ],
+    };
+
+    return requestParameters;
+  }
+
+  // Execute the request to the Runware REST API
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = this.getRequestParameters(
+      text,
+      parameters,
+      prompt,
+    );
+
+    cortexRequest.data = requestParameters.data;
+    cortexRequest.params = requestParameters.params;
+
+    return this.executeRequest(cortexRequest);
+  }
+
+  // Parse the response from the Azure Translate API
+  parseResponse(data) {
+    if (data.data) {
+      return JSON.stringify(data.data);
+    }
+    return JSON.stringify(data);
+  }
+
+  // Override the logging function to display the request and response
+  logRequestData(data, responseData, prompt) {
+    const modelInput = data[1].positivePrompt;
+
+    logger.verbose(`${modelInput}`);
+    logger.verbose(`${this.parseResponse(responseData)}`);
+
+    prompt &&
+      prompt.debugInfo &&
+      (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+  }
+}
+
+export default RunwareAiPlugin;

package/server/rest.js

@@ -5,7 +5,20 @@ import pubsub from './pubsub.js';
 import { requestState } from './requestState.js';
 import { v4 as uuidv4 } from 'uuid';
 import logger from '../lib/logger.js';
-
+import { getSingleTokenChunks } from './chunker.js';
+
+const chunkTextIntoTokens = (() => {
+    let partialToken = '';
+    return (text, isLast = false, useSingleTokenStream = false) => {
+        const tokens = useSingleTokenStream ? getSingleTokenChunks(partialToken + text) : [text];
+        if (isLast) {
+            partialToken = '';
+            return tokens;
+        }
+        partialToken = useSingleTokenStream ? tokens.pop() : '';
+        return tokens;
+    };
+})();
 
 const processRestRequest = async (server, req, pathway, name, parameterMap = {}) => {
     const fieldVariableDefs = pathway.typeDef(pathway).restDefinition || [];
@@ -50,7 +63,8 @@ const processRestRequest = async (server, req, pathway, name, parameterMap = {})
     return resultText;
 };
 
-const processIncomingStream = (requestId, res, jsonResponse) => {
+const processIncomingStream = (requestId, res, jsonResponse, pathway) => {
+    const useSingleTokenStream = pathway.useSingleTokenStream || false;
 
     const startStream = (res) => {
         // Set the headers for streaming
@@ -61,6 +75,14 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     }
     
     const finishStream = (res, jsonResponse) => {
+        // Send the last partial token if it exists
+        const lastTokens = chunkTextIntoTokens('', true, useSingleTokenStream);
+        if (lastTokens.length > 0) {
+            lastTokens.forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
+        }
 
         // If we haven't sent the stop message yet, do it now
         if (jsonResponse.choices?.[0]?.finish_reason !== "stop") {
@@ -85,11 +107,11 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     }
 
     const sendStreamData = (data) => {
-        logger.debug(`REST SEND: data: ${JSON.stringify(data)}`);
         const dataString = (data==='[DONE]') ? data : JSON.stringify(data);
 
         if (!res.writableEnded) {
             res.write(`data: ${dataString}\n\n`);
+            logger.debug(`REST SEND: data: ${dataString}`);
         }
     }
 
@@ -115,62 +137,85 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
             if (subscription) {
                 try {
                     const subPromiseResult = await subscription;
-                    if (subPromiseResult) {
+                    if (subPromiseResult && pubsub.subscriptions?.[subPromiseResult]) {
                         pubsub.unsubscribe(subPromiseResult);
                     }
                 } catch (error) {
-                    logger.error(`Error unsubscribing from pubsub: ${error}`);
+                    logger.warn(`Pubsub unsubscribe threw error: ${error}`);
                 }
             }
         }
 
-        if (data.requestProgress.requestId === requestId) {
-            logger.debug(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
-            
-            const progress = data.requestProgress.progress;
-            const progressData = data.requestProgress.data;
-
-            try {
-                const messageJson = JSON.parse(progressData);
-                if (messageJson.error) {
-                    logger.error(`Stream error REST: ${messageJson?.error?.message || 'unknown error'}`);
-                    safeUnsubscribe();
-                    finishStream(res, jsonResponse);
-                    return;
-                } else if (messageJson.choices) {
-                    const { text, delta, finish_reason } = messageJson.choices[0];
-
-                    if (messageJson.object === 'text_completion') {
-                        fillJsonResponse(jsonResponse, text, finish_reason);
-                    } else {
-                        fillJsonResponse(jsonResponse, delta.content, finish_reason);
-                    }
-                } else if (messageJson.candidates) {
-                    const { content, finishReason } = messageJson.candidates[0];
-                    fillJsonResponse(jsonResponse, content.parts[0].text, finishReason);
-                } else if (messageJson.content) {
-                    const text = messageJson.content?.[0]?.text || '';
-                    const finishReason = messageJson.stop_reason;
-                    fillJsonResponse(jsonResponse, text, finishReason);
-                } else {
-                    fillJsonResponse(jsonResponse, messageJson, null);
-                }
-            } catch (error) {
-                //logger.info(`progressData not JSON: ${progressData}`);
-                fillJsonResponse(jsonResponse, progressData, "stop");
-            }
-            if (progress === 1 && progressData.trim() === "[DONE]") {
+        const processStringData = (stringData) => {
+            if (progress === 1 && stringData.trim() === "[DONE]") {
+                fillJsonResponse(jsonResponse, stringData, "stop");
                 safeUnsubscribe();
                 finishStream(res, jsonResponse);
                 return;
             }
 
-            sendStreamData(jsonResponse);
+            chunkTextIntoTokens(stringData, false, useSingleTokenStream).forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
 
             if (progress === 1) {
                 safeUnsubscribe();
                 finishStream(res, jsonResponse);
             }
+
+        }
+
+        if (data.requestProgress.requestId !== requestId) return;
+
+        logger.debug(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
+        
+        const { progress, data: progressData } = data.requestProgress;
+
+        try {
+            const messageJson = JSON.parse(progressData);
+
+            if (typeof messageJson === 'string') {
+                processStringData(messageJson);
+                return;
+            }
+
+            if (messageJson.error) {
+                logger.error(`Stream error REST: ${messageJson?.error?.message || 'unknown error'}`);
+                safeUnsubscribe();
+                finishStream(res, jsonResponse);
+                return;
+            }
+
+            let content = '';
+            if (messageJson.choices) {
+                const { text, delta } = messageJson.choices[0];
+                content = messageJson.object === 'text_completion' ? text : delta.content;
+            } else if (messageJson.candidates) {
+                content = messageJson.candidates[0].content.parts[0].text;
+            } else if (messageJson.content) {
+                content = messageJson.content?.[0]?.text || '';
+            } else {
+                content = messageJson;
+            }
+
+            chunkTextIntoTokens(content, false, useSingleTokenStream).forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
+        } catch (error) {
+            logger.debug(`progressData not JSON: ${progressData}`);
+            if (typeof progressData === 'string') {
+                processStringData(progressData);
+            } else {
+                fillJsonResponse(jsonResponse, progressData, "stop");
+                sendStreamData(jsonResponse);
+            }
+        }
+
+        if (progress === 1) {
+            safeUnsubscribe();
+            finishStream(res, jsonResponse);
         }
     });
 
@@ -254,7 +299,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 jsonResponse.choices[0].finish_reason = null;
                 //jsonResponse.object = "text_completion.chunk";
 
-                processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse, pathway);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `cmpl-${requestId}`;
@@ -306,7 +351,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 }
                 jsonResponse.object = "chat.completion.chunk";
 
-                processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse, pathway);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `chatcmpl-${requestId}`;
@@ -346,4 +391,4 @@ function buildRestEndpoints(pathways, app, server, config) {
     }
 }
 
-export { buildRestEndpoints };
+export { buildRestEndpoints };

package/server/typeDef.js

@@ -28,7 +28,26 @@ const getGraphQlType = (value) => {
   }
 };
 
-const typeDef = (pathway) => {
+const getMessageTypeDefs = () => {
+  const messageType = `input Message { role: String, content: String, name: String }`;
+  const multiMessageType = `input MultiMessage { role: String, content: [String], name: String }`;
+  
+  return `${messageType}\n\n${multiMessageType}`;
+};
+
+const getPathwayTypeDef = (name, returnType) => {
+  return `type ${name} {
+    debug: String
+    result: ${returnType}
+    previousResult: String
+    warnings: [String]
+    errors: [String]
+    contextId: String
+    tool: String
+  }`
+};
+
+const getPathwayTypeDefAndExtendQuery = (pathway) => {
   const { name, objName, defaultInputParameters, inputParameters, format } = pathway;
 
   const fields = format ? format.match(/\b(\w+)\b/g) : null;
@@ -36,24 +55,13 @@ const typeDef = (pathway) => {
 
   const typeName = fields ? `${objName}Result` : `String`;
 
-  const messageType = `input Message { role: String, content: String, name: String }`;
-  const multiMessageType = `input MultiMessage { role: String, content: [String], name: String }`;
-
   const type = fields ? `type ${typeName} {
     ${fieldsStr}
     }` : ``;
 
-  const resultStr = pathway.list ? `[${typeName}]` : typeName;
+  const returnType = pathway.list ? `[${typeName}]` : typeName;
 
-  const responseType = `type ${objName} {
-        debug: String
-        result: ${resultStr}
-        previousResult: String
-        warnings: [String]
-        errors: [String]
-        contextId: String
-        tool: String
-}`;
+  const responseType = getPathwayTypeDef(objName, returnType);
 
   const params = { ...defaultInputParameters, ...inputParameters };
 
@@ -71,7 +79,7 @@ const typeDef = (pathway) => {
     };
   });
 
-  const gqlDefinition = `${messageType}\n\n${multiMessageType}\n\n${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
+  const gqlDefinition = `${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
 
   return {
     gqlDefinition,
@@ -79,6 +87,16 @@ const typeDef = (pathway) => {
   };
 };
 
+const typeDef = (pathway) => {
+  return getPathwayTypeDefAndExtendQuery(pathway);
+};
+
+const userPathwayInputParameters = `text: String`;
+
+
 export {
   typeDef,
+  getMessageTypeDefs,
+  getPathwayTypeDef,
+  userPathwayInputParameters,
 };

package/tests/chunkfunction.test.js

@@ -1,5 +1,5 @@
 import test from 'ava';
-import { getSemanticChunks, determineTextFormat } from '../server/chunker.js';
+import { getSemanticChunks, determineTextFormat, getSingleTokenChunks } from '../server/chunker.js';
 import { encode } from '../lib/encodeCache.js';
 
 const testText = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. In id erat sem. Phasellus ac dapibus purus, in fermentum nunc. Mauris quis rutrum magna. Quisque rutrum, augue vel blandit posuere, augue magna convallis turpis, nec elementum augue mauris sit amet nunc. Aenean sit amet leo est. Nunc ante ex, blandit et felis ut, iaculis lacinia est. Phasellus dictum orci id libero ullamcorper tempor.
@@ -207,4 +207,18 @@ test('should return identical text that chunker was passed, given weird spaces a
     t.assert(chunks.every(chunk => encode(chunk).length <= maxChunkToken)); //check chunk size
     const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
     t.assert(recomposedText === testTextShortWeirdSpaces); //check recomposition
+});
+
+test('should correctly split text into single token chunks', t => {
+    const testString = 'Hello, world!';
+    const chunks = getSingleTokenChunks(testString);
+    
+    // Check that each chunk is a single token
+    t.true(chunks.every(chunk => encode(chunk).length === 1));
+    
+    // Check that joining the chunks recreates the original string
+    t.is(chunks.join(''), testString);
+    
+    // Check specific tokens (this may need adjustment based on your tokenizer)
+    t.deepEqual(chunks, ['Hello', ',', ' world', '!']);
 });

package/tests/claude3VertexPlugin.test.js

@@ -199,7 +199,7 @@ test('convertMessagesToClaudeVertex system message with user message', async (t)
 test('convertMessagesToClaudeVertex user message with unsupported image type', async (t) => {
   const plugin = new Claude3VertexPlugin(pathway, model);
   // Test with unsupported image type
-  const messages = [{ role: 'user', content: { type: 'image_url', image_url: 'http://example.com/image.svg' } }];
+  const messages = [{ role: 'user', content: { type: 'image_url', image_url: 'https://unec.edu.az/application/uploads/2014/12/pdf-sample.pdf' } }];
   const output = await plugin.convertMessagesToClaudeVertex(messages);
   t.deepEqual(output, { system: '', modifiedMessages: [{role: 'user', content: [] }] });
 });