@aj-archipelago/cortex 1.1.19 → 1.1.20

package/config/default.example.json

@@ -1,5 +1,5 @@
 {
-    "defaultModelName": "oai-td3",
+    "defaultModelName": "oai-gpturbo",
     "models": {
         "azure-translate": {
             "type": "AZURE-TRANSLATE",
@@ -12,19 +12,6 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 2000
         },
-        "oai-td3": {
-            "type": "OPENAI-COMPLETION",
-            "url": "https://api.openai.com/v1/completions",
-            "headers": {
-                "Authorization": "Bearer {{OPENAI_API_KEY}}",
-                "Content-Type": "application/json"
-            },
-            "params": {
-                "model": "text-davinci-003"
-            },
-            "requestsPerSecond": 10,
-            "maxTokenLength": 4096
-        },
         "oai-gpturbo": {
             "type": "OPENAI-CHAT",
             "url": "https://api.openai.com/v1/chat/completions",

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -11,6 +11,7 @@ import { join } from "path";
 import { Storage } from "@google-cloud/storage";
 import axios from "axios";
 import { publicFolder, port, ipAddress } from "./start.js";
+import mime from "mime-types";
 
 const IMAGE_EXTENSIONS = [
   ".jpg",
@@ -21,6 +22,7 @@ const IMAGE_EXTENSIONS = [
   ".webp",
   ".tiff",
   ".svg",
+  ".pdf"
 ];
 
 const VIDEO_EXTENSIONS = [
@@ -74,7 +76,7 @@ if (!GCP_PROJECT_ID || !GCP_SERVICE_ACCOUNT) {
 
 const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
 
-const getBlobClient = () => {
+const getBlobClient = async () => {
   const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
   const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
   if (!connectionString || !containerName) {
@@ -83,15 +85,21 @@ const getBlobClient = () => {
     );
   }
 
-  const blobServiceClient =
-    BlobServiceClient.fromConnectionString(connectionString);
+  const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
+
+  const serviceProperties = await blobServiceClient.getProperties();
+  if(!serviceProperties.defaultServiceVersion) {
+    serviceProperties.defaultServiceVersion = '2020-02-10';
+    await blobServiceClient.setProperties(serviceProperties);
+  }
+
   const containerClient = blobServiceClient.getContainerClient(containerName);
 
   return { blobServiceClient, containerClient };
 };
 
 async function saveFileToBlob(chunkPath, requestId) {
-  const { containerClient } = getBlobClient();
+  const { containerClient } = await getBlobClient();
   // Use the filename with a UUID as the blob name
   const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
 
@@ -110,7 +118,7 @@ async function saveFileToBlob(chunkPath, requestId) {
 //deletes blob that has the requestId
 async function deleteBlob(requestId) {
   if (!requestId) throw new Error("Missing requestId parameter");
-  const { containerClient } = getBlobClient();
+  const { containerClient } = await getBlobClient();
   // List the blobs in the container with the specified prefix
   const blobs = containerClient.listBlobsFlat({ prefix: `${requestId}/` });
 
@@ -181,14 +189,20 @@ async function uploadBlob(
           resolve(body); // Resolve the promise
         } else {
           const filename = `${requestId}/${uuidv4()}_${info.filename}`;
-          const { containerClient } = getBlobClient();
+          const { containerClient } = await getBlobClient();
+
+          const contentType = mime.lookup(filename);  // content type based on file extension
+          const options = {};
+          if (contentType) {
+            options.blobHTTPHeaders = { blobContentType: contentType };
+          }
 
           const blockBlobClient = containerClient.getBlockBlobClient(filename);
 
           const passThroughStream = new PassThrough();
           file.pipe(passThroughStream);
 
-          await blockBlobClient.uploadStream(passThroughStream);
+          await blockBlobClient.uploadStream(passThroughStream, undefined, undefined, options);
 
           const message = `File '${filename}' uploaded successfully.`;
           const url = blockBlobClient.url;
@@ -250,7 +264,7 @@ async function uploadBlob(
 
 // Function to delete files that haven't been used in more than a month
 async function cleanup(urls=null) {
-  const { containerClient } = getBlobClient();
+  const { containerClient } = await getBlobClient();
 
   if(!urls) {
     const xMonthAgo = new Date();

package/helper-apps/cortex-file-handler/package.json

@@ -5,6 +5,7 @@
   "type": "module",
   "scripts": {
     "start": "node start.js",
+    "dev": "node -r dotenv/config start.js",
     "test": "echo \"No tests yet...\""
   },
   "dependencies": {

package/lib/pathwayTools.js

@@ -3,7 +3,11 @@ import { encode, decode } from '../lib/encodeCache.js';
 import { config } from '../config.js';
 
 // callPathway - call a pathway from another pathway
-const callPathway = async (pathwayName, args) => {
+const callPathway = async (pathwayName, inArgs) => {
+
+    // Clone the args object to avoid modifying the original
+    const args = JSON.parse(JSON.stringify(inArgs));
+    
     const pathway = config.get(`pathways.${pathwayName}`);
     if (!pathway) {
         throw new Error(`Pathway ${pathwayName} not found`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.19",
+  "version": "1.1.20",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/server/plugins/claude3VertexPlugin.js

@@ -1,172 +1,305 @@
-import OpenAIVisionPlugin from './openAiVisionPlugin.js';
-import logger from '../../lib/logger.js';
+import OpenAIVisionPlugin from "./openAiVisionPlugin.js";
+import logger from "../../lib/logger.js";
+import mime from 'mime-types';
 
-class Claude3VertexPlugin extends OpenAIVisionPlugin {
+async function convertContentItem(item) {
 
-    parseResponse(data) 
-    {
-        if (!data) {
-            return data;
-        }
+  let imageUrl = "";
+  let isDataURL = false;
+  let urlData = "";
+  let mimeTypeMatch = "";
+  let mimeType = "";
+  let base64Image = "";
+  const allowedMIMETypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
+
+  try {
+    switch (typeof item) {
+      case "string":
+        return item ? { type: "text", text: item } : null;
+
+      case "object":
+        switch (item.type) {
+          case "text":
+            return item.text ? { type: "text", text: item.text } : null;
+
+          case "image_url":
+            imageUrl = item.image_url.url || item.image_url;
+            if (!imageUrl) {
+              logger.warn("Could not parse image URL from content - skipping image content.");
+              return null;
+            }
 
-        const { content } = data;
+            if (!allowedMIMETypes.includes(mime.lookup(imageUrl) || "")) {
+                logger.warn("Unsupported image type - skipping image content.");
+                return null;
+            }
 
-        // if the response is an array, return the text property of the first item
-        // if the type property is 'text'
-        if (content && Array.isArray(content) && content[0].type === 'text') {
-            return content[0].text;
-        } else {
-            return data;
+            isDataURL = imageUrl.startsWith("data:");
+            urlData = isDataURL ? item.image_url.url : await fetchImageAsDataURL(imageUrl);
+            mimeTypeMatch = urlData.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/);
+            mimeType = mimeTypeMatch && mimeTypeMatch[1] ? mimeTypeMatch[1] : "image/jpeg";
+            base64Image = urlData.split(",")[1];
+
+            return {
+              type: "image",
+              source: {
+                type: "base64",
+                media_type: mimeType,
+                data: base64Image,
+              },
+            };
+
+          default:
+            return null;
         }
+
+      default:
+        return null;
     }
+  }
+  catch (e) {
+    logger.warn(`Error converting content item: ${e}`);
+    return null;
+  }
+}
 
-    // This code converts messages to the format required by the Claude Vertex API
-    convertMessagesToClaudeVertex(messages) {
-        let modifiedMessages = [];
-        let system = '';
-        let lastAuthor = '';
-
-        // Claude needs system messages in a separate field
-        const systemMessages = messages.filter(message => message.role === 'system');
-        if (systemMessages.length > 0) {
-            system = systemMessages.map(message => message.content).join('\n');
-            modifiedMessages = messages.filter(message => message.role !== 'system');
-        } else {
-            modifiedMessages = messages;
-        }
+// Fetch image and convert to base 64 data URL
+async function fetchImageAsDataURL(imageUrl) {
+  try {
+    const response = await fetch(imageUrl);
 
-        // remove any empty messages
-        modifiedMessages = modifiedMessages.filter(message => message.content);
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
 
-        // combine any consecutive messages from the same author
-        var combinedMessages = [];
+    const buffer = await response.arrayBuffer();
+    const base64Image = Buffer.from(buffer).toString("base64");
+    const mimeType = mime.lookup(imageUrl) || "image/jpeg";
+    return `data:${mimeType};base64,${base64Image}`;
+  }
+  catch (e) {
+    logger.error(`Failed to fetch image: ${imageUrl}. ${e}`);
+    throw e;
+  }
+}
 
-        modifiedMessages.forEach((message) => {
-        if (message.role === lastAuthor) {
-            combinedMessages[combinedMessages.length - 1].content += '\n' + message.content;
-        } else {
-            combinedMessages.push(message);
-            lastAuthor = message.role;
-        }
-        });
+class Claude3VertexPlugin extends OpenAIVisionPlugin {
+  
+  parseResponse(data) {
+    if (!data) {
+      return data;
+    }
 
-        modifiedMessages = combinedMessages;
+    const { content } = data;
 
-        // Claude vertex requires an even number of messages
-        if (modifiedMessages.length % 2 === 0) {
-            modifiedMessages = modifiedMessages.slice(1);
-        }
+    // if the response is an array, return the text property of the first item
+    // if the type property is 'text'
+    if (content && Array.isArray(content) && content[0].type === "text") {
+      return content[0].text;
+    } else {
+      return data;
+    }
+  }
+
+  // This code converts messages to the format required by the Claude Vertex API
+  async convertMessagesToClaudeVertex(messages) {
+    let modifiedMessages = [];
+    let system = "";
+    let lastAuthor = "";
 
+    // Claude needs system messages in a separate field
+    const systemMessages = messages.filter(
+      (message) => message.role === "system"
+    );
+    if (systemMessages.length > 0) {
+      system = systemMessages.map((message) => message.content).join("\n");
+      modifiedMessages = messages.filter(
+        (message) => message.role !== "system"
+      );
+    } else {
+      modifiedMessages = messages;
+    }
+
+    // remove any empty messages
+    modifiedMessages = modifiedMessages.filter((message) => message.content);
+
+    // combine any consecutive messages from the same author
+    var combinedMessages = [];
+
+    modifiedMessages.forEach((message) => {
+      if (message.role === lastAuthor) {
+        combinedMessages[combinedMessages.length - 1].content +=
+          "\n" + message.content;
+      } else {
+        combinedMessages.push(message);
+        lastAuthor = message.role;
+      }
+    });
+
+    modifiedMessages = combinedMessages;
+
+    // Claude vertex requires an even number of messages
+    if (modifiedMessages.length % 2 === 0) {
+      modifiedMessages = modifiedMessages.slice(1);
+    }
+
+    const claude3Messages = await Promise.all(
+      modifiedMessages.map(async (message) => {
+        const contentArray = Array.isArray(message.content) ? message.content : [message.content];
+        const claude3Content = await Promise.all(contentArray.map(convertContentItem));
         return {
-            system,
-            modifiedMessages,
+          role: message.role,
+          content: claude3Content.filter(Boolean),
         };
-    }
+      })
+    );
 
-    getRequestParameters(text, parameters, prompt, cortexRequest) {
-        const requestParameters = super.getRequestParameters(text, parameters, prompt, cortexRequest);
-        const { system, modifiedMessages } = this.convertMessagesToClaudeVertex(requestParameters.messages);
-        requestParameters.system = system;
-        requestParameters.messages = modifiedMessages;  
-        requestParameters.max_tokens = this.getModelMaxReturnTokens();
-        requestParameters.anthropic_version = 'vertex-2023-10-16';
-        return requestParameters;
+    return {
+      system,
+      modifiedMessages: claude3Messages,
+    };
+  }
+
+  async getRequestParameters(text, parameters, prompt, cortexRequest) {
+    const requestParameters = await super.getRequestParameters(
+      text,
+      parameters,
+      prompt,
+      cortexRequest
+    );
+    const { system, modifiedMessages } =
+      await this.convertMessagesToClaudeVertex(requestParameters.messages);
+    requestParameters.system = system;
+    requestParameters.messages = modifiedMessages;
+    requestParameters.max_tokens = this.getModelMaxReturnTokens();
+    requestParameters.anthropic_version = "vertex-2023-10-16";
+    return requestParameters;
+  }
+
+  // Override the logging function to display the messages and responses
+  logRequestData(data, responseData, prompt) {
+    const { stream, messages, system } = data;
+    if (system) {
+      const { length, units } = this.getLength(system);
+      logger.info(`[system messages sent containing ${length} ${units}]`);
+      logger.debug(`${system}`);
     }
 
-    // Override the logging function to display the messages and responses
-    logRequestData(data, responseData, prompt) {
-        const { stream, messages, system } = data;
-        if (system) {
-            const { length, units } = this.getLength(system);
-            logger.info(`[system messages sent containing ${length} ${units}]`);
-            logger.debug(`${system}`);
-        }
-        
-        if (messages && messages.length > 1) {
-            logger.info(`[chat request sent containing ${messages.length} messages]`);
-            let totalLength = 0;
-            let totalUnits;
-            messages.forEach((message, index) => {
-                //message.content string or array
-                const content = Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content;
-                const words = content.split(" ");
-                const { length, units } = this.getLength(content);
-                const preview = words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
-    
-                logger.debug(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${preview}"`);
-                totalLength += length;
-                totalUnits = units;
-            });
-            logger.info(`[chat request contained ${totalLength} ${totalUnits}]`);
-        } else {
-            const message = messages[0];
-            const content = Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content;
-            const { length, units } = this.getLength(content);
-            logger.info(`[request sent containing ${length} ${units}]`);
-            logger.debug(`${content}`);
-        }
-    
-        if (stream) {
-            logger.info(`[response received as an SSE stream]`);
-        } else {
-            const responseText = this.parseResponse(responseData);
-            const { length, units } = this.getLength(responseText);
-            logger.info(`[response received containing ${length} ${units}]`);
-            logger.debug(`${responseText}`);
-        }
+    if (messages && messages.length > 1) {
+      logger.info(`[chat request sent containing ${messages.length} messages]`);
+      let totalLength = 0;
+      let totalUnits;
+      messages.forEach((message, index) => {
+        //message.content string or array
+        const content = Array.isArray(message.content)
+          ? message.content.map((item) => {
+              if (item.source && item.source.type === 'base64') {
+                item.source.data = '* base64 data truncated for log *';
+              }
+              return JSON.stringify(item);
+            }).join(", ")
+          : message.content;
+        const words = content.split(" ");
+        const { length, units } = this.getLength(content);
+        const preview =
+          words.length < 41
+            ? content
+            : words.slice(0, 20).join(" ") +
+              " ... " +
+              words.slice(-20).join(" ");
 
-        prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+        logger.debug(
+          `message ${index + 1}: role: ${
+            message.role
+          }, ${units}: ${length}, content: "${preview}"`
+        );
+        totalLength += length;
+        totalUnits = units;
+      });
+      logger.info(`[chat request contained ${totalLength} ${totalUnits}]`);
+    } else {
+      const message = messages[0];
+      const content = Array.isArray(message.content)
+        ? message.content.map((item) => JSON.stringify(item)).join(", ")
+        : message.content;
+      const { length, units } = this.getLength(content);
+      logger.info(`[request sent containing ${length} ${units}]`);
+      logger.debug(`${content}`);
     }
 
-    async execute(text, parameters, prompt, cortexRequest) {
-        const requestParameters = this.getRequestParameters(text, parameters, prompt, cortexRequest);
-        const { stream } = parameters;
+    if (stream) {
+      logger.info(`[response received as an SSE stream]`);
+    } else {
+      const responseText = this.parseResponse(responseData);
+      const { length, units } = this.getLength(responseText);
+      logger.info(`[response received containing ${length} ${units}]`);
+      logger.debug(`${responseText}`);
+    }
 
-        cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
-        cortexRequest.params = {}; // query params
-        cortexRequest.stream = stream;
-        cortexRequest.urlSuffix = cortexRequest.stream ? ':streamRawPredict' : ':rawPredict';
+    prompt &&
+      prompt.debugInfo &&
+      (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+  }
 
-        const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
-        const authToken = await gcpAuthTokenHelper.getAccessToken();
-        cortexRequest.headers.Authorization = `Bearer ${authToken}`;
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = await this.getRequestParameters(
+      text,
+      parameters,
+      prompt,
+      cortexRequest
+    );
+    const { stream } = parameters;
 
-        return this.executeRequest(cortexRequest);
-    }
+    cortexRequest.data = {
+      ...(cortexRequest.data || {}),
+      ...requestParameters,
+    };
+    cortexRequest.params = {}; // query params
+    cortexRequest.stream = stream;
+    cortexRequest.urlSuffix = cortexRequest.stream
+      ? ":streamRawPredict"
+      : ":rawPredict";
 
-    processStreamEvent(event, requestProgress) {
-        const eventData = JSON.parse(event.data);
-        switch (eventData.type) {
-            case 'message_start':
-                requestProgress.data = JSON.stringify(eventData.message);
-                break;
-            case 'content_block_start':
-                break;
-            case 'ping':
-                break;
-            case 'content_block_delta':
-                if (eventData.delta.type === 'text_delta') {
-                    requestProgress.data = JSON.stringify(eventData.delta.text);
-                }
-                break;
-            case 'content_block_stop':
-                break;
-            case 'message_delta':
-                break;
-            case 'message_stop':
-                requestProgress.data = '[DONE]';
-                requestProgress.progress = 1;
-                break;
-            case 'error':
-                requestProgress.data = `\n\n*** ${eventData.error.message || eventData.error} ***`;
-                requestProgress.progress = 1;
-                break;
-        }
+    const gcpAuthTokenHelper = this.config.get("gcpAuthTokenHelper");
+    const authToken = await gcpAuthTokenHelper.getAccessToken();
+    cortexRequest.headers.Authorization = `Bearer ${authToken}`;
 
-        return requestProgress;
+    return this.executeRequest(cortexRequest);
+  }
 
+  processStreamEvent(event, requestProgress) {
+    const eventData = JSON.parse(event.data);
+    switch (eventData.type) {
+      case "message_start":
+        requestProgress.data = JSON.stringify(eventData.message);
+        break;
+      case "content_block_start":
+        break;
+      case "ping":
+        break;
+      case "content_block_delta":
+        if (eventData.delta.type === "text_delta") {
+          requestProgress.data = JSON.stringify(eventData.delta.text);
+        }
+        break;
+      case "content_block_stop":
+        break;
+      case "message_delta":
+        break;
+      case "message_stop":
+        requestProgress.data = "[DONE]";
+        requestProgress.progress = 1;
+        break;
+      case "error":
+        requestProgress.data = `\n\n*** ${
+          eventData.error.message || eventData.error
+        } ***`;
+        requestProgress.progress = 1;
+        break;
     }
 
+    return requestProgress;
+  }
 }
 
 export default Claude3VertexPlugin;

package/server/plugins/gemini15VisionPlugin.js

@@ -25,9 +25,10 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                 }
     
                 // Convert content to Gemini format, trying to maintain compatibility
-                const convertPartToGemini = (partString) => {
+                const convertPartToGemini = (inputPart) => {
                     try {
-                        const part = JSON.parse(partString);
+                        const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart;
+
                         if (typeof part === 'string') {
                             return { text: part };
                         } else if (part.type === 'text') {
@@ -50,9 +51,9 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                             }
                         }
                     } catch (e) {
-                        logger.warn(`Unable to parse part - including as string: ${partString}`);
+                        logger.warn(`Unable to parse part - including as string: ${inputPart}`);
                     }
-                    return { text: partString };
+                    return { text: inputPart };
                 };
     
                 const addPartToMessages = (geminiPart) => {

package/server/plugins/openAiVisionPlugin.js

@@ -8,9 +8,15 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
         messages.map(message => {
             try {
                 // message.content can be array or string
-                if (typeof message.content === 'string') message.content = JSON.parse(message.content);
-                else if (Array.isArray(message.content)) message.content = message.content.map(item => JSON.parse(item));
-                
+                if (typeof message.content === 'string') {
+                    message.content = JSON.parse(message.content);
+                } else if (Array.isArray(message.content)) {
+                    message.content = message.content.map(item => {
+                        const parsedItem = JSON.parse(item);
+                        const { type, text, image_url, url } = parsedItem;
+                        return { type, text, image_url: url || image_url};
+                    });
+                }     
             } catch (e) {
                 return message;
             }

package/server/resolver.js

@@ -1,5 +1,6 @@
 import { fulfillWithTimeout } from '../lib/promiser.js';
 import { PathwayResolver } from './pathwayResolver.js';
+import logger from '../lib/logger.js';
 
 // This resolver uses standard parameters required by Apollo server:
 // (parent, args, contextValue, info)
@@ -16,7 +17,15 @@ const rootResolver = async (parent, args, contextValue, info) => {
     contextValue.pathwayResolver = pathwayResolver;
 
     // Execute the request with timeout
-    const result = await fulfillWithTimeout(pathway.resolver(parent, args, contextValue, info), pathway.timeout);
+    let result = null;
+
+    try {
+        result = await fulfillWithTimeout(pathway.resolver(parent, args, contextValue, info), pathway.timeout);
+    } catch (error) {
+        logger.error(`Request failed with error: ${error}`);
+        result = error.message || error.toString();
+    }
+    
     const { warnings, previousResult, savedContextId, tool } = pathwayResolver;
     
     // Add request parameters back as debug

package/tests/vision.test.js

@@ -131,7 +131,7 @@ test('vision multi single long text', async t => {
         },
     });
 
-    t.is(response.body?.singleResult?.errors?.[0]?.message, 'Unable to process your request as your single message content is too long. Please try again with a shorter message.');
+    t.is(response.body?.singleResult?.errors?.[0]?.message || response.body?.singleResult?.data?.vision?.result, 'Unable to process your request as your single message content is too long. Please try again with a shorter message.');
 });