@aj-archipelago/cortex 1.1.19 → 1.1.21

package/server/plugins/claude3VertexPlugin.js

@@ -1,172 +1,306 @@
-import OpenAIVisionPlugin from './openAiVisionPlugin.js';
-import logger from '../../lib/logger.js';
+import OpenAIVisionPlugin from "./openAiVisionPlugin.js";
+import logger from "../../lib/logger.js";
+import mime from 'mime-types';
 
-class Claude3VertexPlugin extends OpenAIVisionPlugin {
+async function convertContentItem(item) {
 
-    parseResponse(data) 
-    {
-        if (!data) {
-            return data;
-        }
+  let imageUrl = "";
+  let isDataURL = false;
+  let urlData = "";
+  let mimeTypeMatch = "";
+  let mimeType = "";
+  let base64Image = "";
+  const allowedMIMETypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
+
+  try {
+    switch (typeof item) {
+      case "string":
+        return item ? { type: "text", text: item } : null;
+
+      case "object":
+        switch (item.type) {
+          case "text":
+            return item.text ? { type: "text", text: item.text } : null;
+
+          case "image_url":
+            imageUrl = item.image_url.url || item.image_url;
+            if (!imageUrl) {
+              logger.warn("Could not parse image URL from content - skipping image content.");
+              return null;
+            }
 
-        const { content } = data;
+            if (!allowedMIMETypes.includes(mime.lookup(imageUrl) || "")) {
+                logger.warn("Unsupported image type - skipping image content.");
+                return null;
+            }
 
-        // if the response is an array, return the text property of the first item
-        // if the type property is 'text'
-        if (content && Array.isArray(content) && content[0].type === 'text') {
-            return content[0].text;
-        } else {
-            return data;
+            isDataURL = imageUrl.startsWith("data:");
+            urlData = isDataURL ? item.image_url.url : await fetchImageAsDataURL(imageUrl);
+            mimeTypeMatch = urlData.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/);
+            mimeType = mimeTypeMatch && mimeTypeMatch[1] ? mimeTypeMatch[1] : "image/jpeg";
+            base64Image = urlData.split(",")[1];
+
+            return {
+              type: "image",
+              source: {
+                type: "base64",
+                media_type: mimeType,
+                data: base64Image,
+              },
+            };
+
+          default:
+            return null;
         }
+
+      default:
+        return null;
     }
+  }
+  catch (e) {
+    logger.warn(`Error converting content item: ${e}`);
+    return null;
+  }
+}
 
-    // This code converts messages to the format required by the Claude Vertex API
-    convertMessagesToClaudeVertex(messages) {
-        let modifiedMessages = [];
-        let system = '';
-        let lastAuthor = '';
-
-        // Claude needs system messages in a separate field
-        const systemMessages = messages.filter(message => message.role === 'system');
-        if (systemMessages.length > 0) {
-            system = systemMessages.map(message => message.content).join('\n');
-            modifiedMessages = messages.filter(message => message.role !== 'system');
-        } else {
-            modifiedMessages = messages;
-        }
+// Fetch image and convert to base 64 data URL
+async function fetchImageAsDataURL(imageUrl) {
+  try {
+    const response = await fetch(imageUrl);
 
-        // remove any empty messages
-        modifiedMessages = modifiedMessages.filter(message => message.content);
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
 
-        // combine any consecutive messages from the same author
-        var combinedMessages = [];
+    const buffer = await response.arrayBuffer();
+    const base64Image = Buffer.from(buffer).toString("base64");
+    const mimeType = mime.lookup(imageUrl) || "image/jpeg";
+    return `data:${mimeType};base64,${base64Image}`;
+  }
+  catch (e) {
+    logger.error(`Failed to fetch image: ${imageUrl}. ${e}`);
+    throw e;
+  }
+}
 
-        modifiedMessages.forEach((message) => {
-        if (message.role === lastAuthor) {
-            combinedMessages[combinedMessages.length - 1].content += '\n' + message.content;
-        } else {
-            combinedMessages.push(message);
-            lastAuthor = message.role;
-        }
-        });
+class Claude3VertexPlugin extends OpenAIVisionPlugin {
+  
+  parseResponse(data) {
+    if (!data) {
+      return data;
+    }
 
-        modifiedMessages = combinedMessages;
+    const { content } = data;
 
-        // Claude vertex requires an even number of messages
-        if (modifiedMessages.length % 2 === 0) {
-            modifiedMessages = modifiedMessages.slice(1);
-        }
+    // if the response is an array, return the text property of the first item
+    // if the type property is 'text'
+    if (content && Array.isArray(content) && content[0].type === "text") {
+      return content[0].text;
+    } else {
+      return data;
+    }
+  }
+
+  // This code converts messages to the format required by the Claude Vertex API
+  async convertMessagesToClaudeVertex(messages) {
+    let modifiedMessages = [];
+    let system = "";
+    let lastAuthor = "";
 
+    // Claude needs system messages in a separate field
+    const systemMessages = messages.filter(
+      (message) => message.role === "system"
+    );
+    if (systemMessages.length > 0) {
+      system = systemMessages.map((message) => message.content).join("\n");
+      modifiedMessages = messages.filter(
+        (message) => message.role !== "system"
+      );
+    } else {
+      modifiedMessages = messages;
+    }
+
+    // remove any empty messages
+    modifiedMessages = modifiedMessages.filter((message) => message.content);
+
+    // combine any consecutive messages from the same author
+    var combinedMessages = [];
+
+    modifiedMessages.forEach((message) => {
+      if (message.role === lastAuthor) {
+        combinedMessages[combinedMessages.length - 1].content +=
+          "\n" + message.content;
+      } else {
+        combinedMessages.push(message);
+        lastAuthor = message.role;
+      }
+    });
+
+    modifiedMessages = combinedMessages;
+
+    // Claude vertex requires an odd number of messages
+    // for proper conversation turn-taking
+    if (modifiedMessages.length % 2 === 0) {
+      modifiedMessages = modifiedMessages.slice(1);
+    }
+
+    const claude3Messages = await Promise.all(
+      modifiedMessages.map(async (message) => {
+        const contentArray = Array.isArray(message.content) ? message.content : [message.content];
+        const claude3Content = await Promise.all(contentArray.map(convertContentItem));
         return {
-            system,
-            modifiedMessages,
+          role: message.role,
+          content: claude3Content.filter(Boolean),
         };
-    }
+      })
+    );
 
-    getRequestParameters(text, parameters, prompt, cortexRequest) {
-        const requestParameters = super.getRequestParameters(text, parameters, prompt, cortexRequest);
-        const { system, modifiedMessages } = this.convertMessagesToClaudeVertex(requestParameters.messages);
-        requestParameters.system = system;
-        requestParameters.messages = modifiedMessages;  
-        requestParameters.max_tokens = this.getModelMaxReturnTokens();
-        requestParameters.anthropic_version = 'vertex-2023-10-16';
-        return requestParameters;
+    return {
+      system,
+      modifiedMessages: claude3Messages,
+    };
+  }
+
+  async getRequestParameters(text, parameters, prompt, cortexRequest) {
+    const requestParameters = await super.getRequestParameters(
+      text,
+      parameters,
+      prompt,
+      cortexRequest
+    );
+    const { system, modifiedMessages } =
+      await this.convertMessagesToClaudeVertex(requestParameters.messages);
+    requestParameters.system = system;
+    requestParameters.messages = modifiedMessages;
+    requestParameters.max_tokens = this.getModelMaxReturnTokens();
+    requestParameters.anthropic_version = "vertex-2023-10-16";
+    return requestParameters;
+  }
+
+  // Override the logging function to display the messages and responses
+  logRequestData(data, responseData, prompt) {
+    const { stream, messages, system } = data;
+    if (system) {
+      const { length, units } = this.getLength(system);
+      logger.info(`[system messages sent containing ${length} ${units}]`);
+      logger.debug(`${system}`);
     }
 
-    // Override the logging function to display the messages and responses
-    logRequestData(data, responseData, prompt) {
-        const { stream, messages, system } = data;
-        if (system) {
-            const { length, units } = this.getLength(system);
-            logger.info(`[system messages sent containing ${length} ${units}]`);
-            logger.debug(`${system}`);
-        }
-        
-        if (messages && messages.length > 1) {
-            logger.info(`[chat request sent containing ${messages.length} messages]`);
-            let totalLength = 0;
-            let totalUnits;
-            messages.forEach((message, index) => {
-                //message.content string or array
-                const content = Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content;
-                const words = content.split(" ");
-                const { length, units } = this.getLength(content);
-                const preview = words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
-    
-                logger.debug(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${preview}"`);
-                totalLength += length;
-                totalUnits = units;
-            });
-            logger.info(`[chat request contained ${totalLength} ${totalUnits}]`);
-        } else {
-            const message = messages[0];
-            const content = Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content;
-            const { length, units } = this.getLength(content);
-            logger.info(`[request sent containing ${length} ${units}]`);
-            logger.debug(`${content}`);
-        }
-    
-        if (stream) {
-            logger.info(`[response received as an SSE stream]`);
-        } else {
-            const responseText = this.parseResponse(responseData);
-            const { length, units } = this.getLength(responseText);
-            logger.info(`[response received containing ${length} ${units}]`);
-            logger.debug(`${responseText}`);
-        }
+    if (messages && messages.length > 1) {
+      logger.info(`[chat request sent containing ${messages.length} messages]`);
+      let totalLength = 0;
+      let totalUnits;
+      messages.forEach((message, index) => {
+        //message.content string or array
+        const content = Array.isArray(message.content)
+          ? message.content.map((item) => {
+              if (item.source && item.source.type === 'base64') {
+                item.source.data = '* base64 data truncated for log *';
+              }
+              return JSON.stringify(item);
+            }).join(", ")
+          : message.content;
+        const words = content.split(" ");
+        const { length, units } = this.getLength(content);
+        const preview =
+          words.length < 41
+            ? content
+            : words.slice(0, 20).join(" ") +
+              " ... " +
+              words.slice(-20).join(" ");
 
-        prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+        logger.debug(
+          `message ${index + 1}: role: ${
+            message.role
+          }, ${units}: ${length}, content: "${preview}"`
+        );
+        totalLength += length;
+        totalUnits = units;
+      });
+      logger.info(`[chat request contained ${totalLength} ${totalUnits}]`);
+    } else {
+      const message = messages[0];
+      const content = Array.isArray(message.content)
+        ? message.content.map((item) => JSON.stringify(item)).join(", ")
+        : message.content;
+      const { length, units } = this.getLength(content);
+      logger.info(`[request sent containing ${length} ${units}]`);
+      logger.debug(`${content}`);
     }
 
-    async execute(text, parameters, prompt, cortexRequest) {
-        const requestParameters = this.getRequestParameters(text, parameters, prompt, cortexRequest);
-        const { stream } = parameters;
+    if (stream) {
+      logger.info(`[response received as an SSE stream]`);
+    } else {
+      const responseText = this.parseResponse(responseData);
+      const { length, units } = this.getLength(responseText);
+      logger.info(`[response received containing ${length} ${units}]`);
+      logger.debug(`${responseText}`);
+    }
 
-        cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
-        cortexRequest.params = {}; // query params
-        cortexRequest.stream = stream;
-        cortexRequest.urlSuffix = cortexRequest.stream ? ':streamRawPredict' : ':rawPredict';
+    prompt &&
+      prompt.debugInfo &&
+      (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+  }
 
-        const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
-        const authToken = await gcpAuthTokenHelper.getAccessToken();
-        cortexRequest.headers.Authorization = `Bearer ${authToken}`;
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = await this.getRequestParameters(
+      text,
+      parameters,
+      prompt,
+      cortexRequest
+    );
+    const { stream } = parameters;
 
-        return this.executeRequest(cortexRequest);
-    }
+    cortexRequest.data = {
+      ...(cortexRequest.data || {}),
+      ...requestParameters,
+    };
+    cortexRequest.params = {}; // query params
+    cortexRequest.stream = stream;
+    cortexRequest.urlSuffix = cortexRequest.stream
+      ? ":streamRawPredict"
+      : ":rawPredict";
 
-    processStreamEvent(event, requestProgress) {
-        const eventData = JSON.parse(event.data);
-        switch (eventData.type) {
-            case 'message_start':
-                requestProgress.data = JSON.stringify(eventData.message);
-                break;
-            case 'content_block_start':
-                break;
-            case 'ping':
-                break;
-            case 'content_block_delta':
-                if (eventData.delta.type === 'text_delta') {
-                    requestProgress.data = JSON.stringify(eventData.delta.text);
-                }
-                break;
-            case 'content_block_stop':
-                break;
-            case 'message_delta':
-                break;
-            case 'message_stop':
-                requestProgress.data = '[DONE]';
-                requestProgress.progress = 1;
-                break;
-            case 'error':
-                requestProgress.data = `\n\n*** ${eventData.error.message || eventData.error} ***`;
-                requestProgress.progress = 1;
-                break;
-        }
+    const gcpAuthTokenHelper = this.config.get("gcpAuthTokenHelper");
+    const authToken = await gcpAuthTokenHelper.getAccessToken();
+    cortexRequest.headers.Authorization = `Bearer ${authToken}`;
 
-        return requestProgress;
+    return this.executeRequest(cortexRequest);
+  }
 
+  processStreamEvent(event, requestProgress) {
+    const eventData = JSON.parse(event.data);
+    switch (eventData.type) {
+      case "message_start":
+        requestProgress.data = JSON.stringify(eventData.message);
+        break;
+      case "content_block_start":
+        break;
+      case "ping":
+        break;
+      case "content_block_delta":
+        if (eventData.delta.type === "text_delta") {
+          requestProgress.data = JSON.stringify(eventData.delta.text);
+        }
+        break;
+      case "content_block_stop":
+        break;
+      case "message_delta":
+        break;
+      case "message_stop":
+        requestProgress.data = "[DONE]";
+        requestProgress.progress = 1;
+        break;
+      case "error":
+        requestProgress.data = `\n\n*** ${
+          eventData.error.message || eventData.error
+        } ***`;
+        requestProgress.progress = 1;
+        break;
     }
 
+    return requestProgress;
+  }
 }
 
 export default Claude3VertexPlugin;

package/server/plugins/gemini15ChatPlugin.js

@@ -76,7 +76,7 @@ class Gemini15ChatPlugin extends ModelPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }
@@ -135,7 +135,13 @@ class Gemini15ChatPlugin extends ModelPlugin {
         if (data && data.contents && Array.isArray(data.contents)) {
             dataToMerge = data.contents;
         } else if (data && data.candidates && Array.isArray(data.candidates)) {
-            return data.candidates[0].content.parts[0].text;
+            const { content, finishReason, safetyRatings } = data.candidates[0];
+            if (finishReason === 'STOP') {
+                return content?.parts?.[0]?.text ?? '';
+            } else {
+                const returnString = `Response was not completed.  Finish reason: ${finishReason}, Safety ratings: ${JSON.stringify(safetyRatings, null, 2)}`;
+                throw new Error(returnString);
+            }
         } else if (Array.isArray(data)) {
             dataToMerge = data;
         } else {
@@ -154,7 +160,6 @@ class Gemini15ChatPlugin extends ModelPlugin {
         cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
         cortexRequest.params = {}; // query params
         cortexRequest.stream = stream;
-        cortexRequest.stream = stream;
         cortexRequest.urlSuffix = cortexRequest.stream ? ':streamGenerateContent?alt=sse' : ':generateContent';
 
         const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');

package/server/plugins/gemini15VisionPlugin.js

@@ -1,6 +1,5 @@
 import Gemini15ChatPlugin from './gemini15ChatPlugin.js';
 import mime from 'mime-types';
-import logger from '../../lib/logger.js';
 
 class Gemini15VisionPlugin extends Gemini15ChatPlugin {
 
@@ -25,9 +24,10 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                 }
     
                 // Convert content to Gemini format, trying to maintain compatibility
-                const convertPartToGemini = (partString) => {
+                const convertPartToGemini = (inputPart) => {
                     try {
-                        const part = JSON.parse(partString);
+                        const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart;
+
                         if (typeof part === 'string') {
                             return { text: part };
                         } else if (part.type === 'text') {
@@ -50,9 +50,10 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                             }
                         }
                     } catch (e) {
-                        logger.warn(`Unable to parse part - including as string: ${partString}`);
+                        // this space intentionally left blank
                     }
-                    return { text: partString };
+
+                    return { text: inputPart };
                 };
     
                 const addPartToMessages = (geminiPart) => {
@@ -82,7 +83,7 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }
@@ -95,6 +96,22 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
         };
     }
 
+    async execute(text, parameters, prompt, cortexRequest) {
+        let result = null;
+        try {
+            result = await super.execute(text, parameters, prompt, cortexRequest);
+        } catch (e) {
+            const { data } = e;
+            if (data && data.error) {
+                if (data.error.code === 400 && data.error.message === 'Precondition check failed.') {
+                    throw new Error('One or more of the included files is too large to process. Please try again with a smaller file.');
+                }
+            throw e;
+            }
+        }
+        return result; 
+    }
+
 }
 
 export default Gemini15VisionPlugin;

package/server/plugins/geminiChatPlugin.js

@@ -81,7 +81,7 @@ class GeminiChatPlugin extends ModelPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }

package/server/plugins/geminiVisionPlugin.js

@@ -1,6 +1,5 @@
 import GeminiChatPlugin from './geminiChatPlugin.js';
 import mime from 'mime-types';
-import logger from '../../lib/logger.js';
 
 class GeminiVisionPlugin extends GeminiChatPlugin {
 
@@ -55,7 +54,7 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
                             }
                         }
                     } catch (e) {
-                        logger.warn(`Unable to parse part - including as string: ${partString}`);
+                        // this space intentionally left blank
                     }
                     return { text: partString };
                 };
@@ -87,7 +86,7 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }