@aj-archipelago/cortex 1.3.21 → 1.3.23

package/server/plugins/claude3VertexPlugin.js

@@ -15,6 +15,21 @@ async function convertContentItem(item, maxImageSize, plugin) {
           case "text":
             return item.text ? { type: "text", text: item.text } : null;
 
+          case "tool_use":
+            return {
+              type: "tool_use",
+              id: item.id,
+              name: item.name,
+              input: typeof item.input === 'string' ? { query: item.input } : item.input
+            };
+
+          case "tool_result":
+            return {
+              type: "tool_result",
+              tool_use_id: item.tool_use_id,
+              content: item.content
+            };
+
           case "image_url":
             imageUrl = item.url || item.image_url?.url || item.image_url;
 
@@ -126,9 +141,42 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
   
     // Filter out system messages and empty messages
     let modifiedMessages = messagesCopy
-      .filter(message => message.role !== "system" && message.content)
-      .map(message => ({ ...message }));
-  
+      .filter(message => message.role !== "system")
+      .map(message => {
+        // Handle OpenAI tool calls format conversion to Claude format
+        if (message.tool_calls) {
+          return {
+            role: message.role,
+            content: message.tool_calls.map(toolCall => ({
+              type: "tool_use",
+              id: toolCall.id,
+              name: toolCall.function.name,
+              input: JSON.parse(toolCall.function.arguments)
+            }))
+          };
+        }
+        
+        // Handle OpenAI tool response format conversion to Claude format
+        if (message.role === "tool") {
+          return {
+            role: "user",
+            content: [{
+              type: "tool_result",
+              tool_use_id: message.tool_call_id,
+              content: message.content
+            }]
+          };
+        }
+
+        return { ...message };
+      })
+      .filter(message => {
+        // Filter out messages with empty content
+        if (!message.content) return false;
+        if (Array.isArray(message.content) && message.content.length === 0) return false;
+        return true;
+      });
+
     // Combine consecutive messages from the same author
     const combinedMessages = modifiedMessages.reduce((acc, message) => {
       if (acc.length === 0 || message.role !== acc[acc.length - 1].role) {
@@ -191,10 +239,68 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
       prompt,
       cortexRequest
     );
+
     const { system, modifiedMessages } =
       await this.convertMessagesToClaudeVertex(requestParameters.messages);
     requestParameters.system = system;
     requestParameters.messages = modifiedMessages;
+
+    // Convert OpenAI tools format to Claude format if present
+    if (parameters.tools) {
+      requestParameters.tools = parameters.tools.map(tool => {
+        if (tool.type === 'function') {
+          return {
+            name: tool.function.name,
+            description: tool.function.description,
+                  input_schema: {
+              type: "object",
+              properties: tool.function.parameters.properties,
+              required: tool.function.parameters.required || []
+            }
+          };
+        }
+        return tool;
+      });
+    }
+
+    // If there are function calls in messages, generate tools block
+    if (modifiedMessages?.some(msg => 
+      Array.isArray(msg.content) && msg.content.some(item => item.type === 'tool_use')
+    )) {
+      const toolsMap = new Map();
+      
+      // Collect all unique tool uses from messages
+      modifiedMessages.forEach(msg => {
+        if (Array.isArray(msg.content)) {
+          msg.content.forEach(item => {
+            if (item.type === 'tool_use') {
+              toolsMap.set(item.name, {
+                name: item.name,
+                description: `Tool for ${item.name}`,
+                input_schema: {
+                  type: "object",
+                  properties: item.input ? Object.keys(item.input).reduce((acc, key) => {
+                    acc[key] = {
+                      type: typeof item.input[key] === 'string' ? 'string' : 'object',
+                      description: `Parameter ${key} for ${item.name}`
+                    };
+                    return acc;
+                  }, {}) : {},
+                  required: item.input ? Object.keys(item.input) : []
+                }
+              });
+            }
+          });
+        }
+      });
+      
+      if (requestParameters.tools) {
+        requestParameters.tools.push(...Array.from(toolsMap.values()));
+      } else {
+        requestParameters.tools = Array.from(toolsMap.values());
+      }
+    }
+
     requestParameters.max_tokens = this.getModelMaxReturnTokens();
     requestParameters.anthropic_version = "vertex-2023-10-16";
     return requestParameters;
@@ -274,7 +380,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     cortexRequest.params = {}; // query params
     cortexRequest.stream = stream;
     cortexRequest.urlSuffix = cortexRequest.stream
-      ? ":streamRawPredict"
+      ? ":streamRawPredict?alt=sse"
       : ":rawPredict";
 
     const gcpAuthTokenHelper = this.config.get("gcpAuthTokenHelper");
@@ -286,33 +392,59 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
   processStreamEvent(event, requestProgress) {
     const eventData = JSON.parse(event.data);
+    const baseOpenAIResponse = {
+      id: eventData.message?.id || `chatcmpl-${Date.now()}`,
+      object: "chat.completion.chunk",
+      created: Math.floor(Date.now() / 1000),
+      model: this.modelName,
+      choices: [{
+        index: 0,
+        delta: {},
+        finish_reason: null
+      }]
+    };
+
     switch (eventData.type) {
       case "message_start":
-        requestProgress.data = JSON.stringify(eventData.message);
-        break;
-      case "content_block_start":
-        break;
-      case "ping":
+        // Initial message with role
+        baseOpenAIResponse.choices[0].delta = {
+          role: "assistant",
+          content: ""
+        };
+        requestProgress.data = JSON.stringify(baseOpenAIResponse);
         break;
+
       case "content_block_delta":
         if (eventData.delta.type === "text_delta") {
-          requestProgress.data = JSON.stringify(eventData.delta.text);
+          baseOpenAIResponse.choices[0].delta = {
+            content: eventData.delta.text
+          };
+          requestProgress.data = JSON.stringify(baseOpenAIResponse);
         }
         break;
-      case "content_block_stop":
-        break;
-      case "message_delta":
-        break;
+
       case "message_stop":
-        requestProgress.data = "[DONE]";
+        baseOpenAIResponse.choices[0].delta = {};
+        baseOpenAIResponse.choices[0].finish_reason = "stop";
+        requestProgress.data = JSON.stringify(baseOpenAIResponse);
         requestProgress.progress = 1;
         break;
+
       case "error":
-        requestProgress.data = `\n\n*** ${
-          eventData.error.message || eventData.error
-        } ***`;
+        baseOpenAIResponse.choices[0].delta = {
+          content: `\n\n*** ${eventData.error.message || eventData.error} ***`
+        };
+        baseOpenAIResponse.choices[0].finish_reason = "error";
+        requestProgress.data = JSON.stringify(baseOpenAIResponse);
         requestProgress.progress = 1;
         break;
+
+      // Ignore other event types as they don't map to OpenAI format
+      case "content_block_start":
+      case "content_block_stop":
+      case "message_delta":
+      case "ping":
+        break;
     }
 
     return requestProgress;

package/server/plugins/gemini15ChatPlugin.js

@@ -56,7 +56,11 @@ class Gemini15ChatPlugin extends ModelPlugin {
                 const { role, author, content } = message;
         
                 if (role === 'system') {
-                    systemParts.push({ text: content });
+                    if (Array.isArray(content)) {
+                        content.forEach(item => systemParts.push({ text: item }));
+                    } else {
+                        systemParts.push({ text: content });
+                    }
                     return;
                 }
         
@@ -169,6 +173,91 @@ class Gemini15ChatPlugin extends ModelPlugin {
         return this.executeRequest(cortexRequest);
     }
 
+    processStreamEvent(event, requestProgress) {
+        const eventData = JSON.parse(event.data);
+        
+        // Initialize requestProgress if needed
+        requestProgress = requestProgress || {};
+        requestProgress.data = requestProgress.data || null;
+        
+        // Create a helper function to generate message chunks
+        const createChunk = (delta) => ({
+            id: eventData.responseId || `chatcmpl-${Date.now()}`,
+            object: "chat.completion.chunk",
+            created: Math.floor(Date.now() / 1000),
+            model: this.modelName,
+            choices: [{
+                index: 0,
+                delta,
+                finish_reason: null
+            }]
+        });
+
+        // Handle content chunks - do this first before handling any finish conditions
+        if (eventData.candidates?.[0]?.content?.parts?.[0]?.text) {
+            if (!requestProgress.started) {
+                // First chunk - send role
+                requestProgress.data = JSON.stringify(createChunk({ role: "assistant" }));
+                requestProgress.started = true;
+                
+                // Immediately follow up with the first content chunk
+                requestProgress.data = JSON.stringify(createChunk({ 
+                    content: eventData.candidates[0].content.parts[0].text 
+                }));
+            } else {
+                // Send content chunk
+                requestProgress.data = JSON.stringify(createChunk({ 
+                    content: eventData.candidates[0].content.parts[0].text 
+                }));
+            }
+
+            // If this message also has STOP, mark it for completion but don't overwrite the content
+            if (eventData.candidates[0].finishReason === "STOP") {
+                requestProgress.progress = 1;
+            }
+        } else if (eventData.candidates?.[0]?.finishReason === "STOP") {
+            // Only send DONE if there was no content in this message
+            requestProgress.data = '[DONE]';
+            requestProgress.progress = 1;
+        }
+
+        // Handle safety blocks
+        if (eventData.candidates?.[0]?.safetyRatings?.some(rating => rating.blocked)) {
+            requestProgress.data = JSON.stringify({
+                id: eventData.responseId || `chatcmpl-${Date.now()}`,
+                object: "chat.completion.chunk",
+                created: Math.floor(Date.now() / 1000),
+                model: this.modelName,
+                choices: [{
+                    index: 0,
+                    delta: { content: "\n\n*** Response blocked due to safety ratings ***" },
+                    finish_reason: "content_filter"
+                }]
+            });
+            requestProgress.progress = 1;
+            return requestProgress;
+        }
+
+        // Handle prompt feedback blocks
+        if (eventData.promptFeedback?.blockReason) {
+            requestProgress.data = JSON.stringify({
+                id: eventData.responseId || `chatcmpl-${Date.now()}`,
+                object: "chat.completion.chunk",
+                created: Math.floor(Date.now() / 1000),
+                model: this.modelName,
+                choices: [{
+                    index: 0,
+                    delta: { content: `\n\n*** Response blocked: ${eventData.promptFeedback.blockReason} ***` },
+                    finish_reason: "content_filter"
+                }]
+            });
+            requestProgress.progress = 1;
+            return requestProgress;
+        }
+
+        return requestProgress;
+    }
+
     // Override the logging function to display the messages and responses
     logRequestData(data, responseData, prompt) {
         const messages = data && data.contents;

package/server/plugins/gemini15VisionPlugin.js

@@ -24,19 +24,24 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                 const { role, author, content } = message;
     
                 if (role === 'system') {
-                    systemParts.push({ text: content });
+                    if (Array.isArray(content)) {
+                        content.forEach(item => systemParts.push({ text: item }));
+                    } else {
+                        systemParts.push({ text: content });
+                    }
                     return;
                 }
     
                 // Convert content to Gemini format, trying to maintain compatibility
                 const convertPartToGemini = (inputPart) => {
                     try {
+                        // First try to parse as JSON if it's a string
                         const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart;
                         const {type, text, image_url, gcs} = part;
                         let fileUrl = gcs || image_url?.url;
 
                         if (typeof part === 'string') {
-                            return { text: text };
+                            return { text: inputPart };
                         } else if (type === 'text') {
                             return { text: text };
                         } else if (type === 'image_url') {
@@ -66,11 +71,19 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                                         data: base64Data
                                     }
                                 };
+                            } else if (fileUrl.includes('youtube.com/') || fileUrl.includes('youtu.be/')) {
+                                return {
+                                    fileData: {
+                                        mimeType: 'video/youtube',
+                                        fileUri: fileUrl
+                                    }
+                                };
                             }
                             return null;
                         }
                     } catch (e) {
-                        // this space intentionally left blank
+                        // If JSON parsing fails or any other error, treat as plain text
+                        return inputPart ? { text: inputPart } : null;
                     }
                     return inputPart ? { text: inputPart } : null;
                 };

package/server/plugins/modelPlugin.js

@@ -210,7 +210,7 @@ class ModelPlugin {
     
         // First run handlebars compile on the pathway messages
         const compiledMessages = modelPrompt.messages.map((message) => {
-            if (message.content) {
+            if (message.content && typeof message.content === 'string') {
                 const compileText = HandleBars.compile(message.content);
                 return {
                     ...message,
@@ -381,14 +381,17 @@ class ModelPlugin {
 
             // finish reason can be in different places in the message
             const finishReason = parsedMessage?.choices?.[0]?.finish_reason || parsedMessage?.candidates?.[0]?.finishReason;
-            if (finishReason?.toLowerCase() === 'stop') {
-                requestProgress.progress = 1;
-            } else {
-                if (finishReason?.toLowerCase() === 'safety') {
-                    const safetyRatings = JSON.stringify(parsedMessage?.candidates?.[0]?.safetyRatings) || '';
-                    logger.warn(`Request ${this.requestId} was blocked by the safety filter. ${safetyRatings}`);
-                    requestProgress.data = `\n\nResponse blocked by safety filter: ${safetyRatings}`;
-                    requestProgress.progress = 1;
+            if (finishReason) {
+                switch (finishReason.toLowerCase()) {
+                    case 'safety':
+                        const safetyRatings = JSON.stringify(parsedMessage?.candidates?.[0]?.safetyRatings) || '';
+                        logger.warn(`Request ${this.requestId} was blocked by the safety filter. ${safetyRatings}`);
+                        requestProgress.data = `\n\nResponse blocked by safety filter: ${safetyRatings}`;
+                        requestProgress.progress = 1;
+                        break;
+                    default:
+                        requestProgress.progress = 1;
+                        break;
                 }
             }
         }

package/server/plugins/ollamaChatPlugin.js

@@ -0,0 +1,158 @@
+import ModelPlugin from './modelPlugin.js';
+import logger from '../../lib/logger.js';
+import { Transform } from 'stream';
+
+class OllamaChatPlugin extends ModelPlugin {
+
+  getRequestParameters(text, parameters, prompt) {
+    const { modelPromptMessages } = this.getCompiledPrompt(text, parameters, prompt);
+    return {
+      data: {
+        model: parameters.ollamaModel,
+        messages: modelPromptMessages,
+        stream: parameters.stream
+      },
+      params: {}
+    };
+  }
+
+  logRequestData(data, responseData, prompt) {
+    const { stream, messages, model } = data;
+
+    if (messages && messages.length > 0) {
+      logger.info(`[ollama chat request sent to model ${model} containing ${messages.length} messages]`);
+      let totalLength = 0;
+      let totalUnits;
+      messages.forEach((message, index) => {
+        const content = message.content;
+        const { length, units } = this.getLength(content);
+        const preview = this.shortenContent(content);
+
+        logger.verbose(
+          `message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${preview}"`
+        );
+        totalLength += length;
+        totalUnits = units;
+      });
+      logger.info(`[chat request contained ${totalLength} ${totalUnits}]`);
+    }
+
+    if (stream) {
+      logger.info(`[response received as an SSE stream]`);
+    } else if (responseData) {
+      const responseText = this.parseResponse(responseData);
+      const { length, units } = this.getLength(responseText);
+      logger.info(`[response received containing ${length} ${units}]`);
+      logger.verbose(`${this.shortenContent(responseText)}`);
+    }
+
+    prompt &&
+      prompt.debugInfo &&
+      (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+  }
+
+  parseResponse(data) {
+    // If data is not a string (e.g. streaming), return as is
+    if (typeof data !== 'string') {
+      return data;
+    }
+
+    // Split into lines and filter empty ones
+    const lines = data.split('\n').filter(line => line.trim());
+    
+    let fullResponse = '';
+    
+    for (const line of lines) {
+      try {
+        const jsonObj = JSON.parse(line);
+        
+        if (jsonObj.message && jsonObj.message.content) {
+          // Unescape special sequences
+          const content = jsonObj.message.content
+            .replace(/\\n/g, '\n')
+            .replace(/\\"/g, '"')
+            .replace(/\\\\/g, '\\')
+            .replace(/\\u003c/g, '<')
+            .replace(/\\u003e/g, '>');
+            
+          fullResponse += content;
+        }
+      } catch (err) {
+        // If we can't parse the line as JSON, just skip it
+        continue;
+      }
+    }
+  
+    return fullResponse;
+  }
+
+  processStreamEvent(event, requestProgress) {
+    try {
+      const data = JSON.parse(event.data);
+      
+      // Handle the streaming response
+      if (data.message?.content) {
+        // Unescape special sequences in the content
+        const content = data.message.content
+          .replace(/\\n/g, '\n')
+          .replace(/\\"/g, '"')
+          .replace(/\\\\/g, '\\')
+          .replace(/\\u003c/g, '<')
+          .replace(/\\u003e/g, '>');
+          
+        requestProgress.data = JSON.stringify(content);
+      }
+
+      // Check if this is the final message
+      if (data.done) {
+        requestProgress.data = '[DONE]';
+        requestProgress.progress = 1;
+      }
+
+      return requestProgress;
+    } catch (err) {
+      // If we can't parse the event data, return the progress as is
+      return requestProgress;
+    }
+  }
+
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = this.getRequestParameters(text, parameters, prompt);
+    cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters.data };
+    cortexRequest.params = { ...(cortexRequest.params || {}), ...requestParameters.params };
+    
+    // For Ollama streaming, transform NDJSON to SSE format
+    if (parameters.stream) {
+      const response = await this.executeRequest(cortexRequest);
+      
+      // Create a transform stream that converts NDJSON to SSE format
+      const transformer = new Transform({
+        decodeStrings: false, // Keep as string
+        transform(chunk, encoding, callback) {
+          try {
+            const lines = chunk.toString().split('\n');
+            for (const line of lines) {
+              if (line.trim()) {
+                // Format as SSE data
+                this.push(`data: ${line}\n\n`);
+              }
+            }
+            callback();
+          } catch (err) {
+            callback(err);
+          }
+        }
+      });
+
+      // Pipe the response through our transformer
+      response.pipe(transformer);
+      
+      // Return the transformed stream
+      return transformer;
+    }
+    
+    return this.executeRequest(cortexRequest);
+  }
+}
+
+export default OllamaChatPlugin;