@aj-archipelago/cortex 1.3.22 → 1.3.24

package/server/pathwayResolver.js

@@ -79,6 +79,13 @@ class PathwayResolver {
         let streamErrorOccurred = false;
         let responseData = null;
 
+        const publishNestedRequestProgress = (requestProgress) => {
+            if (requestProgress.progress === 1 && this.rootRequestId) {
+                delete requestProgress.progress;
+            }
+            publishRequestProgress(requestProgress);
+        }
+
         try {
             responseData = await this.executePathway(args);
         }
@@ -89,8 +96,13 @@ class PathwayResolver {
                     progress: 1,
                     data: '[DONE]',
                 });
+            } else {
+                publishRequestProgress({
+                    requestId: this.rootRequestId || this.requestId,
+                    progress: 1,
+                    data: error.message || error.toString(),
+                });
             }
-            return;
         }
 
         // If the response is a string, it's a regular long running response
@@ -100,7 +112,7 @@ class PathwayResolver {
             
             // some models don't support progress updates
             if (!modelTypesExcludedFromProgressUpdates.includes(this.model.type)) {
-                await publishRequestProgress({
+                await publishNestedRequestProgress({
                         requestId: this.rootRequestId || this.requestId,
                         progress: Math.min(completedCount,totalCount) / totalCount,
                         data: JSON.stringify(responseData),
@@ -139,10 +151,7 @@ class PathwayResolver {
 
                     try {
                         if (!streamEnded && requestProgress.data) {
-                            if (!(this.rootRequestId && requestProgress.progress === 1)) {
-                                logger.debug(`Publishing stream message to requestId ${this.requestId}: ${requestProgress.data}`);
-                                publishRequestProgress(requestProgress);
-                            }
+                            publishNestedRequestProgress(requestProgress);
                             streamEnded = requestProgress.progress === 1;
                         }
                     } catch (error) {

package/server/plugins/claude3VertexPlugin.js

@@ -136,7 +136,16 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     // Extract system messages
     const systemMessages = messagesCopy.filter(message => message.role === "system");
     if (systemMessages.length > 0) {
-      system = systemMessages.map(message => message.content).join("\n");
+      system = systemMessages.map(message => {
+        if (Array.isArray(message.content)) {
+          // For content arrays, extract text content and join
+          return message.content
+            .filter(item => item.type === 'text')
+            .map(item => item.text)
+            .join("\n");
+        }
+        return message.content;
+      }).join("\n");
     }
   
     // Filter out system messages and empty messages
@@ -380,7 +389,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     cortexRequest.params = {}; // query params
     cortexRequest.stream = stream;
     cortexRequest.urlSuffix = cortexRequest.stream
-      ? ":streamRawPredict"
+      ? ":streamRawPredict?alt=sse"
       : ":rawPredict";
 
     const gcpAuthTokenHelper = this.config.get("gcpAuthTokenHelper");
@@ -392,33 +401,59 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
   processStreamEvent(event, requestProgress) {
     const eventData = JSON.parse(event.data);
+    const baseOpenAIResponse = {
+      id: eventData.message?.id || `chatcmpl-${Date.now()}`,
+      object: "chat.completion.chunk",
+      created: Math.floor(Date.now() / 1000),
+      model: this.modelName,
+      choices: [{
+        index: 0,
+        delta: {},
+        finish_reason: null
+      }]
+    };
+
     switch (eventData.type) {
       case "message_start":
-        requestProgress.data = JSON.stringify(eventData.message);
-        break;
-      case "content_block_start":
-        break;
-      case "ping":
+        // Initial message with role
+        baseOpenAIResponse.choices[0].delta = {
+          role: "assistant",
+          content: ""
+        };
+        requestProgress.data = JSON.stringify(baseOpenAIResponse);
         break;
+
       case "content_block_delta":
         if (eventData.delta.type === "text_delta") {
-          requestProgress.data = JSON.stringify(eventData.delta.text);
+          baseOpenAIResponse.choices[0].delta = {
+            content: eventData.delta.text
+          };
+          requestProgress.data = JSON.stringify(baseOpenAIResponse);
         }
         break;
-      case "content_block_stop":
-        break;
-      case "message_delta":
-        break;
+
       case "message_stop":
-        requestProgress.data = "[DONE]";
+        baseOpenAIResponse.choices[0].delta = {};
+        baseOpenAIResponse.choices[0].finish_reason = "stop";
+        requestProgress.data = JSON.stringify(baseOpenAIResponse);
         requestProgress.progress = 1;
         break;
+
       case "error":
-        requestProgress.data = `\n\n*** ${
-          eventData.error.message || eventData.error
-        } ***`;
+        baseOpenAIResponse.choices[0].delta = {
+          content: `\n\n*** ${eventData.error.message || eventData.error} ***`
+        };
+        baseOpenAIResponse.choices[0].finish_reason = "error";
+        requestProgress.data = JSON.stringify(baseOpenAIResponse);
         requestProgress.progress = 1;
         break;
+
+      // Ignore other event types as they don't map to OpenAI format
+      case "content_block_start":
+      case "content_block_stop":
+      case "message_delta":
+      case "ping":
+        break;
     }
 
     return requestProgress;

package/server/plugins/gemini15ChatPlugin.js

@@ -56,7 +56,11 @@ class Gemini15ChatPlugin extends ModelPlugin {
                 const { role, author, content } = message;
         
                 if (role === 'system') {
-                    systemParts.push({ text: content });
+                    if (Array.isArray(content)) {
+                        content.forEach(item => systemParts.push({ text: item }));
+                    } else {
+                        systemParts.push({ text: content });
+                    }
                     return;
                 }
         
@@ -169,6 +173,95 @@ class Gemini15ChatPlugin extends ModelPlugin {
         return this.executeRequest(cortexRequest);
     }
 
+    processStreamEvent(event, requestProgress) {
+        const eventData = JSON.parse(event.data);
+        
+        // Initialize requestProgress if needed
+        requestProgress = requestProgress || {};
+        requestProgress.data = requestProgress.data || null;
+        
+        // Create a helper function to generate message chunks
+        const createChunk = (delta) => ({
+            id: eventData.responseId || `chatcmpl-${Date.now()}`,
+            object: "chat.completion.chunk",
+            created: Math.floor(Date.now() / 1000),
+            model: this.modelName,
+            choices: [{
+                index: 0,
+                delta,
+                finish_reason: null
+            }]
+        });
+
+        // Handle content chunks - do this first before handling any finish conditions
+        if (eventData.candidates?.[0]?.content?.parts?.[0]?.text) {
+            if (!requestProgress.started) {
+                // First chunk - send role
+                requestProgress.data = JSON.stringify(createChunk({ role: "assistant" }));
+                requestProgress.started = true;
+                
+                // Immediately follow up with the first content chunk
+                requestProgress.data = JSON.stringify(createChunk({ 
+                    content: eventData.candidates[0].content.parts[0].text 
+                }));
+            } else {
+                // Send content chunk
+                requestProgress.data = JSON.stringify(createChunk({ 
+                    content: eventData.candidates[0].content.parts[0].text 
+                }));
+            }
+
+            // If this message also has STOP, mark it for completion but don't overwrite the content
+            if (eventData.candidates[0].finishReason === "STOP") {
+                // Send the content first
+                requestProgress.data = JSON.stringify(createChunk({ 
+                    content: eventData.candidates[0].content.parts[0].text
+                }));
+                requestProgress.progress = 1;
+            }
+        } else if (eventData.candidates?.[0]?.finishReason === "STOP") {
+            // Only send DONE if there was no content in this message
+            requestProgress.data = '[DONE]';
+            requestProgress.progress = 1;
+        }
+
+        // Handle safety blocks
+        if (eventData.candidates?.[0]?.safetyRatings?.some(rating => rating.blocked)) {
+            requestProgress.data = JSON.stringify({
+                id: eventData.responseId || `chatcmpl-${Date.now()}`,
+                object: "chat.completion.chunk",
+                created: Math.floor(Date.now() / 1000),
+                model: this.modelName,
+                choices: [{
+                    index: 0,
+                    delta: { content: "\n\n*** Response blocked due to safety ratings ***" },
+                    finish_reason: "content_filter"
+                }]
+            });
+            requestProgress.progress = 1;
+            return requestProgress;
+        }
+
+        // Handle prompt feedback blocks
+        if (eventData.promptFeedback?.blockReason) {
+            requestProgress.data = JSON.stringify({
+                id: eventData.responseId || `chatcmpl-${Date.now()}`,
+                object: "chat.completion.chunk",
+                created: Math.floor(Date.now() / 1000),
+                model: this.modelName,
+                choices: [{
+                    index: 0,
+                    delta: { content: `\n\n*** Response blocked: ${eventData.promptFeedback.blockReason} ***` },
+                    finish_reason: "content_filter"
+                }]
+            });
+            requestProgress.progress = 1;
+            return requestProgress;
+        }
+
+        return requestProgress;
+    }
+
     // Override the logging function to display the messages and responses
     logRequestData(data, responseData, prompt) {
         const messages = data && data.contents;

package/server/plugins/gemini15VisionPlugin.js

@@ -24,19 +24,24 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                 const { role, author, content } = message;
     
                 if (role === 'system') {
-                    systemParts.push({ text: content });
+                    if (Array.isArray(content)) {
+                        content.forEach(item => systemParts.push({ text: item }));
+                    } else {
+                        systemParts.push({ text: content });
+                    }
                     return;
                 }
     
                 // Convert content to Gemini format, trying to maintain compatibility
                 const convertPartToGemini = (inputPart) => {
                     try {
+                        // First try to parse as JSON if it's a string
                         const part = typeof inputPart === 'string' ? JSON.parse(inputPart) : inputPart;
                         const {type, text, image_url, gcs} = part;
                         let fileUrl = gcs || image_url?.url;
 
                         if (typeof part === 'string') {
-                            return { text: text };
+                            return { text: inputPart };
                         } else if (type === 'text') {
                             return { text: text };
                         } else if (type === 'image_url') {
@@ -77,7 +82,8 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                             return null;
                         }
                     } catch (e) {
-                        // this space intentionally left blank
+                        // If JSON parsing fails or any other error, treat as plain text
+                        return inputPart ? { text: inputPart } : null;
                     }
                     return inputPart ? { text: inputPart } : null;
                 };

package/server/plugins/modelPlugin.js

@@ -381,14 +381,17 @@ class ModelPlugin {
 
             // finish reason can be in different places in the message
             const finishReason = parsedMessage?.choices?.[0]?.finish_reason || parsedMessage?.candidates?.[0]?.finishReason;
-            if (finishReason?.toLowerCase() === 'stop') {
-                requestProgress.progress = 1;
-            } else {
-                if (finishReason?.toLowerCase() === 'safety') {
-                    const safetyRatings = JSON.stringify(parsedMessage?.candidates?.[0]?.safetyRatings) || '';
-                    logger.warn(`Request ${this.requestId} was blocked by the safety filter. ${safetyRatings}`);
-                    requestProgress.data = `\n\nResponse blocked by safety filter: ${safetyRatings}`;
-                    requestProgress.progress = 1;
+            if (finishReason) {
+                switch (finishReason.toLowerCase()) {
+                    case 'safety':
+                        const safetyRatings = JSON.stringify(parsedMessage?.candidates?.[0]?.safetyRatings) || '';
+                        logger.warn(`Request ${this.requestId} was blocked by the safety filter. ${safetyRatings}`);
+                        requestProgress.data = `\n\nResponse blocked by safety filter: ${safetyRatings}`;
+                        requestProgress.progress = 1;
+                        break;
+                    default:
+                        requestProgress.progress = 1;
+                        break;
                 }
             }
         }

package/server/plugins/ollamaChatPlugin.js

@@ -0,0 +1,158 @@
+import ModelPlugin from './modelPlugin.js';
+import logger from '../../lib/logger.js';
+import { Transform } from 'stream';
+
+class OllamaChatPlugin extends ModelPlugin {
+
+  getRequestParameters(text, parameters, prompt) {
+    const { modelPromptMessages } = this.getCompiledPrompt(text, parameters, prompt);
+    return {
+      data: {
+        model: parameters.ollamaModel,
+        messages: modelPromptMessages,
+        stream: parameters.stream
+      },
+      params: {}
+    };
+  }
+
+  logRequestData(data, responseData, prompt) {
+    const { stream, messages, model } = data;
+
+    if (messages && messages.length > 0) {
+      logger.info(`[ollama chat request sent to model ${model} containing ${messages.length} messages]`);
+      let totalLength = 0;
+      let totalUnits;
+      messages.forEach((message, index) => {
+        const content = message.content;
+        const { length, units } = this.getLength(content);
+        const preview = this.shortenContent(content);
+
+        logger.verbose(
+          `message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${preview}"`
+        );
+        totalLength += length;
+        totalUnits = units;
+      });
+      logger.info(`[chat request contained ${totalLength} ${totalUnits}]`);
+    }
+
+    if (stream) {
+      logger.info(`[response received as an SSE stream]`);
+    } else if (responseData) {
+      const responseText = this.parseResponse(responseData);
+      const { length, units } = this.getLength(responseText);
+      logger.info(`[response received containing ${length} ${units}]`);
+      logger.verbose(`${this.shortenContent(responseText)}`);
+    }
+
+    prompt &&
+      prompt.debugInfo &&
+      (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+  }
+
+  parseResponse(data) {
+    // If data is not a string (e.g. streaming), return as is
+    if (typeof data !== 'string') {
+      return data;
+    }
+
+    // Split into lines and filter empty ones
+    const lines = data.split('\n').filter(line => line.trim());
+    
+    let fullResponse = '';
+    
+    for (const line of lines) {
+      try {
+        const jsonObj = JSON.parse(line);
+        
+        if (jsonObj.message && jsonObj.message.content) {
+          // Unescape special sequences
+          const content = jsonObj.message.content
+            .replace(/\\n/g, '\n')
+            .replace(/\\"/g, '"')
+            .replace(/\\\\/g, '\\')
+            .replace(/\\u003c/g, '<')
+            .replace(/\\u003e/g, '>');
+            
+          fullResponse += content;
+        }
+      } catch (err) {
+        // If we can't parse the line as JSON, just skip it
+        continue;
+      }
+    }
+  
+    return fullResponse;
+  }
+
+  processStreamEvent(event, requestProgress) {
+    try {
+      const data = JSON.parse(event.data);
+      
+      // Handle the streaming response
+      if (data.message?.content) {
+        // Unescape special sequences in the content
+        const content = data.message.content
+          .replace(/\\n/g, '\n')
+          .replace(/\\"/g, '"')
+          .replace(/\\\\/g, '\\')
+          .replace(/\\u003c/g, '<')
+          .replace(/\\u003e/g, '>');
+          
+        requestProgress.data = JSON.stringify(content);
+      }
+
+      // Check if this is the final message
+      if (data.done) {
+        requestProgress.data = '[DONE]';
+        requestProgress.progress = 1;
+      }
+
+      return requestProgress;
+    } catch (err) {
+      // If we can't parse the event data, return the progress as is
+      return requestProgress;
+    }
+  }
+
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = this.getRequestParameters(text, parameters, prompt);
+    cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters.data };
+    cortexRequest.params = { ...(cortexRequest.params || {}), ...requestParameters.params };
+    
+    // For Ollama streaming, transform NDJSON to SSE format
+    if (parameters.stream) {
+      const response = await this.executeRequest(cortexRequest);
+      
+      // Create a transform stream that converts NDJSON to SSE format
+      const transformer = new Transform({
+        decodeStrings: false, // Keep as string
+        transform(chunk, encoding, callback) {
+          try {
+            const lines = chunk.toString().split('\n');
+            for (const line of lines) {
+              if (line.trim()) {
+                // Format as SSE data
+                this.push(`data: ${line}\n\n`);
+              }
+            }
+            callback();
+          } catch (err) {
+            callback(err);
+          }
+        }
+      });
+
+      // Pipe the response through our transformer
+      response.pipe(transformer);
+      
+      // Return the transformed stream
+      return transformer;
+    }
+    
+    return this.executeRequest(cortexRequest);
+  }
+}
+
+export default OllamaChatPlugin;

package/server/plugins/ollamaCompletionPlugin.js

@@ -0,0 +1,147 @@
+import ModelPlugin from './modelPlugin.js';
+import logger from '../../lib/logger.js';
+import { Transform } from 'stream';
+
+class OllamaCompletionPlugin extends ModelPlugin {
+
+  getRequestParameters(text, parameters, prompt) {
+    return {
+      data: {
+        model: parameters.ollamaModel,
+        prompt: text,
+        stream: parameters.stream
+      },
+      params: {}
+    };
+  }
+
+  logRequestData(data, responseData, prompt) {
+    const { stream, prompt: promptText, model } = data;
+
+    if (promptText) {
+      logger.info(`[ollama completion request sent to model ${model}]`);
+      const { length, units } = this.getLength(promptText);
+      const preview = this.shortenContent(promptText);
+      logger.verbose(`prompt ${units}: ${length}, content: "${preview}"`);
+      logger.info(`[completion request contained ${length} ${units}]`);
+    }
+
+    if (stream) {
+      logger.info(`[response received as an SSE stream]`);
+    } else if (responseData) {
+      const responseText = this.parseResponse(responseData);
+      const { length, units } = this.getLength(responseText);
+      logger.info(`[response received containing ${length} ${units}]`);
+      logger.verbose(`${this.shortenContent(responseText)}`);
+    }
+
+    prompt &&
+      prompt.debugInfo &&
+      (prompt.debugInfo += `\n${JSON.stringify(data)}`);
+  }
+
+  parseResponse(data) {
+    // If data is not a string (e.g. streaming), return as is
+    if (typeof data !== 'string') {
+      return data;
+    }
+
+    // Split into lines and filter empty ones
+    const lines = data.split('\n').filter(line => line.trim());
+    
+    let fullResponse = '';
+    
+    for (const line of lines) {
+      try {
+        const jsonObj = JSON.parse(line);
+        
+        if (jsonObj.response) {
+          // Unescape special sequences
+          const content = jsonObj.response
+            .replace(/\\n/g, '\n')
+            .replace(/\\"/g, '"')
+            .replace(/\\\\/g, '\\')
+            .replace(/\\u003c/g, '<')
+            .replace(/\\u003e/g, '>');
+            
+          fullResponse += content;
+        }
+      } catch (err) {
+        // If we can't parse the line as JSON, just skip it
+        continue;
+      }
+    }
+  
+    return fullResponse;
+  }
+
+  processStreamEvent(event, requestProgress) {
+    try {
+      const data = JSON.parse(event.data);
+      
+      // Handle the streaming response
+      if (data.response) {
+        // Unescape special sequences in the content
+        const content = data.response
+          .replace(/\\n/g, '\n')
+          .replace(/\\"/g, '"')
+          .replace(/\\\\/g, '\\')
+          .replace(/\\u003c/g, '<')
+          .replace(/\\u003e/g, '>');
+          
+        requestProgress.data = JSON.stringify(content);
+      }
+
+      // Check if this is the final message
+      if (data.done) {
+        requestProgress.data = '[DONE]';
+        requestProgress.progress = 1;
+      }
+
+      return requestProgress;
+    } catch (err) {
+      // If we can't parse the event data, return the progress as is
+      return requestProgress;
+    }
+  }
+
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = this.getRequestParameters(text, parameters, prompt);
+    cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters.data };
+    cortexRequest.params = { ...(cortexRequest.params || {}), ...requestParameters.params };
+    
+    // For Ollama streaming, transform NDJSON to SSE format
+    if (parameters.stream) {
+      const response = await this.executeRequest(cortexRequest);
+      
+      // Create a transform stream that converts NDJSON to SSE format
+      const transformer = new Transform({
+        decodeStrings: false, // Keep as string
+        transform(chunk, encoding, callback) {
+          try {
+            const lines = chunk.toString().split('\n');
+            for (const line of lines) {
+              if (line.trim()) {
+                // Format as SSE data
+                this.push(`data: ${line}\n\n`);
+              }
+            }
+            callback();
+          } catch (err) {
+            callback(err);
+          }
+        }
+      });
+
+      // Pipe the response through our transformer
+      response.pipe(transformer);
+      
+      // Return the transformed stream
+      return transformer;
+    }
+    
+    return this.executeRequest(cortexRequest);
+  }
+}
+
+export default OllamaCompletionPlugin;