@aj-archipelago/cortex 1.4.32 → 1.4.33

package/server/pathwayResolver.js

@@ -274,6 +274,14 @@ class PathwayResolver {
 
     async handleStream(response) {
         let streamErrorOccurred = false;
+        let streamErrorMessage = null;
+        let completionSent = false;
+        let receivedSSEData = false; // Track if we actually received SSE events
+        let receivedAnyData = false; // Track if we received ANY data from the stream
+        let toolCallbackInvoked = false; // Track if a tool callback was invoked (stream close is expected)
+        // Accumulate streamed content for continuity memory
+        this.streamedContent = '';
+        const requestId = this.rootRequestId || this.requestId;
 
         if (response && typeof response.on === 'function') {
             try {
@@ -282,7 +290,7 @@ class PathwayResolver {
 
                 const onParse = (event) => {
                     let requestProgress = {
-                        requestId: this.rootRequestId || this.requestId
+                        requestId
                     };
 
                     logger.debug(`Received event: ${event.type}`);
@@ -292,15 +300,34 @@ class PathwayResolver {
                         logger.debug(`id: ${event.id || '<none>'}`)
                         logger.debug(`name: ${event.name || '<none>'}`)
                         logger.debug(`data: ${event.data}`)
+
+                        receivedSSEData = true; // Only mark SSE data when we get actual 'event' type
+
+                        // Check for error events in the stream data
+                        try {
+                            const eventData = JSON.parse(event.data);
+                            if (eventData.error) {
+                                streamErrorOccurred = true;
+                                streamErrorMessage = eventData.error.message || JSON.stringify(eventData.error);
+                                logger.error(`Stream contained error event: ${streamErrorMessage}`);
+                            }
+                        } catch {
+                            // Not JSON or no error field, continue normal processing
+                        }
                     } else if (event.type === 'reconnect-interval') {
                         logger.debug(`We should set reconnect interval to ${event.value} milliseconds`)
                     }
 
                     try {
                         requestProgress = this.modelExecutor.plugin.processStreamEvent(event, requestProgress);
+                        // Check if plugin signaled a tool callback was invoked
+                        if (requestProgress.toolCallbackInvoked) {
+                            toolCallbackInvoked = true;
+                        }
                     } catch (error) {
                         streamErrorOccurred = true;
-                        logger.error(`Stream error: ${error instanceof Error ? error.stack || error.message : JSON.stringify(error)}`);
+                        streamErrorMessage = error instanceof Error ? error.message : String(error);
+                        logger.error(`Stream processing error: ${error instanceof Error ? error.stack || error.message : JSON.stringify(error)}`);
                         incomingMessage.off('data', processStream);
                         return;
                     }
@@ -309,6 +336,9 @@ class PathwayResolver {
                         if (!streamEnded && requestProgress.data) {
                             this.publishNestedRequestProgress(requestProgress);
                             streamEnded = requestProgress.progress === 1;
+                            if (streamEnded) {
+                                completionSent = true;
+                            }
                         }
                     } catch (error) {
                         logger.error(`Could not publish the stream message: "${event.data}", ${error instanceof Error ? error.stack || error.message : JSON.stringify(error)}`);
@@ -319,7 +349,7 @@ class PathwayResolver {
                 const sseParser = createParser(onParse);
 
                 const processStream = (data) => {
-                    //logger.warn(`RECEIVED DATA: ${JSON.stringify(data.toString())}`);
+                    receivedAnyData = true; // Track that we got data from the stream
                     sseParser.feed(data.toString());
                 }
 
@@ -327,27 +357,71 @@ class PathwayResolver {
                     await new Promise((resolve, reject) => {
                         incomingMessage.on('data', processStream);
                         incomingMessage.on('end', resolve);
-                        incomingMessage.on('error', reject);
+                        incomingMessage.on('error', (err) => {
+                            streamErrorOccurred = true;
+                            streamErrorMessage = err instanceof Error ? err.message : String(err);
+                            reject(err);
+                        });
+                        incomingMessage.on('close', () => {
+                            // Stream closed - detect various incomplete states
+                            if (!receivedAnyData && !streamErrorOccurred && !toolCallbackInvoked) {
+                                // Stream opened but closed with NO data at all - this is likely a provider issue
+                                logger.warn('Stream closed with no data received (empty stream)');
+                            } else if (receivedSSEData && !completionSent && !streamErrorOccurred && !toolCallbackInvoked) {
+                                // Got SSE data but no completion signal
+                                logger.warn('Stream closed without completion signal');
+                            }
+                            resolve();
+                        });
                     });
                 }
 
             } catch (error) {
+                streamErrorOccurred = true;
+                if (!streamErrorMessage) {
+                    streamErrorMessage = error instanceof Error ? error.message : String(error);
+                }
                 logger.error(`Could not subscribe to stream: ${error instanceof Error ? error.stack || error.message : JSON.stringify(error)}`);
             }
 
-            if (streamErrorOccurred) {
-                logger.error(`Stream read failed. Finishing stream...`);
+            // Detect empty stream (opened but closed with no data) - this should be retried
+            const emptyStream = !receivedAnyData && !streamErrorOccurred && !toolCallbackInvoked;
+
+            // Ensure completion is sent if not already done
+            // Send completion if:
+            // 1. Stream error occurred (always notify client of errors)
+            // 2. OR we received SSE data but no completion was sent (and no tool callback)
+            // 3. OR empty stream (will only happen if retry logic exhausted - see executePathway)
+            // Don't send completion if a tool callback was invoked (stream will resume)
+            const shouldSendCompletion = !toolCallbackInvoked && !completionSent &&
+                (streamErrorOccurred || receivedSSEData);
+
+            if (shouldSendCompletion) {
+                if (streamErrorOccurred) {
+                    logger.error(`Stream read failed: ${streamErrorMessage}`);
+                }
+                const errorMessage = streamErrorOccurred
+                    ? (streamErrorMessage || this.errors.join(', ') || 'Stream read failed')
+                    : '';
                 publishRequestProgress({
-                    requestId: this.requestId,
+                    requestId,
                     progress: 1,
                     data: '',
-                    info: '',
-                    error: 'Stream read failed'
+                    info: JSON.stringify(this.pathwayResultData || {}),
+                    error: errorMessage
                 });
-            } else {
-                return;
             }
+
+            // Return stream result for retry logic
+            return {
+                success: completionSent || toolCallbackInvoked,
+                emptyStream,
+                error: streamErrorOccurred ? streamErrorMessage : null
+            };
         }
+
+        // Non-stream response
+        return { success: true, emptyStream: false, error: null };
     }
 
     async resolve(args) {
@@ -479,7 +553,23 @@ class PathwayResolver {
 
             // if data is a stream, handle it
             if (data && typeof data.on === 'function') {
-                await this.handleStream(data);
+                const streamResult = await this.handleStream(data);
+                // Check if stream was empty (opened but closed with no data) - retry if so
+                if (streamResult?.emptyStream) {
+                    logger.warn(`Empty stream received - retrying. Attempt ${retries + 1} of ${MAX_RETRIES}`);
+                    if (retries === MAX_RETRIES - 1) {
+                        // Last retry - send error completion so client doesn't hang
+                        logger.error('All stream retries exhausted - empty stream from provider');
+                        publishRequestProgress({
+                            requestId: this.rootRequestId || this.requestId,
+                            progress: 1,
+                            data: '',
+                            info: JSON.stringify(this.pathwayResultData || {}),
+                            error: 'Provider returned empty stream - please try again'
+                        });
+                    }
+                    continue; // Retry
+                }
                 return data;
             }
 

package/server/plugins/claudeAnthropicPlugin.js

@@ -0,0 +1,84 @@
+import Claude4VertexPlugin from "./claude4VertexPlugin.js";
+
+/**
+ * Plugin for direct Anthropic API access (api.anthropic.com)
+ * 
+ * This plugin extends Claude4VertexPlugin and reuses all the message/content
+ * conversion logic, but uses direct Anthropic API authentication and endpoints
+ * instead of Google Vertex AI.
+ * 
+ * Key differences from Vertex AI:
+ * - Uses x-api-key header instead of Bearer token
+ * - Uses https://api.anthropic.com/v1/messages endpoint
+ * - Model specified in request body, not URL
+ * - anthropic-version specified in header, not body
+ * - Streaming via stream:true in body, not URL suffix
+ */
+class ClaudeAnthropicPlugin extends Claude4VertexPlugin {
+  
+  constructor(pathway, model) {
+    super(pathway, model);
+  }
+  
+  async getRequestParameters(text, parameters, prompt) {
+    // Get base request parameters from parent (includes message conversion)
+    const requestParameters = await super.getRequestParameters(
+      text,
+      parameters,
+      prompt
+    );
+
+    // Remove Vertex-specific anthropic_version from body
+    delete requestParameters.anthropic_version;
+    
+    // Add model to request body (required for direct Anthropic API)
+    // The model name should come from the endpoint params or model config
+    const modelName = this.model.params?.model || 
+                      this.model.endpoints?.[0]?.params?.model ||
+                      this.modelName;
+    requestParameters.model = modelName;
+    
+    return requestParameters;
+  }
+
+  async execute(text, parameters, prompt, cortexRequest) {
+    const requestParameters = await this.getRequestParameters(
+      text,
+      parameters,
+      prompt,
+      cortexRequest
+    );
+    const { stream } = parameters;
+
+    // Add stream parameter to request body for Anthropic API
+    if (stream) {
+      requestParameters.stream = true;
+    }
+
+    cortexRequest.data = {
+      ...(cortexRequest.data || {}),
+      ...requestParameters,
+    };
+    cortexRequest.params = {}; // query params
+    cortexRequest.stream = stream;
+    
+    // Direct Anthropic API doesn't use URL suffix for streaming
+    cortexRequest.urlSuffix = "";
+
+    // Set Anthropic-specific headers
+    // The x-api-key should already be in the model config headers
+    // but we need to add the anthropic-version header
+    cortexRequest.headers = {
+      ...(cortexRequest.headers || {}),
+      "anthropic-version": "2023-06-01"
+    };
+
+    // For direct Anthropic API, authentication is handled via headers in config
+    // (x-api-key: {{CLAUDE_API_KEY}})
+    // No need for GCP auth token
+
+    return this.executeRequest(cortexRequest);
+  }
+}
+
+export default ClaudeAnthropicPlugin;

package/server/plugins/gemini15ChatPlugin.js

@@ -178,6 +178,16 @@ class Gemini15ChatPlugin extends ModelPlugin {
             const { content, finishReason, safetyRatings } = data.candidates[0];
             if (finishReason === 'STOP' || finishReason === 'MAX_TOKENS') {
                 return content?.parts?.[0]?.text ?? '';
+            } else if (finishReason === 'MALFORMED_FUNCTION_CALL') {
+                // Model attempted a function call but generated invalid JSON
+                // Return any partial text content if available, otherwise return an error message
+                const textContent = content?.parts?.[0]?.text;
+                if (textContent) {
+                    logger.warn(`Gemini returned MALFORMED_FUNCTION_CALL but had text content, returning text`);
+                    return textContent;
+                }
+                logger.warn(`Gemini returned MALFORMED_FUNCTION_CALL with no text content`);
+                return 'I encountered an issue processing that request. Please try rephrasing your question.';
             } else {
                 const returnString = `Response was not completed.  Finish reason: ${finishReason}, Safety ratings: ${JSON.stringify(safetyRatings, null, 2)}`;
                 throw new Error(returnString);
@@ -259,6 +269,13 @@ class Gemini15ChatPlugin extends ModelPlugin {
             // Only send DONE if there was no content in this message
             requestProgress.data = '[DONE]';
             requestProgress.progress = 1;
+        } else if (eventData.candidates?.[0]?.finishReason === "MALFORMED_FUNCTION_CALL") {
+            // Model attempted a function call but generated invalid JSON
+            logger.warn(`Gemini streaming returned MALFORMED_FUNCTION_CALL`);
+            requestProgress.data = JSON.stringify(createChunk({
+                content: '\n\nI encountered an issue processing that request. Please try rephrasing your question.'
+            }));
+            requestProgress.progress = 1;
         }
 
         // Handle safety blocks

package/server/plugins/gemini15VisionPlugin.js

@@ -2,6 +2,7 @@ import Gemini15ChatPlugin from './gemini15ChatPlugin.js';
 import CortexResponse from '../../lib/cortexResponse.js';
 import { requestState } from '../requestState.js';
 import { addCitationsToResolver } from '../../lib/pathwayTools.js';
+import logger from '../../lib/logger.js';
 import mime from 'mime-types';
 
 class Gemini15VisionPlugin extends Gemini15ChatPlugin {
@@ -414,6 +415,18 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
 
                 return cortexResponse;
             }
+
+            // Handle MALFORMED_FUNCTION_CALL - model tried to call a function but generated invalid JSON
+            if (finishReason === 'MALFORMED_FUNCTION_CALL') {
+                const textContent = content?.parts?.[0]?.text || '';
+                logger.warn(`Gemini returned MALFORMED_FUNCTION_CALL, returning graceful response`);
+                return new CortexResponse({
+                    output_text: textContent || 'I encountered an issue processing that request. Please try rephrasing your question.',
+                    finishReason: "stop",
+                    usage: data.usageMetadata || null,
+                    metadata: { model: this.modelName }
+                });
+            }
         }
 
         // Fallback to parent implementation
@@ -500,20 +513,38 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
             const finishReason = this.hadToolCalls ? "tool_calls" : "stop";
 
             // Check if there's any remaining content in the final chunk that needs to be published
+            let sentFinalChunk = false;
             if (eventData.candidates?.[0]?.content?.parts) {
                 const parts = eventData.candidates[0].content.parts;
                 for (const part of parts) {
                     if (part.text && part.text.trim()) {
                         // Send the final content chunk with finish reason
-                        requestProgress.data = JSON.stringify(createChunk({ 
-                            content: part.text 
+                        requestProgress.data = JSON.stringify(createChunk({
+                            content: part.text
                         }, finishReason));
+                        sentFinalChunk = true;
                         break; // Only process the first text part
                     }
                 }
-            } else {
-                // No content, just send finish chunk
-                requestProgress.data = JSON.stringify(createChunk({}, finishReason));
+            }
+            if (!sentFinalChunk) {
+                // If we have tool calls, include them in the finish chunk
+                // (Gemini often sends functionCall and finishReason in the same event)
+                // Filter out undefined elements before mapping
+                const validToolCallsForChunk = this.toolCallsBuffer.filter(tc => tc && tc.function);
+                if (this.hadToolCalls && validToolCallsForChunk.length > 0) {
+                    requestProgress.data = JSON.stringify(createChunk({
+                        tool_calls: validToolCallsForChunk.map((tc, index) => ({
+                            index,
+                            id: tc.id,
+                            type: tc.type,
+                            function: tc.function
+                        }))
+                    }, finishReason));
+                } else {
+                    // No final text content, just send finish chunk
+                    requestProgress.data = JSON.stringify(createChunk({}, finishReason));
+                }
             }
 
             const pathwayResolver = requestState[this.requestId]?.pathwayResolver;
@@ -528,6 +559,8 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                     tool_calls: validToolCalls,
                 };
                 this.pathwayToolCallback(pathwayResolver?.args, toolMessage, pathwayResolver);
+                // Signal to pathwayResolver that stream close is expected (tool callback invoked)
+                requestProgress.toolCallbackInvoked = true;
                 // Clear tool buffer after processing; keep content for citations/continuations
                 this.toolCallsBuffer = [];
             } else {
@@ -539,6 +572,19 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
             }
         }
 
+        // Handle MALFORMED_FUNCTION_CALL - model tried to call a function but generated invalid JSON
+        if (eventData.candidates?.[0]?.finishReason === "MALFORMED_FUNCTION_CALL") {
+            logger.warn(`Gemini streaming returned MALFORMED_FUNCTION_CALL`);
+            requestProgress.data = JSON.stringify(createChunk({
+                content: '\n\nI encountered an issue processing that request. Please try rephrasing your question.'
+            }, "stop"));
+            requestProgress.progress = 1;
+            // Clear buffers
+            this.toolCallsBuffer = [];
+            this.contentBuffer = '';
+            return requestProgress;
+        }
+
         // Handle safety blocks
         if (eventData.candidates?.[0]?.safetyRatings?.some(rating => rating.blocked)) {
             requestProgress.data = JSON.stringify(createChunk({ 

package/server/plugins/grokResponsesPlugin.js

@@ -642,6 +642,8 @@ class GrokResponsesPlugin extends OpenAIVisionPlugin {
                                     tool_calls: validToolCalls,
                                 };
                                 this.pathwayToolCallback(pathwayResolver?.args, toolMessage, pathwayResolver);
+                                // Signal to pathwayResolver that tool callback was invoked - prevents [DONE] from ending stream
+                                requestProgress.toolCallbackInvoked = true;
                             }
                             this.toolCallsBuffer = [];
                             break;

package/server/plugins/openAiVisionPlugin.js

@@ -374,14 +374,16 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
                             const validToolCalls = this.toolCallsBuffer.filter(tc => tc && tc.function && tc.function.name);
                             const toolMessage = {
                                 role: 'assistant',
-                                content: delta?.content || '', 
+                                content: delta?.content || '',
                                 tool_calls: validToolCalls,
                             };
                             this.pathwayToolCallback(pathwayResolver?.args, toolMessage, pathwayResolver);
+                            // Signal to pathwayResolver that tool callback was invoked - prevents [DONE] from ending stream
+                            requestProgress.toolCallbackInvoked = true;
                         }
                         // Don't set progress to 1 for tool calls to keep stream open
                         // Clear tool buffer after processing, but keep content buffer
-                        this.toolCallsBuffer = []; 
+                        this.toolCallsBuffer = [];
                         break;
                     case 'safety':
                         const safetyRatings = JSON.stringify(parsedMessage?.candidates?.[0]?.safetyRatings) || '';