@llumiverse/drivers 0.23.0 → 0.24.0-dev.202601221707

package/src/vertexai/models/claude.ts

@@ -321,11 +321,17 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
             driver.logger.warn({ options: options.model_options }, "Invalid model options");
         }
 
-        const { payload, requestOptions } = getClaudePayload(options, prompt);
+        // Include conversation history (same as non-streaming)
+        const conversation = updateConversation(options.conversation as ClaudePrompt, prompt);
+
+        const { payload, requestOptions } = getClaudePayload(options, conversation);
         const streamingPayload: MessageStreamParams = { ...payload, stream: true };
 
         const response_stream = await client.messages.stream(streamingPayload, requestOptions);
 
+        // Track current tool use being built from streaming
+        let currentToolUse: { id: string; name: string; inputJson: string } | null = null;
+
         const stream = asyncMap(response_stream, async (streamEvent: RawMessageStreamEvent) => {
             switch (streamEvent.type) {
                 case "message_start":
@@ -345,6 +351,22 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
                         finish_reason: claudeFinishReason(streamEvent.delta.stop_reason ?? undefined),
                     } satisfies CompletionChunkObject;
                 case "content_block_start":
+                    // Handle tool_use blocks
+                    if (streamEvent.content_block.type === "tool_use") {
+                        currentToolUse = {
+                            id: streamEvent.content_block.id,
+                            name: streamEvent.content_block.name,
+                            inputJson: ''
+                        };
+                        return {
+                            result: [],
+                            tool_use: [{
+                                id: streamEvent.content_block.id,
+                                tool_name: streamEvent.content_block.name,
+                                tool_input: '' as any // Will be accumulated via input_json_delta
+                            }]
+                        } satisfies CompletionChunkObject;
+                    }
                     // Handle redacted thinking blocks
                     if (streamEvent.content_block.type === "redacted_thinking" && model_options?.include_thoughts) {
                         return {
@@ -359,6 +381,19 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
                             return {
                                 result: streamEvent.delta.text ? [{ type: "text", value: streamEvent.delta.text }] : []
                             } satisfies CompletionChunkObject;
+                        case "input_json_delta":
+                            // Accumulate tool input JSON
+                            if (currentToolUse && streamEvent.delta.partial_json) {
+                                return {
+                                    result: [],
+                                    tool_use: [{
+                                        id: currentToolUse.id,
+                                        tool_name: '', // Name already sent in content_block_start
+                                        tool_input: streamEvent.delta.partial_json as any
+                                    }]
+                                } satisfies CompletionChunkObject;
+                            }
+                            break;
                         case "thinking_delta":
                             if (model_options?.include_thoughts) {
                                 return {
@@ -377,6 +412,10 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
                     }
                     break;
                 case "content_block_stop":
+                    // Reset current tool use tracking when block ends
+                    if (currentToolUse) {
+                        currentToolUse = null;
+                    }
                     // Handle the end of content blocks, for redacted thinking blocks
                     if (model_options?.include_thoughts) {
                         return {
@@ -406,6 +445,60 @@ function createPromptFromResponse(response: Message): ClaudePrompt {
     }
 }
 
+/**
+ * Merge consecutive user messages in the conversation.
+ * This is required because Anthropic's API expects all tool_result blocks
+ * from a single assistant turn to be in one user message.
+ * When multiple tool results are added as separate user messages,
+ * we need to merge them before sending to the API.
+ */
+export function mergeConsecutiveUserMessages(messages: MessageParam[]): MessageParam[] {
+    if (messages.length === 0) return [];
+
+    // Check if any merging is needed
+    const needsMerging = messages.some((msg, i) =>
+        i < messages.length - 1 &&
+        msg.role === 'user' &&
+        messages[i + 1].role === 'user'
+    );
+
+    if (!needsMerging) {
+        return messages;
+    }
+
+    const result: MessageParam[] = [];
+    let i = 0;
+
+    while (i < messages.length) {
+        const current = messages[i];
+
+        if (current.role === 'user') {
+            // Collect all consecutive user messages
+            const mergedContent: MessageParam['content'] = [];
+
+            while (i < messages.length && messages[i].role === 'user') {
+                const userMsg = messages[i];
+                if (Array.isArray(userMsg.content)) {
+                    mergedContent.push(...userMsg.content);
+                } else if (typeof userMsg.content === 'string') {
+                    mergedContent.push({ type: 'text', text: userMsg.content });
+                }
+                i++;
+            }
+
+            result.push({
+                role: 'user',
+                content: mergedContent
+            });
+        } else {
+            result.push(current);
+            i++;
+        }
+    }
+
+    return result;
+}
+
 /**
  * Update the conversation messages
  * @param prompt
@@ -416,11 +509,143 @@ function updateConversation(conversation: ClaudePrompt | undefined | null, promp
     const baseSystemMessages = conversation?.system || [];
     const baseMessages = conversation?.messages || [];
     const system = baseSystemMessages.concat(prompt.system || []);
+    // Merge consecutive user messages to ensure tool_result blocks are properly grouped
+    const mergedMessages = mergeConsecutiveUserMessages(baseMessages.concat(prompt.messages || []));
     return {
-        messages: baseMessages.concat(prompt.messages || []),
+        messages: mergedMessages,
         system: system.length > 0 ? system : undefined // If system is empty, set to undefined
     };
 }
+
+/**
+ * Sanitize messages by removing empty text blocks.
+ * Claude API rejects messages with empty text content blocks ("text content blocks must be non-empty").
+ * This handles cases where streaming was interrupted and left empty text blocks.
+ *
+ * - Filters out empty text blocks from each message's content
+ * - Removes messages entirely if they have no content after filtering
+ */
+function sanitizeMessages(messages: MessageParam[]): MessageParam[] {
+    const result: MessageParam[] = [];
+
+    for (const message of messages) {
+        if (typeof message.content === 'string') {
+            // String content - keep only if non-empty
+            if (message.content.trim()) {
+                result.push(message);
+            }
+            continue;
+        }
+
+        // Array content - filter out empty text blocks
+        const filteredContent = message.content.filter(block => {
+            if (block.type === 'text') {
+                return block.text && block.text.trim().length > 0;
+            }
+            // Keep all non-text blocks (tool_use, tool_result, image, etc.)
+            return true;
+        });
+
+        // Only include message if it has content after filtering
+        if (filteredContent.length > 0) {
+            result.push({
+                ...message,
+                content: filteredContent
+            });
+        }
+    }
+
+    return result;
+}
+
+/**
+ * Fix orphaned tool_use blocks in the conversation.
+ * @exported for testing
+ *
+ * When an agent is stopped mid-tool-execution, the assistant message contains tool_use blocks
+ * but no corresponding tool_result was added. The Anthropic API requires that every tool_use
+ * must be followed by a tool_result in the next user message.
+ *
+ * This function detects such cases and injects synthetic tool_result blocks indicating
+ * the tools were interrupted, allowing the conversation to continue.
+ */
+export function fixOrphanedToolUse(messages: MessageParam[]): MessageParam[] {
+    if (messages.length < 2) return messages;
+
+    const result: MessageParam[] = [];
+
+    for (let i = 0; i < messages.length; i++) {
+        const current = messages[i];
+        result.push(current);
+
+        // Check if this is an assistant message with tool_use blocks
+        if (current.role === 'assistant' && Array.isArray(current.content)) {
+            const toolUseBlocks = current.content.filter(
+                (block): block is ContentBlockParam & { type: 'tool_use'; id: string; name: string } =>
+                    block.type === 'tool_use'
+            );
+
+            if (toolUseBlocks.length > 0) {
+                // Check if the next message is a user message with matching tool_results
+                const nextMessage = messages[i + 1];
+
+                if (nextMessage && nextMessage.role === 'user' && Array.isArray(nextMessage.content)) {
+                    // Get tool_result IDs from the next message
+                    const toolResultIds = new Set(
+                        nextMessage.content
+                            .filter((block): block is ToolResultBlockParam => block.type === 'tool_result')
+                            .map(block => block.tool_use_id)
+                    );
+
+                    // Find orphaned tool_use blocks (no matching tool_result)
+                    const orphanedToolUse = toolUseBlocks.filter(block => !toolResultIds.has(block.id));
+
+                    if (orphanedToolUse.length > 0) {
+                        // Inject synthetic tool_results for orphaned tool_use
+                        const syntheticResults: ToolResultBlockParam[] = orphanedToolUse.map(block => ({
+                            type: 'tool_result',
+                            tool_use_id: block.id,
+                            content: `[Tool interrupted: The user stopped the operation before "${block.name}" could execute.]`
+                        }));
+
+                        // Prepend synthetic results to the next user message
+                        const updatedNextMessage: MessageParam = {
+                            ...nextMessage,
+                            content: [...syntheticResults, ...nextMessage.content]
+                        };
+
+                        // Replace the next message in our iteration
+                        messages[i + 1] = updatedNextMessage;
+                    }
+                } else if (nextMessage && nextMessage.role === 'user') {
+                    // Next message is a user message but not array content (plain text)
+                    // We need to convert it and add tool_results
+                    const syntheticResults: ToolResultBlockParam[] = toolUseBlocks.map(block => ({
+                        type: 'tool_result',
+                        tool_use_id: block.id,
+                        content: `[Tool interrupted: The user stopped the operation before "${block.name}" could execute.]`
+                    }));
+
+                    const textContent: TextBlockParam = typeof nextMessage.content === 'string'
+                        ? { type: 'text', text: nextMessage.content }
+                        : { type: 'text', text: '' };
+
+                    const updatedNextMessage: MessageParam = {
+                        role: 'user',
+                        content: [...syntheticResults, textContent]
+                    };
+
+                    messages[i + 1] = updatedNextMessage;
+                }
+                // Note: If there's no nextMessage, we leave the conversation as-is.
+                // The tool_use blocks are expected to be there - the next turn will provide tool_results.
+            }
+        }
+    }
+
+    return result;
+}
+
 interface RequestOptions {
     headers?: Record<string, string>;
 }
@@ -440,8 +665,13 @@ function getClaudePayload(options: ExecutionOptions, prompt: ClaudePrompt): { pa
         };
     }
 
+    // Fix orphaned tool_use blocks (can occur when agent is stopped mid-tool-execution)
+    const fixedMessages = fixOrphanedToolUse(prompt.messages);
+    // Sanitize messages to remove empty text blocks (can occur from interrupted streaming)
+    const sanitizedMessages = sanitizeMessages(fixedMessages);
+
     const payload = {
-        messages: prompt.messages,
+        messages: sanitizedMessages,
         system: prompt.system,
         tools: options.tools, // we are using the same shape as claude for tools
         temperature: model_options?.temperature,

package/src/vertexai/models/gemini.ts

@@ -5,12 +5,20 @@ import {
 } from "@google/genai";
 import {
     AIModel, Completion, CompletionChunkObject, CompletionResult, ExecutionOptions,
-    ExecutionTokenUsage, getMaxTokensLimitVertexAi, JSONObject, JSONSchema, ModelType, PromptOptions, PromptRole,
-    PromptSegment, readStreamAsBase64, StatelessExecutionOptions, ToolDefinition, ToolUse,
+    ExecutionTokenUsage,
+    getConversationMeta,
+    getMaxTokensLimitVertexAi,
+    incrementConversationTurn,
+    JSONObject, JSONSchema, ModelType, PromptOptions, PromptRole,
+    PromptSegment, readStreamAsBase64, StatelessExecutionOptions,
+    stripBase64ImagesFromConversation,
+    ToolDefinition, ToolUse,
+    truncateLargeTextInConversation,
+    unwrapConversationArray,
     VertexAIGeminiOptions
 } from "@llumiverse/core";
 import { asyncMap } from "@llumiverse/core/async";
-import { VertexAIDriver, GenerateContentPrompt } from "../index.js";
+import { GenerateContentPrompt, VertexAIDriver } from "../index.js";
 import { ModelDefinition } from "../models.js";
 
 function supportsStructuredOutput(options: PromptOptions): boolean {
@@ -467,11 +475,17 @@ function collectToolUseParts(content: Content): ToolUse[] | undefined {
     const parts = content.parts ?? [];
     for (const part of parts) {
         if (part.functionCall) {
-            out.push({
+            const toolUse: ToolUse = {
                 id: part.functionCall.name ?? '',
                 tool_name: part.functionCall.name ?? '',
                 tool_input: part.functionCall.args as JSONObject,
-            });
+            };
+            // Capture thought_signature for Gemini thinking models (2.5+/3.0+)
+            // This must be passed back with the function response
+            if (part.thoughtSignature) {
+                toolUse.thought_signature = part.thoughtSignature;
+            }
+            out.push(toolUse);
         }
     }
     return out.length > 0 ? out : undefined;
@@ -545,7 +559,7 @@ function geminiThinkingConfig(option: StatelessExecutionOptions): ThinkingConfig
     const model_options = option.model_options as VertexAIGeminiOptions | undefined;
     const include_thoughts = model_options?.include_thoughts ?? false;
     if (model_options?.thinking_budget_tokens) {
-        return {includeThoughts: include_thoughts, thinkingBudget: model_options.thinking_budget_tokens};
+        return { includeThoughts: include_thoughts, thinkingBudget: model_options.thinking_budget_tokens };
     }
 
     // Set minimum thinking level by default.
@@ -623,16 +637,18 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
                 if (!msg.tool_use_id) {
                     throw new Error("Tool response missing tool_use_id");
                 }
+                // Build functionResponse part with optional thought_signature for Gemini thinking models
+                const functionResponsePart: Part = {
+                    functionResponse: {
+                        name: msg.tool_use_id,
+                        response: formatFunctionResponse(msg.content || ''),
+                    },
+                    // Include thought_signature if provided (required for Gemini 2.5+/3.0+ thinking models)
+                    thoughtSignature: msg.thought_signature,
+                };
                 contents.push({
                     role: 'user',
-                    parts: [
-                        {
-                            functionResponse: {
-                                name: msg.tool_use_id,
-                                response: formatFunctionResponse(msg.content || ''),
-                            }
-                        }
-                    ]
+                    parts: [functionResponsePart]
                 });
             } else {    // PromptRole.user, PromptRole.assistant, PromptRole.safety
                 const parts: Part[] = [];
@@ -646,14 +662,27 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
                 // File content handling
                 if (msg.files) {
                     for (const f of msg.files) {
-                        const stream = await f.getStream();
-                        const data = await readStreamAsBase64(stream);
-                        parts.push({
-                            inlineData: {
-                                data,
-                                mimeType: f.mime_type
-                            }
-                        });
+                        let fileUrl = await f.getURL();                     
+                        const isGsUrl = fileUrl.startsWith('gs://') || fileUrl.startsWith('https://storage.googleapis.com/');
+
+                        if (isGsUrl) {
+                            parts.push({
+                                fileData: {
+                                    fileUri: fileUrl,
+                                    mimeType: f.mime_type
+                                }
+                            });
+                        } else {
+                            // Inline data handling
+                            const stream = await f.getStream();
+                            const data = await readStreamAsBase64(stream); 
+                            parts.push({
+                                inlineData: {
+                                    data,
+                                    mimeType: f.mime_type
+                                }
+                            });
+                        }
                     }
                 }
 
@@ -742,7 +771,7 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
         const modelName = splits[splits.length - 1];
         options = { ...options, model: modelName };
 
-        let conversation = updateConversation(options.conversation as Content[], prompt.contents);
+        let conversation = updateConversation(options.conversation, prompt.contents);
         prompt.contents = conversation;
 
         // TODO: Remove hack, use global endpoint manually if needed.
@@ -792,12 +821,27 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
             finish_reason = "tool_use";
         }
 
+        // Increment turn counter for deferred stripping
+        conversation = incrementConversationTurn(conversation) as Content[];
+
+        // Strip large base64 image data based on options.stripImagesAfterTurns
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+
+        // Truncate large text content if configured
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
         return {
             result: result && result.length > 0 ? result : [{ type: "text" as const, value: '' }],
             token_usage: token_usage,
             finish_reason: finish_reason,
             original_response: options.include_original_response ? response : undefined,
-            conversation,
+            conversation: processedConversation,
             tool_use
         } satisfies Completion;
     }
@@ -811,6 +855,10 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
         const modelName = splits[splits.length - 1];
         options = { ...options, model: modelName };
 
+        // Include conversation history in prompt contents (same as non-streaming)
+        const conversation = updateConversation(options.conversation, prompt.contents);
+        prompt.contents = conversation;
+
         if (options.model.includes("gemini-2.5-flash-image")) {
             region = "global"; // Gemini Flash Image only available in global region, this is for nano-banana model
         }
@@ -897,16 +945,19 @@ function getToolFunction(tool: ToolDefinition): FunctionDeclaration {
     };
 }
 
-
 /**
  * Update the conversation messages
  * @param prompt
  * @param response
  * @returns
  */
-function updateConversation(conversation: Content[], prompt: Content[]): Content[] {
-    return (conversation || [] satisfies Content[]).concat(prompt);
+function updateConversation(conversation: unknown, prompt: Content[]): Content[] {
+    // Unwrap array if wrapped, otherwise treat as array
+    const unwrapped = unwrapConversationArray<Content>(conversation);
+    const convArray = unwrapped ?? (conversation as Content[] || []);
+    return convArray.concat(prompt);
 }
+
 /**
  *
  * Gemini supports JSON output in the response. so we test if the response is a valid JSON object. otherwise we treat the response as a string.

package/src/xai/index.ts

@@ -1,4 +1,4 @@
-import { AIModel, Completion, DriverOptions, ExecutionOptions, PromptOptions, PromptSegment } from "@llumiverse/core";
+import { AIModel, DriverOptions, PromptOptions, PromptSegment, Providers } from "@llumiverse/core";
 import { formatOpenAILikeMultimodalPrompt, OpenAIPromptFormatterOptions } from "../openai/openai_format.js";
 import { FetchClient } from "@vertesia/api-fetch-client";
 import OpenAI from "openai";
@@ -15,7 +15,7 @@ export interface xAiDriverOptions extends DriverOptions {
 export class xAIDriver extends BaseOpenAIDriver {
 
     service: OpenAI;
-    provider: "xai";
+    readonly provider = Providers.xai;
     xai_service: FetchClient;
     DEFAULT_ENDPOINT = "https://api.x.ai/v1";
 
@@ -31,7 +31,6 @@ export class xAIDriver extends BaseOpenAIDriver {
             baseURL: opts.endpoint ?? this.DEFAULT_ENDPOINT,
         });
         this.xai_service = new FetchClient(opts.endpoint ?? this.DEFAULT_ENDPOINT).withAuthCallback(async () => `Bearer ${opts.apiKey}`);
-        this.provider = "xai";
         //this.formatPrompt = this._formatPrompt; //TODO: fix xai prompt formatting
     }
 
@@ -49,17 +48,9 @@ export class xAIDriver extends BaseOpenAIDriver {
 
     }
 
-    extractDataFromResponse(_options: ExecutionOptions, result: OpenAI.Chat.Completions.ChatCompletion): Completion {
-        return {
-            result: result.choices[0].message.content ? [{ type: "text", value: result.choices[0].message.content }] : [],
-            finish_reason: result.choices[0].finish_reason,
-            token_usage: {
-                prompt: result.usage?.prompt_tokens,
-                result: result.usage?.completion_tokens,
-                total: result.usage?.total_tokens,
-            }
-        }
-    }
+    // Note: We intentionally do NOT override extractDataFromResponse here.
+    // The base class implementation properly handles tool_calls extraction.
+    // xAI's API is OpenAI-compatible and returns tool_calls in the same format.
 
     async listModels(): Promise<AIModel[]> {
         const [lm, em] = await Promise.all([
@@ -76,10 +67,12 @@ export class xAIDriver extends BaseOpenAIDriver {
             return {
                 id: model.id,
                 provider: this.provider,
-                name: model.object,
-                description: model.object,
+                name: model.id,
+                description: `${model.id} by ${model.owned_by}`,
                 is_multimodal: model.input_modalities.length > 1,
-                tags: [...model.input_modalities.map(m => `ì:${m}`), ...model.output_modalities.map(m => `ì:${m}`)],
+                input_modalities: model.input_modalities,
+                output_modalities: model.output_modalities,
+                tags: [...model.input_modalities.map(m => `i:${m}`), ...model.output_modalities.map(m => `o:${m}`)],
             } satisfies AIModel;
         });