@llumiverse/drivers 0.23.0 → 0.24.0-dev.20260203.164053Z

package/src/openai/openai_compatible.ts

@@ -0,0 +1,74 @@
+import { AIModel, DriverOptions, ModelType, Providers, getModelCapabilities, modelModalitiesToArray } from "@llumiverse/core";
+import OpenAI from "openai";
+import { BaseOpenAIDriver } from "./index.js";
+
+export interface OpenAICompatibleDriverOptions extends DriverOptions {
+    /**
+     * The API key for the OpenAI-compatible service
+     */
+    apiKey: string;
+
+    /**
+     * The base URL of the OpenAI-compatible API endpoint
+     * Example: https://api.example.com/v1
+     */
+    endpoint: string;
+}
+
+/**
+ * A generic driver for OpenAI-compatible APIs.
+ * This can be used with any service that implements the OpenAI API spec,
+ * such as xAI (Grok), LM Studio, Ollama, vLLM, LocalAI, etc.
+ */
+export class OpenAICompatibleDriver extends BaseOpenAIDriver {
+    service: OpenAI;
+    readonly provider = Providers.openai_compatible;
+
+    constructor(opts: OpenAICompatibleDriverOptions) {
+        super(opts);
+
+        if (!opts.apiKey) {
+            throw new Error("apiKey is required");
+        }
+
+        if (!opts.endpoint) {
+            throw new Error("endpoint is required for OpenAI-compatible driver");
+        }
+
+        this.service = new OpenAI({
+            apiKey: opts.apiKey,
+            baseURL: opts.endpoint,
+        });
+    }
+
+    async listModels(): Promise<AIModel[]> {
+        try {
+            const result = (await this.service.models.list()).data;
+
+            const models = result.map((m) => {
+                const modelCapability = getModelCapabilities(m.id, "openai");
+                let owner = m.owned_by;
+                if (owner === "system") {
+                    owner = "unknown";
+                }
+                return {
+                    id: m.id,
+                    name: m.id,
+                    provider: this.provider,
+                    owner: owner,
+                    type: ModelType.Text,
+                    can_stream: true,
+                    is_multimodal: false,
+                    input_modalities: modelModalitiesToArray(modelCapability.input),
+                    output_modalities: modelModalitiesToArray(modelCapability.output),
+                    tool_support: modelCapability.tool_support,
+                } satisfies AIModel<string>;
+            }).sort((a, b) => a.id.localeCompare(b.id));
+
+            return models;
+        } catch (error) {
+            this.logger.warn({ error }, "[OpenAICompatible] Failed to list models, returning empty list");
+            return [];
+        }
+    }
+}

package/src/openai/openai_format.ts

@@ -3,16 +3,12 @@
 
 import { PromptRole, PromptOptions, PromptSegment } from "@llumiverse/common";
 import { readStreamAsBase64 } from "@llumiverse/core";
+import type OpenAI from "openai";
 
-import type {
-    ChatCompletionMessageParam,
-    ChatCompletionContentPartText,
-    ChatCompletionContentPartImage,
-    ChatCompletionUserMessageParam,
-    ChatCompletionSystemMessageParam,
-    ChatCompletionAssistantMessageParam,
-    ChatCompletionToolMessageParam
-} from 'openai/resources/chat/completions';
+// Types for Response API
+type ResponseInputItem = OpenAI.Responses.ResponseInputItem;
+type ResponseInputContent = OpenAI.Responses.ResponseInputContent;
+type EasyInputMessage = OpenAI.Responses.EasyInputMessage;
 
 export interface OpenAITextMessage {
     content: string;
@@ -47,14 +43,14 @@ export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITex
 }
 
 
-export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[], opts: PromptOptions & OpenAIPromptFormatterOptions): Promise<ChatCompletionMessageParam[]> {
-    const system: ChatCompletionMessageParam[] = [];
-    const safety: ChatCompletionMessageParam[] = [];
-    const others: ChatCompletionMessageParam[] = [];
+export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[], opts: PromptOptions & OpenAIPromptFormatterOptions): Promise<ResponseInputItem[]> {
+    const system: ResponseInputItem[] = [];
+    const safety: ResponseInputItem[] = [];
+    const others: ResponseInputItem[] = [];
 
     for (const msg of segments) {
 
-        const parts: (ChatCompletionContentPartImage | ChatCompletionContentPartText)[] = [];
+        const parts: ResponseInputContent[] = [];
 
         //generate the parts based on PromptSegment
         if (msg.files) {
@@ -62,54 +58,56 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
                 const stream = await file.getStream();
                 const data = await readStreamAsBase64(stream);
                 parts.push({
-                    type: "image_url",
-                    image_url: {
-                        url: `data:${file.mime_type || "image/jpeg"};base64,${data}`,
-                        //detail: "auto"  //This is modified just before execution to "low" | "high" | "auto"
-                    },
+                    type: "input_image",
+                    image_url: `data:${file.mime_type || "image/jpeg"};base64,${data}`,
+                    detail: "auto",
                 })
             }
         }
 
         if (msg.content) {
             parts.push({
+                type: "input_text",
                 text: msg.content,
-                type: "text"
             })
         }
 
 
         if (msg.role === PromptRole.system) {
             // For system messages, filter to only text parts
-            const textParts = parts.filter((part): part is ChatCompletionContentPartText => part.type === 'text');
-            const systemMsg: ChatCompletionSystemMessageParam = {
+            const textParts = parts.filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text');
+            const textContent = textParts.length === 1 && !msg.files ? textParts[0].text : textParts;
+            const systemMsg: EasyInputMessage = {
                 role: "system",
-                content: textParts.length === 1 && !msg.files ? textParts[0].text : textParts
+                content: textContent,
             };
             system.push(systemMsg);
 
             if (opts.useToolForFormatting && opts.schema) {
                 system.forEach(s => {
-                    if (typeof s.content === 'string') {
-                        s.content = "TOOL: " + s.content;
-                    } else if (Array.isArray(s.content)) {
-                        s.content.forEach((c: any) => {
-                            if (c.type === "text") c.text = "TOOL: " + c.text;
-                        });
+                    if ((s as EasyInputMessage).role === 'system') {
+                        const sysMsg = s as EasyInputMessage;
+                        if (typeof sysMsg.content === 'string') {
+                            sysMsg.content = "TOOL: " + sysMsg.content;
+                        } else if (Array.isArray(sysMsg.content)) {
+                            sysMsg.content.forEach((c: any) => {
+                                if (c.type === "input_text") c.text = "TOOL: " + c.text;
+                            });
+                        }
                     }
                 });
             }
 
         } else if (msg.role === PromptRole.safety) {
-            const textParts = parts.filter((part): part is ChatCompletionContentPartText => part.type === 'text');
-            const safetyMsg: ChatCompletionSystemMessageParam = {
+            const textParts = parts.filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text');
+            const safetyMsg: EasyInputMessage = {
                 role: "system",
-                content: textParts
+                content: textParts,
             };
 
             if (Array.isArray(safetyMsg.content)) {
                 safetyMsg.content.forEach((c: any) => {
-                    if (c.type === "text") c.text = "DO NOT IGNORE - IMPORTANT: " + c.text;
+                    if (c.type === "input_text") c.text = "DO NOT IGNORE - IMPORTANT: " + c.text;
                 });
             }
 
@@ -118,35 +116,27 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
             if (!msg.tool_use_id) {
                 throw new Error("Tool use id is required for tool messages")
             }
-            const toolMsg: ChatCompletionToolMessageParam = {
-                role: "tool",
-                tool_call_id: msg.tool_use_id,
-                content: msg.content || ""
+            const toolOutputMsg: OpenAI.Responses.ResponseInputItem.FunctionCallOutput = {
+                type: "function_call_output",
+                call_id: msg.tool_use_id,
+                output: msg.content || ""
             };
-            others.push(toolMsg);
+            others.push(toolOutputMsg);
         } else if (msg.role !== PromptRole.negative && msg.role !== PromptRole.mask) {
-            if (msg.role === 'assistant') {
-                const assistantMsg: ChatCompletionAssistantMessageParam = {
-                    role: 'assistant',
-                    content: parts as (ChatCompletionContentPartText)[]
-                };
-                others.push(assistantMsg);
-            } else {
-                const userMsg: ChatCompletionUserMessageParam = {
-                    role: 'user',
-                    content: parts
-                };
-                others.push(userMsg);
-            }
+            const inputMsg: EasyInputMessage = {
+                role: msg.role === 'assistant' ? 'assistant' : 'user',
+                content: parts,
+            };
+            others.push(inputMsg);
         }
 
     }
 
     if (opts.result_schema && !opts.useToolForFormatting) {
-        const schemaMsg: ChatCompletionSystemMessageParam = {
+        const schemaMsg: EasyInputMessage = {
             role: "system",
             content: [{
-                type: "text",
+                type: "input_text",
                 text: "IMPORTANT: only answer using JSON, and respecting the schema included below, between the <response_schema> tags. " + `<response_schema>${JSON.stringify(opts.result_schema)}</response_schema>`
             }]
         };
@@ -154,7 +144,7 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
     }
 
     // put system messages first and safety last
-    return ([] as ChatCompletionMessageParam[]).concat(system).concat(others).concat(safety);
+    return ([] as ResponseInputItem[]).concat(system).concat(others).concat(safety);
 
 }
 

package/src/vertexai/index.ts

@@ -6,6 +6,7 @@ import {
     AbstractDriver,
     Completion,
     CompletionChunkObject,
+    CompletionResult,
     DriverOptions,
     EmbeddingsOptions,
     EmbeddingsResult,
@@ -14,6 +15,11 @@ import {
     PromptSegment,
     getModelCapabilities,
     modelModalitiesToArray,
+    stripBase64ImagesFromConversation,
+    truncateLargeTextInConversation,
+    getConversationMeta,
+    incrementConversationTurn,
+    unwrapConversationArray,
 } from "@llumiverse/core";
 import { FetchClient } from "@vertesia/api-fetch-client";
 import { GoogleAuth, GoogleAuthOptions, AuthClient } from "google-auth-library";
@@ -251,6 +257,186 @@ export class VertexAIDriver extends AbstractDriver<VertexAIDriverOptions, Vertex
         return getModelDefinition(options.model).requestTextCompletionStream(this, prompt, options);
     }
 
+    /**
+     * Build conversation context after streaming completion.
+     * Reconstructs the assistant message from accumulated results and applies stripping.
+     * Handles both Gemini (Content[]) and Claude (ClaudePrompt) formats.
+     */
+    buildStreamingConversation(
+        prompt: VertexAIPrompt,
+        result: unknown[],
+        toolUse: unknown[] | undefined,
+        options: ExecutionOptions
+    ): Content[] | unknown | undefined {
+        // Handle Claude-style prompts (has 'messages' array)
+        if ('messages' in prompt && Array.isArray((prompt as any).messages)) {
+            return this.buildClaudeStreamingConversation(prompt as any, result, toolUse, options);
+        }
+
+        // Only handle Gemini-style prompts with contents array
+        if (!('contents' in prompt) || !Array.isArray(prompt.contents)) {
+            return undefined;
+        }
+
+        const completionResults = result as CompletionResult[];
+
+        // Convert accumulated results to text content for assistant message
+        const textContent = completionResults
+            .map(r => {
+                switch (r.type) {
+                    case 'text':
+                        return r.value;
+                    case 'json':
+                        return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+                    case 'image':
+                        // Skip images in conversation - they're in the result
+                        return '';
+                    default:
+                        return String((r as any).value || '');
+                }
+            })
+            .join('');
+
+        // Build parts array for assistant message
+        const parts: any[] = [];
+        if (textContent) {
+            parts.push({ text: textContent });
+        }
+        // Add function calls if present (Gemini format)
+        if (toolUse && toolUse.length > 0) {
+            for (const tool of toolUse as any[]) {
+                parts.push({
+                    functionCall: {
+                        name: tool.tool_name,
+                        args: tool.tool_input,
+                    }
+                });
+            }
+        }
+
+        // Build assistant message in Gemini Content format
+        const assistantContent: Content = {
+            role: 'model',
+            parts: parts.length > 0 ? parts : [{ text: '' }]
+        };
+
+        // Unwrap array if wrapped, otherwise treat as array
+        const unwrapped = unwrapConversationArray<Content>(options.conversation);
+        const existingConversation = unwrapped ?? (options.conversation as Content[] || []);
+
+        // Combine existing conversation + prompt contents + assistant response
+        let conversation: Content[] = [
+            ...existingConversation,
+            ...prompt.contents,
+            assistantContent
+        ];
+
+        // Increment turn counter
+        conversation = incrementConversationTurn(conversation) as Content[];
+
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
+        return processedConversation as Content[];
+    }
+
+    /**
+     * Build conversation for Claude streaming.
+     * Creates assistant message with tool_use blocks in Claude's ContentBlock format.
+     */
+    private buildClaudeStreamingConversation(
+        prompt: { messages: unknown[]; system?: unknown[] },
+        result: unknown[],
+        toolUse: unknown[] | undefined,
+        options: ExecutionOptions
+    ): unknown {
+        const completionResults = result as CompletionResult[];
+
+        // Convert accumulated results to text content
+        const textContent = completionResults
+            .map(r => {
+                switch (r.type) {
+                    case 'text':
+                        return r.value;
+                    case 'json':
+                        return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+                    case 'image':
+                        return '';
+                    default:
+                        return String((r as any).value || '');
+                }
+            })
+            .join('');
+
+        // Build Claude-style ContentBlock array for assistant message
+        const content: unknown[] = [];
+
+        // Add text block if there's text content
+        if (textContent) {
+            content.push({
+                type: 'text',
+                text: textContent
+            });
+        }
+
+        // Add tool_use blocks in Claude format
+        if (toolUse && toolUse.length > 0) {
+            for (const tool of toolUse as any[]) {
+                content.push({
+                    type: 'tool_use',
+                    id: tool.id,
+                    name: tool.tool_name,
+                    input: tool.tool_input ?? {}
+                });
+            }
+        }
+
+        // Build assistant message
+        const assistantMessage = {
+            role: 'assistant',
+            content: content.length > 0 ? content : [{ type: 'text', text: '' }]
+        };
+
+        // Get existing conversation or start fresh
+        const existingMessages = (options.conversation as any)?.messages ?? [];
+        const existingSystem = (options.conversation as any)?.system ?? prompt.system;
+
+        // Combine: existing conversation + new prompt messages + assistant response
+        const newMessages = [
+            ...existingMessages,
+            ...prompt.messages,
+            assistantMessage
+        ];
+
+        // Build the new conversation in ClaudePrompt format
+        const conversation = {
+            messages: newMessages,
+            system: existingSystem
+        };
+
+        // Increment turn counter
+        const withTurn = incrementConversationTurn(conversation);
+
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(withTurn).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(withTurn, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
+        return processedConversation;
+    }
+
     async requestImageGeneration(
         _prompt: ImagenPrompt,
         _options: ExecutionOptions,

package/src/vertexai/models/claude.ts

@@ -321,11 +321,17 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
             driver.logger.warn({ options: options.model_options }, "Invalid model options");
         }
 
-        const { payload, requestOptions } = getClaudePayload(options, prompt);
+        // Include conversation history (same as non-streaming)
+        const conversation = updateConversation(options.conversation as ClaudePrompt, prompt);
+
+        const { payload, requestOptions } = getClaudePayload(options, conversation);
         const streamingPayload: MessageStreamParams = { ...payload, stream: true };
 
         const response_stream = await client.messages.stream(streamingPayload, requestOptions);
 
+        // Track current tool use being built from streaming
+        let currentToolUse: { id: string; name: string; inputJson: string } | null = null;
+
         const stream = asyncMap(response_stream, async (streamEvent: RawMessageStreamEvent) => {
             switch (streamEvent.type) {
                 case "message_start":
@@ -345,6 +351,22 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
                         finish_reason: claudeFinishReason(streamEvent.delta.stop_reason ?? undefined),
                     } satisfies CompletionChunkObject;
                 case "content_block_start":
+                    // Handle tool_use blocks
+                    if (streamEvent.content_block.type === "tool_use") {
+                        currentToolUse = {
+                            id: streamEvent.content_block.id,
+                            name: streamEvent.content_block.name,
+                            inputJson: ''
+                        };
+                        return {
+                            result: [],
+                            tool_use: [{
+                                id: streamEvent.content_block.id,
+                                tool_name: streamEvent.content_block.name,
+                                tool_input: '' as any // Will be accumulated via input_json_delta
+                            }]
+                        } satisfies CompletionChunkObject;
+                    }
                     // Handle redacted thinking blocks
                     if (streamEvent.content_block.type === "redacted_thinking" && model_options?.include_thoughts) {
                         return {
@@ -359,6 +381,19 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
                             return {
                                 result: streamEvent.delta.text ? [{ type: "text", value: streamEvent.delta.text }] : []
                             } satisfies CompletionChunkObject;
+                        case "input_json_delta":
+                            // Accumulate tool input JSON
+                            if (currentToolUse && streamEvent.delta.partial_json) {
+                                return {
+                                    result: [],
+                                    tool_use: [{
+                                        id: currentToolUse.id,
+                                        tool_name: '', // Name already sent in content_block_start
+                                        tool_input: streamEvent.delta.partial_json as any
+                                    }]
+                                } satisfies CompletionChunkObject;
+                            }
+                            break;
                         case "thinking_delta":
                             if (model_options?.include_thoughts) {
                                 return {
@@ -377,6 +412,10 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
                     }
                     break;
                 case "content_block_stop":
+                    // Reset current tool use tracking when block ends
+                    if (currentToolUse) {
+                        currentToolUse = null;
+                    }
                     // Handle the end of content blocks, for redacted thinking blocks
                     if (model_options?.include_thoughts) {
                         return {
@@ -406,6 +445,60 @@ function createPromptFromResponse(response: Message): ClaudePrompt {
     }
 }
 
+/**
+ * Merge consecutive user messages in the conversation.
+ * This is required because Anthropic's API expects all tool_result blocks
+ * from a single assistant turn to be in one user message.
+ * When multiple tool results are added as separate user messages,
+ * we need to merge them before sending to the API.
+ */
+export function mergeConsecutiveUserMessages(messages: MessageParam[]): MessageParam[] {
+    if (messages.length === 0) return [];
+
+    // Check if any merging is needed
+    const needsMerging = messages.some((msg, i) =>
+        i < messages.length - 1 &&
+        msg.role === 'user' &&
+        messages[i + 1].role === 'user'
+    );
+
+    if (!needsMerging) {
+        return messages;
+    }
+
+    const result: MessageParam[] = [];
+    let i = 0;
+
+    while (i < messages.length) {
+        const current = messages[i];
+
+        if (current.role === 'user') {
+            // Collect all consecutive user messages
+            const mergedContent: MessageParam['content'] = [];
+
+            while (i < messages.length && messages[i].role === 'user') {
+                const userMsg = messages[i];
+                if (Array.isArray(userMsg.content)) {
+                    mergedContent.push(...userMsg.content);
+                } else if (typeof userMsg.content === 'string') {
+                    mergedContent.push({ type: 'text', text: userMsg.content });
+                }
+                i++;
+            }
+
+            result.push({
+                role: 'user',
+                content: mergedContent
+            });
+        } else {
+            result.push(current);
+            i++;
+        }
+    }
+
+    return result;
+}
+
 /**
  * Update the conversation messages
  * @param prompt
@@ -416,8 +509,10 @@ function updateConversation(conversation: ClaudePrompt | undefined | null, promp
     const baseSystemMessages = conversation?.system || [];
     const baseMessages = conversation?.messages || [];
     const system = baseSystemMessages.concat(prompt.system || []);
+    // Merge consecutive user messages to ensure tool_result blocks are properly grouped
+    const mergedMessages = mergeConsecutiveUserMessages(baseMessages.concat(prompt.messages || []));
     return {
-        messages: baseMessages.concat(prompt.messages || []),
+        messages: mergedMessages,
         system: system.length > 0 ? system : undefined // If system is empty, set to undefined
     };
 }