@llumiverse/drivers 0.23.0 → 0.24.0-dev.202601221707

package/src/openai/openai_compatible.ts

@@ -0,0 +1,74 @@
+import { AIModel, DriverOptions, ModelType, Providers, getModelCapabilities, modelModalitiesToArray } from "@llumiverse/core";
+import OpenAI from "openai";
+import { BaseOpenAIDriver } from "./index.js";
+
+export interface OpenAICompatibleDriverOptions extends DriverOptions {
+    /**
+     * The API key for the OpenAI-compatible service
+     */
+    apiKey: string;
+
+    /**
+     * The base URL of the OpenAI-compatible API endpoint
+     * Example: https://api.example.com/v1
+     */
+    endpoint: string;
+}
+
+/**
+ * A generic driver for OpenAI-compatible APIs.
+ * This can be used with any service that implements the OpenAI API spec,
+ * such as xAI (Grok), LM Studio, Ollama, vLLM, LocalAI, etc.
+ */
+export class OpenAICompatibleDriver extends BaseOpenAIDriver {
+    service: OpenAI;
+    readonly provider = Providers.openai_compatible;
+
+    constructor(opts: OpenAICompatibleDriverOptions) {
+        super(opts);
+
+        if (!opts.apiKey) {
+            throw new Error("apiKey is required");
+        }
+
+        if (!opts.endpoint) {
+            throw new Error("endpoint is required for OpenAI-compatible driver");
+        }
+
+        this.service = new OpenAI({
+            apiKey: opts.apiKey,
+            baseURL: opts.endpoint,
+        });
+    }
+
+    async listModels(): Promise<AIModel[]> {
+        try {
+            const result = (await this.service.models.list()).data;
+
+            const models = result.map((m) => {
+                const modelCapability = getModelCapabilities(m.id, "openai");
+                let owner = m.owned_by;
+                if (owner === "system") {
+                    owner = "unknown";
+                }
+                return {
+                    id: m.id,
+                    name: m.id,
+                    provider: this.provider,
+                    owner: owner,
+                    type: ModelType.Text,
+                    can_stream: true,
+                    is_multimodal: false,
+                    input_modalities: modelModalitiesToArray(modelCapability.input),
+                    output_modalities: modelModalitiesToArray(modelCapability.output),
+                    tool_support: modelCapability.tool_support,
+                } satisfies AIModel<string>;
+            }).sort((a, b) => a.id.localeCompare(b.id));
+
+            return models;
+        } catch (error) {
+            this.logger.warn({ error }, "[OpenAICompatible] Failed to list models, returning empty list");
+            return [];
+        }
+    }
+}

package/src/openai/openai_format.ts

@@ -3,16 +3,12 @@
 
 import { PromptRole, PromptOptions, PromptSegment } from "@llumiverse/common";
 import { readStreamAsBase64 } from "@llumiverse/core";
+import type OpenAI from "openai";
 
-import type {
-    ChatCompletionMessageParam,
-    ChatCompletionContentPartText,
-    ChatCompletionContentPartImage,
-    ChatCompletionUserMessageParam,
-    ChatCompletionSystemMessageParam,
-    ChatCompletionAssistantMessageParam,
-    ChatCompletionToolMessageParam
-} from 'openai/resources/chat/completions';
+// Types for Response API
+type ResponseInputItem = OpenAI.Responses.ResponseInputItem;
+type ResponseInputContent = OpenAI.Responses.ResponseInputContent;
+type EasyInputMessage = OpenAI.Responses.EasyInputMessage;
 
 export interface OpenAITextMessage {
     content: string;
@@ -47,14 +43,14 @@ export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITex
 }
 
 
-export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[], opts: PromptOptions & OpenAIPromptFormatterOptions): Promise<ChatCompletionMessageParam[]> {
-    const system: ChatCompletionMessageParam[] = [];
-    const safety: ChatCompletionMessageParam[] = [];
-    const others: ChatCompletionMessageParam[] = [];
+export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[], opts: PromptOptions & OpenAIPromptFormatterOptions): Promise<ResponseInputItem[]> {
+    const system: ResponseInputItem[] = [];
+    const safety: ResponseInputItem[] = [];
+    const others: ResponseInputItem[] = [];
 
     for (const msg of segments) {
 
-        const parts: (ChatCompletionContentPartImage | ChatCompletionContentPartText)[] = [];
+        const parts: ResponseInputContent[] = [];
 
         //generate the parts based on PromptSegment
         if (msg.files) {
@@ -62,54 +58,56 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
                 const stream = await file.getStream();
                 const data = await readStreamAsBase64(stream);
                 parts.push({
-                    type: "image_url",
-                    image_url: {
-                        url: `data:${file.mime_type || "image/jpeg"};base64,${data}`,
-                        //detail: "auto"  //This is modified just before execution to "low" | "high" | "auto"
-                    },
+                    type: "input_image",
+                    image_url: `data:${file.mime_type || "image/jpeg"};base64,${data}`,
+                    detail: "auto",
                 })
             }
         }
 
         if (msg.content) {
             parts.push({
+                type: "input_text",
                 text: msg.content,
-                type: "text"
             })
         }
 
 
         if (msg.role === PromptRole.system) {
             // For system messages, filter to only text parts
-            const textParts = parts.filter((part): part is ChatCompletionContentPartText => part.type === 'text');
-            const systemMsg: ChatCompletionSystemMessageParam = {
+            const textParts = parts.filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text');
+            const textContent = textParts.length === 1 && !msg.files ? textParts[0].text : textParts;
+            const systemMsg: EasyInputMessage = {
                 role: "system",
-                content: textParts.length === 1 && !msg.files ? textParts[0].text : textParts
+                content: textContent,
             };
             system.push(systemMsg);
 
             if (opts.useToolForFormatting && opts.schema) {
                 system.forEach(s => {
-                    if (typeof s.content === 'string') {
-                        s.content = "TOOL: " + s.content;
-                    } else if (Array.isArray(s.content)) {
-                        s.content.forEach((c: any) => {
-                            if (c.type === "text") c.text = "TOOL: " + c.text;
-                        });
+                    if ((s as EasyInputMessage).role === 'system') {
+                        const sysMsg = s as EasyInputMessage;
+                        if (typeof sysMsg.content === 'string') {
+                            sysMsg.content = "TOOL: " + sysMsg.content;
+                        } else if (Array.isArray(sysMsg.content)) {
+                            sysMsg.content.forEach((c: any) => {
+                                if (c.type === "input_text") c.text = "TOOL: " + c.text;
+                            });
+                        }
                     }
                 });
             }
 
         } else if (msg.role === PromptRole.safety) {
-            const textParts = parts.filter((part): part is ChatCompletionContentPartText => part.type === 'text');
-            const safetyMsg: ChatCompletionSystemMessageParam = {
+            const textParts = parts.filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text');
+            const safetyMsg: EasyInputMessage = {
                 role: "system",
-                content: textParts
+                content: textParts,
             };
 
             if (Array.isArray(safetyMsg.content)) {
                 safetyMsg.content.forEach((c: any) => {
-                    if (c.type === "text") c.text = "DO NOT IGNORE - IMPORTANT: " + c.text;
+                    if (c.type === "input_text") c.text = "DO NOT IGNORE - IMPORTANT: " + c.text;
                 });
             }
 
@@ -118,35 +116,27 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
             if (!msg.tool_use_id) {
                 throw new Error("Tool use id is required for tool messages")
             }
-            const toolMsg: ChatCompletionToolMessageParam = {
-                role: "tool",
-                tool_call_id: msg.tool_use_id,
-                content: msg.content || ""
+            const toolOutputMsg: OpenAI.Responses.ResponseInputItem.FunctionCallOutput = {
+                type: "function_call_output",
+                call_id: msg.tool_use_id,
+                output: msg.content || ""
             };
-            others.push(toolMsg);
+            others.push(toolOutputMsg);
         } else if (msg.role !== PromptRole.negative && msg.role !== PromptRole.mask) {
-            if (msg.role === 'assistant') {
-                const assistantMsg: ChatCompletionAssistantMessageParam = {
-                    role: 'assistant',
-                    content: parts as (ChatCompletionContentPartText)[]
-                };
-                others.push(assistantMsg);
-            } else {
-                const userMsg: ChatCompletionUserMessageParam = {
-                    role: 'user',
-                    content: parts
-                };
-                others.push(userMsg);
-            }
+            const inputMsg: EasyInputMessage = {
+                role: msg.role === 'assistant' ? 'assistant' : 'user',
+                content: parts,
+            };
+            others.push(inputMsg);
         }
 
     }
 
     if (opts.result_schema && !opts.useToolForFormatting) {
-        const schemaMsg: ChatCompletionSystemMessageParam = {
+        const schemaMsg: EasyInputMessage = {
             role: "system",
             content: [{
-                type: "text",
+                type: "input_text",
                 text: "IMPORTANT: only answer using JSON, and respecting the schema included below, between the <response_schema> tags. " + `<response_schema>${JSON.stringify(opts.result_schema)}</response_schema>`
             }]
         };
@@ -154,7 +144,7 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
     }
 
     // put system messages first and safety last
-    return ([] as ChatCompletionMessageParam[]).concat(system).concat(others).concat(safety);
+    return ([] as ResponseInputItem[]).concat(system).concat(others).concat(safety);
 
 }
 

package/src/vertexai/index.ts

@@ -6,6 +6,7 @@ import {
     AbstractDriver,
     Completion,
     CompletionChunkObject,
+    CompletionResult,
     DriverOptions,
     EmbeddingsOptions,
     EmbeddingsResult,
@@ -14,6 +15,11 @@ import {
     PromptSegment,
     getModelCapabilities,
     modelModalitiesToArray,
+    stripBase64ImagesFromConversation,
+    truncateLargeTextInConversation,
+    getConversationMeta,
+    incrementConversationTurn,
+    unwrapConversationArray,
 } from "@llumiverse/core";
 import { FetchClient } from "@vertesia/api-fetch-client";
 import { GoogleAuth, GoogleAuthOptions, AuthClient } from "google-auth-library";
@@ -251,6 +257,196 @@ export class VertexAIDriver extends AbstractDriver<VertexAIDriverOptions, Vertex
         return getModelDefinition(options.model).requestTextCompletionStream(this, prompt, options);
     }
 
+    /**
+     * Build conversation context after streaming completion.
+     * Reconstructs the assistant message from accumulated results and applies stripping.
+     * Handles both Gemini (Content[]) and Claude (ClaudePrompt) formats.
+     */
+    buildStreamingConversation(
+        prompt: VertexAIPrompt,
+        result: unknown[],
+        toolUse: unknown[] | undefined,
+        options: ExecutionOptions
+    ): Content[] | unknown | undefined {
+        // Handle Claude-style prompts (has 'messages' array)
+        if ('messages' in prompt && Array.isArray((prompt as any).messages)) {
+            return this.buildClaudeStreamingConversation(prompt as any, result, toolUse, options);
+        }
+
+        // Only handle Gemini-style prompts with contents array
+        if (!('contents' in prompt) || !Array.isArray(prompt.contents)) {
+            return undefined;
+        }
+
+        const completionResults = result as CompletionResult[];
+
+        // Convert accumulated results to text content for assistant message
+        const textContent = completionResults
+            .map(r => {
+                switch (r.type) {
+                    case 'text':
+                        return r.value;
+                    case 'json':
+                        return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+                    case 'image':
+                        // Skip images in conversation - they're in the result
+                        return '';
+                    default:
+                        return String((r as any).value || '');
+                }
+            })
+            .join('');
+
+        // Build parts array for assistant message
+        const parts: any[] = [];
+        if (textContent) {
+            parts.push({ text: textContent });
+        }
+        // Add function calls if present (Gemini format)
+        if (toolUse && toolUse.length > 0) {
+            for (const tool of toolUse as any[]) {
+                const functionCallPart: any = {
+                    functionCall: {
+                        name: tool.tool_name,
+                        args: tool.tool_input,
+                    }
+                };
+                // Include thought_signature for Gemini thinking models (2.5+/3.0+)
+                // This must be preserved in the conversation for subsequent API calls
+                if (tool.thought_signature) {
+                    functionCallPart.thoughtSignature = tool.thought_signature;
+                }
+                parts.push(functionCallPart);
+            }
+        }
+
+        // Unwrap array if wrapped, otherwise treat as array
+        const unwrapped = unwrapConversationArray<Content>(options.conversation);
+        const existingConversation = unwrapped ?? (options.conversation as Content[] || []);
+
+        // Combine existing conversation + prompt contents
+        let conversation: Content[] = [
+            ...existingConversation,
+            ...prompt.contents,
+        ];
+
+        // Only add assistant message if there's actual content
+        // (Empty text parts can cause API errors)
+        if (parts.length > 0) {
+            conversation.push({
+                role: 'model',
+                parts: parts
+            });
+        }
+
+        // Increment turn counter
+        conversation = incrementConversationTurn(conversation) as Content[];
+
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
+        return processedConversation as Content[];
+    }
+
+    /**
+     * Build conversation for Claude streaming.
+     * Creates assistant message with tool_use blocks in Claude's ContentBlock format.
+     */
+    private buildClaudeStreamingConversation(
+        prompt: { messages: unknown[]; system?: unknown[] },
+        result: unknown[],
+        toolUse: unknown[] | undefined,
+        options: ExecutionOptions
+    ): unknown {
+        const completionResults = result as CompletionResult[];
+
+        // Convert accumulated results to text content
+        const textContent = completionResults
+            .map(r => {
+                switch (r.type) {
+                    case 'text':
+                        return r.value;
+                    case 'json':
+                        return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+                    case 'image':
+                        return '';
+                    default:
+                        return String((r as any).value || '');
+                }
+            })
+            .join('');
+
+        // Build Claude-style ContentBlock array for assistant message
+        const content: unknown[] = [];
+
+        // Add text block if there's text content
+        if (textContent) {
+            content.push({
+                type: 'text',
+                text: textContent
+            });
+        }
+
+        // Add tool_use blocks in Claude format
+        if (toolUse && toolUse.length > 0) {
+            for (const tool of toolUse as any[]) {
+                content.push({
+                    type: 'tool_use',
+                    id: tool.id,
+                    name: tool.tool_name,
+                    input: tool.tool_input ?? {}
+                });
+            }
+        }
+
+        // Get existing conversation or start fresh
+        const existingMessages = (options.conversation as any)?.messages ?? [];
+        const existingSystem = (options.conversation as any)?.system ?? prompt.system;
+
+        // Build the new messages array
+        const newMessages = [
+            ...existingMessages,
+            ...prompt.messages,
+        ];
+
+        // Only add assistant message if there's actual content
+        // (Claude API rejects empty text content blocks)
+        if (content.length > 0) {
+            newMessages.push({
+                role: 'assistant',
+                content: content
+            });
+        }
+
+        // Build the new conversation in ClaudePrompt format
+        const conversation = {
+            messages: newMessages,
+            system: existingSystem
+        };
+
+        // Increment turn counter
+        const withTurn = incrementConversationTurn(conversation);
+
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(withTurn).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(withTurn, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
+        return processedConversation;
+    }
+
     async requestImageGeneration(
         _prompt: ImagenPrompt,
         _options: ExecutionOptions,
@@ -496,6 +692,15 @@ export class VertexAIDriver extends AbstractDriver<VertexAIDriverOptions, Vertex
         };
         return getEmbeddingsForText(this, text_options);
     }
+
+    /**
+     * Cleanup Google Cloud clients when the driver is evicted from the cache.
+     */
+    destroy(): void {
+        this.aiplatform?.close();
+        this.modelGarden?.close();
+        this.imagenClient?.close();
+    }
 }
 
 //'us-central1-aiplatform.googleapis.com',