@llumiverse/drivers 0.23.0 → 0.24.0

package/src/bedrock/index.ts

@@ -6,13 +6,26 @@ import { BedrockRuntime, ConverseRequest, ConverseResponse, ConverseStreamOutput
 import { S3Client } from "@aws-sdk/client-s3";
 import { AwsCredentialIdentity, Provider } from "@aws-sdk/types";
 import {
-    AbstractDriver, AIModel, Completion, CompletionChunkObject, DataSource, DriverOptions, EmbeddingsOptions, EmbeddingsResult,
-    ExecutionOptions, ExecutionTokenUsage, PromptSegment,
-    TextFallbackOptions, ToolDefinition, ToolUse, TrainingJob, TrainingJobStatus, TrainingOptions,
-    BedrockClaudeOptions, BedrockPalmyraOptions, BedrockGptOssOptions, getMaxTokensLimitBedrock, NovaCanvasOptions,
-    modelModalitiesToArray, getModelCapabilities,
+    AbstractDriver, AIModel,
+    BedrockClaudeOptions,
+    BedrockGptOssOptions,
+    BedrockPalmyraOptions,
+    Completion, CompletionChunkObject, DataSource, DriverOptions, EmbeddingsOptions, EmbeddingsResult,
+    ExecutionOptions, ExecutionTokenUsage,
+    getMaxTokensLimitBedrock,
+    getModelCapabilities,
+    modelModalitiesToArray,
+    ModelOptions,
+    NovaCanvasOptions,
+    PromptSegment,
     StatelessExecutionOptions,
-    ModelOptions
+    stripBinaryFromConversation,
+    truncateLargeTextInConversation,
+    deserializeBinaryFromStorage,
+    getConversationMeta,
+    incrementConversationTurn,
+    TextFallbackOptions, ToolDefinition, ToolUse, TrainingJob, TrainingJobStatus, TrainingOptions,
+    CompletionResult
 } from "@llumiverse/core";
 import { transformAsyncIterator } from "@llumiverse/core/async";
 import { formatNovaPrompt, NovaMessagesPrompt } from "@llumiverse/core/formatters";
@@ -22,9 +35,9 @@ import { formatNovaImageGenerationPayload, NovaImageGenerationTaskType } from ".
 import { forceUploadFile } from "./s3.js";
 import {
     formatTwelvelabsPegasusPrompt,
-    TwelvelabsPegasusRequest,
     TwelvelabsMarengoRequest,
-    TwelvelabsMarengoResponse
+    TwelvelabsMarengoResponse,
+    TwelvelabsPegasusRequest
 } from "./twelvelabs.js";
 
 const supportStreamingCache = new LRUCache<string, boolean>(4096);
@@ -114,7 +127,6 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             this._executor = new BedrockRuntime({
                 region: this.options.region,
                 credentials: this.options.credentials,
-
             });
         }
         return this._executor;
@@ -350,6 +362,91 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return canStream;
     }
 
+    /**
+     * Build conversation context after streaming completion.
+     * Reconstructs the assistant message from accumulated results and applies stripping.
+     */
+    buildStreamingConversation(
+        prompt: BedrockPrompt,
+        result: unknown[],
+        toolUse: unknown[] | undefined,
+        options: ExecutionOptions
+    ): ConverseRequest | undefined {
+        // Only handle ConverseRequest prompts (not NovaMessagesPrompt or TwelvelabsPegasusRequest)
+        if (options.model.includes("canvas") || options.model.includes("twelvelabs.pegasus")) {
+            return undefined;
+        }
+
+        const conversePrompt = prompt as ConverseRequest;
+        const completionResults = result as CompletionResult[];
+
+        // Convert accumulated results to text content for assistant message
+        const textContent = completionResults
+            .map(r => {
+                switch (r.type) {
+                    case 'text':
+                        return r.value;
+                    case 'json':
+                        return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+                    case 'image':
+                        // Skip images in conversation - they're in the result
+                        return '';
+                    default:
+                        return String((r as any).value || '');
+                }
+            })
+            .join('');
+
+        // Deserialize any base64-encoded binary data back to Uint8Array
+        const incomingConversation = deserializeBinaryFromStorage(options.conversation) as ConverseRequest;
+
+        // Start with the conversation from options combined with the prompt
+        let conversation = updateConversation(incomingConversation, conversePrompt);
+
+        // Build assistant message content
+        const messageContent: any[] = [];
+        if (textContent) {
+            messageContent.push({ text: textContent });
+        }
+        // Add tool use blocks if present
+        if (toolUse && toolUse.length > 0) {
+            for (const tool of toolUse as ToolUse[]) {
+                messageContent.push({
+                    toolUse: {
+                        toolUseId: tool.id,
+                        name: tool.tool_name,
+                        input: tool.tool_input,
+                    }
+                });
+            }
+        }
+
+        // Add assistant message
+        const assistantMessage: ConverseRequest = {
+            messages: [{
+                content: messageContent.length > 0 ? messageContent : [{ text: '' }],
+                role: "assistant"
+            }],
+            modelId: conversePrompt.modelId,
+        };
+        conversation = updateConversation(conversation, assistantMessage);
+
+        // Increment turn counter
+        conversation = incrementConversationTurn(conversation) as ConverseRequest;
+
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBinaryFromConversation(conversation, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
+        return processedConversation as ConverseRequest;
+    }
+
     async requestTextCompletion(prompt: BedrockPrompt, options: ExecutionOptions): Promise<Completion> {
         // Handle Twelvelabs Pegasus models
         if (options.model.includes("twelvelabs.pegasus")) {
@@ -358,7 +455,10 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         // Handle other Bedrock models that use Converse API
         const conversePrompt = prompt as ConverseRequest;
-        let conversation = updateConversation(options.conversation as ConverseRequest, conversePrompt);
+
+        // Deserialize any base64-encoded binary data back to Uint8Array before API call
+        const incomingConversation = deserializeBinaryFromStorage(options.conversation) as ConverseRequest;
+        let conversation = updateConversation(incomingConversation, conversePrompt);
 
         const payload = this.preparePayload(conversation, options);
         const executor = this.getExecutor();
@@ -372,6 +472,9 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             modelId: conversePrompt.modelId,
         });
 
+        // Increment turn counter for deferred stripping
+        conversation = incrementConversationTurn(conversation) as ConverseRequest;
+
         let tool_use: ToolUse[] | undefined = undefined;
         //Get tool requests, we check tool use regardless of finish reason, as you can hit length and still get a valid response.
         tool_use = res.output?.message?.content?.reduce((tools: ToolUse[], c) => {
@@ -389,10 +492,22 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             tool_use = undefined;
         }
 
+        // Strip/serialize binary data based on options.stripImagesAfterTurns
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBinaryFromConversation(conversation, stripOptions);
+
+        // Truncate large text content if configured
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
         const completion = {
             ...this.getExtractedExecution(res, conversePrompt, options),
             original_response: options.include_original_response ? res : undefined,
-            conversation: conversation,
+            conversation: processedConversation,
             tool_use: tool_use,
         };
 
@@ -496,7 +611,13 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         // Handle other Bedrock models that use Converse API
         const conversePrompt = prompt as ConverseRequest;
-        const payload = this.preparePayload(conversePrompt, options);
+
+        // Include conversation history (same as non-streaming)
+        // Deserialize any base64-encoded binary data back to Uint8Array before API call
+        const incomingConversation = deserializeBinaryFromStorage(options.conversation) as ConverseRequest;
+        const conversation = updateConversation(incomingConversation, conversePrompt);
+
+        const payload = this.preparePayload(conversation, options);
         const executor = this.getExecutor();
         return executor.converseStream({
             ...payload,
@@ -642,22 +763,38 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             prompt.messages = converseJSONprefill(prompt.messages);
         }
 
+        // Clean undefined values from additionalField since AWS Bedrock requires valid JSON
+        // and will throw an exception for unrecognized parameters
+        const cleanedAdditionalFields = removeUndefinedValues(additionalField);
+        const cleanedModelOptions = removeUndefinedValues({
+            maxTokens: model_options.max_tokens,
+            temperature: model_options.temperature,
+            topP: model_options.top_p,
+            stopSequences: model_options.stop_sequence,
+        } satisfies InferenceConfiguration);
+
+        //Construct the final request payload
+        // We only add fields that are defined to avoid AWS errors
         const request: ConverseRequest = {
-            messages: prompt.messages,
-            system: prompt.system,
             modelId: options.model,
-            inferenceConfig: {
-                maxTokens: model_options.max_tokens,
-                temperature: model_options.temperature,
-                topP: model_options.top_p,
-                stopSequences: model_options.stop_sequence,
-            } satisfies InferenceConfiguration,
-            additionalModelRequestFields: {
-                ...additionalField,
-            }
         };
 
-        //Only add tools if they are defined and not empty
+        if (prompt.messages) {
+            request.messages = prompt.messages;
+        }
+
+        if (prompt.system) {
+            request.system = prompt.system;
+        }
+
+        if (Object.keys(cleanedModelOptions).length > 0) {
+            request.inferenceConfig = cleanedModelOptions
+        }
+
+        if (Object.keys(cleanedAdditionalFields).length > 0) {
+            request.additionalModelRequestFields = cleanedAdditionalFields;
+        }
+
         if (tool_defs?.length) {
             request.toolConfig = {
                 tools: tool_defs,
@@ -1087,6 +1224,33 @@ function getToolDefinition(tool: ToolDefinition): Tool.ToolSpecMember {
     }
 }
 
+/**
+ * Recursively removes undefined values from an object.
+ * AWS Bedrock's additionalModelRequestFields must be valid JSON, and undefined is not valid JSON.
+ * Any unrecognized parameters will cause an exception.
+ */
+function removeUndefinedValues<T extends Record<string, any>>(obj: T): Partial<T> {
+    if (obj === null || typeof obj !== 'object' || Array.isArray(obj)) {
+        return obj;
+    }
+
+    const cleaned: any = {};
+    for (const [key, value] of Object.entries(obj)) {
+        if (value !== undefined) {
+            if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+                const cleanedNested = removeUndefinedValues(value);
+                // Only include nested objects if they have properties after cleaning
+                if (Object.keys(cleanedNested).length > 0) {
+                    cleaned[key] = cleanedNested;
+                }
+            } else {
+                cleaned[key] = value;
+            }
+        }
+    }
+    return cleaned;
+}
+
 /**
  * Update the conversation messages
  * @param prompt

package/src/groq/index.ts

@@ -3,9 +3,13 @@ import { transformAsyncIterator } from "@llumiverse/core/async";
 import { formatOpenAILikeMultimodalPrompt } from "../openai/openai_format.js";
 
 import Groq from "groq-sdk";
+import type OpenAI from "openai";
 import type { ChatCompletionMessageParam, ChatCompletionTool } from "groq-sdk/resources/chat/completions";
 import type { FunctionParameters } from "groq-sdk/resources/shared";
 
+type ResponseInputItem = OpenAI.Responses.ResponseInputItem;
+type EasyInputMessage = OpenAI.Responses.EasyInputMessage;
+
 interface GroqDriverOptions extends DriverOptions {
     apiKey: string;
     endpoint_url?: string;
@@ -49,104 +53,13 @@ export class GroqDriver extends AbstractDriver<GroqDriverOptions, ChatCompletion
 
     protected async formatPrompt(segments: PromptSegment[], opts: ExecutionOptions): Promise<ChatCompletionMessageParam[]> {
         // Use OpenAI's multimodal formatter as base then convert to Groq types
-        const openaiMessages = await formatOpenAILikeMultimodalPrompt(segments, {
+        const responseItems = await formatOpenAILikeMultimodalPrompt(segments, {
             ...opts,
             multimodal: true,
         });
 
-        // Convert OpenAI ChatCompletionMessageParam[] to Groq ChatCompletionMessageParam[]
-        // Handle differences between OpenAI and Groq SDK types
-        const groqMessages: ChatCompletionMessageParam[] = openaiMessages.map(msg => {
-            // Handle OpenAI developer messages - convert to system messages for Groq
-            if (msg.role === 'developer' || msg.role === 'system') {
-                const systemMsg: ChatCompletionMessageParam = {
-                    role: 'system',
-                    content: Array.isArray(msg.content)
-                        ? msg.content.map(part => part.text).join('\n')
-                        : msg.content,
-                    // Preserve name if present
-                    ...(msg.name && { name: msg.name })
-                };
-                return systemMsg;
-            }
-
-            // Handle user messages - filter content parts to only supported types
-            if (msg.role === 'user') {
-                let content: string | Array<{ type: 'text', text: string } | { type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }> | undefined = undefined;
-
-                if (typeof msg.content === 'string') {
-                    content = msg.content;
-                } else if (Array.isArray(msg.content)) {
-                    // Filter to only text and image_url parts that Groq supports
-                    const supportedParts = msg.content.filter(part =>
-                        part.type === 'text' || part.type === 'image_url'
-                    ).map(part => {
-                        if (part.type === 'text') {
-                            return { type: 'text' as const, text: part.text };
-                        } else if (part.type === 'image_url') {
-                            return {
-                                type: 'image_url' as const,
-                                image_url: {
-                                    url: part.image_url.url,
-                                    ...(part.image_url.detail && { detail: part.image_url.detail })
-                                }
-                            };
-                        }
-                        return null;
-                    }).filter(Boolean) as Array<{ type: 'text', text: string } | { type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }>;
-
-                    content = supportedParts.length > 0 ? supportedParts : 'Content not supported';
-                }
-
-                const userMsg: ChatCompletionMessageParam = {
-                    role: 'user',
-                    content: content ?? "",
-                    // Preserve name if present
-                    ...(msg.name && { name: msg.name })
-                };
-                return userMsg;
-            }
-
-            // Handle assistant messages - handle content arrays if needed
-            if (msg.role === 'assistant') {
-                const assistantMsg: ChatCompletionMessageParam = {
-                    role: 'assistant',
-                    content: Array.isArray(msg.content)
-                        ? msg.content.map(part => 'text' in part ? part.text : '').filter(Boolean).join('\n') || null
-                        : msg.content,
-                    // Preserve other assistant message properties
-                    ...(msg.tool_calls && { tool_calls: msg.tool_calls }),
-                    ...(msg.name && { name: msg.name })
-                };
-                return assistantMsg;
-            }
-
-            // For tool and function messages, they should be compatible
-            if (msg.role === 'tool') {
-                const toolMsg: ChatCompletionMessageParam = {
-                    role: 'tool',
-                    tool_call_id: msg.tool_call_id,
-                    content: Array.isArray(msg.content)
-                        ? msg.content.map(part => part.text).join('\n')
-                        : msg.content
-                };
-                return toolMsg;
-            }
-
-            if (msg.role === 'function') {
-                const functionMsg: ChatCompletionMessageParam = {
-                    role: 'function',
-                    name: msg.name,
-                    content: msg.content
-                };
-                return functionMsg;
-            }
-
-            // Fallback - should not reach here but provides type safety
-            throw new Error(`Unsupported message role: ${(msg as any).role}`);
-        });
-
-        return groqMessages;
+        // Convert ResponseInputItem[] to Groq ChatCompletionMessageParam[]
+        return convertResponseItemsToGroqMessages(responseItems);
     }
 
     private getToolDefinitions(tools: ToolDefinition[] | undefined): ChatCompletionTool[] | undefined {
@@ -342,4 +255,117 @@ function updateConversation(
     messages: ChatCompletionMessageParam[]
 ): ChatCompletionMessageParam[] {
     return (conversation || []).concat(messages);
+}
+
+/**
+ * Convert ResponseInputItem[] to Groq ChatCompletionMessageParam[]
+ */
+function convertResponseItemsToGroqMessages(items: ResponseInputItem[]): ChatCompletionMessageParam[] {
+    const messages: ChatCompletionMessageParam[] = [];
+
+    for (const item of items) {
+        // Handle EasyInputMessage (has role and content)
+        if ('role' in item && 'content' in item) {
+            const msg = item as EasyInputMessage;
+            const role = msg.role;
+
+            // Handle system/developer messages
+            if (role === 'system' || role === 'developer') {
+                let content: string;
+                if (typeof msg.content === 'string') {
+                    content = msg.content;
+                } else if (Array.isArray(msg.content)) {
+                    content = msg.content
+                        .filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text')
+                        .map(part => part.text)
+                        .join('\n');
+                } else {
+                    content = '';
+                }
+                messages.push({ role: 'system', content });
+                continue;
+            }
+
+            // Handle user messages
+            if (role === 'user') {
+                let content: string | Array<{ type: 'text', text: string } | { type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }>;
+                if (typeof msg.content === 'string') {
+                    content = msg.content;
+                } else if (Array.isArray(msg.content)) {
+                    const parts: Array<{ type: 'text', text: string } | { type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }> = [];
+                    for (const part of msg.content) {
+                        if (part.type === 'input_text') {
+                            parts.push({ type: 'text', text: part.text });
+                        } else if (part.type === 'input_image') {
+                            const imgPart = part as OpenAI.Responses.ResponseInputImage;
+                            if (imgPart.image_url) {
+                                parts.push({
+                                    type: 'image_url',
+                                    image_url: {
+                                        url: imgPart.image_url,
+                                        ...(imgPart.detail && { detail: imgPart.detail })
+                                    }
+                                });
+                            }
+                        }
+                    }
+                    content = parts.length > 0 ? parts : '';
+                } else {
+                    content = '';
+                }
+                messages.push({ role: 'user', content });
+                continue;
+            }
+
+            // Handle assistant messages
+            if (role === 'assistant') {
+                let content: string | null;
+                if (typeof msg.content === 'string') {
+                    content = msg.content;
+                } else if (Array.isArray(msg.content)) {
+                    content = msg.content
+                        .filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text')
+                        .map(part => part.text)
+                        .join('\n') || null;
+                } else {
+                    content = null;
+                }
+                messages.push({ role: 'assistant', content });
+                continue;
+            }
+        }
+
+        // Handle function_call_output (tool response)
+        if ('type' in item && item.type === 'function_call_output') {
+            const output = item as OpenAI.Responses.ResponseInputItem.FunctionCallOutput;
+            messages.push({
+                role: 'tool',
+                tool_call_id: output.call_id,
+                content: typeof output.output === 'string' ? output.output : JSON.stringify(output.output),
+            });
+            continue;
+        }
+
+        // Handle function_call (assistant tool call)
+        if ('type' in item && item.type === 'function_call') {
+            const call = item as OpenAI.Responses.ResponseFunctionToolCall;
+            // Groq expects tool_calls in assistant message, but we handle them separately
+            // This is a simplification - in practice tool_calls come from model responses
+            messages.push({
+                role: 'assistant',
+                content: null,
+                tool_calls: [{
+                    id: call.call_id,
+                    type: 'function',
+                    function: {
+                        name: call.name,
+                        arguments: call.arguments,
+                    }
+                }]
+            });
+            continue;
+        }
+    }
+
+    return messages;
 }

package/src/index.ts

@@ -5,6 +5,7 @@ export * from "./huggingface_ie.js";
 export * from "./mistral/index.js";
 export * from "./openai/azure_openai.js";
 export * from "./openai/openai.js";
+export * from "./openai/openai_compatible.js";
 export * from "./replicate.js";
 export * from "./test-driver/index.js";
 export * from "./togetherai/index.js";