@llumiverse/drivers 0.23.0 → 0.24.0-dev.202601221707

package/src/azure/azure_foundry.ts

@@ -2,7 +2,7 @@ import { DefaultAzureCredential, getBearerTokenProvider, TokenCredential } from
 import { AbstractDriver, AIModel, Completion, CompletionChunkObject, DriverOptions, EmbeddingsOptions, EmbeddingsResult, ExecutionOptions, getModelCapabilities, modelModalitiesToArray, Providers } from "@llumiverse/core";
 import { AIProjectClient, DeploymentUnion, ModelDeployment } from '@azure/ai-projects';
 import { isUnexpected } from "@azure-rest/ai-inference";
-import { ChatCompletionMessageParam } from "openai/resources";
+import type OpenAI from "openai";
 import type {
     ChatCompletionsOutput,
     ChatCompletionsToolCall,
@@ -11,6 +11,9 @@ import type {
 import { AzureOpenAIDriver } from "../openai/azure_openai.js";
 import { createSseStream, NodeJSReadableStream } from "@azure/core-sse";
 import { formatOpenAILikeMultimodalPrompt } from "../openai/openai_format.js";
+
+type ResponseInputItem = OpenAI.Responses.ResponseInputItem;
+type EasyInputMessage = OpenAI.Responses.EasyInputMessage;
 export interface AzureFoundryDriverOptions extends DriverOptions {
     /**
      * The credentials to use to access Azure AI Foundry
@@ -27,12 +30,12 @@ export interface AzureFoundryInferencePrompt {
 }
 
 export interface AzureFoundryOpenAIPrompt {
-    messages: ChatCompletionMessageParam[]
+    messages: ResponseInputItem[]
 }
 
 export type AzureFoundryPrompt = AzureFoundryInferencePrompt | AzureFoundryOpenAIPrompt
 
-export class AzureFoundryDriver extends AbstractDriver<AzureFoundryDriverOptions, ChatCompletionMessageParam[]> {
+export class AzureFoundryDriver extends AbstractDriver<AzureFoundryDriverOptions, ResponseInputItem[]> {
     service: AIProjectClient;
     readonly provider = Providers.azure_foundry;
 
@@ -99,7 +102,7 @@ export class AzureFoundryDriver extends AbstractDriver<AzureFoundryDriverOptions
         return Promise.resolve(true);
     }
 
-    async requestTextCompletion(prompt: ChatCompletionMessageParam[], options: ExecutionOptions): Promise<Completion> {
+    async requestTextCompletion(prompt: ResponseInputItem[], options: ExecutionOptions): Promise<Completion> {
         const { deploymentName } = parseAzureFoundryModelId(options.model);
         const model_options = options.model_options as any;
         const isOpenAI = await this.isOpenAIDeployment(options.model);
@@ -116,10 +119,12 @@ export class AzureFoundryDriver extends AbstractDriver<AzureFoundryDriverOptions
 
         } else {
             // Use the chat completions client from the inference operations
+            // Convert ResponseInputItem[] to ChatRequestMessage[] for non-OpenAI inference
+            const messages = convertToInferenceMessages(prompt);
             const chatClient = this.service.inference.chatCompletions({ apiVersion: this.INFERENCE_API_VERSION });
             response = await chatClient.post({
                 body: {
-                    messages: prompt,
+                    messages,
                     max_tokens: model_options?.max_tokens,
                     model: deploymentName,
                     stream: true,
@@ -139,7 +144,7 @@ export class AzureFoundryDriver extends AbstractDriver<AzureFoundryDriverOptions
         }
     }
 
-    async requestTextCompletionStream(prompt: ChatCompletionMessageParam[], options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
+    async requestTextCompletionStream(prompt: ResponseInputItem[], options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
         const { deploymentName } = parseAzureFoundryModelId(options.model);
         const model_options = options.model_options as any;
         const isOpenAI = await this.isOpenAIDeployment(options.model);
@@ -151,10 +156,12 @@ export class AzureFoundryDriver extends AbstractDriver<AzureFoundryDriverOptions
             const stream = await subDriver.requestTextCompletionStream(prompt, modifiedOptions);
             return stream;
         } else {
+            // Convert ResponseInputItem[] to ChatRequestMessage[] for non-OpenAI inference
+            const messages = convertToInferenceMessages(prompt);
             const chatClient = this.service.inference.chatCompletions({ apiVersion: this.INFERENCE_API_VERSION });
             const response = await chatClient.post({
                 body: {
-                    messages: prompt,
+                    messages,
                     max_tokens: model_options?.max_tokens,
                     model: deploymentName,
                     stream: true,
@@ -456,3 +463,45 @@ export function parseAzureFoundryModelId(compositeId: string): { deploymentName:
 export function isCompositeModelId(modelId: string): boolean {
     return modelId.includes('::');
 }
+
+/**
+ * Convert ResponseInputItem[] to ChatRequestMessage[] for Azure AI Inference API
+ */
+function convertToInferenceMessages(items: ResponseInputItem[]): ChatRequestMessage[] {
+    const messages: ChatRequestMessage[] = [];
+
+    for (const item of items) {
+        // Handle EasyInputMessage (has role and content)
+        if ('role' in item && 'content' in item) {
+            const msg = item as EasyInputMessage;
+            let content: string;
+            if (typeof msg.content === 'string') {
+                content = msg.content;
+            } else if (Array.isArray(msg.content)) {
+                // Extract text from content array
+                content = msg.content
+                    .filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text')
+                    .map(part => part.text)
+                    .join('\n');
+            } else {
+                content = '';
+            }
+
+            messages.push({
+                role: msg.role as 'system' | 'user' | 'assistant',
+                content
+            });
+        }
+        // Handle function_call_output
+        else if ('type' in item && item.type === 'function_call_output') {
+            const output = item as OpenAI.Responses.ResponseInputItem.FunctionCallOutput;
+            messages.push({
+                role: 'tool',
+                content: typeof output.output === 'string' ? output.output : JSON.stringify(output.output),
+                tool_call_id: output.call_id,
+            } as ChatRequestMessage);
+        }
+    }
+
+    return messages;
+}

package/src/bedrock/index.ts

@@ -2,17 +2,30 @@ import {
     Bedrock, CreateModelCustomizationJobCommand, FoundationModelSummary, GetModelCustomizationJobCommand,
     GetModelCustomizationJobCommandOutput, ModelCustomizationJobStatus, ModelModality, StopModelCustomizationJobCommand
 } from "@aws-sdk/client-bedrock";
-import { BedrockRuntime, ConverseRequest, ConverseResponse, ConverseStreamOutput, InferenceConfiguration, Tool } from "@aws-sdk/client-bedrock-runtime";
+import { BedrockRuntime, ContentBlock, ConverseRequest, ConverseResponse, ConverseStreamOutput, InferenceConfiguration, Message, Tool } from "@aws-sdk/client-bedrock-runtime";
 import { S3Client } from "@aws-sdk/client-s3";
 import { AwsCredentialIdentity, Provider } from "@aws-sdk/types";
 import {
-    AbstractDriver, AIModel, Completion, CompletionChunkObject, DataSource, DriverOptions, EmbeddingsOptions, EmbeddingsResult,
-    ExecutionOptions, ExecutionTokenUsage, PromptSegment,
-    TextFallbackOptions, ToolDefinition, ToolUse, TrainingJob, TrainingJobStatus, TrainingOptions,
-    BedrockClaudeOptions, BedrockPalmyraOptions, BedrockGptOssOptions, getMaxTokensLimitBedrock, NovaCanvasOptions,
-    modelModalitiesToArray, getModelCapabilities,
+    AbstractDriver, AIModel,
+    BedrockClaudeOptions,
+    BedrockGptOssOptions,
+    BedrockPalmyraOptions,
+    Completion, CompletionChunkObject, DataSource, DriverOptions, EmbeddingsOptions, EmbeddingsResult,
+    ExecutionOptions, ExecutionTokenUsage,
+    getMaxTokensLimitBedrock,
+    getModelCapabilities,
+    modelModalitiesToArray,
+    ModelOptions,
+    NovaCanvasOptions,
+    PromptSegment,
     StatelessExecutionOptions,
-    ModelOptions
+    stripBinaryFromConversation,
+    truncateLargeTextInConversation,
+    deserializeBinaryFromStorage,
+    getConversationMeta,
+    incrementConversationTurn,
+    TextFallbackOptions, ToolDefinition, ToolUse, TrainingJob, TrainingJobStatus, TrainingOptions,
+    CompletionResult
 } from "@llumiverse/core";
 import { transformAsyncIterator } from "@llumiverse/core/async";
 import { formatNovaPrompt, NovaMessagesPrompt } from "@llumiverse/core/formatters";
@@ -22,9 +35,9 @@ import { formatNovaImageGenerationPayload, NovaImageGenerationTaskType } from ".
 import { forceUploadFile } from "./s3.js";
 import {
     formatTwelvelabsPegasusPrompt,
-    TwelvelabsPegasusRequest,
     TwelvelabsMarengoRequest,
-    TwelvelabsMarengoResponse
+    TwelvelabsMarengoResponse,
+    TwelvelabsPegasusRequest
 } from "./twelvelabs.js";
 
 const supportStreamingCache = new LRUCache<string, boolean>(4096);
@@ -114,7 +127,6 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             this._executor = new BedrockRuntime({
                 region: this.options.region,
                 credentials: this.options.credentials,
-
             });
         }
         return this._executor;
@@ -350,6 +362,91 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return canStream;
     }
 
+    /**
+     * Build conversation context after streaming completion.
+     * Reconstructs the assistant message from accumulated results and applies stripping.
+     */
+    buildStreamingConversation(
+        prompt: BedrockPrompt,
+        result: unknown[],
+        toolUse: unknown[] | undefined,
+        options: ExecutionOptions
+    ): ConverseRequest | undefined {
+        // Only handle ConverseRequest prompts (not NovaMessagesPrompt or TwelvelabsPegasusRequest)
+        if (options.model.includes("canvas") || options.model.includes("twelvelabs.pegasus")) {
+            return undefined;
+        }
+
+        const conversePrompt = prompt as ConverseRequest;
+        const completionResults = result as CompletionResult[];
+
+        // Convert accumulated results to text content for assistant message
+        const textContent = completionResults
+            .map(r => {
+                switch (r.type) {
+                    case 'text':
+                        return r.value;
+                    case 'json':
+                        return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+                    case 'image':
+                        // Skip images in conversation - they're in the result
+                        return '';
+                    default:
+                        return String((r as any).value || '');
+                }
+            })
+            .join('');
+
+        // Deserialize any base64-encoded binary data back to Uint8Array
+        const incomingConversation = deserializeBinaryFromStorage(options.conversation) as ConverseRequest;
+
+        // Start with the conversation from options combined with the prompt
+        let conversation = updateConversation(incomingConversation, conversePrompt);
+
+        // Build assistant message content
+        const messageContent: any[] = [];
+        if (textContent) {
+            messageContent.push({ text: textContent });
+        }
+        // Add tool use blocks if present
+        if (toolUse && toolUse.length > 0) {
+            for (const tool of toolUse as ToolUse[]) {
+                messageContent.push({
+                    toolUse: {
+                        toolUseId: tool.id,
+                        name: tool.tool_name,
+                        input: tool.tool_input,
+                    }
+                });
+            }
+        }
+
+        // Add assistant message
+        const assistantMessage: ConverseRequest = {
+            messages: [{
+                content: messageContent.length > 0 ? messageContent : [{ text: '' }],
+                role: "assistant"
+            }],
+            modelId: conversePrompt.modelId,
+        };
+        conversation = updateConversation(conversation, assistantMessage);
+
+        // Increment turn counter
+        conversation = incrementConversationTurn(conversation) as ConverseRequest;
+
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBinaryFromConversation(conversation, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
+        return processedConversation as ConverseRequest;
+    }
+
     async requestTextCompletion(prompt: BedrockPrompt, options: ExecutionOptions): Promise<Completion> {
         // Handle Twelvelabs Pegasus models
         if (options.model.includes("twelvelabs.pegasus")) {
@@ -358,7 +455,10 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         // Handle other Bedrock models that use Converse API
         const conversePrompt = prompt as ConverseRequest;
-        let conversation = updateConversation(options.conversation as ConverseRequest, conversePrompt);
+
+        // Deserialize any base64-encoded binary data back to Uint8Array before API call
+        const incomingConversation = deserializeBinaryFromStorage(options.conversation) as ConverseRequest;
+        let conversation = updateConversation(incomingConversation, conversePrompt);
 
         const payload = this.preparePayload(conversation, options);
         const executor = this.getExecutor();
@@ -372,6 +472,9 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             modelId: conversePrompt.modelId,
         });
 
+        // Increment turn counter for deferred stripping
+        conversation = incrementConversationTurn(conversation) as ConverseRequest;
+
         let tool_use: ToolUse[] | undefined = undefined;
         //Get tool requests, we check tool use regardless of finish reason, as you can hit length and still get a valid response.
         tool_use = res.output?.message?.content?.reduce((tools: ToolUse[], c) => {
@@ -389,10 +492,22 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             tool_use = undefined;
         }
 
+        // Strip/serialize binary data based on options.stripImagesAfterTurns
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBinaryFromConversation(conversation, stripOptions);
+
+        // Truncate large text content if configured
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
         const completion = {
             ...this.getExtractedExecution(res, conversePrompt, options),
             original_response: options.include_original_response ? res : undefined,
-            conversation: conversation,
+            conversation: processedConversation,
             tool_use: tool_use,
         };
 
@@ -496,7 +611,13 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         // Handle other Bedrock models that use Converse API
         const conversePrompt = prompt as ConverseRequest;
-        const payload = this.preparePayload(conversePrompt, options);
+
+        // Include conversation history (same as non-streaming)
+        // Deserialize any base64-encoded binary data back to Uint8Array before API call
+        const incomingConversation = deserializeBinaryFromStorage(options.conversation) as ConverseRequest;
+        const conversation = updateConversation(incomingConversation, conversePrompt);
+
+        const payload = this.preparePayload(conversation, options);
         const executor = this.getExecutor();
         return executor.converseStream({
             ...payload,
@@ -642,22 +763,38 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             prompt.messages = converseJSONprefill(prompt.messages);
         }
 
+        // Clean undefined values from additionalField since AWS Bedrock requires valid JSON
+        // and will throw an exception for unrecognized parameters
+        const cleanedAdditionalFields = removeUndefinedValues(additionalField);
+        const cleanedModelOptions = removeUndefinedValues({
+            maxTokens: model_options.max_tokens,
+            temperature: model_options.temperature,
+            topP: model_options.top_p,
+            stopSequences: model_options.stop_sequence,
+        } satisfies InferenceConfiguration);
+
+        //Construct the final request payload
+        // We only add fields that are defined to avoid AWS errors
         const request: ConverseRequest = {
-            messages: prompt.messages,
-            system: prompt.system,
             modelId: options.model,
-            inferenceConfig: {
-                maxTokens: model_options.max_tokens,
-                temperature: model_options.temperature,
-                topP: model_options.top_p,
-                stopSequences: model_options.stop_sequence,
-            } satisfies InferenceConfiguration,
-            additionalModelRequestFields: {
-                ...additionalField,
-            }
         };
 
-        //Only add tools if they are defined and not empty
+        if (prompt.messages) {
+            request.messages = prompt.messages;
+        }
+
+        if (prompt.system) {
+            request.system = prompt.system;
+        }
+
+        if (Object.keys(cleanedModelOptions).length > 0) {
+            request.inferenceConfig = cleanedModelOptions
+        }
+
+        if (Object.keys(cleanedAdditionalFields).length > 0) {
+            request.additionalModelRequestFields = cleanedAdditionalFields;
+        }
+
         if (tool_defs?.length) {
             request.toolConfig = {
                 tools: tool_defs,
@@ -1045,6 +1182,14 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             token_count: undefined
         };
     }
+
+    /**
+     * Cleanup AWS SDK clients when the driver is evicted from the cache.
+     */
+    destroy(): void {
+        this._executor?.destroy();
+        this._service?.destroy();
+    }
 }
 
 function jobInfo(job: GetModelCustomizationJobCommandOutput, jobId: string): TrainingJob {
@@ -1087,6 +1232,33 @@ function getToolDefinition(tool: ToolDefinition): Tool.ToolSpecMember {
     }
 }
 
+/**
+ * Recursively removes undefined values from an object.
+ * AWS Bedrock's additionalModelRequestFields must be valid JSON, and undefined is not valid JSON.
+ * Any unrecognized parameters will cause an exception.
+ */
+function removeUndefinedValues<T extends Record<string, any>>(obj: T): Partial<T> {
+    if (obj === null || typeof obj !== 'object' || Array.isArray(obj)) {
+        return obj;
+    }
+
+    const cleaned: any = {};
+    for (const [key, value] of Object.entries(obj)) {
+        if (value !== undefined) {
+            if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+                const cleanedNested = removeUndefinedValues(value);
+                // Only include nested objects if they have properties after cleaning
+                if (Object.keys(cleanedNested).length > 0) {
+                    cleaned[key] = cleanedNested;
+                }
+            } else {
+                cleaned[key] = value;
+            }
+        }
+    }
+    return cleaned;
+}
+
 /**
  * Update the conversation messages
  * @param prompt
@@ -1097,13 +1269,112 @@ function updateConversation(conversation: ConverseRequest, prompt: ConverseReque
     const combinedMessages = [...(conversation?.messages || []), ...(prompt.messages || [])];
     const combinedSystem = prompt.system || conversation?.system;
 
+    // Fix orphaned toolUse blocks before returning
+    const fixedMessages = fixOrphanedToolUse(combinedMessages);
+
     return {
         modelId: prompt?.modelId || conversation?.modelId,
-        messages: combinedMessages.length > 0 ? combinedMessages : [],
+        messages: fixedMessages.length > 0 ? fixedMessages : [],
         system: combinedSystem && combinedSystem.length > 0 ? combinedSystem : undefined,
     };
 }
 
+/**
+ * Fix orphaned toolUse blocks in the conversation.
+ *
+ * When an agent is stopped mid-tool-execution, the assistant message contains toolUse blocks
+ * but no corresponding toolResult was added. The AWS Converse API requires that every toolUse
+ * must be followed by a toolResult in the next user message.
+ *
+ * This function detects such cases and injects synthetic toolResult blocks indicating
+ * the tools were interrupted, allowing the conversation to continue.
+ */
+export function fixOrphanedToolUse(messages: Message[]): Message[] {
+    if (messages.length < 2) return messages;
+
+    const result: Message[] = [];
+
+    for (let i = 0; i < messages.length; i++) {
+        const current = messages[i];
+        result.push(current);
+
+        // Check if this is an assistant message with toolUse blocks
+        if (current.role === 'assistant' && current.content) {
+            // Extract toolUse blocks using simple property check (same pattern as existing Bedrock code)
+            const toolUseBlocks: Array<{ toolUseId: string; name: string }> = [];
+            for (const block of current.content) {
+                if (block.toolUse?.toolUseId) {
+                    toolUseBlocks.push({
+                        toolUseId: block.toolUse.toolUseId,
+                        name: block.toolUse.name ?? 'unknown'
+                    });
+                }
+            }
+
+            if (toolUseBlocks.length > 0) {
+                // Check if the next message is a user message with matching toolResults
+                const nextMessage = messages[i + 1];
+
+                if (nextMessage && nextMessage.role === 'user' && nextMessage.content) {
+                    // Get toolResult IDs from the next message using simple property check
+                    const toolResultIds = new Set<string>();
+                    for (const block of nextMessage.content) {
+                        if (block.toolResult?.toolUseId) {
+                            toolResultIds.add(block.toolResult.toolUseId);
+                        }
+                    }
+
+                    // Find orphaned toolUse blocks (no matching toolResult)
+                    const orphanedToolUse = toolUseBlocks.filter(tu => !toolResultIds.has(tu.toolUseId));
+
+                    if (orphanedToolUse.length > 0) {
+                        // Inject synthetic toolResults for orphaned toolUse
+                        const syntheticResults: ContentBlock[] = orphanedToolUse.map(tu => ({
+                            toolResult: {
+                                toolUseId: tu.toolUseId,
+                                content: [{
+                                    text: `[Tool interrupted: The user stopped the operation before "${tu.name}" could execute.]`
+                                }]
+                            }
+                        }));
+
+                        // Prepend synthetic results to the next user message
+                        const updatedNextMessage: Message = {
+                            ...nextMessage,
+                            content: [...syntheticResults, ...nextMessage.content]
+                        };
+
+                        // Replace the next message in our iteration
+                        messages[i + 1] = updatedNextMessage;
+                    }
+                } else if (nextMessage && nextMessage.role === 'user' && !nextMessage.content) {
+                    // Next message is a user message but has no content
+                    // We need to add toolResults
+                    const syntheticResults: ContentBlock[] = toolUseBlocks.map(tu => ({
+                        toolResult: {
+                            toolUseId: tu.toolUseId,
+                            content: [{
+                                text: `[Tool interrupted: The user stopped the operation before "${tu.name}" could execute.]`
+                            }]
+                        }
+                    }));
+
+                    const updatedNextMessage: Message = {
+                        role: 'user',
+                        content: syntheticResults
+                    };
+
+                    messages[i + 1] = updatedNextMessage;
+                }
+                // Note: If there's no nextMessage, we leave the conversation as-is.
+                // The toolUse blocks are expected to be there - the next turn will provide toolResults.
+            }
+        }
+    }
+
+    return result;
+}
+
 function formatAmazonModalities(modalities: ModelModality[]): string[] {
     const standardizedModalities: string[] = [];
     for (const modality of modalities) {