@llumiverse/drivers 0.20.0 → 0.22.0-dev.1

package/src/bedrock/index.ts

@@ -7,9 +7,9 @@ import { S3Client } from "@aws-sdk/client-s3";
 import { AwsCredentialIdentity, Provider } from "@aws-sdk/types";
 import {
     AbstractDriver, AIModel, Completion, CompletionChunkObject, DataSource, DriverOptions, EmbeddingsOptions, EmbeddingsResult,
-    ExecutionOptions, ExecutionTokenUsage, ImageGeneration, Modalities, PromptSegment,
+    ExecutionOptions, ExecutionTokenUsage, Modalities, PromptSegment,
     TextFallbackOptions, ToolDefinition, ToolUse, TrainingJob, TrainingJobStatus, TrainingOptions,
-    BedrockClaudeOptions, BedrockPalmyraOptions, getMaxTokensLimitBedrock, NovaCanvasOptions,
+    BedrockClaudeOptions, BedrockPalmyraOptions, BedrockGptOssOptions, getMaxTokensLimitBedrock, NovaCanvasOptions,
     modelModalitiesToArray, getModelCapabilities,
     StatelessExecutionOptions,
     ModelOptions
@@ -74,11 +74,13 @@ function maxTokenFallbackClaude(option: StatelessExecutionOptions): number {
     if (modelOptions && typeof modelOptions.max_tokens === "number") {
         return modelOptions.max_tokens;
     } else {
+        const thinking_budget = modelOptions?.thinking_budget_tokens ?? 0;
+        let maxSupportedTokens = getMaxTokensLimitBedrock(option.model) ?? 8192; // Should always return a number for claude, 8192 is to satisfy the TypeScript type checker;
         // Fallback to the default max tokens limit for the model
-        if (option.model.includes('claude-3-7-sonnet') && (modelOptions?.thinking_budget_tokens ?? 0) < 64000) {
-            return 64000; // Claude 3.7 can go up to 128k with a beta header, but when no max tokens is specified, we default to 64k.
+        if (option.model.includes('claude-3-7-sonnet') && (modelOptions?.thinking_budget_tokens ?? 0) < 48000) {
+            maxSupportedTokens = 64000; // Claude 3.7 can go up to 128k with a beta header, but when no max tokens is specified, we default to 64k.
         }
-        return getMaxTokensLimitBedrock(option.model) ?? 8192; // Should always return a number for claude, 8192 is to satisfy the TypeScript type checker
+        return Math.min(16000 + thinking_budget, maxSupportedTokens); // Cap to 16k, to avoid taking up too much context window and quota.
     }
 }
 
@@ -130,7 +132,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return await formatConversePrompt(segments, opts);
     }
 
-    static getExtractedExecution(result: ConverseResponse, _prompt?: BedrockPrompt, options?: ExecutionOptions): CompletionChunkObject {
+    getExtractedExecution(result: ConverseResponse, _prompt?: BedrockPrompt, options?: ExecutionOptions): CompletionChunkObject {
         let resultText = "";
         let reasoning = "";
 
@@ -139,10 +141,9 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                 // Get text output
                 if (content.text) {
                     resultText += content.text;
-                }
-                // Get reasoning content only if include_thoughts is true
-                if (content.reasoningContent && options) {
-                    const claudeOptions = options.model_options as BedrockClaudeOptions;
+                } else if (content.reasoningContent) {
+                    // Get reasoning content only if include_thoughts is true
+                    const claudeOptions = options?.model_options as BedrockClaudeOptions;
                     if (claudeOptions?.include_thoughts) {
                         if (content.reasoningContent.reasoningText) {
                             reasoning += content.reasoningContent.reasoningText.text;
@@ -151,7 +152,15 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                             const redactedData = new TextDecoder().decode(content.reasoningContent.redactedContent);
                             reasoning += `[Redacted thinking: ${redactedData}]`;
                         }
+                    } else {
+                        this.logger.info("[Bedrock] Not outputting reasoning content as include_thoughts is false");
                     }
+                } else {
+                    // Get content block type
+                    const type = Object.keys(content).find(
+                        key => key !== '$unknown' && content[key as keyof typeof content] !== undefined
+                    );
+                    this.logger.info("[Bedrock] Unsupported content response type:", type);
                 }
             }
 
@@ -162,7 +171,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         }
 
         const completionResult: CompletionChunkObject = {
-            result: reasoning + resultText,
+            result: reasoning + resultText ? [{ type: "text", value: reasoning + resultText }] : [],
             token_usage: {
                 prompt: result.usage?.inputTokens,
                 result: result.usage?.outputTokens,
@@ -174,7 +183,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return completionResult;
     };
 
-    static getExtractedStream(result: ConverseStreamOutput, _prompt?: BedrockPrompt, options?: ExecutionOptions): CompletionChunkObject {
+    getExtractedStream(result: ConverseStreamOutput, _prompt?: BedrockPrompt, options?: ExecutionOptions): CompletionChunkObject {
         let output: string = "";
         let reasoning: string = "";
         let stop_reason = "";
@@ -209,7 +218,15 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                 } else if (delta.reasoningContent.signature) {
                     // Handle signature updates for reasoning content - end of thinking
                     reasoning = "\n\n";
+                    // Putting logging here so it only triggers once.
+                    this.logger.info("[Bedrock] Not outputting reasoning content as include_thoughts is false");
                 }
+            } else if (delta) {
+                // Get content block type
+                const type = Object.keys(delta).find(
+                    key => key !== '$unknown' && (delta as any)[key] !== undefined
+                );
+                this.logger.info("[Bedrock] Unsupported content response type:", type);
             }
         }
 
@@ -235,7 +252,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         }
 
         const completionResult: CompletionChunkObject = {
-            result: reasoning + output,
+            result: reasoning + output ? [{ type: "text", value: reasoning + output }] : [],
             token_usage: token_usage,
             finish_reason: converseFinishReason(stop_reason),
         };
@@ -335,26 +352,24 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         });
 
         let tool_use: ToolUse[] | undefined = undefined;
-        //Get tool requests
-        if (res.stopReason == "tool_use") {
-            tool_use = res.output?.message?.content?.reduce((tools: ToolUse[], c) => {
-                if (c.toolUse) {
-                    tools.push({
-                        tool_name: c.toolUse.name ?? "",
-                        tool_input: c.toolUse.input as any,
-                        id: c.toolUse.toolUseId ?? "",
-                    } satisfies ToolUse);
-                }
-                return tools;
-            }, []);
-            //If no tools were used, set to undefined
-            if (tool_use && tool_use.length == 0) {
-                tool_use = undefined;
+        //Get tool requests, we check tool use regardless of finish reason, as you can hit length and still get a valid response.
+        tool_use = res.output?.message?.content?.reduce((tools: ToolUse[], c) => {
+            if (c.toolUse) {
+                tools.push({
+                    tool_name: c.toolUse.name ?? "",
+                    tool_input: c.toolUse.input as any,
+                    id: c.toolUse.toolUseId ?? "",
+                } satisfies ToolUse);
             }
+            return tools;
+        }, []);
+        //If no tools were used, set to undefined
+        if (tool_use && tool_use.length == 0) {
+            tool_use = undefined;
         }
 
         const completion = {
-            ...BedrockDriver.getExtractedExecution(res, prompt, options),
+            ...this.getExtractedExecution(res, prompt, options),
             original_response: options.include_original_response ? res : undefined,
             conversation: conversation,
             tool_use: tool_use,
@@ -376,11 +391,11 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             }
 
             return transformAsyncIterator(stream, (streamSegment: ConverseStreamOutput) => {
-                return BedrockDriver.getExtractedStream(streamSegment, prompt, options);
+                return this.getExtractedStream(streamSegment, prompt, options);
             });
 
         }).catch((err) => {
-            this.logger.error("[Bedrock] Failed to stream", err);
+            this.logger.error("[Bedrock] Failed to stream", { error: err });
             throw err;
         });
     }
@@ -428,7 +443,6 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         } else if (options.model.includes("mistral")) {
             //7B instruct and 8x7B instruct
             if (options.model.includes("7b")) {
-                supportsJSONPrefill = true;
                 additionalField = { top_k: model_options.top_k };
                 //Does not support system messages
                 if (prompt.system && prompt.system?.length != 0) {
@@ -484,12 +498,17 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             }
         } else if (options.model.includes("deepseek")) {
             //DeepSeek models support no additional options
+        } else if (options.model.includes("gpt-oss")) {
+            const gptOssOptions = model_options as ModelOptions as BedrockGptOssOptions;
+            additionalField = {
+                reasoning_effort: gptOssOptions?.reasoning_effort,
+            };
         }
 
         //If last message is "```json", add corresponding ``` as a stop sequence.
         if (prompt.messages && prompt.messages.length > 0) {
             if (prompt.messages[prompt.messages.length - 1].content?.[0].text === "```json") {
-                let stopSeq = model_options.stop_sequence;
+                const stopSeq = model_options.stop_sequence;
                 if (!stopSeq) {
                     model_options.stop_sequence = ["```"];
                 } else if (!stopSeq.includes("```")) {
@@ -521,8 +540,8 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             }
         };
 
-        //Only add tools if they are defined
-        if (tool_defs) {
+        //Only add tools if they are defined and not empty
+        if (tool_defs?.length) {
             request.toolConfig = {
                 tools: tool_defs,
             }
@@ -532,7 +551,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
     }
 
 
-    async requestImageGeneration(prompt: NovaMessagesPrompt, options: ExecutionOptions): Promise<Completion<ImageGeneration>> {
+    async requestImageGeneration(prompt: NovaMessagesPrompt, options: ExecutionOptions): Promise<Completion> {
         if (options.output_modality !== Modalities.image) {
             throw new Error(`Image generation requires image output_modality`);
         }
@@ -564,13 +583,14 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         const decoder = new TextDecoder();
         const body = decoder.decode(res.body);
-        const result = JSON.parse(body);
+        const bedrockResult = JSON.parse(body);
 
         return {
-            error: result.error,
-            result: {
-                images: result.images,
-            }
+            error: bedrockResult.error,
+            result: bedrockResult.images.map((image: any) => ({
+                type: "image" as const,
+                value: image
+            }))
         }
     }
 
@@ -683,7 +703,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             foundationModels = foundationModels.filter(foundationFilter);
         }
 
-        const supportedPublishers = ["amazon", "anthropic", "cohere", "ai21", "mistral", "meta", "deepseek", "writer"];
+        const supportedPublishers = ["amazon", "anthropic", "cohere", "ai21", "mistral", "meta", "deepseek", "writer", "openai"];
         const unsupportedModelsByPublisher = {
             amazon: ["titan-image-generator", "nova-reel", "nova-sonic", "rerank"],
             anthropic: [],
@@ -693,6 +713,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             meta: [],
             deepseek: [],
             writer: [],
+            openai: [],
         };
 
         // Helper function to check if model should be filtered out
@@ -737,7 +758,6 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                 id: m.modelArn ?? m.modelId,
                 name: `${m.providerName} ${m.modelName}`,
                 provider: this.provider,
-                //description: ``,
                 owner: m.providerName,
                 can_stream: m.responseStreamingSupported ?? false,
                 input_modalities: m.inputModalities ? formatAmazonModalities(m.inputModalities) : modelModalitiesToArray(modelCapability.input),
@@ -762,6 +782,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                     id: m.modelArn,
                     name: m.modelName ?? m.modelArn,
                     provider: this.provider,
+                    owner: "custom",
                     description: `Custom model from ${m.baseModelName}`,
                     is_custom: true,
                     input_modalities: modelModalitiesToArray(modelCapability.input),
@@ -801,6 +822,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                         id: p.inferenceProfileArn ?? p.inferenceProfileId,
                         name: p.inferenceProfileName ?? p.inferenceProfileArn,
                         provider: this.provider,
+                        owner: providerName,
                         input_modalities: modelModalitiesToArray(modelCapability.input),
                         output_modalities: modelModalitiesToArray(modelCapability.output),
                         tool_support: modelCapability.tool_support,

package/src/groq/index.ts

@@ -1,16 +1,16 @@
-import { AIModel, AbstractDriver, Completion, CompletionChunkObject, DriverOptions, EmbeddingsOptions, EmbeddingsResult, ExecutionOptions, PromptSegment, TextFallbackOptions } from "@llumiverse/core";
+import { AIModel, AbstractDriver, Completion, CompletionChunkObject, DriverOptions, EmbeddingsOptions, EmbeddingsResult, ExecutionOptions, PromptSegment, TextFallbackOptions, ToolDefinition, ToolUse } from "@llumiverse/core";
 import { transformAsyncIterator } from "@llumiverse/core/async";
-import { OpenAITextMessage, formatOpenAILikeTextPrompt, getJSONSafetyNotice } from "@llumiverse/core/formatters";
-import Groq from "groq-sdk";
+import { formatOpenAILikeMultimodalPrompt } from "../openai/openai_format.js";
 
+import Groq from "groq-sdk";
+import type { ChatCompletionMessageParam, ChatCompletionTool } from "groq-sdk/resources/chat/completions";
 
 interface GroqDriverOptions extends DriverOptions {
     apiKey: string;
     endpoint_url?: string;
 }
 
-
-export class GroqDriver extends AbstractDriver<GroqDriverOptions, OpenAITextMessage[]> {
+export class GroqDriver extends AbstractDriver<GroqDriverOptions, ChatCompletionMessageParam[]> {
     static PROVIDER = "groq";
     provider = GroqDriver.PROVIDER;
     apiKey: string;
@@ -46,27 +46,169 @@ export class GroqDriver extends AbstractDriver<GroqDriverOptions, OpenAITextMess
         return undefined;
     }
 
-    protected async formatPrompt(segments: PromptSegment[], opts: ExecutionOptions): Promise<OpenAITextMessage[]> {
-        const messages = formatOpenAILikeTextPrompt(segments);
-        //Add JSON instruction is schema is provided
-        if (opts.result_schema) {
-            messages.push({
-                role: "user",
-                content: "IMPORTANT: " + getJSONSafetyNotice(opts.result_schema)
-            });
+    protected async formatPrompt(segments: PromptSegment[], opts: ExecutionOptions): Promise<ChatCompletionMessageParam[]> {
+        // Use OpenAI's multimodal formatter as base then convert to Groq types
+        const openaiMessages = await formatOpenAILikeMultimodalPrompt(segments, {
+            ...opts,
+            multimodal: true,
+        });
+
+        // Convert OpenAI ChatCompletionMessageParam[] to Groq ChatCompletionMessageParam[]
+        // Handle differences between OpenAI and Groq SDK types
+        const groqMessages: ChatCompletionMessageParam[] = openaiMessages.map(msg => {
+            // Handle OpenAI developer messages - convert to system messages for Groq
+            if (msg.role === 'developer' || msg.role === 'system') {
+                const systemMsg: ChatCompletionMessageParam = {
+                    role: 'system',
+                    content: Array.isArray(msg.content)
+                        ? msg.content.map(part => part.text).join('\n')
+                        : msg.content,
+                    // Preserve name if present
+                    ...(msg.name && { name: msg.name })
+                };
+                return systemMsg;
+            }
+
+            // Handle user messages - filter content parts to only supported types
+            if (msg.role === 'user') {
+                let content: string | Array<{ type: 'text', text: string } | { type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }> | undefined = undefined;
+
+                if (typeof msg.content === 'string') {
+                    content = msg.content;
+                } else if (Array.isArray(msg.content)) {
+                    // Filter to only text and image_url parts that Groq supports
+                    const supportedParts = msg.content.filter(part =>
+                        part.type === 'text' || part.type === 'image_url'
+                    ).map(part => {
+                        if (part.type === 'text') {
+                            return { type: 'text' as const, text: part.text };
+                        } else if (part.type === 'image_url') {
+                            return {
+                                type: 'image_url' as const,
+                                image_url: {
+                                    url: part.image_url.url,
+                                    ...(part.image_url.detail && { detail: part.image_url.detail })
+                                }
+                            };
+                        }
+                        return null;
+                    }).filter(Boolean) as Array<{ type: 'text', text: string } | { type: 'image_url', image_url: { url: string, detail?: 'auto' | 'low' | 'high' } }>;
+
+                    content = supportedParts.length > 0 ? supportedParts : 'Content not supported';
+                }
+
+                const userMsg: ChatCompletionMessageParam = {
+                    role: 'user',
+                    content: content ?? "",
+                    // Preserve name if present
+                    ...(msg.name && { name: msg.name })
+                };
+                return userMsg;
+            }
+
+            // Handle assistant messages - handle content arrays if needed
+            if (msg.role === 'assistant') {
+                const assistantMsg: ChatCompletionMessageParam = {
+                    role: 'assistant',
+                    content: Array.isArray(msg.content)
+                        ? msg.content.map(part => 'text' in part ? part.text : '').filter(Boolean).join('\n') || null
+                        : msg.content,
+                    // Preserve other assistant message properties
+                    ...(msg.tool_calls && { tool_calls: msg.tool_calls }),
+                    ...(msg.name && { name: msg.name })
+                };
+                return assistantMsg;
+            }
+
+            // For tool and function messages, they should be compatible
+            if (msg.role === 'tool') {
+                const toolMsg: ChatCompletionMessageParam = {
+                    role: 'tool',
+                    tool_call_id: msg.tool_call_id,
+                    content: Array.isArray(msg.content)
+                        ? msg.content.map(part => part.text).join('\n')
+                        : msg.content
+                };
+                return toolMsg;
+            }
+
+            if (msg.role === 'function') {
+                const functionMsg: ChatCompletionMessageParam = {
+                    role: 'function',
+                    name: msg.name,
+                    content: msg.content
+                };
+                return functionMsg;
+            }
+
+            // Fallback - should not reach here but provides type safety
+            throw new Error(`Unsupported message role: ${(msg as any).role}`);
+        });
+
+        return groqMessages;
+    }
+
+    private getToolDefinitions(tools: ToolDefinition[] | undefined): ChatCompletionTool[] | undefined {
+        if (!tools || tools.length === 0) {
+            return undefined;
+        }
+
+        return tools.map(tool => ({
+            type: 'function' as const,
+            function: {
+                name: tool.name,
+                description: tool.description,
+                parameters: tool.input_schema as any,
+            }
+        }));
+    }
+
+    private extractToolUse(message: any): ToolUse[] | undefined {
+        if (!message.tool_calls || message.tool_calls.length === 0) {
+            return undefined;
         }
-        return messages;
+
+        return message.tool_calls.map((toolCall: any) => ({
+            id: toolCall.id,
+            tool_name: toolCall.function.name,
+            tool_input: JSON.parse(toolCall.function.arguments || '{}'),
+        }));
+    }
+
+    private sanitizeMessagesForGroq(messages: ChatCompletionMessageParam[]): ChatCompletionMessageParam[] {
+        return messages.map(message => {
+            // Remove any reasoning field from message objects
+            const { reasoning, ...sanitizedMessage } = message as any;
+
+            // If message has content array, filter out reasoning content types
+            if (Array.isArray(sanitizedMessage.content)) {
+                sanitizedMessage.content = sanitizedMessage.content.filter((part: any) => {
+                    // Filter out any reasoning-related content parts
+                    return part.type !== 'reasoning' && !('reasoning' in part);
+                });
+            }
+
+            return sanitizedMessage as ChatCompletionMessageParam;
+        });
     }
 
-    async requestTextCompletion(messages: OpenAITextMessage[], options: ExecutionOptions): Promise<Completion<any>> {
+    async requestTextCompletion(messages: ChatCompletionMessageParam[], options: ExecutionOptions): Promise<Completion> {
         if (options.model_options?._option_id !== "text-fallback" && options.model_options?._option_id !== "groq-deepseek-thinking") {
-            this.logger.warn("Invalid model options", {options: options.model_options });
+            this.logger.warn("Invalid model options", { options: options.model_options });
         }
         options.model_options = options.model_options as TextFallbackOptions;
 
+        // Update conversation with current messages
+        let conversation = updateConversation(options.conversation as ChatCompletionMessageParam[], messages);
+
+        // Filter out any reasoning content that Groq doesn't support
+        conversation = this.sanitizeMessagesForGroq(conversation);
+
+        const tools = this.getToolDefinitions(options.tools);
+
         const res = await this.client.chat.completions.create({
             model: options.model,
-            messages: messages,
+            messages: conversation,
             max_completion_tokens: options.model_options?.max_tokens,
             temperature: options.model_options?.temperature,
             top_p: options.model_options?.top_p,
@@ -75,33 +217,54 @@ export class GroqDriver extends AbstractDriver<GroqDriverOptions, OpenAITextMess
             presence_penalty: options.model_options?.presence_penalty,
             frequency_penalty: options.model_options?.frequency_penalty,
             response_format: this.getResponseFormat(options),
+            tools: tools,
         });
 
-
         const choice = res.choices[0];
         const result = choice.message.content;
 
+        // Extract tool use from the response
+        const tool_use = this.extractToolUse(choice.message);
+
+        // Update conversation with the response
+        conversation = updateConversation(conversation, [choice.message]);
+
+        let finish_reason = choice.finish_reason;
+        if (tool_use && tool_use.length > 0) {
+            finish_reason = "tool_calls";
+        }
+
         return {
-            result: result,
+            result: result ? [{ type: "text", value: result }] : [],
             token_usage: {
                 prompt: res.usage?.prompt_tokens,
                 result: res.usage?.completion_tokens,
                 total: res.usage?.total_tokens,
             },
-            finish_reason: choice.finish_reason,
+            finish_reason: finish_reason,
             original_response: options.include_original_response ? res : undefined,
+            conversation,
+            tool_use,
         };
     }
 
-    async requestTextCompletionStream(messages: OpenAITextMessage[], options: ExecutionOptions): Promise < AsyncIterable < CompletionChunkObject >> {
+    async requestTextCompletionStream(messages: ChatCompletionMessageParam[], options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
         if (options.model_options?._option_id !== "text-fallback") {
-            this.logger.warn("Invalid model options", {options: options.model_options });
+            this.logger.warn("Invalid model options", { options: options.model_options });
         }
         options.model_options = options.model_options as TextFallbackOptions;
 
+        // Update conversation with current messages
+        let conversation = updateConversation(options.conversation as ChatCompletionMessageParam[], messages);
+
+        // Filter out any reasoning content that Groq doesn't support
+        conversation = this.sanitizeMessagesForGroq(conversation);
+
+        const tools = this.getToolDefinitions(options.tools);
+
         const res = await this.client.chat.completions.create({
             model: options.model,
-            messages: messages,
+            messages: conversation,
             max_completion_tokens: options.model_options?.max_tokens,
             temperature: options.model_options?.temperature,
             top_p: options.model_options?.top_p,
@@ -110,17 +273,28 @@ export class GroqDriver extends AbstractDriver<GroqDriverOptions, OpenAITextMess
             presence_penalty: options.model_options?.presence_penalty,
             frequency_penalty: options.model_options?.frequency_penalty,
             stream: true,
+            tools: tools,
         });
 
-        return transformAsyncIterator(res, (res) => ({
-            result: res.choices[0].delta.content ?? '',
-            finish_reason: res.choices[0].finish_reason,
-            token_usage: {
-                prompt: res.x_groq?.usage?.prompt_tokens,
-                result: res.x_groq?.usage?.completion_tokens,
-                total: res.x_groq?.usage?.total_tokens,
-            },
-            } as CompletionChunkObject));
+        return transformAsyncIterator(res, (chunk) => {
+            const choice = chunk.choices[0];
+            let finish_reason = choice.finish_reason;
+
+            // Check for tool calls in the delta
+            if (choice.delta.tool_calls && choice.delta.tool_calls.length > 0) {
+                finish_reason = "tool_calls";
+            }
+
+            return {
+                result: choice.delta.content ? [{ type: "text", value: choice.delta.content }] : [],
+                finish_reason: finish_reason,
+                token_usage: {
+                    prompt: chunk.x_groq?.usage?.prompt_tokens,
+                    result: chunk.x_groq?.usage?.completion_tokens,
+                    total: chunk.x_groq?.usage?.total_tokens,
+                },
+            } as CompletionChunkObject;
+        });
     }
 
     async listModels(): Promise<AIModel<string>[]> {
@@ -154,4 +328,17 @@ export class GroqDriver extends AbstractDriver<GroqDriverOptions, OpenAITextMess
         throw new Error("Method not implemented.");
     }
 
+}
+
+/**
+ * Update the conversation messages by combining existing conversation with new messages
+ * @param conversation Existing conversation messages
+ * @param messages New messages to add
+ * @returns Combined conversation
+ */
+function updateConversation(
+    conversation: ChatCompletionMessageParam[] | undefined,
+    messages: ChatCompletionMessageParam[]
+): ChatCompletionMessageParam[] {
+    return (conversation || []).concat(messages);
 }