@llumiverse/drivers 0.23.0 → 0.24.0

package/lib/esm/index.js

@@ -5,6 +5,7 @@ export * from "./huggingface_ie.js";
 export * from "./mistral/index.js";
 export * from "./openai/azure_openai.js";
 export * from "./openai/openai.js";
+export * from "./openai/openai_compatible.js";
 export * from "./replicate.js";
 export * from "./test-driver/index.js";
 export * from "./togetherai/index.js";

package/lib/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC;AACzC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC"}

package/lib/esm/openai/index.js

@@ -1,5 +1,4 @@
-import { AbstractDriver, ModelType, TrainingJobStatus, getModelCapabilities, modelModalitiesToArray, supportsToolUse, } from "@llumiverse/core";
-import { asyncMap } from "@llumiverse/core/async";
+import { AbstractDriver, ModelType, TrainingJobStatus, getConversationMeta, getModelCapabilities, incrementConversationTurn, modelModalitiesToArray, stripBase64ImagesFromConversation, supportsToolUse, truncateLargeTextInConversation, unwrapConversationArray, } from "@llumiverse/core";
 import { formatOpenAILikeMultimodalPrompt } from "./openai_format.js";
 // Helper function to convert string to CompletionResult[]
 function textToCompletionResult(text) {
@@ -17,17 +16,11 @@ export class BaseOpenAIDriver extends AbstractDriver {
     constructor(opts) {
         super(opts);
         this.formatPrompt = formatOpenAILikeMultimodalPrompt;
-        //TODO: better type, we send back OpenAI.Chat.Completions.ChatCompletionMessageParam[] but just not compatible with Function call that we don't use here
     }
     extractDataFromResponse(_options, result) {
-        const tokenInfo = {
-            prompt: result.usage?.prompt_tokens,
-            result: result.usage?.completion_tokens,
-            total: result.usage?.total_tokens,
-        };
-        const choice = result.choices[0];
-        const tools = collectTools(choice.message.tool_calls);
-        const data = choice.message.content ?? undefined;
+        const tokenInfo = mapUsage(result.usage);
+        const tools = collectTools(result.output);
+        const data = extractTextFromResponse(result);
         if (!data && !tools) {
             this.logger.error({ result }, "[OpenAI] Response is not valid");
             throw new Error("Response is not valid: no data");
@@ -35,7 +28,7 @@ export class BaseOpenAIDriver extends AbstractDriver {
         return {
             result: textToCompletionResult(data || ''),
             token_usage: tokenInfo,
-            finish_reason: openAiFinishReason(choice.finish_reason),
+            finish_reason: responseFinishReason(result, tools),
             tool_use: tools,
         };
     }
@@ -43,26 +36,10 @@ export class BaseOpenAIDriver extends AbstractDriver {
         if (options.model_options?._option_id !== "openai-text" && options.model_options?._option_id !== "openai-thinking") {
             this.logger.warn({ options: options.model_options }, "Invalid model options");
         }
+        // Include conversation history (same as non-streaming)
+        const conversation = updateConversation(options.conversation, prompt);
         const toolDefs = getToolDefinitions(options.tools);
-        const useTools = toolDefs ? supportsToolUse(options.model, "openai", true) : false;
-        const mapFn = (chunk) => {
-            let result = undefined;
-            if (useTools && this.provider !== "xai" && options.result_schema) {
-                result = chunk.choices[0]?.delta?.tool_calls?.[0].function?.arguments ?? "";
-            }
-            else {
-                result = chunk.choices[0]?.delta.content ?? "";
-            }
-            return {
-                result: textToCompletionResult(result),
-                finish_reason: openAiFinishReason(chunk.choices[0]?.finish_reason ?? undefined), //Uses expected "stop" , "length" format
-                token_usage: {
-                    prompt: chunk.usage?.prompt_tokens,
-                    result: chunk.usage?.completion_tokens,
-                    total: (chunk.usage?.prompt_tokens ?? 0) + (chunk.usage?.completion_tokens ?? 0),
-                }
-            };
-        };
+        const useTools = toolDefs ? supportsToolUse(options.model, this.provider, true) : false;
         convertRoles(prompt, options.model);
         const model_options = options.model_options;
         insert_image_detail(prompt, model_options?.image_detail ?? "auto");
@@ -78,30 +55,26 @@ export class BaseOpenAIDriver extends AbstractDriver {
                 strictMode = false;
             }
         }
-        const stream = await this.service.chat.completions.create({
+        const reasoning = model_options?.reasoning_effort ? { effort: model_options.reasoning_effort } : undefined;
+        const stream = await this.service.responses.create({
             stream: true,
-            stream_options: { include_usage: true },
             model: options.model,
-            messages: prompt,
-            reasoning_effort: model_options?.reasoning_effort,
+            input: conversation,
+            reasoning,
             temperature: model_options?.temperature,
             top_p: model_options?.top_p,
-            presence_penalty: model_options?.presence_penalty,
-            frequency_penalty: model_options?.frequency_penalty,
-            n: 1,
-            max_completion_tokens: model_options?.max_tokens, //TODO: use max_tokens for older models, currently relying on OpenAI to handle it
+            max_output_tokens: model_options?.max_tokens,
             tools: useTools ? toolDefs : undefined,
-            stop: model_options?.stop_sequence,
-            response_format: parsedSchema ? {
-                type: "json_schema",
-                json_schema: {
+            text: parsedSchema ? {
+                format: {
+                    type: "json_schema",
                     name: "format_output",
                     schema: parsedSchema,
                     strict: strictMode,
                 }
             } : undefined,
         });
-        return asyncMap(stream, mapFn);
+        return mapResponseStream(stream);
     }
     async requestTextCompletion(prompt, options) {
         if (options.model_options?._option_id !== "openai-text" && options.model_options?._option_id !== "openai-thinking") {
@@ -111,7 +84,7 @@ export class BaseOpenAIDriver extends AbstractDriver {
         const model_options = options.model_options;
         insert_image_detail(prompt, model_options?.image_detail ?? "auto");
         const toolDefs = getToolDefinitions(options.tools);
-        const useTools = toolDefs ? supportsToolUse(options.model, "openai") : false;
+        const useTools = toolDefs ? supportsToolUse(options.model, this.provider) : false;
         let conversation = updateConversation(options.conversation, prompt);
         let parsedSchema = undefined;
         let strictMode = false;
@@ -125,22 +98,19 @@ export class BaseOpenAIDriver extends AbstractDriver {
                 strictMode = false;
             }
         }
-        const res = await this.service.chat.completions.create({
+        const reasoning = model_options?.reasoning_effort ? { effort: model_options.reasoning_effort } : undefined;
+        const res = await this.service.responses.create({
             stream: false,
             model: options.model,
-            messages: conversation,
-            reasoning_effort: model_options?.reasoning_effort,
+            input: conversation,
+            reasoning,
             temperature: model_options?.temperature,
             top_p: model_options?.top_p,
-            presence_penalty: model_options?.presence_penalty,
-            frequency_penalty: model_options?.frequency_penalty,
-            n: 1,
-            max_completion_tokens: model_options?.max_tokens, //TODO: use max_tokens for older models, currently relying on OpenAI to handle it
+            max_output_tokens: model_options?.max_tokens, //TODO: use max_tokens for older models, currently relying on OpenAI to handle it
             tools: useTools ? toolDefs : undefined,
-            stop: model_options?.stop_sequence,
-            response_format: parsedSchema ? {
-                type: "json_schema",
-                json_schema: {
+            text: parsedSchema ? {
+                format: {
+                    type: "json_schema",
                     name: "format_output",
                     schema: parsedSchema,
                     strict: strictMode,
@@ -151,8 +121,20 @@ export class BaseOpenAIDriver extends AbstractDriver {
         if (options.include_original_response) {
             completion.original_response = res;
         }
-        conversation = updateConversation(conversation, createPromptFromResponse(res.choices[0].message));
-        completion.conversation = conversation;
+        conversation = updateConversation(conversation, createAssistantMessageFromCompletion(completion));
+        // Increment turn counter for deferred stripping
+        conversation = incrementConversationTurn(conversation);
+        // Strip large base64 image data based on options.stripImagesAfterTurns
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+        // Truncate large text content if configured
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+        completion.conversation = processedConversation;
         return completion;
     }
     canStream(_options) {
@@ -164,6 +146,47 @@ export class BaseOpenAIDriver extends AbstractDriver {
         }
         return Promise.resolve(true);
     }
+    /**
+     * Build conversation context after streaming completion.
+     * Reconstructs the assistant message from accumulated results and applies stripping.
+     */
+    buildStreamingConversation(prompt, result, toolUse, options) {
+        // Build assistant message from accumulated CompletionResult[]
+        const completionResults = result;
+        const textContent = completionResultsToText(completionResults);
+        // Start with the conversation from options or the prompt
+        let conversation = updateConversation(options.conversation, prompt);
+        // Add assistant message as EasyInputMessage
+        if (textContent) {
+            const assistantMessage = {
+                role: 'assistant',
+                content: textContent,
+            };
+            conversation = updateConversation(conversation, [assistantMessage]);
+        }
+        // Add function calls as separate items (Response API format)
+        if (toolUse && toolUse.length > 0) {
+            const functionCalls = toolUse.map(t => ({
+                type: 'function_call',
+                call_id: t.id,
+                name: t.tool_name,
+                arguments: typeof t.tool_input === 'string' ? t.tool_input : JSON.stringify(t.tool_input ?? {}),
+            }));
+            conversation = updateConversation(conversation, functionCalls);
+        }
+        // Increment turn counter
+        conversation = incrementConversationTurn(conversation);
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+        return processedConversation;
+    }
     createTrainingPrompt(options) {
         if (options.model.includes("gpt")) {
             return super.createTrainingPrompt(options);
@@ -286,44 +309,169 @@ function jobInfo(job) {
         details
     };
 }
-function insert_image_detail(messages, detail_level) {
-    if (detail_level == "auto" || detail_level == "low" || detail_level == "high") {
-        for (const message of messages) {
-            if (message.role !== 'assistant' && message.content) {
-                for (const part of message.content) {
-                    if (typeof part === "string") {
-                        continue;
+function mapUsage(usage) {
+    if (!usage) {
+        return undefined;
+    }
+    return {
+        prompt: usage.input_tokens,
+        result: usage.output_tokens,
+        total: usage.total_tokens,
+    };
+}
+function completionResultsToText(completionResults) {
+    if (!completionResults) {
+        return '';
+    }
+    return completionResults
+        .map(r => {
+        switch (r.type) {
+            case 'text':
+                return r.value;
+            case 'json':
+                return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+            case 'image':
+                // Skip images in conversation - they're in the result
+                return '';
+            default:
+                return String(r.value || '');
+        }
+    })
+        .join('');
+}
+function createAssistantMessageFromCompletion(completion) {
+    const textContent = completionResultsToText(completion.result);
+    const result = [];
+    // Add assistant text message if present
+    if (textContent) {
+        const assistantMessage = {
+            role: 'assistant',
+            content: textContent,
+        };
+        result.push(assistantMessage);
+    }
+    // Add function calls as separate items (Response API format)
+    if (completion.tool_use && completion.tool_use.length > 0) {
+        for (const t of completion.tool_use) {
+            const functionCall = {
+                type: 'function_call',
+                call_id: t.id,
+                name: t.tool_name,
+                arguments: typeof t.tool_input === 'string'
+                    ? t.tool_input
+                    : JSON.stringify(t.tool_input ?? {}),
+            };
+            result.push(functionCall);
+        }
+    }
+    return result;
+}
+function mapResponseStream(stream) {
+    const toolCallMetadata = new Map();
+    return {
+        async *[Symbol.asyncIterator]() {
+            for await (const event of stream) {
+                if (event.type === 'response.output_item.added' && event.item.type === 'function_call') {
+                    const syntheticId = `tool_${event.output_index}`;
+                    const actualId = event.item.id ?? event.item.call_id;
+                    if (actualId) {
+                        toolCallMetadata.set(actualId, { syntheticId, name: event.item.name });
                     }
-                    if (part.type === 'image_url') {
-                        part.image_url = { ...part.image_url, detail: detail_level };
+                    const toolUse = {
+                        id: syntheticId,
+                        _actual_id: actualId,
+                        tool_name: event.item.name,
+                        tool_input: '',
+                    };
+                    yield {
+                        result: [],
+                        tool_use: [toolUse],
+                    };
+                }
+                else if (event.type === 'response.function_call_arguments.delta') {
+                    const metadata = toolCallMetadata.get(event.item_id);
+                    const syntheticId = metadata?.syntheticId ?? `tool_${event.output_index}`;
+                    const toolUse = {
+                        id: syntheticId,
+                        _actual_id: event.item_id,
+                        tool_name: metadata?.name ?? '',
+                        tool_input: event.delta,
+                    };
+                    yield {
+                        result: [],
+                        tool_use: [toolUse],
+                    };
+                }
+                // Note: We don't emit response.function_call_arguments.done because the arguments were already
+                // streamed via delta events. Emitting it again would duplicate the tool_input content.
+                // We only update the metadata to ensure the tool name is captured.
+                else if (event.type === 'response.function_call_arguments.done') {
+                    // Just update metadata, don't yield (arguments already accumulated from delta events)
+                    const metadata = toolCallMetadata.get(event.item_id);
+                    const syntheticId = metadata?.syntheticId ?? `tool_${event.output_index}`;
+                    const tool_name = metadata?.name ?? event.name ?? '';
+                    if (event.item_id) {
+                        toolCallMetadata.set(event.item_id, { syntheticId, name: tool_name });
+                    }
+                }
+                else if (event.type === 'response.output_text.delta') {
+                    yield {
+                        result: textToCompletionResult(event.delta),
+                    };
+                }
+                // Note: We don't emit response.output_text.done because the text was already
+                // streamed via delta events. Emitting it again would duplicate the content.
+                else if (event.type === 'response.completed' || event.type === 'response.incomplete' || event.type === 'response.failed') {
+                    const finalTools = collectTools(event.response.output);
+                    yield {
+                        result: [],
+                        finish_reason: responseFinishReason(event.response, finalTools),
+                        token_usage: mapUsage(event.response.usage),
+                    };
+                }
+            }
+        }
+    };
+}
+function insert_image_detail(items, detail_level) {
+    if (detail_level === "auto" || detail_level === "low" || detail_level === "high") {
+        for (const item of items) {
+            // Check if it's an EasyInputMessage or Message with content array
+            if ('role' in item && 'content' in item && item.role !== 'assistant') {
+                const content = item.content;
+                if (Array.isArray(content)) {
+                    for (const part of content) {
+                        if (typeof part === 'object' && part.type === 'input_image') {
+                            part.detail = detail_level;
+                        }
                     }
                 }
             }
         }
     }
-    return messages;
+    return items;
 }
-function convertRoles(messages, model) {
+function convertRoles(items, model) {
     //New openai models use developer role instead of system
     if (model.includes("o1") || model.includes("o3")) {
         if (model.includes("o1-mini") || model.includes("o1-preview")) {
             //o1-mini and o1-preview support neither system nor developer
-            for (const message of messages) {
-                if (message.role === 'system') {
-                    message.role = 'user';
+            for (const item of items) {
+                if ('role' in item && item.role === 'system') {
+                    item.role = 'user';
                 }
             }
         }
         else {
             //Models newer than o1 use developer role
-            for (const message of messages) {
-                if (message.role === 'system') {
-                    message.role = 'developer';
+            for (const item of items) {
+                if ('role' in item && item.role === 'system') {
+                    item.role = 'developer';
                 }
             }
         }
     }
-    return messages;
+    return items;
 }
 //Structured output support is typically aligned with tool use support
 //Not true for realtime models, which do not support structured output, but do support tool use.
@@ -352,62 +500,63 @@ function getToolDefinition(toolDef) {
     }
     return {
         type: "function",
-        function: {
-            name: toolDef.name,
-            description: toolDef.description,
-            parameters: parsedSchema,
-            strict: strictMode,
-        },
+        name: toolDef.name,
+        description: toolDef.description,
+        parameters: parsedSchema ?? null,
+        strict: strictMode,
     };
 }
-function openAiFinishReason(finish_reason) {
-    if (finish_reason === "tool_calls") {
-        return "tool_use";
-    }
-    return finish_reason;
-}
-function updateConversation(conversation, message) {
-    if (!message) {
-        return conversation;
+function updateConversation(conversation, items) {
+    if (!items) {
+        // Unwrap array if wrapped, otherwise treat as array
+        const unwrapped = unwrapConversationArray(conversation);
+        return unwrapped ?? (conversation || []);
     }
     if (!conversation) {
-        return message;
+        return items;
     }
-    return [...conversation, ...message];
+    // Unwrap array if wrapped, otherwise treat as array
+    const unwrapped = unwrapConversationArray(conversation);
+    const convArray = unwrapped ?? conversation;
+    return [...convArray, ...items];
 }
-export function collectTools(toolCalls) {
-    if (!toolCalls) {
+export function collectTools(output) {
+    if (!output) {
         return undefined;
     }
     const tools = [];
-    for (const call of toolCalls) {
-        tools.push({
-            id: call.id,
-            tool_name: call.function.name,
-            tool_input: JSON.parse(call.function.arguments),
-        });
+    for (const item of output) {
+        if (item.type === 'function_call') {
+            const id = item.call_id || item.id;
+            if (!id) {
+                continue;
+            }
+            tools.push({
+                id,
+                tool_name: item.name ?? '',
+                tool_input: safeJsonParse(item.arguments),
+            });
+        }
     }
     return tools.length > 0 ? tools : undefined;
 }
-function createPromptFromResponse(response) {
-    const messages = [];
-    if (response) {
-        messages.push({
-            role: response.role,
-            content: [{
-                    type: "text",
-                    text: response.content ?? ""
-                }],
-            tool_calls: response.tool_calls,
-        });
-    }
-    return messages;
-}
 //For strict mode false
 function limitedSchemaFormat(schema) {
     const formattedSchema = { ...schema };
     // Defaults not supported
     delete formattedSchema.default;
+    // OpenAI requires type field even in non-strict mode
+    // If no type is specified, default to 'object' for properties with format/editor hints,
+    // otherwise 'string' as a safe fallback
+    if (!formattedSchema.type && formattedSchema.description) {
+        // Properties with format: "document" or editor hints are typically objects
+        if (formattedSchema.format === 'document' || formattedSchema.editor) {
+            formattedSchema.type = 'object';
+        }
+        else {
+            formattedSchema.type = 'string';
+        }
+    }
     if (formattedSchema?.properties) {
         // Process each property recursively
         for (const propName of Object.keys(formattedSchema.properties)) {
@@ -443,6 +592,10 @@ function openAISchemaFormat(schema, nesting = 0) {
         // Process each property recursively
         for (const propName of Object.keys(formattedSchema.properties)) {
             const property = formattedSchema.properties[propName];
+            // OpenAI strict mode requires all properties to have a type
+            if (!property?.type) {
+                throw new Error(`Property '${propName}' is missing required 'type' field for OpenAI strict mode`);
+            }
             // Recursively process properties
             formattedSchema.properties[propName] = openAISchemaFormat(property, nesting + 1);
             // Process arrays with items of type object
@@ -461,4 +614,47 @@ function openAISchemaFormat(schema, nesting = 0) {
     }
     return formattedSchema;
 }
+function extractTextFromResponse(response) {
+    if (response.output_text) {
+        return response.output_text;
+    }
+    const collected = [];
+    for (const item of response.output ?? []) {
+        if (item.type === 'message') {
+            const text = item.content
+                .map(part => part.type === 'output_text' ? part.text : '')
+                .join('');
+            if (text) {
+                collected.push(text);
+            }
+        }
+    }
+    return collected.join("\n");
+}
+function responseFinishReason(response, tools) {
+    if (tools && tools.length > 0) {
+        return "tool_use";
+    }
+    if (response.status === 'incomplete') {
+        if (response.incomplete_details?.reason === 'max_output_tokens') {
+            return 'length';
+        }
+        return response.incomplete_details?.reason ?? 'incomplete';
+    }
+    if (response.status && response.status !== 'completed') {
+        return response.status;
+    }
+    return 'stop';
+}
+function safeJsonParse(value) {
+    if (typeof value !== 'string') {
+        return value;
+    }
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return value;
+    }
+}
 //# sourceMappingURL=index.js.map