@llumiverse/drivers 1.3.0 → 1.4.0

package/src/openai/openai.ts

@@ -1,30 +1,23 @@
-
-
-import { DriverOptions, Providers } from "@llumiverse/core";
-import OpenAI from "openai";
-import { BaseOpenAIDriver } from "./index.js";
+import { type DriverOptions, Providers } from '@llumiverse/core';
+import OpenAI from 'openai';
+import { BaseOpenAIDriver } from './index.js';
 
 export interface OpenAIDriverOptions extends DriverOptions {
-
     /**
      * The OpenAI api key
      */
     apiKey?: string; //type with azure credentials
-    
 }
 
-
-
-
 export class OpenAIDriver extends BaseOpenAIDriver {
-
     service: OpenAI;
     readonly provider = Providers.openai;
 
     constructor(opts: OpenAIDriverOptions) {
         super(opts);
         this.service = new OpenAI({
-            apiKey: opts.apiKey
+            apiKey: opts.apiKey,
+            fetch: this.getDriverFetch(),
         });
     }
-}
+}

package/src/openai/openai_compatible.ts

@@ -1,6 +1,13 @@
-import { AIModel, DriverOptions, ModelType, Providers, getModelCapabilities, modelModalitiesToArray } from "@llumiverse/core";
-import OpenAI from "openai";
-import { BaseOpenAIDriver } from "./index.js";
+import {
+    type AIModel,
+    type DriverOptions,
+    getModelCapabilities,
+    ModelType,
+    modelModalitiesToArray,
+    Providers,
+} from '@llumiverse/core';
+import OpenAI from 'openai';
+import { BaseOpenAIDriver } from './index.js';
 
 export interface OpenAICompatibleDriverOptions extends DriverOptions {
     /**
@@ -34,17 +41,18 @@ export class OpenAICompatibleDriver extends BaseOpenAIDriver {
         super(opts);
 
         if (!opts.apiKey) {
-            throw new Error("apiKey is required");
+            throw new Error('apiKey is required');
         }
 
         if (!opts.endpoint) {
-            throw new Error("endpoint is required for OpenAI-compatible driver");
+            throw new Error('endpoint is required for OpenAI-compatible driver');
         }
 
         this.service = new OpenAI({
             apiKey: opts.apiKey,
             baseURL: opts.endpoint,
             defaultHeaders: opts.default_headers,
+            fetch: this.getDriverFetch(),
         });
     }
 
@@ -52,29 +60,31 @@ export class OpenAICompatibleDriver extends BaseOpenAIDriver {
         try {
             const result = (await this.service.models.list()).data;
 
-            const models = result.map((m) => {
-                const modelCapability = getModelCapabilities(m.id, "openai");
-                let owner = m.owned_by;
-                if (owner === "system") {
-                    owner = "unknown";
-                }
-                return {
-                    id: m.id,
-                    name: m.id,
-                    provider: this.provider,
-                    owner: owner,
-                    type: ModelType.Text,
-                    can_stream: true,
-                    is_multimodal: false,
-                    input_modalities: modelModalitiesToArray(modelCapability.input),
-                    output_modalities: modelModalitiesToArray(modelCapability.output),
-                    tool_support: modelCapability.tool_support,
-                } satisfies AIModel<string>;
-            }).sort((a, b) => a.id.localeCompare(b.id));
+            const models = result
+                .map((m) => {
+                    const modelCapability = getModelCapabilities(m.id, 'openai');
+                    let owner = m.owned_by;
+                    if (owner === 'system') {
+                        owner = 'unknown';
+                    }
+                    return {
+                        id: m.id,
+                        name: m.id,
+                        provider: this.provider,
+                        owner: owner,
+                        type: ModelType.Text,
+                        can_stream: true,
+                        is_multimodal: false,
+                        input_modalities: modelModalitiesToArray(modelCapability.input),
+                        output_modalities: modelModalitiesToArray(modelCapability.output),
+                        tool_support: modelCapability.tool_support,
+                    } satisfies AIModel<string>;
+                })
+                .sort((a, b) => a.id.localeCompare(b.id));
 
             return models;
         } catch (error) {
-            this.logger.warn({ error }, "[OpenAICompatible] Failed to list models, returning empty list");
+            this.logger.warn({ error }, '[OpenAICompatible] Failed to list models, returning empty list');
             return [];
         }
     }

package/src/openai/openai_format.ts

@@ -1,15 +1,58 @@
 // This file is used by multiple drivers
 // to format prompts in a way that is compatible with OpenAI's API.
 
-import { PromptRole, PromptOptions, PromptSegment } from "@llumiverse/common";
-import { readStreamAsBase64 } from "@llumiverse/core";
-import type OpenAI from "openai";
+import { type PromptOptions, PromptRole, type PromptSegment } from '@llumiverse/common';
+import { readStreamAsBase64 } from '@llumiverse/core';
+import type OpenAI from 'openai';
+import { truncateDataUrlForDebug } from '../shared/debug-prompt.js';
 
 // Types for Response API
 type ResponseInputItem = OpenAI.Responses.ResponseInputItem;
 type ResponseInputContent = OpenAI.Responses.ResponseInputContent;
 type EasyInputMessage = OpenAI.Responses.EasyInputMessage;
 
+function isResponseInputContent(value: unknown): value is ResponseInputContent {
+    return (
+        typeof value === 'object' &&
+        value !== null &&
+        'type' in value &&
+        (value.type === 'input_text' || value.type === 'input_image' || value.type === 'input_file')
+    );
+}
+
+function isEasyInputMessageWithInputContent(
+    item: ResponseInputItem,
+): item is EasyInputMessage & { content: ResponseInputContent[] } {
+    return 'content' in item && Array.isArray(item.content) && item.content.every(isResponseInputContent);
+}
+
+export function formatOpenAIDebugPrompt(prompt: ResponseInputItem[]): ResponseInputItem[] {
+    return prompt.map((item) => {
+        if (!isEasyInputMessageWithInputContent(item)) {
+            return item;
+        }
+        const content = item.content.map((content) => {
+            if (content.type === 'input_image' && 'image_url' in content && typeof content.image_url === 'string') {
+                return {
+                    ...content,
+                    image_url: truncateDataUrlForDebug(content.image_url),
+                } satisfies ResponseInputContent;
+            }
+            if (content.type === 'input_file' && 'file_data' in content && typeof content.file_data === 'string') {
+                return {
+                    ...content,
+                    file_data: truncateDataUrlForDebug(content.file_data),
+                } satisfies ResponseInputContent;
+            }
+            return content;
+        });
+        return {
+            ...item,
+            content,
+        } satisfies EasyInputMessage;
+    });
+}
+
 export interface OpenAITextMessage {
     content: string;
     role: 'system' | 'user' | 'assistant' | 'developer';
@@ -27,14 +70,14 @@ export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITex
 
     for (const msg of segments) {
         if (msg.role === PromptRole.system) {
-            system.push({ content: msg.content, role: "system" });
+            system.push({ content: msg.content, role: 'system' });
         } else if (msg.role === PromptRole.safety) {
-            safety.push({ content: "IMPORTANT: " + msg.content, role: "system" });
+            safety.push({ content: `IMPORTANT: ${msg.content}`, role: 'system' });
         } else if (msg.role !== PromptRole.negative && msg.role !== PromptRole.mask && msg.role !== PromptRole.tool) {
             user.push({
                 content: msg.content,
                 role: msg.role || 'user',
-            })
+            });
         }
     }
 
@@ -42,14 +85,15 @@ export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITex
     return system.concat(user).concat(safety);
 }
 
-
-export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[], opts: PromptOptions & OpenAIPromptFormatterOptions): Promise<ResponseInputItem[]> {
+export async function formatOpenAILikeMultimodalPrompt(
+    segments: PromptSegment[],
+    opts: PromptOptions & OpenAIPromptFormatterOptions,
+): Promise<ResponseInputItem[]> {
     const system: ResponseInputItem[] = [];
     const safety: ResponseInputItem[] = [];
     const others: ResponseInputItem[] = [];
 
     for (const msg of segments) {
-
         const parts: ResponseInputContent[] = [];
 
         //generate the parts based on PromptSegment
@@ -58,98 +102,104 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
                 const stream = await file.getStream();
                 const data = await readStreamAsBase64(stream);
                 parts.push({
-                    type: "input_image",
-                    image_url: `data:${file.mime_type || "image/jpeg"};base64,${data}`,
-                    detail: "auto",
-                })
+                    type: 'input_image',
+                    image_url: `data:${file.mime_type || 'image/jpeg'};base64,${data}`,
+                    detail: 'auto',
+                });
             }
         }
 
         if (msg.content) {
             parts.push({
-                type: "input_text",
+                type: 'input_text',
                 text: msg.content,
-            })
+            });
         }
 
-
         if (msg.role === PromptRole.system) {
             // For system messages, filter to only text parts
-            const textParts = parts.filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text');
+            const textParts = parts.filter(
+                (part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text',
+            );
             const textContent = textParts.length === 1 && !msg.files ? textParts[0].text : textParts;
             const systemMsg: EasyInputMessage = {
-                role: "system",
+                type: 'message',
+                role: 'system',
                 content: textContent,
             };
             system.push(systemMsg);
 
             if (opts.useToolForFormatting && opts.schema) {
-                system.forEach(s => {
+                system.forEach((s) => {
                     if ((s as EasyInputMessage).role === 'system') {
                         const sysMsg = s as EasyInputMessage;
                         if (typeof sysMsg.content === 'string') {
-                            sysMsg.content = "TOOL: " + sysMsg.content;
+                            sysMsg.content = `TOOL: ${sysMsg.content}`;
                         } else if (Array.isArray(sysMsg.content)) {
-                            sysMsg.content.forEach((c: any) => {
-                                if (c.type === "input_text") c.text = "TOOL: " + c.text;
+                            sysMsg.content.forEach((c) => {
+                                if (c.type === 'input_text') c.text = `TOOL: ${c.text}`;
                             });
                         }
                     }
                 });
             }
-
         } else if (msg.role === PromptRole.safety) {
-            const textParts = parts.filter((part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text');
+            const textParts = parts.filter(
+                (part): part is OpenAI.Responses.ResponseInputText => part.type === 'input_text',
+            );
             const safetyMsg: EasyInputMessage = {
-                role: "system",
+                type: 'message',
+                role: 'system',
                 content: textParts,
             };
 
             if (Array.isArray(safetyMsg.content)) {
-                safetyMsg.content.forEach((c: any) => {
-                    if (c.type === "input_text") c.text = "DO NOT IGNORE - IMPORTANT: " + c.text;
+                safetyMsg.content.forEach((c) => {
+                    if (c.type === 'input_text') c.text = `DO NOT IGNORE - IMPORTANT: ${c.text}`;
                 });
             }
 
             system.push(safetyMsg);
         } else if (msg.role === PromptRole.tool) {
             if (!msg.tool_use_id) {
-                throw new Error("Tool use id is required for tool messages")
+                throw new Error('Tool use id is required for tool messages');
             }
             const toolOutputMsg: OpenAI.Responses.ResponseInputItem.FunctionCallOutput = {
-                type: "function_call_output",
+                type: 'function_call_output',
                 call_id: msg.tool_use_id,
-                output: msg.content || ""
+                output: msg.content || '',
             };
             others.push(toolOutputMsg);
         } else if (msg.role !== PromptRole.negative && msg.role !== PromptRole.mask) {
             const inputMsg: EasyInputMessage = {
+                type: 'message',
                 role: msg.role === 'assistant' ? 'assistant' : 'user',
                 content: parts,
             };
             others.push(inputMsg);
         }
-
     }
 
     if (opts.result_schema && !opts.useToolForFormatting) {
         const schemaMsg: EasyInputMessage = {
-            role: "system",
-            content: [{
-                type: "input_text",
-                text: "IMPORTANT: only answer using JSON, and respecting the schema included below, between the <response_schema> tags. " + `<response_schema>${JSON.stringify(opts.result_schema)}</response_schema>`
-            }]
+            type: 'message',
+            role: 'system',
+            content: [
+                {
+                    type: 'input_text',
+                    text: `IMPORTANT: only answer using JSON, and respecting the schema included below, between the <response_schema> tags. <response_schema>${JSON.stringify(opts.result_schema)}</response_schema>`,
+                },
+            ],
         };
         system.push(schemaMsg);
     }
 
     // put system messages first and safety last
     return ([] as ResponseInputItem[]).concat(system).concat(others).concat(safety);
-
 }
 
 export interface OpenAIPromptFormatterOptions {
-    multimodal?: boolean
-    useToolForFormatting?: boolean
-    schema?: object
+    multimodal?: boolean;
+    useToolForFormatting?: boolean;
+    schema?: object;
 }