modelfusion 0.83.0 → 0.85.0

package/README.md

@@ -58,7 +58,7 @@ const text = await generateText(
 );
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Hugging Face](https://modelfusion.dev/integration/model-provider/huggingface), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [OpenAI compatible](https://modelfusion.dev/integration/model-provider/openaicompatible), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Hugging Face](https://modelfusion.dev/integration/model-provider/huggingface), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic)
 
 #### streamText
 
@@ -75,7 +75,7 @@ for await (const textPart of textStream) {
 }
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai),[OpenAI compatible](https://modelfusion.dev/integration/model-provider/openaicompatible), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic)
 
 #### streamText with multi-modal prompt
 
@@ -95,7 +95,7 @@ const textStream = await streamText(
 );
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [OpenAI compatible](https://modelfusion.dev/integration/model-provider/openaicompatible), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
 
 ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
 
@@ -345,14 +345,12 @@ const result = await guard(
   fixStructure({
     modifyInputForRetry: async ({ input, error }) => [
       ...input,
-      {
-        role: "assistant",
-        content: null,
-        function_call: {
+      OpenAIChatMessage.assistant(null, {
+        functionCall: {
           name: "sentiment",
           arguments: JSON.stringify(error.valueText),
         },
-      } satisfies OpenAIChatMessage,
+      }),
       OpenAIChatMessage.user(error.message),
       OpenAIChatMessage.user("Please fix the error and try again."),
     ],
@@ -736,9 +734,11 @@ invokeFlow({
 - [Troubleshooting](https://modelfusion.dev/guide/troubleshooting)
   - [Bundling](https://modelfusion.dev/guide/troubleshooting/bundling)
 
+### [Integrations](https://modelfusion.dev/integration/model-provider)
+
 ### [Examples & Tutorials](https://modelfusion.dev/tutorial)
 
-### [Integrations](https://modelfusion.dev/integration/model-provider)
+### [Showcase](https://modelfusion.dev/showcase)
 
 ### [API Reference](https://modelfusion.dev/api/modules)
 

package/guard/fixStructure.cjs

@@ -45,14 +45,12 @@ const StructureValidationError_js_1 = require("../model-function/generate-struct
  *  fixStructure({
  *    modifyInputForRetry: async ({ input, error }) => [
  *      ...input,
- *      {
- *        role: "assistant",
-  *       content: null,
-          function_call: {
+ *      OpenAIChatMessage.assistant(null, {
+ *        functionCall: {
  *          name: "sentiment",
  *          arguments: JSON.stringify(error.valueText),
  *        },
- *      } satisfies OpenAIChatMessage,
+ *      }),
  *      OpenAIChatMessage.user(error.message),
  *      OpenAIChatMessage.user("Please fix the error and try again."),
  *    ],

package/guard/fixStructure.d.ts

@@ -43,14 +43,12 @@ import { Guard } from "./guard.js";
  *  fixStructure({
  *    modifyInputForRetry: async ({ input, error }) => [
  *      ...input,
- *      {
- *        role: "assistant",
-  *       content: null,
-          function_call: {
+ *      OpenAIChatMessage.assistant(null, {
+ *        functionCall: {
  *          name: "sentiment",
  *          arguments: JSON.stringify(error.valueText),
  *        },
- *      } satisfies OpenAIChatMessage,
+ *      }),
  *      OpenAIChatMessage.user(error.message),
  *      OpenAIChatMessage.user("Please fix the error and try again."),
  *    ],

package/guard/fixStructure.js

@@ -42,14 +42,12 @@ import { StructureValidationError } from "../model-function/generate-structure/S
  *  fixStructure({
  *    modifyInputForRetry: async ({ input, error }) => [
  *      ...input,
- *      {
- *        role: "assistant",
-  *       content: null,
-          function_call: {
+ *      OpenAIChatMessage.assistant(null, {
+ *        functionCall: {
  *          name: "sentiment",
  *          arguments: JSON.stringify(error.valueText),
  *        },
- *      } satisfies OpenAIChatMessage,
+ *      }),
  *      OpenAIChatMessage.user(error.message),
  *      OpenAIChatMessage.user("Please fix the error and try again."),
  *    ],

package/model-function/AbstractModel.d.ts

@@ -1,6 +1,7 @@
 import { ModelInformation } from "./ModelInformation.js";
 import { Model, ModelSettings } from "./Model.js";
 export declare abstract class AbstractModel<SETTINGS extends ModelSettings> implements Model<SETTINGS> {
+    readonly settings: SETTINGS;
     constructor({ settings }: {
         settings: SETTINGS;
     });
@@ -8,6 +9,5 @@ export declare abstract class AbstractModel<SETTINGS extends ModelSettings> impl
     abstract readonly modelName: string | null;
     get modelInformation(): ModelInformation;
     abstract get settingsForEvent(): Partial<SETTINGS>;
-    readonly settings: SETTINGS;
     abstract withSettings(additionalSettings: Partial<SETTINGS>): this;
 }

package/model-provider/index.cjs

@@ -23,4 +23,5 @@ __exportStar(require("./llamacpp/index.cjs"), exports);
 __exportStar(require("./lmnt/index.cjs"), exports);
 __exportStar(require("./ollama/index.cjs"), exports);
 __exportStar(require("./openai/index.cjs"), exports);
+__exportStar(require("./openai-compatible/index.cjs"), exports);
 __exportStar(require("./stability/index.cjs"), exports);

package/model-provider/index.d.ts

@@ -7,4 +7,5 @@ export * from "./llamacpp/index.js";
 export * from "./lmnt/index.js";
 export * from "./ollama/index.js";
 export * from "./openai/index.js";
+export * from "./openai-compatible/index.js";
 export * from "./stability/index.js";

package/model-provider/index.js

@@ -7,4 +7,5 @@ export * from "./llamacpp/index.js";
 export * from "./lmnt/index.js";
 export * from "./ollama/index.js";
 export * from "./openai/index.js";
+export * from "./openai-compatible/index.js";
 export * from "./stability/index.js";

package/model-provider/openai/AzureOpenAIApiConfiguration.cjs

@@ -4,7 +4,7 @@ exports.AzureOpenAIApiConfiguration = void 0;
 const AbstractApiConfiguration_js_1 = require("../../core/api/AbstractApiConfiguration.cjs");
 const loadApiKey_js_1 = require("../../core/api/loadApiKey.cjs");
 /**
- * Cconfiguration for the Azure OpenAI API. This class is responsible for constructing URLs specific to the Azure OpenAI deployment.
+ * Configuration for the Azure OpenAI API. This class is responsible for constructing URLs specific to the Azure OpenAI deployment.
  * It creates URLs of the form
  * `https://[resourceName].openai.azure.com/openai/deployments/[deploymentId]/[path]?api-version=[apiVersion]`
  *

package/model-provider/openai/AzureOpenAIApiConfiguration.d.ts

@@ -2,7 +2,7 @@ import { AbstractApiConfiguration } from "../../core/api/AbstractApiConfiguratio
 import { RetryFunction } from "../../core/api/RetryFunction.js";
 import { ThrottleFunction } from "../../core/api/ThrottleFunction.js";
 /**
- * Cconfiguration for the Azure OpenAI API. This class is responsible for constructing URLs specific to the Azure OpenAI deployment.
+ * Configuration for the Azure OpenAI API. This class is responsible for constructing URLs specific to the Azure OpenAI deployment.
  * It creates URLs of the form
  * `https://[resourceName].openai.azure.com/openai/deployments/[deploymentId]/[path]?api-version=[apiVersion]`
  *

package/model-provider/openai/AzureOpenAIApiConfiguration.js

@@ -1,7 +1,7 @@
 import { AbstractApiConfiguration } from "../../core/api/AbstractApiConfiguration.js";
 import { loadApiKey } from "../../core/api/loadApiKey.js";
 /**
- * Cconfiguration for the Azure OpenAI API. This class is responsible for constructing URLs specific to the Azure OpenAI deployment.
+ * Configuration for the Azure OpenAI API. This class is responsible for constructing URLs specific to the Azure OpenAI deployment.
  * It creates URLs of the form
  * `https://[resourceName].openai.azure.com/openai/deployments/[deploymentId]/[path]?api-version=[apiVersion]`
  *

package/model-provider/openai/chat/AbstractOpenAIChatModel.cjs

@@ -0,0 +1,228 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIChatResponseFormat = exports.AbstractOpenAIChatModel = void 0;
+const zod_1 = require("zod");
+const callWithRetryAndThrottle_js_1 = require("../../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../../core/api/postToApi.cjs");
+const parseJSON_js_1 = require("../../../core/schema/parseJSON.cjs");
+const AbstractModel_js_1 = require("../../../model-function/AbstractModel.cjs");
+const parsePartialJson_js_1 = require("../../../model-function/generate-structure/parsePartialJson.cjs");
+const OpenAIApiConfiguration_js_1 = require("../OpenAIApiConfiguration.cjs");
+const OpenAIError_js_1 = require("../OpenAIError.cjs");
+const OpenAIChatStreamIterable_js_1 = require("./OpenAIChatStreamIterable.cjs");
+/**
+ * Abstract text generation model that calls an API that is compatible with the OpenAI chat API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/chat/create
+ */
+class AbstractOpenAIChatModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings) {
+        super({ settings });
+    }
+    async callAPI(messages, options) {
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => callOpenAIChatCompletionAPI({
+                ...this.settings,
+                // function & tool calling:
+                functions: options.functions ?? this.settings.functions,
+                functionCall: options.functionCall ?? this.settings.functionCall,
+                tools: options.tools ?? this.settings.tools,
+                toolChoice: options.toolChoice ?? this.settings.toolChoice,
+                // map to OpenAI API names:
+                stop: this.settings.stopSequences,
+                maxTokens: this.settings.maxCompletionTokens,
+                openAIResponseFormat: this.settings.responseFormat,
+                // other settings:
+                user: this.settings.isUserIdForwardingEnabled
+                    ? options.run?.userId
+                    : undefined,
+                abortSignal: options.run?.abortSignal,
+                responseFormat: options.responseFormat,
+                messages,
+            }),
+        });
+    }
+    async doGenerateText(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OpenAIChatResponseFormat.json,
+        });
+        return {
+            response,
+            text: response.choices[0].message.content,
+            usage: this.extractUsage(response),
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OpenAIChatResponseFormat.textDeltaIterable,
+        });
+    }
+    async doGenerateToolCall(tool, prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OpenAIChatResponseFormat.json,
+            toolChoice: {
+                type: "function",
+                function: { name: tool.name },
+            },
+            tools: [
+                {
+                    type: "function",
+                    function: {
+                        name: tool.name,
+                        description: tool.description,
+                        parameters: tool.parameters.getJsonSchema(),
+                    },
+                },
+            ],
+        });
+        const toolCalls = response.choices[0]?.message.tool_calls;
+        return {
+            response,
+            toolCall: toolCalls == null || toolCalls.length === 0
+                ? null
+                : {
+                    id: toolCalls[0].id,
+                    args: (0, parseJSON_js_1.parseJSON)({ text: toolCalls[0].function.arguments }),
+                },
+            usage: this.extractUsage(response),
+        };
+    }
+    async doGenerateToolCallsOrText(tools, prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OpenAIChatResponseFormat.json,
+            toolChoice: "auto",
+            tools: tools.map((tool) => ({
+                type: "function",
+                function: {
+                    name: tool.name,
+                    description: tool.description,
+                    parameters: tool.parameters.getJsonSchema(),
+                },
+            })),
+        });
+        const message = response.choices[0]?.message;
+        return {
+            response,
+            text: message.content ?? null,
+            toolCalls: message.tool_calls?.map((toolCall) => ({
+                id: toolCall.id,
+                name: toolCall.function.name,
+                args: (0, parseJSON_js_1.parseJSON)({ text: toolCall.function.arguments }),
+            })) ?? null,
+            usage: this.extractUsage(response),
+        };
+    }
+    extractUsage(response) {
+        return {
+            promptTokens: response.usage.prompt_tokens,
+            completionTokens: response.usage.completion_tokens,
+            totalTokens: response.usage.total_tokens,
+        };
+    }
+}
+exports.AbstractOpenAIChatModel = AbstractOpenAIChatModel;
+const openAIChatResponseSchema = zod_1.z.object({
+    id: zod_1.z.string(),
+    choices: zod_1.z.array(zod_1.z.object({
+        message: zod_1.z.object({
+            role: zod_1.z.literal("assistant"),
+            content: zod_1.z.string().nullable(),
+            function_call: zod_1.z
+                .object({
+                name: zod_1.z.string(),
+                arguments: zod_1.z.string(),
+            })
+                .optional(),
+            tool_calls: zod_1.z
+                .array(zod_1.z.object({
+                id: zod_1.z.string(),
+                type: zod_1.z.literal("function"),
+                function: zod_1.z.object({
+                    name: zod_1.z.string(),
+                    arguments: zod_1.z.string(),
+                }),
+            }))
+                .optional(),
+        }),
+        index: zod_1.z.number(),
+        logprobs: zod_1.z.nullable(zod_1.z.any()),
+        finish_reason: zod_1.z
+            .enum([
+            "stop",
+            "length",
+            "tool_calls",
+            "content_filter",
+            "function_call",
+        ])
+            .optional()
+            .nullable(),
+    })),
+    created: zod_1.z.number(),
+    model: zod_1.z.string(),
+    system_fingerprint: zod_1.z.string().optional(),
+    object: zod_1.z.literal("chat.completion"),
+    usage: zod_1.z.object({
+        prompt_tokens: zod_1.z.number(),
+        completion_tokens: zod_1.z.number(),
+        total_tokens: zod_1.z.number(),
+    }),
+});
+async function callOpenAIChatCompletionAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, responseFormat, model, messages, functions, functionCall, tools, toolChoice, temperature, topP, n, stop, maxTokens, presencePenalty, frequencyPenalty, logitBias, user, openAIResponseFormat, seed, }) {
+    // empty arrays are not allowed for stop:
+    if (stop != null && Array.isArray(stop) && stop.length === 0) {
+        stop = undefined;
+    }
+    return (0, postToApi_js_1.postJsonToApi)({
+        url: api.assembleUrl("/chat/completions"),
+        headers: api.headers,
+        body: {
+            stream: responseFormat.stream,
+            model,
+            messages,
+            functions,
+            function_call: functionCall,
+            tools,
+            tool_choice: toolChoice,
+            temperature,
+            top_p: topP,
+            n,
+            stop,
+            max_tokens: maxTokens,
+            presence_penalty: presencePenalty,
+            frequency_penalty: frequencyPenalty,
+            logit_bias: logitBias,
+            seed,
+            response_format: openAIResponseFormat,
+            user,
+        },
+        failedResponseHandler: OpenAIError_js_1.failedOpenAICallResponseHandler,
+        successfulResponseHandler: responseFormat.handler,
+        abortSignal,
+    });
+}
+exports.OpenAIChatResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: (0, postToApi_js_1.createJsonResponseHandler)(openAIChatResponseSchema),
+    },
+    /**
+     * Returns an async iterable over the text deltas (only the tex different of the first choice).
+     */
+    textDeltaIterable: {
+        stream: true,
+        handler: async ({ response }) => (0, OpenAIChatStreamIterable_js_1.createOpenAIChatDeltaIterableQueue)(response.body, (delta) => delta[0]?.delta.content ?? ""),
+    },
+    structureDeltaIterable: {
+        stream: true,
+        handler: async ({ response }) => (0, OpenAIChatStreamIterable_js_1.createOpenAIChatDeltaIterableQueue)(response.body, (delta) => (0, parsePartialJson_js_1.parsePartialJson)(delta[0]?.function_call?.arguments)),
+    },
+};