modelfusion 0.70.0 → 0.71.0

package/README.md

@@ -52,7 +52,7 @@ const text = await generateText(
 );
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Hugging Face](https://modelfusion.dev/integration/model-provider/huggingface)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Hugging Face](https://modelfusion.dev/integration/model-provider/huggingface), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic)
 
 #### streamText
 
@@ -67,7 +67,7 @@ for await (const textPart of textStream) {
 }
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic)
 
 #### streamText with multi-modal prompt
 
@@ -304,7 +304,7 @@ const embeddings = await embedMany(
 );
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Hugging Face](https://modelfusion.dev/integration/model-provider/huggingface)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama), [Hugging Face](https://modelfusion.dev/integration/model-provider/huggingface), [Cohere](https://modelfusion.dev/integration/model-provider/cohere)
 
 ### [Tokenize Text](https://modelfusion.dev/guide/function/tokenize-text)
 
@@ -322,7 +322,7 @@ const tokensAndTokenTexts = await tokenizer.tokenizeWithTexts(text);
 const reconstructedText = await tokenizer.detokenize(tokens);
 ```
 
-Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Cohere](https://modelfusion.dev/integration/model-provider/cohere)
 
 ### [Guards](https://modelfusion.dev/guide/guard)
 

package/model-function/ModelCallEvent.d.ts

@@ -7,6 +7,7 @@ import { StructureGenerationFinishedEvent, StructureGenerationStartedEvent } fro
 import { StructureStreamingFinishedEvent, StructureStreamingStartedEvent } from "./generate-structure/StructureStreamingEvent.js";
 import { TextGenerationFinishedEvent, TextGenerationStartedEvent, TextStreamingFinishedEvent, TextStreamingStartedEvent } from "./generate-text/TextGenerationEvent.js";
 import { ToolCallGenerationFinishedEvent, ToolCallGenerationStartedEvent } from "./generate-tool-call/ToolCallGenerationEvent.js";
+import { ToolCallsOrTextGenerationFinishedEvent, ToolCallsOrTextGenerationStartedEvent } from "./generate-tool-call/ToolCallsOrTextGenerationEvent.js";
 import { TranscriptionFinishedEvent, TranscriptionStartedEvent } from "./generate-transcription/TranscriptionEvent.js";
 export interface BaseModelCallStartedEvent extends BaseFunctionStartedEvent {
     model: ModelInformation;
@@ -48,5 +49,5 @@ export interface BaseModelCallFinishedEvent extends BaseFunctionFinishedEvent {
      */
     result: BaseModelCallFinishedEventResult;
 }
-export type ModelCallStartedEvent = EmbeddingStartedEvent | ImageGenerationStartedEvent | SpeechGenerationStartedEvent | SpeechStreamingStartedEvent | StructureGenerationStartedEvent | StructureStreamingStartedEvent | TextGenerationStartedEvent | TextStreamingStartedEvent | ToolCallGenerationStartedEvent | TranscriptionStartedEvent;
-export type ModelCallFinishedEvent = EmbeddingFinishedEvent | ImageGenerationFinishedEvent | SpeechGenerationFinishedEvent | SpeechStreamingFinishedEvent | StructureGenerationFinishedEvent | StructureStreamingFinishedEvent | TextGenerationFinishedEvent | TextStreamingFinishedEvent | ToolCallGenerationFinishedEvent | TranscriptionFinishedEvent;
+export type ModelCallStartedEvent = EmbeddingStartedEvent | ImageGenerationStartedEvent | SpeechGenerationStartedEvent | SpeechStreamingStartedEvent | StructureGenerationStartedEvent | StructureStreamingStartedEvent | TextGenerationStartedEvent | TextStreamingStartedEvent | ToolCallGenerationStartedEvent | ToolCallsOrTextGenerationStartedEvent | TranscriptionStartedEvent;
+export type ModelCallFinishedEvent = EmbeddingFinishedEvent | ImageGenerationFinishedEvent | SpeechGenerationFinishedEvent | SpeechStreamingFinishedEvent | StructureGenerationFinishedEvent | StructureStreamingFinishedEvent | TextGenerationFinishedEvent | TextStreamingFinishedEvent | ToolCallGenerationFinishedEvent | ToolCallsOrTextGenerationFinishedEvent | TranscriptionFinishedEvent;

package/model-function/generate-tool-call/NoSuchToolDefinitionError.cjs

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NoSuchToolDefinitionError = void 0;
+class NoSuchToolDefinitionError extends Error {
+    constructor({ toolName, parameters, }) {
+        super(`Tool definition '${toolName}' not found. ` +
+            `Parameters: ${JSON.stringify(parameters)}.`);
+        Object.defineProperty(this, "toolName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cause", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "parameters", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.name = "NoSuchToolDefinitionError";
+        this.toolName = toolName;
+        this.parameters = parameters;
+    }
+    toJSON() {
+        return {
+            name: this.name,
+            message: this.message,
+            cause: this.cause,
+            stack: this.stack,
+            toolName: this.toolName,
+            parameter: this.parameters,
+        };
+    }
+}
+exports.NoSuchToolDefinitionError = NoSuchToolDefinitionError;

package/model-function/generate-tool-call/NoSuchToolDefinitionError.d.ts

@@ -0,0 +1,17 @@
+export declare class NoSuchToolDefinitionError extends Error {
+    readonly toolName: string;
+    readonly cause: unknown;
+    readonly parameters: unknown;
+    constructor({ toolName, parameters, }: {
+        toolName: string;
+        parameters: unknown;
+    });
+    toJSON(): {
+        name: string;
+        message: string;
+        cause: unknown;
+        stack: string | undefined;
+        toolName: string;
+        parameter: unknown;
+    };
+}

package/model-function/generate-tool-call/NoSuchToolDefinitionError.js

@@ -0,0 +1,37 @@
+export class NoSuchToolDefinitionError extends Error {
+    constructor({ toolName, parameters, }) {
+        super(`Tool definition '${toolName}' not found. ` +
+            `Parameters: ${JSON.stringify(parameters)}.`);
+        Object.defineProperty(this, "toolName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cause", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "parameters", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.name = "NoSuchToolDefinitionError";
+        this.toolName = toolName;
+        this.parameters = parameters;
+    }
+    toJSON() {
+        return {
+            name: this.name,
+            message: this.message,
+            cause: this.cause,
+            stack: this.stack,
+            toolName: this.toolName,
+            parameter: this.parameters,
+        };
+    }
+}

package/model-function/generate-tool-call/ToolCall.d.ts

@@ -0,0 +1,5 @@
+export interface ToolCall<NAME extends string, PARAMETERS> {
+    id: string;
+    name: NAME;
+    parameters: PARAMETERS;
+}

package/model-function/generate-tool-call/ToolCallGenerationModel.d.ts

@@ -1,12 +1,12 @@
 import { FunctionOptions } from "../../core/FunctionOptions.js";
 import { Model, ModelSettings } from "../Model.js";
-import { ToolCallDefinition } from "./ToolCallDefinition.js";
+import { ToolDefinition } from "./ToolDefinition.js";
 export interface ToolCallGenerationModelSettings extends ModelSettings {
 }
 export interface ToolCallGenerationModel<PROMPT, SETTINGS extends ToolCallGenerationModelSettings = ToolCallGenerationModelSettings> extends Model<SETTINGS> {
-    doGenerateToolCall(tool: ToolCallDefinition<string, unknown>, prompt: PROMPT, options?: FunctionOptions): PromiseLike<{
+    doGenerateToolCall(tool: ToolDefinition<string, unknown>, prompt: PROMPT, options?: FunctionOptions): PromiseLike<{
         response: unknown;
-        value: {
+        toolCall: {
             id: string;
             parameters: unknown;
         } | null;

package/model-function/generate-tool-call/ToolCallParametersValidationError.cjs

@@ -5,7 +5,7 @@ const getErrorMessage_js_1 = require("../../util/getErrorMessage.cjs");
 class ToolCallParametersValidationError extends Error {
     constructor({ toolName, parameters, cause, }) {
         super(`Parameter validation failed for tool '${toolName}'. ` +
-            `Value: ${JSON.stringify(parameters)}.\n` +
+            `Parameters: ${JSON.stringify(parameters)}.\n` +
             `Error message: ${(0, getErrorMessage_js_1.getErrorMessage)(cause)}`);
         Object.defineProperty(this, "toolName", {
             enumerable: true,

package/model-function/generate-tool-call/ToolCallParametersValidationError.js

@@ -2,7 +2,7 @@ import { getErrorMessage } from "../../util/getErrorMessage.js";
 export class ToolCallParametersValidationError extends Error {
     constructor({ toolName, parameters, cause, }) {
         super(`Parameter validation failed for tool '${toolName}'. ` +
-            `Value: ${JSON.stringify(parameters)}.\n` +
+            `Parameters: ${JSON.stringify(parameters)}.\n` +
             `Error message: ${getErrorMessage(cause)}`);
         Object.defineProperty(this, "toolName", {
             enumerable: true,

package/model-function/generate-tool-call/ToolCallsOrTextGenerationEvent.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-tool-call/ToolCallsOrTextGenerationEvent.d.ts

@@ -0,0 +1,23 @@
+import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
+export interface ToolCallsOrTextGenerationStartedEvent extends BaseModelCallStartedEvent {
+    functionType: "generate-tool-calls-or-text";
+}
+export type ToolCallsOrTextGenerationFinishedEventResult = {
+    status: "success";
+    response: unknown;
+    value: unknown;
+    usage?: {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+    };
+} | {
+    status: "error";
+    error: unknown;
+} | {
+    status: "abort";
+};
+export type ToolCallsOrTextGenerationFinishedEvent = BaseModelCallFinishedEvent & {
+    functionType: "generate-tool-calls-or-text";
+    result: ToolCallsOrTextGenerationFinishedEventResult;
+};

package/model-function/generate-tool-call/ToolCallsOrTextGenerationEvent.js

@@ -0,0 +1 @@
+export {};

package/model-function/generate-tool-call/ToolCallsOrTextGenerationModel.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-tool-call/ToolCallsOrTextGenerationModel.d.ts

@@ -0,0 +1,21 @@
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { Model, ModelSettings } from "../Model.js";
+import { ToolDefinition } from "./ToolDefinition.js";
+export interface ToolCallsOrTextGenerationModelSettings extends ModelSettings {
+}
+export interface ToolCallsOrTextGenerationModel<PROMPT, SETTINGS extends ToolCallsOrTextGenerationModelSettings> extends Model<SETTINGS> {
+    doGenerateToolCallsOrText(tools: Array<ToolDefinition<string, unknown>>, prompt: PROMPT, options?: FunctionOptions): PromiseLike<{
+        response: unknown;
+        text: string | null;
+        toolCalls: Array<{
+            id: string;
+            name: string;
+            parameters: unknown;
+        }> | null;
+        usage?: {
+            promptTokens: number;
+            completionTokens: number;
+            totalTokens: number;
+        };
+    }>;
+}

package/model-function/generate-tool-call/ToolCallsOrTextGenerationModel.js

@@ -0,0 +1 @@
+export {};

package/model-function/generate-tool-call/ToolDefinition.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-tool-call/{ToolCallDefinition.d.ts → ToolDefinition.d.ts}

@@ -1,6 +1,6 @@
 import { JsonSchemaProducer } from "../../core/schema/JsonSchemaProducer.js";
 import { Schema } from "../../core/schema/Schema.js";
-export interface ToolCallDefinition<NAME extends string, PARAMETERS> {
+export interface ToolDefinition<NAME extends string, PARAMETERS> {
     name: NAME;
     description?: string;
     parameters: Schema<PARAMETERS> & JsonSchemaProducer;

package/model-function/generate-tool-call/ToolDefinition.js

@@ -0,0 +1 @@
+export {};

package/model-function/generate-tool-call/generateToolCall.cjs

@@ -17,7 +17,7 @@ async function generateToolCall(model, tool, prompt, options) {
         generateResponse: async (options) => {
             try {
                 const result = await model.doGenerateToolCall(tool, expandedPrompt, options);
-                const toolCall = result.value;
+                const toolCall = result.toolCall;
                 if (toolCall === null) {
                     throw new ToolCallGenerationError_js_1.ToolCallsGenerationError({
                         toolName: tool.name,
@@ -36,6 +36,7 @@ async function generateToolCall(model, tool, prompt, options) {
                     response: result.response,
                     extractedValue: {
                         id: toolCall.id,
+                        name: tool.name,
                         parameters: parseResult.data,
                     },
                     usage: result.usage,

package/model-function/generate-tool-call/generateToolCall.d.ts

@@ -1,20 +1,15 @@
 import { FunctionOptions } from "../../core/FunctionOptions.js";
 import { ModelCallMetadata } from "../ModelCallMetadata.js";
-import { ToolCallDefinition } from "./ToolCallDefinition.js";
+import { ToolDefinition } from "./ToolDefinition.js";
 import { ToolCallGenerationModel, ToolCallGenerationModelSettings } from "./ToolCallGenerationModel.js";
-export declare function generateToolCall<PARAMETERS, PROMPT, NAME extends string, SETTINGS extends ToolCallGenerationModelSettings>(model: ToolCallGenerationModel<PROMPT, SETTINGS>, tool: ToolCallDefinition<NAME, PARAMETERS>, prompt: PROMPT | ((tool: ToolCallDefinition<NAME, PARAMETERS>) => PROMPT), options?: FunctionOptions & {
+import { ToolCall } from "./ToolCall.js";
+export declare function generateToolCall<PARAMETERS, PROMPT, NAME extends string, SETTINGS extends ToolCallGenerationModelSettings>(model: ToolCallGenerationModel<PROMPT, SETTINGS>, tool: ToolDefinition<NAME, PARAMETERS>, prompt: PROMPT | ((tool: ToolDefinition<NAME, PARAMETERS>) => PROMPT), options?: FunctionOptions & {
     returnType?: "structure";
-}): Promise<{
-    id: string;
-    parameters: PARAMETERS;
-}>;
-export declare function generateToolCall<PARAMETERS, PROMPT, NAME extends string, SETTINGS extends ToolCallGenerationModelSettings>(model: ToolCallGenerationModel<PROMPT, SETTINGS>, tool: ToolCallDefinition<NAME, PARAMETERS>, prompt: PROMPT | ((tool: ToolCallDefinition<NAME, PARAMETERS>) => PROMPT), options: FunctionOptions & {
+}): Promise<ToolCall<NAME, PARAMETERS>>;
+export declare function generateToolCall<PARAMETERS, PROMPT, NAME extends string, SETTINGS extends ToolCallGenerationModelSettings>(model: ToolCallGenerationModel<PROMPT, SETTINGS>, tool: ToolDefinition<NAME, PARAMETERS>, prompt: PROMPT | ((tool: ToolDefinition<NAME, PARAMETERS>) => PROMPT), options: FunctionOptions & {
     returnType: "full";
 }): Promise<{
-    value: {
-        id: string;
-        parameters: PARAMETERS;
-    };
+    value: ToolCall<NAME, PARAMETERS>;
     response: unknown;
     metadata: ModelCallMetadata;
 }>;

package/model-function/generate-tool-call/generateToolCall.js

@@ -14,7 +14,7 @@ export async function generateToolCall(model, tool, prompt, options) {
         generateResponse: async (options) => {
             try {
                 const result = await model.doGenerateToolCall(tool, expandedPrompt, options);
-                const toolCall = result.value;
+                const toolCall = result.toolCall;
                 if (toolCall === null) {
                     throw new ToolCallsGenerationError({
                         toolName: tool.name,
@@ -33,6 +33,7 @@ export async function generateToolCall(model, tool, prompt, options) {
                     response: result.response,
                     extractedValue: {
                         id: toolCall.id,
+                        name: tool.name,
                         parameters: parseResult.data,
                     },
                     usage: result.usage,

package/model-function/generate-tool-call/generateToolCallsOrText.cjs

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateToolCallsOrText = void 0;
+const executeStandardCall_js_1 = require("../executeStandardCall.cjs");
+const NoSuchToolDefinitionError_js_1 = require("./NoSuchToolDefinitionError.cjs");
+const ToolCallParametersValidationError_js_1 = require("./ToolCallParametersValidationError.cjs");
+async function generateToolCallsOrText(model, tools, prompt, options) {
+    // Note: PROMPT must not be a function.
+    const expandedPrompt = typeof prompt === "function"
+        ? prompt(tools)
+        : prompt;
+    const fullResponse = await (0, executeStandardCall_js_1.executeStandardCall)({
+        functionType: "generate-tool-calls-or-text",
+        input: expandedPrompt,
+        model,
+        options,
+        generateResponse: async (options) => {
+            const result = await model.doGenerateToolCallsOrText(tools, expandedPrompt, options);
+            const { text, toolCalls: rawToolCalls } = result;
+            // no tool calls:
+            if (rawToolCalls == null) {
+                return {
+                    response: result.response,
+                    extractedValue: { text, toolCalls: null },
+                    usage: result.usage,
+                };
+            }
+            // map tool calls:
+            const toolCalls = rawToolCalls.map((rawToolCall) => {
+                const tool = tools.find((tool) => tool.name === rawToolCall.name);
+                if (tool == undefined) {
+                    throw new NoSuchToolDefinitionError_js_1.NoSuchToolDefinitionError({
+                        toolName: rawToolCall.name,
+                        parameters: rawToolCall.parameters,
+                    });
+                }
+                const parseResult = tool.parameters.validate(rawToolCall.parameters);
+                if (!parseResult.success) {
+                    throw new ToolCallParametersValidationError_js_1.ToolCallParametersValidationError({
+                        toolName: tool.name,
+                        parameters: rawToolCall.parameters,
+                        cause: parseResult.error,
+                    });
+                }
+                return {
+                    id: rawToolCall.id,
+                    name: tool.name,
+                    parameters: parseResult.data,
+                };
+            });
+            return {
+                response: result.response,
+                extractedValue: {
+                    text,
+                    toolCalls: toolCalls,
+                },
+                usage: result.usage,
+            };
+        },
+    });
+    return options?.returnType === "full" ? fullResponse : fullResponse.value;
+}
+exports.generateToolCallsOrText = generateToolCallsOrText;

package/model-function/generate-tool-call/generateToolCallsOrText.d.ts

@@ -0,0 +1,33 @@
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { ModelCallMetadata } from "../ModelCallMetadata.js";
+import { ToolCallsOrTextGenerationModel, ToolCallsOrTextGenerationModelSettings } from "./ToolCallsOrTextGenerationModel.js";
+import { ToolDefinition } from "./ToolDefinition.js";
+type ToolCallDefinitionArray<T extends ToolDefinition<any, any>[]> = T;
+type ToToolCallDefinitionMap<T extends ToolCallDefinitionArray<ToolDefinition<any, any>[]>> = {
+    [K in T[number]["name"]]: Extract<T[number], ToolDefinition<K, any>>;
+};
+type ToToolCallUnion<T> = {
+    [KEY in keyof T]: T[KEY] extends ToolDefinition<any, infer PARAMETERS> ? {
+        id: string;
+        name: KEY;
+        parameters: PARAMETERS;
+    } : never;
+}[keyof T];
+type ToOutputValue<TOOL_CALLS extends ToolCallDefinitionArray<ToolDefinition<any, any>[]>> = ToToolCallUnion<ToToolCallDefinitionMap<TOOL_CALLS>>;
+export declare function generateToolCallsOrText<TOOLS extends Array<ToolDefinition<any, any>>, PROMPT>(model: ToolCallsOrTextGenerationModel<PROMPT, ToolCallsOrTextGenerationModelSettings>, tools: TOOLS, prompt: PROMPT | ((structureDefinitions: TOOLS) => PROMPT), options?: FunctionOptions & {
+    returnType?: "structure";
+}): Promise<{
+    text: string | null;
+    toolCalls: Array<ToOutputValue<TOOLS>> | null;
+}>;
+export declare function generateToolCallsOrText<TOOLS extends ToolDefinition<any, any>[], PROMPT>(model: ToolCallsOrTextGenerationModel<PROMPT, ToolCallsOrTextGenerationModelSettings>, tools: TOOLS, prompt: PROMPT | ((structureDefinitions: TOOLS) => PROMPT), options: FunctionOptions & {
+    returnType?: "full";
+}): Promise<{
+    value: {
+        text: string | null;
+        toolCalls: Array<ToOutputValue<TOOLS>>;
+    };
+    response: unknown;
+    metadata: ModelCallMetadata;
+}>;
+export {};

package/model-function/generate-tool-call/generateToolCallsOrText.js

@@ -0,0 +1,59 @@
+import { executeStandardCall } from "../executeStandardCall.js";
+import { NoSuchToolDefinitionError } from "./NoSuchToolDefinitionError.js";
+import { ToolCallParametersValidationError } from "./ToolCallParametersValidationError.js";
+export async function generateToolCallsOrText(model, tools, prompt, options) {
+    // Note: PROMPT must not be a function.
+    const expandedPrompt = typeof prompt === "function"
+        ? prompt(tools)
+        : prompt;
+    const fullResponse = await executeStandardCall({
+        functionType: "generate-tool-calls-or-text",
+        input: expandedPrompt,
+        model,
+        options,
+        generateResponse: async (options) => {
+            const result = await model.doGenerateToolCallsOrText(tools, expandedPrompt, options);
+            const { text, toolCalls: rawToolCalls } = result;
+            // no tool calls:
+            if (rawToolCalls == null) {
+                return {
+                    response: result.response,
+                    extractedValue: { text, toolCalls: null },
+                    usage: result.usage,
+                };
+            }
+            // map tool calls:
+            const toolCalls = rawToolCalls.map((rawToolCall) => {
+                const tool = tools.find((tool) => tool.name === rawToolCall.name);
+                if (tool == undefined) {
+                    throw new NoSuchToolDefinitionError({
+                        toolName: rawToolCall.name,
+                        parameters: rawToolCall.parameters,
+                    });
+                }
+                const parseResult = tool.parameters.validate(rawToolCall.parameters);
+                if (!parseResult.success) {
+                    throw new ToolCallParametersValidationError({
+                        toolName: tool.name,
+                        parameters: rawToolCall.parameters,
+                        cause: parseResult.error,
+                    });
+                }
+                return {
+                    id: rawToolCall.id,
+                    name: tool.name,
+                    parameters: parseResult.data,
+                };
+            });
+            return {
+                response: result.response,
+                extractedValue: {
+                    text,
+                    toolCalls: toolCalls,
+                },
+                usage: result.usage,
+            };
+        },
+    });
+    return options?.returnType === "full" ? fullResponse : fullResponse.value;
+}

package/model-function/generate-tool-call/index.cjs

@@ -14,8 +14,13 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./ToolCallDefinition.cjs"), exports);
-__exportStar(require("./ToolCallParametersValidationError.cjs"), exports);
+__exportStar(require("./ToolCall.cjs"), exports);
+__exportStar(require("./ToolCallGenerationError.cjs"), exports);
 __exportStar(require("./ToolCallGenerationEvent.cjs"), exports);
 __exportStar(require("./ToolCallGenerationModel.cjs"), exports);
+__exportStar(require("./ToolCallParametersValidationError.cjs"), exports);
+__exportStar(require("./ToolCallsOrTextGenerationEvent.cjs"), exports);
+__exportStar(require("./ToolCallsOrTextGenerationModel.cjs"), exports);
+__exportStar(require("./ToolDefinition.cjs"), exports);
 __exportStar(require("./generateToolCall.cjs"), exports);
+__exportStar(require("./generateToolCallsOrText.cjs"), exports);

package/model-function/generate-tool-call/index.d.ts

@@ -1,5 +1,10 @@
-export * from "./ToolCallDefinition.js";
-export * from "./ToolCallParametersValidationError.js";
+export * from "./ToolCall.js";
+export * from "./ToolCallGenerationError.js";
 export * from "./ToolCallGenerationEvent.js";
 export * from "./ToolCallGenerationModel.js";
+export * from "./ToolCallParametersValidationError.js";
+export * from "./ToolCallsOrTextGenerationEvent.js";
+export * from "./ToolCallsOrTextGenerationModel.js";
+export * from "./ToolDefinition.js";
 export * from "./generateToolCall.js";
+export * from "./generateToolCallsOrText.js";

package/model-function/generate-tool-call/index.js

@@ -1,5 +1,10 @@
-export * from "./ToolCallDefinition.js";
-export * from "./ToolCallParametersValidationError.js";
+export * from "./ToolCall.js";
+export * from "./ToolCallGenerationError.js";
 export * from "./ToolCallGenerationEvent.js";
 export * from "./ToolCallGenerationModel.js";
+export * from "./ToolCallParametersValidationError.js";
+export * from "./ToolCallsOrTextGenerationEvent.js";
+export * from "./ToolCallsOrTextGenerationModel.js";
+export * from "./ToolDefinition.js";
 export * from "./generateToolCall.js";
+export * from "./generateToolCallsOrText.js";

package/model-provider/openai/chat/OpenAIChatMessage.d.ts

@@ -9,7 +9,10 @@ export type OpenAIChatMessage = {
         text: string;
     } | {
         type: "image_url";
-        image_url: string;
+        image_url: string | {
+            url: string;
+            detail: "low" | "high" | "auto";
+        };
     }>;
     name?: string;
 } | {

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -383,7 +383,7 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
         const toolCalls = response.choices[0]?.message.tool_calls;
         return {
             response,
-            value: toolCalls == null || toolCalls.length === 0
+            toolCall: toolCalls == null || toolCalls.length === 0
                 ? null
                 : {
                     id: toolCalls[0].id,
@@ -392,6 +392,32 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
             usage: this.extractUsage(response),
         };
     }
+    async doGenerateToolCallsOrText(tools, prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OpenAIChatResponseFormat.json,
+            toolChoice: "auto",
+            tools: tools.map((tool) => ({
+                type: "function",
+                function: {
+                    name: tool.name,
+                    description: tool.description,
+                    parameters: tool.parameters.getJsonSchema(),
+                },
+            })),
+        });
+        const message = response.choices[0]?.message;
+        return {
+            response,
+            text: message.content ?? null,
+            toolCalls: message.tool_calls?.map((toolCall) => ({
+                id: toolCall.id,
+                name: toolCall.function.name,
+                parameters: (0, parseJSON_js_1.parseJSON)({ text: toolCall.function.arguments }),
+            })) ?? null,
+            usage: this.extractUsage(response),
+        };
+    }
     extractUsage(response) {
         return {
             promptTokens: response.usage.prompt_tokens,

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -10,8 +10,9 @@ import { StructureOrTextGenerationModel } from "../../../model-function/generate
 import { PromptFormatTextStreamingModel } from "../../../model-function/generate-text/PromptFormatTextStreamingModel.js";
 import { TextGenerationModelSettings, TextStreamingModel } from "../../../model-function/generate-text/TextGenerationModel.js";
 import { TextGenerationPromptFormat } from "../../../model-function/generate-text/TextGenerationPromptFormat.js";
-import { ToolCallDefinition } from "../../../model-function/generate-tool-call/ToolCallDefinition.js";
 import { ToolCallGenerationModel } from "../../../model-function/generate-tool-call/ToolCallGenerationModel.js";
+import { ToolCallsOrTextGenerationModel } from "../../../model-function/generate-tool-call/ToolCallsOrTextGenerationModel.js";
+import { ToolDefinition } from "../../../model-function/generate-tool-call/ToolDefinition.js";
 import { TikTokenTokenizer } from "../TikTokenTokenizer.js";
 import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
 export declare const OPENAI_CHAT_MODELS: {
@@ -168,7 +169,7 @@ export interface OpenAIChatSettings extends TextGenerationModelSettings, Omit<Op
  *   ),
  * ]);
  */
-export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> implements TextStreamingModel<OpenAIChatMessage[], OpenAIChatSettings>, StructureGenerationModel<OpenAIChatMessage[], OpenAIChatSettings>, StructureOrTextGenerationModel<OpenAIChatMessage[], OpenAIChatSettings>, ToolCallGenerationModel<OpenAIChatMessage[], OpenAIChatSettings> {
+export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> implements TextStreamingModel<OpenAIChatMessage[], OpenAIChatSettings>, StructureGenerationModel<OpenAIChatMessage[], OpenAIChatSettings>, StructureOrTextGenerationModel<OpenAIChatMessage[], OpenAIChatSettings>, ToolCallGenerationModel<OpenAIChatMessage[], OpenAIChatSettings>, ToolCallsOrTextGenerationModel<OpenAIChatMessage[], OpenAIChatSettings> {
     constructor(settings: OpenAIChatSettings);
     readonly provider: "openai";
     get modelName(): OpenAIChatModelType;
@@ -371,7 +372,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             totalTokens: number;
         };
     }>;
-    doGenerateToolCall(tool: ToolCallDefinition<string, unknown>, prompt: OpenAIChatMessage[], options?: FunctionOptions): Promise<{
+    doGenerateToolCall(tool: ToolDefinition<string, unknown>, prompt: OpenAIChatMessage[], options?: FunctionOptions): Promise<{
         response: {
             object: "chat.completion";
             usage: {
@@ -405,7 +406,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             created: number;
             system_fingerprint?: string | undefined;
         };
-        value: {
+        toolCall: {
             id: string;
             parameters: unknown;
         } | null;
@@ -415,6 +416,52 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             totalTokens: number;
         };
     }>;
+    doGenerateToolCallsOrText(tools: Array<ToolDefinition<string, unknown>>, prompt: OpenAIChatMessage[], options?: FunctionOptions): Promise<{
+        response: {
+            object: "chat.completion";
+            usage: {
+                prompt_tokens: number;
+                total_tokens: number;
+                completion_tokens: number;
+            };
+            model: string;
+            id: string;
+            choices: {
+                message: {
+                    role: "assistant";
+                    content: string | null;
+                    function_call?: {
+                        name: string;
+                        arguments: string;
+                    } | undefined;
+                    tool_calls?: {
+                        function: {
+                            name: string;
+                            arguments: string;
+                        };
+                        type: "function";
+                        id: string;
+                    }[] | undefined;
+                };
+                index: number;
+                logprobs?: any;
+                finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
+            }[];
+            created: number;
+            system_fingerprint?: string | undefined;
+        };
+        text: string | null;
+        toolCalls: {
+            id: string;
+            name: string;
+            parameters: unknown;
+        }[] | null;
+        usage: {
+            promptTokens: number;
+            completionTokens: number;
+            totalTokens: number;
+        };
+    }>;
     extractUsage(response: OpenAIChatResponse): {
         promptTokens: number;
         completionTokens: number;

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -374,7 +374,7 @@ export class OpenAIChatModel extends AbstractModel {
         const toolCalls = response.choices[0]?.message.tool_calls;
         return {
             response,
-            value: toolCalls == null || toolCalls.length === 0
+            toolCall: toolCalls == null || toolCalls.length === 0
                 ? null
                 : {
                     id: toolCalls[0].id,
@@ -383,6 +383,32 @@ export class OpenAIChatModel extends AbstractModel {
             usage: this.extractUsage(response),
         };
     }
+    async doGenerateToolCallsOrText(tools, prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: OpenAIChatResponseFormat.json,
+            toolChoice: "auto",
+            tools: tools.map((tool) => ({
+                type: "function",
+                function: {
+                    name: tool.name,
+                    description: tool.description,
+                    parameters: tool.parameters.getJsonSchema(),
+                },
+            })),
+        });
+        const message = response.choices[0]?.message;
+        return {
+            response,
+            text: message.content ?? null,
+            toolCalls: message.tool_calls?.map((toolCall) => ({
+                id: toolCall.id,
+                name: toolCall.function.name,
+                parameters: parseJSON({ text: toolCall.function.arguments }),
+            })) ?? null,
+            usage: this.extractUsage(response),
+        };
+    }
     extractUsage(response) {
         return {
             promptTokens: response.usage.prompt_tokens,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
-  "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.70.0",
+  "description": "The TypeScript library for building multi-modal AI applications.",
+  "version": "0.71.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [

package/tool/Tool.cjs

@@ -71,7 +71,7 @@ class Tool {
         this.execute = execute;
     }
     /**
-     * Provdes a structure definition with the name, description and schema of the input.
+     * Provides a structure definition with the name, description and schema of the input.
      * This is used by `useTool`.
      */
     get inputStructureDefinition() {

package/tool/Tool.d.ts

@@ -38,7 +38,7 @@ export declare class Tool<NAME extends string, INPUT, OUTPUT> {
         execute(input: INPUT, options?: FunctionOptions): PromiseLike<OUTPUT>;
     });
     /**
-     * Provdes a structure definition with the name, description and schema of the input.
+     * Provides a structure definition with the name, description and schema of the input.
      * This is used by `useTool`.
      */
     get inputStructureDefinition(): StructureDefinition<NAME, INPUT>;

package/tool/Tool.js

@@ -68,7 +68,7 @@ export class Tool {
         this.execute = execute;
     }
     /**
-     * Provdes a structure definition with the name, description and schema of the input.
+     * Provides a structure definition with the name, description and schema of the input.
      * This is used by `useTool`.
      */
     get inputStructureDefinition() {