modelfusion 0.41.3 → 0.43.0

package/model-provider/anthropic/AnthropicTextGenerationModel.d.ts

@@ -0,0 +1,102 @@
+import { z } from "zod";
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { ResponseHandler } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { Delta } from "../../model-function/Delta.js";
+import { TextGenerationModelSettings, TextStreamingModel } from "../../model-function/generate-text/TextGenerationModel.js";
+import { PromptFormat } from "../../prompt/PromptFormat.js";
+import { PromptFormatTextStreamingModel } from "../../prompt/PromptFormatTextStreamingModel.js";
+export declare const ANTHROPIC_TEXT_GENERATION_MODELS: {
+    "claude-instant-1": {
+        contextWindowSize: number;
+    };
+    "claude-instant-1.2": {
+        contextWindowSize: number;
+    };
+    "claude-2": {
+        contextWindowSize: number;
+    };
+    "claude-2.0": {
+        contextWindowSize: number;
+    };
+};
+export type AnthropicTextGenerationModelType = keyof typeof ANTHROPIC_TEXT_GENERATION_MODELS;
+export interface AnthropicTextGenerationModelSettings extends TextGenerationModelSettings {
+    api?: ApiConfiguration;
+    model: AnthropicTextGenerationModelType;
+    temperature?: number;
+    topP?: number;
+    topK?: number;
+    userId?: number;
+}
+/**
+ * Create a text generation model that calls the Anthropic API.
+ *
+ * @see https://docs.anthropic.com/claude/reference/complete_post
+ */
+export declare class AnthropicTextGenerationModel extends AbstractModel<AnthropicTextGenerationModelSettings> implements TextStreamingModel<string, AnthropicTextGenerationModelSettings> {
+    constructor(settings: AnthropicTextGenerationModelSettings);
+    readonly provider: "anthropic";
+    get modelName(): "claude-instant-1" | "claude-instant-1.2" | "claude-2" | "claude-2.0";
+    readonly contextWindowSize: number;
+    readonly tokenizer: undefined;
+    readonly countPromptTokens: undefined;
+    callAPI<RESPONSE>(prompt: string, options: {
+        responseFormat: AnthropicTextGenerationResponseFormatType<RESPONSE>;
+    } & FunctionOptions): Promise<RESPONSE>;
+    get settingsForEvent(): Partial<AnthropicTextGenerationModelSettings>;
+    doGenerateText(prompt: string, options?: FunctionOptions): Promise<{
+        response: {
+            model: string;
+            completion: string;
+            stop_reason: string;
+        };
+        text: string;
+    }>;
+    doStreamText(prompt: string, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
+    withPromptFormat<INPUT_PROMPT>(promptFormat: PromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, AnthropicTextGenerationModelSettings, this>;
+    withSettings(additionalSettings: Partial<AnthropicTextGenerationModelSettings>): this;
+}
+declare const anthropicTextGenerationResponseSchema: z.ZodObject<{
+    completion: z.ZodString;
+    stop_reason: z.ZodString;
+    model: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    model: string;
+    completion: string;
+    stop_reason: string;
+}, {
+    model: string;
+    completion: string;
+    stop_reason: string;
+}>;
+export type AnthropicTextGenerationResponse = z.infer<typeof anthropicTextGenerationResponseSchema>;
+export type AnthropicTextGenerationResponseFormatType<T> = {
+    stream: boolean;
+    handler: ResponseHandler<T>;
+};
+export declare const AnthropicTextGenerationResponseFormat: {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false;
+        handler: ResponseHandler<{
+            model: string;
+            completion: string;
+            stop_reason: string;
+        }>;
+    };
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true;
+        handler: ({ response }: {
+            response: Response;
+        }) => Promise<AsyncIterable<Delta<string>>>;
+    };
+};
+export {};

package/model-provider/anthropic/AnthropicTextGenerationModel.js

@@ -0,0 +1,219 @@
+import SecureJSON from "secure-json-parse";
+import { z } from "zod";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { AsyncQueue } from "../../event-source/AsyncQueue.js";
+import { parseEventSourceStream } from "../../event-source/parseEventSourceStream.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { PromptFormatTextStreamingModel } from "../../prompt/PromptFormatTextStreamingModel.js";
+import { AnthropicApiConfiguration } from "./AnthropicApiConfiguration.js";
+import { failedAnthropicCallResponseHandler } from "./AnthropicError.js";
+export const ANTHROPIC_TEXT_GENERATION_MODELS = {
+    "claude-instant-1": {
+        contextWindowSize: 100000,
+    },
+    "claude-instant-1.2": {
+        contextWindowSize: 100000,
+    },
+    "claude-2": {
+        contextWindowSize: 100000,
+    },
+    "claude-2.0": {
+        contextWindowSize: 100000,
+    },
+};
+/**
+ * Create a text generation model that calls the Anthropic API.
+ *
+ * @see https://docs.anthropic.com/claude/reference/complete_post
+ */
+export class AnthropicTextGenerationModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "anthropic"
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        this.contextWindowSize =
+            ANTHROPIC_TEXT_GENERATION_MODELS[this.settings.model].contextWindowSize;
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(prompt, options) {
+        return callWithRetryAndThrottle({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => callAnthropicTextGenerationAPI({
+                ...this.settings,
+                stopSequences: this.settings.stopSequences,
+                maxTokens: this.settings.maxCompletionTokens,
+                abortSignal: options.run?.abortSignal,
+                responseFormat: options.responseFormat,
+                prompt,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        const eventSettingProperties = [
+            "maxCompletionTokens",
+            "stopSequences",
+            "temperature",
+            "topK",
+            "topP",
+            "userId",
+        ];
+        return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
+    }
+    async doGenerateText(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: AnthropicTextGenerationResponseFormat.json,
+        });
+        return {
+            response,
+            text: response.completion,
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: AnthropicTextGenerationResponseFormat.deltaIterable,
+        });
+    }
+    withPromptFormat(promptFormat) {
+        return new PromptFormatTextStreamingModel({
+            model: this.withSettings({
+                stopSequences: [
+                    ...(this.settings.stopSequences ?? []),
+                    ...promptFormat.stopSequences,
+                ],
+            }),
+            promptFormat,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new AnthropicTextGenerationModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+const anthropicTextGenerationResponseSchema = z.object({
+    completion: z.string(),
+    stop_reason: z.string(),
+    model: z.string(),
+});
+async function callAnthropicTextGenerationAPI({ api = new AnthropicApiConfiguration(), abortSignal, responseFormat, model, prompt, maxTokens, stopSequences, temperature, topK, topP, userId, }) {
+    return postJsonToApi({
+        url: api.assembleUrl(`/complete`),
+        headers: api.headers,
+        body: {
+            model,
+            prompt,
+            stream: responseFormat.stream,
+            max_tokens_to_sample: maxTokens,
+            temperature,
+            top_k: topK,
+            top_p: topP,
+            stop_sequences: stopSequences,
+            metadata: userId != null ? { user_id: userId } : undefined,
+        },
+        failedResponseHandler: failedAnthropicCallResponseHandler,
+        successfulResponseHandler: responseFormat.handler,
+        abortSignal,
+    });
+}
+const anthropicTextStreamingResponseSchema = z.object({
+    completion: z.string(),
+    stop_reason: z.string().nullable(),
+    model: z.string(),
+});
+async function createAnthropicFullDeltaIterableQueue(stream) {
+    const queue = new AsyncQueue();
+    let content = "";
+    // process the stream asynchonously (no 'await' on purpose):
+    parseEventSourceStream({ stream })
+        .then(async (events) => {
+        try {
+            for await (const event of events) {
+                if (event.event === "error") {
+                    queue.push({ type: "error", error: event.data });
+                    queue.close();
+                    return;
+                }
+                if (event.event !== "completion") {
+                    continue;
+                }
+                const data = event.data;
+                const json = SecureJSON.parse(data);
+                const parseResult = anthropicTextStreamingResponseSchema.safeParse(json);
+                if (!parseResult.success) {
+                    queue.push({
+                        type: "error",
+                        error: parseResult.error,
+                    });
+                    queue.close();
+                    return;
+                }
+                const eventData = parseResult.data;
+                content += eventData.completion;
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content,
+                        isComplete: eventData.stop_reason != null,
+                        delta: eventData.completion,
+                    },
+                    valueDelta: eventData.completion,
+                });
+                if (eventData.stop_reason != null) {
+                    queue.close();
+                }
+            }
+        }
+        catch (error) {
+            queue.push({ type: "error", error });
+            queue.close();
+        }
+    })
+        .catch((error) => {
+        queue.push({ type: "error", error });
+        queue.close();
+    });
+    return queue;
+}
+export const AnthropicTextGenerationResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: createJsonResponseHandler(anthropicTextGenerationResponseSchema),
+    },
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createAnthropicFullDeltaIterableQueue(response.body),
+    },
+};

package/model-provider/anthropic/index.cjs

@@ -0,0 +1,23 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.anthropicErrorDataSchema = exports.AnthropicError = void 0;
+__exportStar(require("./AnthropicApiConfiguration.cjs"), exports);
+var AnthropicError_js_1 = require("./AnthropicError.cjs");
+Object.defineProperty(exports, "AnthropicError", { enumerable: true, get: function () { return AnthropicError_js_1.AnthropicError; } });
+Object.defineProperty(exports, "anthropicErrorDataSchema", { enumerable: true, get: function () { return AnthropicError_js_1.anthropicErrorDataSchema; } });
+__exportStar(require("./AnthropicPromptFormat.cjs"), exports);
+__exportStar(require("./AnthropicTextGenerationModel.cjs"), exports);

package/model-provider/anthropic/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./AnthropicApiConfiguration.js";
+export { AnthropicError, anthropicErrorDataSchema } from "./AnthropicError.js";
+export * from "./AnthropicPromptFormat.js";
+export * from "./AnthropicTextGenerationModel.js";

package/model-provider/anthropic/index.js

@@ -0,0 +1,4 @@
+export * from "./AnthropicApiConfiguration.js";
+export { AnthropicError, anthropicErrorDataSchema } from "./AnthropicError.js";
+export * from "./AnthropicPromptFormat.js";
+export * from "./AnthropicTextGenerationModel.js";

package/model-provider/huggingface/HuggingFaceTextGenerationModel.d.ts

@@ -44,9 +44,9 @@ export declare class HuggingFaceTextGenerationModel extends AbstractModel<Huggin
     get modelName(): string;
     readonly contextWindowSize: undefined;
     readonly tokenizer: undefined;
+    readonly countPromptTokens: undefined;
     callAPI(prompt: string, options?: FunctionOptions): Promise<HuggingFaceTextGenerationResponse>;
     get settingsForEvent(): Partial<HuggingFaceTextGenerationModelSettings>;
-    readonly countPromptTokens: undefined;
     doGenerateText(prompt: string, options?: FunctionOptions): Promise<{
         response: {
             generated_text: string;

package/model-provider/index.cjs

@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./anthropic/index.cjs"), exports);
 __exportStar(require("./automatic1111/index.cjs"), exports);
 __exportStar(require("./cohere/index.cjs"), exports);
 __exportStar(require("./elevenlabs/index.cjs"), exports);

package/model-provider/index.d.ts

@@ -1,3 +1,4 @@
+export * from "./anthropic/index.js";
 export * from "./automatic1111/index.js";
 export * from "./cohere/index.js";
 export * from "./elevenlabs/index.js";

package/model-provider/index.js

@@ -1,3 +1,4 @@
+export * from "./anthropic/index.js";
 export * from "./automatic1111/index.js";
 export * from "./cohere/index.js";
 export * from "./elevenlabs/index.js";

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -102,9 +102,6 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             responseFormat: exports.LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
-    extractTextDelta(fullDelta) {
-        return fullDelta.delta;
-    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel_js_1.PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -100,7 +100,6 @@ export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends num
         };
     }>;
     doStreamText(prompt: string, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
-    extractTextDelta(fullDelta: LlamaCppTextGenerationDelta): string | undefined;
     withPromptFormat<INPUT_PROMPT>(promptFormat: PromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withSettings(additionalSettings: Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>): this;
 }

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -96,9 +96,6 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
             responseFormat: LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
-    extractTextDelta(fullDelta) {
-        return fullDelta.delta;
-    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -9,6 +9,7 @@ const zod_1 = __importDefault(require("zod"));
 const callWithRetryAndThrottle_js_1 = require("../../../core/api/callWithRetryAndThrottle.cjs");
 const postToApi_js_1 = require("../../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../../model-function/AbstractModel.cjs");
+const StructureParseError_js_1 = require("../../../model-function/generate-structure/StructureParseError.cjs");
 const parsePartialJson_js_1 = require("../../../model-function/generate-structure/parsePartialJson.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../../prompt/PromptFormatTextStreamingModel.cjs");
 const OpenAIApiConfiguration_js_1 = require("../OpenAIApiConfiguration.cjs");
@@ -256,11 +257,22 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
                 },
             ],
         });
-        return {
-            response,
-            structure: secure_json_parse_1.default.parse(response.choices[0].message.function_call.arguments),
-            usage: this.extractUsage(response),
-        };
+        const valueText = response.choices[0].message.function_call.arguments;
+        try {
+            return {
+                response,
+                valueText,
+                value: secure_json_parse_1.default.parse(valueText),
+                usage: this.extractUsage(response),
+            };
+        }
+        catch (error) {
+            throw new StructureParseError_js_1.StructureParseError({
+                structureName: structureDefinition.name,
+                valueText: valueText,
+                cause: error,
+            });
+        }
     }
     async doStreamStructure(structureDefinition, prompt, options) {
         return this.callAPI(prompt, {
@@ -290,22 +302,37 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
         const message = response.choices[0].message;
         const content = message.content;
         const functionCall = message.function_call;
-        const structureAndText = functionCall == null
-            ? {
-                structure: null,
-                value: null,
-                text: content ?? "",
-            }
-            : {
-                structure: functionCall.name,
-                value: secure_json_parse_1.default.parse(functionCall.arguments),
-                text: content,
+        if (functionCall == null) {
+            return {
+                response,
+                structureAndText: {
+                    structure: null,
+                    value: null,
+                    valueText: null,
+                    text: content ?? "",
+                },
+                usage: this.extractUsage(response),
             };
-        return {
-            response,
-            structureAndText,
-            usage: this.extractUsage(response),
-        };
+        }
+        try {
+            return {
+                response,
+                structureAndText: {
+                    structure: functionCall.name,
+                    value: secure_json_parse_1.default.parse(functionCall.arguments),
+                    valueText: functionCall.arguments,
+                    text: content,
+                },
+                usage: this.extractUsage(response),
+            };
+        }
+        catch (error) {
+            throw new StructureParseError_js_1.StructureParseError({
+                structureName: functionCall.name,
+                valueText: functionCall.arguments,
+                cause: error,
+            });
+        }
     }
     extractUsage(response) {
         return {

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -220,7 +220,8 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 logprobs?: any;
             }[];
         };
-        structure: any;
+        valueText: string;
+        value: any;
         usage: {
             promptTokens: number;
             completionTokens: number;
@@ -256,10 +257,43 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
         structureAndText: {
             structure: null;
             value: null;
+            valueText: null;
             text: string;
-        } | {
+        };
+        usage: {
+            promptTokens: number;
+            completionTokens: number;
+            totalTokens: number;
+        };
+    } | {
+        response: {
+            object: "chat.completion";
+            model: string;
+            usage: {
+                prompt_tokens: number;
+                completion_tokens: number;
+                total_tokens: number;
+            };
+            id: string;
+            created: number;
+            choices: {
+                message: {
+                    content: string | null;
+                    role: "assistant";
+                    function_call?: {
+                        name: string;
+                        arguments: string;
+                    } | undefined;
+                };
+                finish_reason: string;
+                index: number;
+                logprobs?: any;
+            }[];
+        };
+        structureAndText: {
             structure: string;
             value: any;
+            valueText: string;
             text: string | null;
         };
         usage: {

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -3,6 +3,7 @@ import z from "zod";
 import { callWithRetryAndThrottle } from "../../../core/api/callWithRetryAndThrottle.js";
 import { createJsonResponseHandler, postJsonToApi, } from "../../../core/api/postToApi.js";
 import { AbstractModel } from "../../../model-function/AbstractModel.js";
+import { StructureParseError } from "../../../model-function/generate-structure/StructureParseError.js";
 import { parsePartialJson } from "../../../model-function/generate-structure/parsePartialJson.js";
 import { PromptFormatTextStreamingModel } from "../../../prompt/PromptFormatTextStreamingModel.js";
 import { OpenAIApiConfiguration } from "../OpenAIApiConfiguration.js";
@@ -247,11 +248,22 @@ export class OpenAIChatModel extends AbstractModel {
                 },
             ],
         });
-        return {
-            response,
-            structure: SecureJSON.parse(response.choices[0].message.function_call.arguments),
-            usage: this.extractUsage(response),
-        };
+        const valueText = response.choices[0].message.function_call.arguments;
+        try {
+            return {
+                response,
+                valueText,
+                value: SecureJSON.parse(valueText),
+                usage: this.extractUsage(response),
+            };
+        }
+        catch (error) {
+            throw new StructureParseError({
+                structureName: structureDefinition.name,
+                valueText: valueText,
+                cause: error,
+            });
+        }
     }
     async doStreamStructure(structureDefinition, prompt, options) {
         return this.callAPI(prompt, {
@@ -281,22 +293,37 @@ export class OpenAIChatModel extends AbstractModel {
         const message = response.choices[0].message;
         const content = message.content;
         const functionCall = message.function_call;
-        const structureAndText = functionCall == null
-            ? {
-                structure: null,
-                value: null,
-                text: content ?? "",
-            }
-            : {
-                structure: functionCall.name,
-                value: SecureJSON.parse(functionCall.arguments),
-                text: content,
+        if (functionCall == null) {
+            return {
+                response,
+                structureAndText: {
+                    structure: null,
+                    value: null,
+                    valueText: null,
+                    text: content ?? "",
+                },
+                usage: this.extractUsage(response),
             };
-        return {
-            response,
-            structureAndText,
-            usage: this.extractUsage(response),
-        };
+        }
+        try {
+            return {
+                response,
+                structureAndText: {
+                    structure: functionCall.name,
+                    value: SecureJSON.parse(functionCall.arguments),
+                    valueText: functionCall.arguments,
+                    text: content,
+                },
+                usage: this.extractUsage(response),
+            };
+        }
+        catch (error) {
+            throw new StructureParseError({
+                structureName: functionCall.name,
+                valueText: functionCall.arguments,
+                cause: error,
+            });
+        }
     }
     extractUsage(response) {
         return {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.41.3",
+  "version": "0.43.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [