modelfusion 0.58.0 → 0.60.0

package/README.md

@@ -39,8 +39,7 @@ You can provide API keys for the different [integrations](https://modelfusion.de
 
 ### [Generate Text](https://modelfusion.dev/guide/function/generate-text)
 
-Generate text using a language model and a prompt.
-You can stream the text if it is supported by the model.
+Generate text using a language model and a prompt. You can stream the text if it is supported by the model. You can use images for multi-modal prompting if the model supports it (e.g. with [llama.cpp](https://modelfusion.dev/guide/)).
 You can use [prompt formats](https://modelfusion.dev/guide/function/generate-text#prompt-format) to change the prompt format of a model.
 
 #### generateText
@@ -118,6 +117,7 @@ const textStream = await streamText(/* ... */);
 
 const speechStream = await streamSpeech(
   new ElevenLabsSpeechModel({
+    model: "eleven_turbo_v2",
     voice: "pNInz6obpgDQGcFmaJgB", // Adam
     optimizeStreamingLatency: 1,
     voiceSettings: { stability: 1, similarityBoost: 0.35 },
@@ -447,7 +447,7 @@ const retrievedTexts = await retrieve(
 );
 ```
 
-Available Vector Stores: [Memory](https://modelfusion.dev/integration/vector-index/memory), [Pinecone](https://modelfusion.dev/integration/vector-index/pinecone)
+Available Vector Stores: [Memory](https://modelfusion.dev/integration/vector-index/memory), [SQLite VSS](https://modelfusion.dev/integration/vector-index/sqlite-vss), [Pinecone](https://modelfusion.dev/integration/vector-index/pinecone)
 
 ### Prompt Formats
 

package/model-provider/elevenlabs/ElevenLabsSpeechModel.cjs

@@ -14,6 +14,7 @@ const elevenLabsModels = [
     "eleven_multilingual_v2",
     "eleven_multilingual_v1",
     "eleven_monolingual_v1",
+    "eleven_turbo_v2",
 ];
 const defaultModel = "eleven_monolingual_v1";
 /**

package/model-provider/elevenlabs/ElevenLabsSpeechModel.d.ts

@@ -4,7 +4,7 @@ import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { Delta } from "../../model-function/Delta.js";
 import { StreamingSpeechGenerationModel, SpeechGenerationModelSettings } from "../../model-function/generate-speech/SpeechGenerationModel.js";
-declare const elevenLabsModels: readonly ["eleven_multilingual_v2", "eleven_multilingual_v1", "eleven_monolingual_v1"];
+declare const elevenLabsModels: readonly ["eleven_multilingual_v2", "eleven_multilingual_v1", "eleven_monolingual_v1", "eleven_turbo_v2"];
 export interface ElevenLabsSpeechModelSettings extends SpeechGenerationModelSettings {
     api?: ApiConfiguration & {
         apiKey: string;

package/model-provider/elevenlabs/ElevenLabsSpeechModel.js

@@ -11,6 +11,7 @@ const elevenLabsModels = [
     "eleven_multilingual_v2",
     "eleven_multilingual_v1",
     "eleven_monolingual_v1",
+    "eleven_turbo_v2",
 ];
 const defaultModel = "eleven_monolingual_v1";
 /**

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -75,7 +75,7 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
     async countPromptTokens(prompt) {
-        const tokens = await this.tokenizer.tokenize(prompt);
+        const tokens = await this.tokenizer.tokenize(prompt.text);
         return tokens.length;
     }
     async doGenerateText(prompt, options) {
@@ -99,6 +99,14 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             responseFormat: exports.LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
+    withTextPrompt() {
+        return this.withPromptFormat({
+            format(prompt) {
+                return { text: prompt };
+            },
+            stopSequences: [],
+        });
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel_js_1.PromptFormatTextStreamingModel({
             model: this.withSettings({
@@ -177,7 +185,7 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
         headers: api.headers,
         body: {
             stream: responseFormat.stream,
-            prompt,
+            prompt: prompt.text,
             temperature,
             top_k: topK,
             top_p: topP,
@@ -195,6 +203,12 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
             seed,
             ignore_eos: ignoreEos,
             logit_bias: logitBias,
+            image_data: prompt.images != null
+                ? Object.entries(prompt.images).map(([id, data]) => ({
+                    id: +id,
+                    data,
+                }))
+                : undefined,
         },
         failedResponseHandler: LlamaCppError_js_1.failedLlamaCppCallResponseHandler,
         successfulResponseHandler: responseFormat.handler,

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -31,18 +31,28 @@ export interface LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE extends
     ignoreEos?: boolean;
     logitBias?: Array<[number, number | false]>;
 }
-export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends number | undefined> extends AbstractModel<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> implements TextStreamingModel<string, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> {
+export interface LlamaCppTextGenerationPrompt {
+    /**
+     * Text prompt. Images can be included through references such as `[img-ID]`, e.g. `[img-1]`.
+     */
+    text: string;
+    /**
+     * Maps image id to image base data.
+     */
+    images?: Record<number, string>;
+}
+export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends number | undefined> extends AbstractModel<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> implements TextStreamingModel<LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> {
     constructor(settings?: LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>);
     readonly provider = "llamacpp";
     get modelName(): null;
     get contextWindowSize(): CONTEXT_WINDOW_SIZE;
     readonly tokenizer: LlamaCppTokenizer;
-    callAPI<RESPONSE>(prompt: string, options: {
+    callAPI<RESPONSE>(prompt: LlamaCppTextGenerationPrompt, options: {
         responseFormat: LlamaCppTextGenerationResponseFormatType<RESPONSE>;
     } & FunctionOptions): Promise<RESPONSE>;
     get settingsForEvent(): Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>;
-    countPromptTokens(prompt: string): Promise<number>;
-    doGenerateText(prompt: string, options?: FunctionOptions): Promise<{
+    countPromptTokens(prompt: LlamaCppTextGenerationPrompt): Promise<number>;
+    doGenerateText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<{
         response: {
             model: string;
             prompt: string;
@@ -99,8 +109,9 @@ export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends num
             totalTokens: number;
         };
     }>;
-    doStreamText(prompt: string, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
-    withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
+    doStreamText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
+    withTextPrompt(): PromptFormatTextStreamingModel<string, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
+    withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, LlamaCppTextGenerationPrompt>): PromptFormatTextStreamingModel<INPUT_PROMPT, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withSettings(additionalSettings: Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>): this;
 }
 declare const llamaCppTextGenerationResponseSchema: z.ZodObject<{

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -72,7 +72,7 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
     async countPromptTokens(prompt) {
-        const tokens = await this.tokenizer.tokenize(prompt);
+        const tokens = await this.tokenizer.tokenize(prompt.text);
         return tokens.length;
     }
     async doGenerateText(prompt, options) {
@@ -96,6 +96,14 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
             responseFormat: LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
+    withTextPrompt() {
+        return this.withPromptFormat({
+            format(prompt) {
+                return { text: prompt };
+            },
+            stopSequences: [],
+        });
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel({
             model: this.withSettings({
@@ -173,7 +181,7 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
         headers: api.headers,
         body: {
             stream: responseFormat.stream,
-            prompt,
+            prompt: prompt.text,
             temperature,
             top_k: topK,
             top_p: topP,
@@ -191,6 +199,12 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
             seed,
             ignore_eos: ignoreEos,
             logit_bias: logitBias,
+            image_data: prompt.images != null
+                ? Object.entries(prompt.images).map(([id, data]) => ({
+                    id: +id,
+                    data,
+                }))
+                : undefined,
         },
         failedResponseHandler: failedLlamaCppCallResponseHandler,
         successfulResponseHandler: responseFormat.handler,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.58.0",
+  "version": "0.60.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [

package/tool/NoSuchToolError.cjs

@@ -13,5 +13,13 @@ class NoSuchToolError extends Error {
         this.name = "NoSuchToolError";
         this.toolName = toolName;
     }
+    toJSON() {
+        return {
+            name: this.name,
+            toolName: this.toolName,
+            message: this.message,
+            stack: this.stack,
+        };
+    }
 }
 exports.NoSuchToolError = NoSuchToolError;

package/tool/NoSuchToolError.d.ts

@@ -1,4 +1,10 @@
 export declare class NoSuchToolError extends Error {
     readonly toolName: string;
     constructor(toolName: string);
+    toJSON(): {
+        name: string;
+        toolName: string;
+        message: string;
+        stack: string | undefined;
+    };
 }

package/tool/NoSuchToolError.js

@@ -10,4 +10,12 @@ export class NoSuchToolError extends Error {
         this.name = "NoSuchToolError";
         this.toolName = toolName;
     }
+    toJSON() {
+        return {
+            name: this.name,
+            toolName: this.toolName,
+            message: this.message,
+            stack: this.stack,
+        };
+    }
 }

package/tool/ToolExecutionError.cjs

@@ -27,5 +27,15 @@ class ToolExecutionError extends Error {
         this.input = input;
         this.cause = cause;
     }
+    toJSON() {
+        return {
+            name: this.name,
+            toolName: this.toolName,
+            input: this.input,
+            cause: this.cause,
+            message: this.message,
+            stack: this.stack,
+        };
+    }
 }
 exports.ToolExecutionError = ToolExecutionError;

package/tool/ToolExecutionError.d.ts

@@ -8,4 +8,12 @@ export declare class ToolExecutionError extends Error {
         message: string | undefined;
         cause: unknown | undefined;
     });
+    toJSON(): {
+        name: string;
+        toolName: string;
+        input: unknown;
+        cause: unknown;
+        message: string;
+        stack: string | undefined;
+    };
 }

package/tool/ToolExecutionError.js

@@ -24,4 +24,14 @@ export class ToolExecutionError extends Error {
         this.input = input;
         this.cause = cause;
     }
+    toJSON() {
+        return {
+            name: this.name,
+            toolName: this.toolName,
+            input: this.input,
+            cause: this.cause,
+            message: this.message,
+            stack: this.stack,
+        };
+    }
 }

package/util/JSONParseError.cjs

@@ -35,6 +35,7 @@ class JSONParseError extends Error {
             message: this.message,
             cause: this.cause,
             valueText: this.valueText,
+            stack: this.stack,
         };
     }
 }

package/util/JSONParseError.d.ts

@@ -11,5 +11,6 @@ export declare class JSONParseError extends Error {
         message: string;
         cause: unknown;
         valueText: string;
+        stack: string | undefined;
     };
 }

package/util/JSONParseError.js

@@ -32,6 +32,7 @@ export class JSONParseError extends Error {
             message: this.message,
             cause: this.cause,
             valueText: this.valueText,
+            stack: this.stack,
         };
     }
 }