modelfusion 0.59.0 → 0.60.0

package/README.md

@@ -39,8 +39,7 @@ You can provide API keys for the different [integrations](https://modelfusion.de
 
 ### [Generate Text](https://modelfusion.dev/guide/function/generate-text)
 
-Generate text using a language model and a prompt.
-You can stream the text if it is supported by the model.
+Generate text using a language model and a prompt. You can stream the text if it is supported by the model. You can use images for multi-modal prompting if the model supports it (e.g. with [llama.cpp](https://modelfusion.dev/guide/)).
 You can use [prompt formats](https://modelfusion.dev/guide/function/generate-text#prompt-format) to change the prompt format of a model.
 
 #### generateText

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -75,7 +75,7 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
     async countPromptTokens(prompt) {
-        const tokens = await this.tokenizer.tokenize(prompt);
+        const tokens = await this.tokenizer.tokenize(prompt.text);
         return tokens.length;
     }
     async doGenerateText(prompt, options) {
@@ -99,6 +99,14 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             responseFormat: exports.LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
+    withTextPrompt() {
+        return this.withPromptFormat({
+            format(prompt) {
+                return { text: prompt };
+            },
+            stopSequences: [],
+        });
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel_js_1.PromptFormatTextStreamingModel({
             model: this.withSettings({
@@ -177,7 +185,7 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
         headers: api.headers,
         body: {
             stream: responseFormat.stream,
-            prompt,
+            prompt: prompt.text,
             temperature,
             top_k: topK,
             top_p: topP,
@@ -195,6 +203,12 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
             seed,
             ignore_eos: ignoreEos,
             logit_bias: logitBias,
+            image_data: prompt.images != null
+                ? Object.entries(prompt.images).map(([id, data]) => ({
+                    id: +id,
+                    data,
+                }))
+                : undefined,
         },
         failedResponseHandler: LlamaCppError_js_1.failedLlamaCppCallResponseHandler,
         successfulResponseHandler: responseFormat.handler,

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -31,18 +31,28 @@ export interface LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE extends
     ignoreEos?: boolean;
     logitBias?: Array<[number, number | false]>;
 }
-export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends number | undefined> extends AbstractModel<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> implements TextStreamingModel<string, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> {
+export interface LlamaCppTextGenerationPrompt {
+    /**
+     * Text prompt. Images can be included through references such as `[img-ID]`, e.g. `[img-1]`.
+     */
+    text: string;
+    /**
+     * Maps image id to image base data.
+     */
+    images?: Record<number, string>;
+}
+export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends number | undefined> extends AbstractModel<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> implements TextStreamingModel<LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> {
     constructor(settings?: LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>);
     readonly provider = "llamacpp";
     get modelName(): null;
     get contextWindowSize(): CONTEXT_WINDOW_SIZE;
     readonly tokenizer: LlamaCppTokenizer;
-    callAPI<RESPONSE>(prompt: string, options: {
+    callAPI<RESPONSE>(prompt: LlamaCppTextGenerationPrompt, options: {
         responseFormat: LlamaCppTextGenerationResponseFormatType<RESPONSE>;
     } & FunctionOptions): Promise<RESPONSE>;
     get settingsForEvent(): Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>;
-    countPromptTokens(prompt: string): Promise<number>;
-    doGenerateText(prompt: string, options?: FunctionOptions): Promise<{
+    countPromptTokens(prompt: LlamaCppTextGenerationPrompt): Promise<number>;
+    doGenerateText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<{
         response: {
             model: string;
             prompt: string;
@@ -99,8 +109,9 @@ export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends num
             totalTokens: number;
         };
     }>;
-    doStreamText(prompt: string, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
-    withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
+    doStreamText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
+    withTextPrompt(): PromptFormatTextStreamingModel<string, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
+    withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, LlamaCppTextGenerationPrompt>): PromptFormatTextStreamingModel<INPUT_PROMPT, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withSettings(additionalSettings: Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>): this;
 }
 declare const llamaCppTextGenerationResponseSchema: z.ZodObject<{

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -72,7 +72,7 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
     async countPromptTokens(prompt) {
-        const tokens = await this.tokenizer.tokenize(prompt);
+        const tokens = await this.tokenizer.tokenize(prompt.text);
         return tokens.length;
     }
     async doGenerateText(prompt, options) {
@@ -96,6 +96,14 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
             responseFormat: LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
+    withTextPrompt() {
+        return this.withPromptFormat({
+            format(prompt) {
+                return { text: prompt };
+            },
+            stopSequences: [],
+        });
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel({
             model: this.withSettings({
@@ -173,7 +181,7 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
         headers: api.headers,
         body: {
             stream: responseFormat.stream,
-            prompt,
+            prompt: prompt.text,
             temperature,
             top_k: topK,
             top_p: topP,
@@ -191,6 +199,12 @@ async function callLlamaCppTextGenerationAPI({ api = new LlamaCppApiConfiguratio
             seed,
             ignore_eos: ignoreEos,
             logit_bias: logitBias,
+            image_data: prompt.images != null
+                ? Object.entries(prompt.images).map(([id, data]) => ({
+                    id: +id,
+                    data,
+                }))
+                : undefined,
         },
         failedResponseHandler: failedLlamaCppCallResponseHandler,
         successfulResponseHandler: responseFormat.handler,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.59.0",
+  "version": "0.60.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [