modelfusion 0.102.0 → 0.104.0

package/model-provider/anthropic/AnthropicTextGenerationModel.d.ts

@@ -7,6 +7,7 @@ import { Delta } from "../../model-function/Delta.js";
 import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
 import { TextGenerationModelSettings, TextStreamingModel } from "../../model-function/generate-text/TextGenerationModel.js";
 import { TextGenerationPromptTemplate } from "../../model-function/generate-text/TextGenerationPromptTemplate.js";
+import { TextGenerationFinishReason } from "../../model-function/generate-text/TextGenerationResult.js";
 export declare const ANTHROPIC_TEXT_GENERATION_MODELS: {
     "claude-instant-1": {
         contextWindowSize: number;
@@ -55,8 +56,12 @@ export declare class AnthropicTextGenerationModel extends AbstractModel<Anthropi
             completion: string;
             stop_reason: string;
         };
-        texts: string[];
+        textGenerationResults: {
+            text: string;
+            finishReason: TextGenerationFinishReason;
+        }[];
     }>;
+    private translateFinishReason;
     doStreamText(prompt: string, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
     /**
      * Returns this model with a text prompt template.
@@ -65,11 +70,11 @@ export declare class AnthropicTextGenerationModel extends AbstractModel<Anthropi
     /**
      * Returns this model with an instruction prompt template.
      */
-    withInstructionPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").TextInstructionPrompt, string, AnthropicTextGenerationModelSettings, this>;
+    withInstructionPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").InstructionPrompt, string, AnthropicTextGenerationModelSettings, this>;
     /**
      * Returns this model with a chat prompt template.
      */
-    withChatPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").TextChatPrompt, string, AnthropicTextGenerationModelSettings, this>;
+    withChatPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").ChatPrompt, string, AnthropicTextGenerationModelSettings, this>;
     withPromptTemplate<INPUT_PROMPT>(promptTemplate: TextGenerationPromptTemplate<INPUT_PROMPT, string>): PromptTemplateTextStreamingModel<INPUT_PROMPT, string, AnthropicTextGenerationModelSettings, this>;
     withSettings(additionalSettings: Partial<AnthropicTextGenerationModelSettings>): this;
 }

package/model-provider/anthropic/AnthropicTextGenerationModel.js

@@ -1,15 +1,16 @@
 import { z } from "zod";
 import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
-import { parseEventSourceStream } from "../../util/streaming/parseEventSourceStream.js";
+import { ZodSchema } from "../../core/schema/ZodSchema.js";
+import { parseJSON } from "../../core/schema/parseJSON.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
+import { textGenerationModelProperties, } from "../../model-function/generate-text/TextGenerationModel.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
-import { ZodSchema } from "../../core/schema/ZodSchema.js";
-import { parseJSON } from "../../core/schema/parseJSON.js";
+import { parseEventSourceStream } from "../../util/streaming/parseEventSourceStream.js";
 import { AnthropicApiConfiguration } from "./AnthropicApiConfiguration.js";
 import { failedAnthropicCallResponseHandler } from "./AnthropicError.js";
-import { instruction, chat, text } from "./AnthropicPromptTemplate.js";
+import { chat, instruction, text } from "./AnthropicPromptTemplate.js";
 export const ANTHROPIC_TEXT_GENERATION_MODELS = {
     "claude-instant-1": {
         contextWindowSize: 100000,
@@ -71,8 +72,8 @@ export class AnthropicTextGenerationModel extends AbstractModel {
         const abortSignal = options.run?.abortSignal;
         const userId = this.settings.userId;
         return callWithRetryAndThrottle({
-            retry: this.settings.api?.retry,
-            throttle: this.settings.api?.throttle,
+            retry: api.retry,
+            throttle: api.throttle,
             call: async () => {
                 return postJsonToApi({
                     url: api.assembleUrl(`/complete`),
@@ -97,8 +98,7 @@ export class AnthropicTextGenerationModel extends AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
-            "maxGenerationTokens",
-            "stopSequences",
+            ...textGenerationModelProperties,
             "temperature",
             "topK",
             "topP",
@@ -113,9 +113,24 @@ export class AnthropicTextGenerationModel extends AbstractModel {
         });
         return {
             response,
-            texts: [response.completion],
+            textGenerationResults: [
+                {
+                    text: response.completion,
+                    finishReason: this.translateFinishReason(response.stop_reason),
+                },
+            ],
         };
     }
+    translateFinishReason(finishReason) {
+        switch (finishReason) {
+            case "stop_sequence":
+                return "stop";
+            case "max_tokens":
+                return "length";
+            default:
+                return "unknown";
+        }
+    }
     doStreamText(prompt, options) {
         return this.callAPI(prompt, {
             ...options,

package/model-provider/automatic1111/Automatic1111ImageGenerationModel.d.ts

@@ -49,7 +49,7 @@ export declare class Automatic1111ImageGenerationModel extends AbstractModel<Aut
     }>;
     withTextPrompt(): PromptTemplateImageGenerationModel<string, Automatic1111ImageGenerationPrompt, Automatic1111ImageGenerationSettings, this>;
     withPromptTemplate<INPUT_PROMPT>(promptTemplate: PromptTemplate<INPUT_PROMPT, Automatic1111ImageGenerationPrompt>): PromptTemplateImageGenerationModel<INPUT_PROMPT, Automatic1111ImageGenerationPrompt, Automatic1111ImageGenerationSettings, this>;
-    withSettings(additionalSettings: Automatic1111ImageGenerationSettings): this;
+    withSettings(additionalSettings: Partial<Automatic1111ImageGenerationSettings>): this;
 }
 declare const Automatic1111ImageGenerationResponseSchema: z.ZodObject<{
     images: z.ZodArray<z.ZodString, "many">;

package/model-provider/cohere/CohereTextGenerationModel.cjs

@@ -7,6 +7,7 @@ const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const ZodSchema_js_1 = require("../../core/schema/ZodSchema.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptTemplateTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptTemplateTextStreamingModel.cjs");
+const TextGenerationModel_js_1 = require("../../model-function/generate-text/TextGenerationModel.cjs");
 const TextPromptTemplate_js_1 = require("../../model-function/generate-text/prompt-template/TextPromptTemplate.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
@@ -84,8 +85,8 @@ class CohereTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         const responseFormat = options.responseFormat;
         const abortSignal = options.run?.abortSignal;
         return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
-            retry: this.settings.api?.retry,
-            throttle: this.settings.api?.throttle,
+            retry: api.retry,
+            throttle: api.throttle,
             call: async () => {
                 return (0, postToApi_js_1.postJsonToApi)({
                     url: api.assembleUrl(`/generate`),
@@ -116,9 +117,7 @@ class CohereTextGenerationModel extends AbstractModel_js_1.AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
-            "maxGenerationTokens",
-            "stopSequences",
-            "numberOfGenerations",
+            ...TextGenerationModel_js_1.textGenerationModelProperties,
             "temperature",
             "k",
             "p",
@@ -138,9 +137,26 @@ class CohereTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         });
         return {
             response,
-            texts: response.generations.map((generation) => generation.text),
+            textGenerationResults: response.generations.map((generation) => ({
+                text: generation.text,
+                finishReason: this.translateFinishReason(generation.finish_reason),
+            })),
         };
     }
+    translateFinishReason(finishReason) {
+        switch (finishReason) {
+            case "COMPLETE":
+                return "stop";
+            case "MAX_TOKENS":
+                return "length";
+            case "ERROR_TOXIC":
+                return "content-filter";
+            case "ERROR":
+                return "error";
+            default:
+                return "unknown";
+        }
+    }
     doStreamText(prompt, options) {
         return this.callAPI(prompt, {
             ...options,

package/model-provider/cohere/CohereTextGenerationModel.d.ts

@@ -7,6 +7,7 @@ import { Delta } from "../../model-function/Delta.js";
 import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
 import { TextGenerationModelSettings, TextStreamingModel } from "../../model-function/generate-text/TextGenerationModel.js";
 import { TextGenerationPromptTemplate } from "../../model-function/generate-text/TextGenerationPromptTemplate.js";
+import { TextGenerationFinishReason } from "../../model-function/generate-text/TextGenerationResult.js";
 import { CohereTokenizer } from "./CohereTokenizer.js";
 export declare const COHERE_TEXT_GENERATION_MODELS: {
     command: {
@@ -79,21 +80,25 @@ export declare class CohereTextGenerationModel extends AbstractModel<CohereTextG
                 };
             } | undefined;
         };
-        texts: string[];
+        textGenerationResults: {
+            text: string;
+            finishReason: TextGenerationFinishReason;
+        }[];
     }>;
+    private translateFinishReason;
     doStreamText(prompt: string, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
     extractTextDelta(fullDelta: CohereTextGenerationDelta): string | undefined;
     /**
      * Returns this model with an instruction prompt template.
      */
-    withInstructionPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").TextInstructionPrompt, string, CohereTextGenerationModelSettings, this>;
+    withInstructionPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").InstructionPrompt, string, CohereTextGenerationModelSettings, this>;
     /**
      * Returns this model with a chat prompt template.
      */
     withChatPrompt(options?: {
         user?: string;
         assistant?: string;
-    }): PromptTemplateTextStreamingModel<import("../../index.js").TextChatPrompt, string, CohereTextGenerationModelSettings, this>;
+    }): PromptTemplateTextStreamingModel<import("../../index.js").ChatPrompt, string, CohereTextGenerationModelSettings, this>;
     withPromptTemplate<INPUT_PROMPT>(promptTemplate: TextGenerationPromptTemplate<INPUT_PROMPT, string>): PromptTemplateTextStreamingModel<INPUT_PROMPT, string, CohereTextGenerationModelSettings, this>;
     withSettings(additionalSettings: Partial<CohereTextGenerationModelSettings>): this;
 }

package/model-provider/cohere/CohereTextGenerationModel.js

@@ -4,6 +4,7 @@ import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postTo
 import { ZodSchema } from "../../core/schema/ZodSchema.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
+import { textGenerationModelProperties, } from "../../model-function/generate-text/TextGenerationModel.js";
 import { chat, instruction, } from "../../model-function/generate-text/prompt-template/TextPromptTemplate.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
@@ -81,8 +82,8 @@ export class CohereTextGenerationModel extends AbstractModel {
         const responseFormat = options.responseFormat;
         const abortSignal = options.run?.abortSignal;
         return callWithRetryAndThrottle({
-            retry: this.settings.api?.retry,
-            throttle: this.settings.api?.throttle,
+            retry: api.retry,
+            throttle: api.throttle,
             call: async () => {
                 return postJsonToApi({
                     url: api.assembleUrl(`/generate`),
@@ -113,9 +114,7 @@ export class CohereTextGenerationModel extends AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
-            "maxGenerationTokens",
-            "stopSequences",
-            "numberOfGenerations",
+            ...textGenerationModelProperties,
             "temperature",
             "k",
             "p",
@@ -135,9 +134,26 @@ export class CohereTextGenerationModel extends AbstractModel {
         });
         return {
             response,
-            texts: response.generations.map((generation) => generation.text),
+            textGenerationResults: response.generations.map((generation) => ({
+                text: generation.text,
+                finishReason: this.translateFinishReason(generation.finish_reason),
+            })),
         };
     }
+    translateFinishReason(finishReason) {
+        switch (finishReason) {
+            case "COMPLETE":
+                return "stop";
+            case "MAX_TOKENS":
+                return "length";
+            case "ERROR_TOXIC":
+                return "content-filter";
+            case "ERROR":
+                return "error";
+            default:
+                return "unknown";
+        }
+    }
     doStreamText(prompt, options) {
         return this.callAPI(prompt, {
             ...options,

package/model-provider/elevenlabs/ElevenLabsSpeechModel.cjs

@@ -101,7 +101,7 @@ class ElevenLabsSpeechModel extends AbstractModel_js_1.AbstractModel {
                 // The JS WebSocket API does not support authorization headers, so we send the API key in the BOS message.
                 // See https://stackoverflow.com/questions/4361173/http-headers-in-websockets-client-api
                 xi_api_key: api.apiKey,
-                text: " ",
+                text: " ", // first message
                 voice_settings: toApiVoiceSettings(this.settings.voiceSettings),
                 generation_config: toGenerationConfig(this.settings.generationConfig),
             }));
@@ -125,7 +125,7 @@ class ElevenLabsSpeechModel extends AbstractModel_js_1.AbstractModel {
             // send remaining text:
             if (textBuffer.length > 0) {
                 socket.send(JSON.stringify({
-                    text: `${textBuffer} `,
+                    text: `${textBuffer} `, // append space
                     try_trigger_generation: true,
                 }));
             }

package/model-provider/elevenlabs/ElevenLabsSpeechModel.js

@@ -98,7 +98,7 @@ export class ElevenLabsSpeechModel extends AbstractModel {
                 // The JS WebSocket API does not support authorization headers, so we send the API key in the BOS message.
                 // See https://stackoverflow.com/questions/4361173/http-headers-in-websockets-client-api
                 xi_api_key: api.apiKey,
-                text: " ",
+                text: " ", // first message
                 voice_settings: toApiVoiceSettings(this.settings.voiceSettings),
                 generation_config: toGenerationConfig(this.settings.generationConfig),
             }));
@@ -122,7 +122,7 @@ export class ElevenLabsSpeechModel extends AbstractModel {
             // send remaining text:
             if (textBuffer.length > 0) {
                 socket.send(JSON.stringify({
-                    text: `${textBuffer} `,
+                    text: `${textBuffer} `, // append space
                     try_trigger_generation: true,
                 }));
             }

package/model-provider/huggingface/HuggingFaceTextGenerationModel.cjs

@@ -6,6 +6,7 @@ const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndTh
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptTemplateTextGenerationModel_js_1 = require("../../model-function/generate-text/PromptTemplateTextGenerationModel.cjs");
+const TextGenerationModel_js_1 = require("../../model-function/generate-text/TextGenerationModel.cjs");
 const HuggingFaceApiConfiguration_js_1 = require("./HuggingFaceApiConfiguration.cjs");
 const HuggingFaceError_js_1 = require("./HuggingFaceError.cjs");
 /**
@@ -61,8 +62,8 @@ class HuggingFaceTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         const api = this.settings.api ?? new HuggingFaceApiConfiguration_js_1.HuggingFaceApiConfiguration();
         const abortSignal = options?.run?.abortSignal;
         return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
-            retry: this.settings.api?.retry,
-            throttle: this.settings.api?.throttle,
+            retry: api.retry,
+            throttle: api.throttle,
             call: async () => {
                 return (0, postToApi_js_1.postJsonToApi)({
                     url: api.assembleUrl(`/${this.settings.model}`),
@@ -91,16 +92,13 @@ class HuggingFaceTextGenerationModel extends AbstractModel_js_1.AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
-            "stopSequences",
-            "maxGenerationTokens",
-            "numberOfGenerations",
+            ...TextGenerationModel_js_1.textGenerationModelProperties,
             "topK",
             "topP",
             "temperature",
             "repetitionPenalty",
             "maxTime",
             "doSample",
-            "options",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -108,12 +106,15 @@ class HuggingFaceTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         const response = await this.callAPI(prompt, options);
         return {
             response,
-            texts: response.map((response) => response.generated_text),
+            textGenerationResults: response.map((response) => ({
+                text: response.generated_text,
+                finishReason: "unknown",
+            })),
         };
     }
     withPromptTemplate(promptTemplate) {
         return new PromptTemplateTextGenerationModel_js_1.PromptTemplateTextGenerationModel({
-            model: this,
+            model: this, // stop tokens are not supported by this model
             promptTemplate,
         });
     }

package/model-provider/huggingface/HuggingFaceTextGenerationModel.d.ts

@@ -14,10 +14,6 @@ export interface HuggingFaceTextGenerationModelSettings extends TextGenerationMo
     repetitionPenalty?: number;
     maxTime?: number;
     doSample?: boolean;
-    options?: {
-        useCache?: boolean;
-        waitForModel?: boolean;
-    };
 }
 /**
  * Create a text generation model that calls a Hugging Face Inference API Text Generation Task.
@@ -50,7 +46,10 @@ export declare class HuggingFaceTextGenerationModel extends AbstractModel<Huggin
         response: {
             generated_text: string;
         }[];
-        texts: string[];
+        textGenerationResults: {
+            text: string;
+            finishReason: "unknown";
+        }[];
     }>;
     withPromptTemplate<INPUT_PROMPT>(promptTemplate: TextGenerationPromptTemplate<INPUT_PROMPT, string>): PromptTemplateTextGenerationModel<INPUT_PROMPT, string, HuggingFaceTextGenerationModelSettings, this>;
     withSettings(additionalSettings: Partial<HuggingFaceTextGenerationModelSettings>): this;

package/model-provider/huggingface/HuggingFaceTextGenerationModel.js

@@ -3,6 +3,7 @@ import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottl
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptTemplateTextGenerationModel } from "../../model-function/generate-text/PromptTemplateTextGenerationModel.js";
+import { textGenerationModelProperties, } from "../../model-function/generate-text/TextGenerationModel.js";
 import { HuggingFaceApiConfiguration } from "./HuggingFaceApiConfiguration.js";
 import { failedHuggingFaceCallResponseHandler } from "./HuggingFaceError.js";
 /**
@@ -58,8 +59,8 @@ export class HuggingFaceTextGenerationModel extends AbstractModel {
         const api = this.settings.api ?? new HuggingFaceApiConfiguration();
         const abortSignal = options?.run?.abortSignal;
         return callWithRetryAndThrottle({
-            retry: this.settings.api?.retry,
-            throttle: this.settings.api?.throttle,
+            retry: api.retry,
+            throttle: api.throttle,
             call: async () => {
                 return postJsonToApi({
                     url: api.assembleUrl(`/${this.settings.model}`),
@@ -88,16 +89,13 @@ export class HuggingFaceTextGenerationModel extends AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
-            "stopSequences",
-            "maxGenerationTokens",
-            "numberOfGenerations",
+            ...textGenerationModelProperties,
             "topK",
             "topP",
             "temperature",
             "repetitionPenalty",
             "maxTime",
             "doSample",
-            "options",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -105,12 +103,15 @@ export class HuggingFaceTextGenerationModel extends AbstractModel {
         const response = await this.callAPI(prompt, options);
         return {
             response,
-            texts: response.map((response) => response.generated_text),
+            textGenerationResults: response.map((response) => ({
+                text: response.generated_text,
+                finishReason: "unknown",
+            })),
         };
     }
     withPromptTemplate(promptTemplate) {
         return new PromptTemplateTextGenerationModel({
-            model: this,
+            model: this, // stop tokens are not supported by this model
             promptTemplate,
         });
     }

package/model-provider/llamacpp/LlamaCppBakLLaVA1PromptTemplate.cjs

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.chat = exports.instruction = void 0;
-const ChatPrompt_js_1 = require("../../model-function/generate-text/prompt-template/ChatPrompt.cjs");
 // default Vicuna 1 system message
 const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
     "The assistant gives helpful, detailed, and polite answers to the user's questions.";
@@ -16,23 +15,28 @@ function instruction() {
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             text += `USER: `;
-            // construct text and image mapping:
-            let imageCounter = 1;
             const images = {};
-            for (const content of prompt.instruction) {
-                switch (content.type) {
-                    case "text": {
-                        text += content.text;
-                        break;
-                    }
-                    case "image": {
-                        text += `[img-${imageCounter}]`;
-                        images[imageCounter.toString()] = content.base64Image;
-                        imageCounter++;
-                        break;
+            if (typeof prompt.instruction === "string") {
+                text += `${prompt.instruction}\n`;
+            }
+            else {
+                // construct text and image mapping:
+                let imageCounter = 1;
+                for (const content of prompt.instruction) {
+                    switch (content.type) {
+                        case "text": {
+                            text += content.text;
+                            break;
+                        }
+                        case "image": {
+                            text += `[img-${imageCounter}]`;
+                            images[imageCounter.toString()] = content.base64Image;
+                            imageCounter++;
+                            break;
+                        }
                     }
+                    text += `${content}\n`;
                 }
-                text += `${content}\n`;
             }
             text += `\nASSISTANT: `;
             return { text, images };
@@ -44,7 +48,6 @@ exports.instruction = instruction;
 function chat() {
     return {
         format(prompt) {
-            (0, ChatPrompt_js_1.validateChatPrompt)(prompt);
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             // construct text and image mapping:
@@ -54,6 +57,10 @@ function chat() {
                 switch (role) {
                     case "user": {
                         text += `USER: `;
+                        if (typeof content === "string") {
+                            text += content;
+                            break;
+                        }
                         for (const part of content) {
                             switch (part.type) {
                                 case "text": {

package/model-provider/llamacpp/LlamaCppBakLLaVA1PromptTemplate.d.ts

@@ -1,11 +1,11 @@
 import { TextGenerationPromptTemplate } from "../../model-function/generate-text/TextGenerationPromptTemplate.js";
-import { MultiModalChatPrompt } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
-import { MultiModalInstructionPrompt } from "../../model-function/generate-text/prompt-template/InstructionPrompt.js";
+import { ChatPrompt } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
+import { InstructionPrompt } from "../../model-function/generate-text/prompt-template/InstructionPrompt.js";
 import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
 /**
  * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompt structure.
  *
  * @see https://github.com/SkunkworksAI/BakLLaVA
  */
-export declare function instruction(): TextGenerationPromptTemplate<MultiModalInstructionPrompt, LlamaCppTextGenerationPrompt>;
-export declare function chat(): TextGenerationPromptTemplate<MultiModalChatPrompt, LlamaCppTextGenerationPrompt>;
+export declare function instruction(): TextGenerationPromptTemplate<InstructionPrompt, LlamaCppTextGenerationPrompt>;
+export declare function chat(): TextGenerationPromptTemplate<ChatPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/LlamaCppBakLLaVA1PromptTemplate.js

@@ -1,4 +1,3 @@
-import { validateChatPrompt, } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
 // default Vicuna 1 system message
 const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
     "The assistant gives helpful, detailed, and polite answers to the user's questions.";
@@ -13,23 +12,28 @@ export function instruction() {
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             text += `USER: `;
-            // construct text and image mapping:
-            let imageCounter = 1;
             const images = {};
-            for (const content of prompt.instruction) {
-                switch (content.type) {
-                    case "text": {
-                        text += content.text;
-                        break;
-                    }
-                    case "image": {
-                        text += `[img-${imageCounter}]`;
-                        images[imageCounter.toString()] = content.base64Image;
-                        imageCounter++;
-                        break;
+            if (typeof prompt.instruction === "string") {
+                text += `${prompt.instruction}\n`;
+            }
+            else {
+                // construct text and image mapping:
+                let imageCounter = 1;
+                for (const content of prompt.instruction) {
+                    switch (content.type) {
+                        case "text": {
+                            text += content.text;
+                            break;
+                        }
+                        case "image": {
+                            text += `[img-${imageCounter}]`;
+                            images[imageCounter.toString()] = content.base64Image;
+                            imageCounter++;
+                            break;
+                        }
                     }
+                    text += `${content}\n`;
                 }
-                text += `${content}\n`;
             }
             text += `\nASSISTANT: `;
             return { text, images };
@@ -40,7 +44,6 @@ export function instruction() {
 export function chat() {
     return {
         format(prompt) {
-            validateChatPrompt(prompt);
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             // construct text and image mapping:
@@ -50,6 +53,10 @@ export function chat() {
                 switch (role) {
                     case "user": {
                         text += `USER: `;
+                        if (typeof content === "string") {
+                            text += content;
+                            break;
+                        }
                         for (const part of content) {
                             switch (part.type) {
                                 case "text": {