modelfusion 0.103.0 → 0.105.0

package/model-provider/cohere/CohereTextGenerationModel.test.cjs

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const streamText_js_1 = require("../../model-function/generate-text/streamText.cjs");
+const StreamingTestServer_js_1 = require("../../test/StreamingTestServer.cjs");
+const arrayFromAsync_js_1 = require("../../test/arrayFromAsync.cjs");
+const CohereApiConfiguration_js_1 = require("./CohereApiConfiguration.cjs");
+const CohereTextGenerationModel_js_1 = require("./CohereTextGenerationModel.cjs");
+describe("streamText", () => {
+    const server = new StreamingTestServer_js_1.StreamingTestServer("https://api.cohere.ai/v1/generate");
+    server.setupTestEnvironment();
+    it("should return a text stream", async () => {
+        server.responseChunks = [
+            `{"text":"Hello","is_finished":false}\n`,
+            `{"text":", ","is_finished":false}\n`,
+            `{"text":"world!","is_finished":false}\n`,
+            `{"is_finished":true,"finish_reason":"COMPLETE",` +
+                `"response":{"id":"40141e4f-2202-4e09-9188-c6ece324b7ba",` +
+                `"generations":[{"id":"c598f9d2-4a6d-46d6-beed-47d55b996540",` +
+                `"text":"Hello, world!","finish_reason":"COMPLETE"}],` +
+                `"prompt":"hello"}}\n`,
+        ];
+        const stream = await (0, streamText_js_1.streamText)(new CohereTextGenerationModel_js_1.CohereTextGenerationModel({
+            api: new CohereApiConfiguration_js_1.CohereApiConfiguration({ apiKey: "test-key" }),
+            model: "command-light",
+        }), "hello");
+        // note: space moved to last chunk bc of trimming
+        expect(await (0, arrayFromAsync_js_1.arrayFromAsync)(stream)).toStrictEqual([
+            "Hello",
+            ",",
+            " world!",
+        ]);
+    });
+});

package/model-provider/cohere/CohereTextGenerationModel.test.js

@@ -0,0 +1,31 @@
+import { streamText } from "../../model-function/generate-text/streamText.js";
+import { StreamingTestServer } from "../../test/StreamingTestServer.js";
+import { arrayFromAsync } from "../../test/arrayFromAsync.js";
+import { CohereApiConfiguration } from "./CohereApiConfiguration.js";
+import { CohereTextGenerationModel } from "./CohereTextGenerationModel.js";
+describe("streamText", () => {
+    const server = new StreamingTestServer("https://api.cohere.ai/v1/generate");
+    server.setupTestEnvironment();
+    it("should return a text stream", async () => {
+        server.responseChunks = [
+            `{"text":"Hello","is_finished":false}\n`,
+            `{"text":", ","is_finished":false}\n`,
+            `{"text":"world!","is_finished":false}\n`,
+            `{"is_finished":true,"finish_reason":"COMPLETE",` +
+                `"response":{"id":"40141e4f-2202-4e09-9188-c6ece324b7ba",` +
+                `"generations":[{"id":"c598f9d2-4a6d-46d6-beed-47d55b996540",` +
+                `"text":"Hello, world!","finish_reason":"COMPLETE"}],` +
+                `"prompt":"hello"}}\n`,
+        ];
+        const stream = await streamText(new CohereTextGenerationModel({
+            api: new CohereApiConfiguration({ apiKey: "test-key" }),
+            model: "command-light",
+        }), "hello");
+        // note: space moved to last chunk bc of trimming
+        expect(await arrayFromAsync(stream)).toStrictEqual([
+            "Hello",
+            ",",
+            " world!",
+        ]);
+    });
+});

package/model-provider/elevenlabs/ElevenLabsSpeechModel.cjs

@@ -149,8 +149,7 @@ class ElevenLabsSpeechModel extends AbstractModel_js_1.AbstractModel {
             if (!response.isFinal) {
                 queue.push({
                     type: "delta",
-                    fullDelta: event,
-                    valueDelta: Buffer.from(response.audio, "base64"),
+                    deltaValue: Buffer.from(response.audio, "base64"),
                 });
             }
         };

package/model-provider/elevenlabs/ElevenLabsSpeechModel.js

@@ -146,8 +146,7 @@ export class ElevenLabsSpeechModel extends AbstractModel {
             if (!response.isFinal) {
                 queue.push({
                     type: "delta",
-                    fullDelta: event,
-                    valueDelta: Buffer.from(response.audio, "base64"),
+                    deltaValue: Buffer.from(response.audio, "base64"),
                 });
             }
         };

package/model-provider/llamacpp/LlamaCppBakLLaVA1PromptTemplate.cjs

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.chat = exports.instruction = void 0;
-const ChatPrompt_js_1 = require("../../model-function/generate-text/prompt-template/ChatPrompt.cjs");
+const ContentPart_js_1 = require("../../model-function/generate-text/prompt-template/ContentPart.cjs");
+const InvalidPromptError_js_1 = require("../../model-function/generate-text/prompt-template/InvalidPromptError.cjs");
 // default Vicuna 1 system message
 const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
     "The assistant gives helpful, detailed, and polite answers to the user's questions.";
@@ -16,23 +17,28 @@ function instruction() {
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             text += `USER: `;
-            // construct text and image mapping:
-            let imageCounter = 1;
             const images = {};
-            for (const content of prompt.instruction) {
-                switch (content.type) {
-                    case "text": {
-                        text += content.text;
-                        break;
-                    }
-                    case "image": {
-                        text += `[img-${imageCounter}]`;
-                        images[imageCounter.toString()] = content.base64Image;
-                        imageCounter++;
-                        break;
+            if (typeof prompt.instruction === "string") {
+                text += `${prompt.instruction}\n`;
+            }
+            else {
+                // construct text and image mapping:
+                let imageCounter = 1;
+                for (const content of prompt.instruction) {
+                    switch (content.type) {
+                        case "text": {
+                            text += content.text;
+                            break;
+                        }
+                        case "image": {
+                            text += `[img-${imageCounter}]`;
+                            images[imageCounter.toString()] = content.base64Image;
+                            imageCounter++;
+                            break;
+                        }
                     }
+                    text += `${content}\n`;
                 }
-                text += `${content}\n`;
             }
             text += `\nASSISTANT: `;
             return { text, images };
@@ -44,7 +50,6 @@ exports.instruction = instruction;
 function chat() {
     return {
         format(prompt) {
-            (0, ChatPrompt_js_1.validateChatPrompt)(prompt);
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             // construct text and image mapping:
@@ -54,6 +59,10 @@ function chat() {
                 switch (role) {
                     case "user": {
                         text += `USER: `;
+                        if (typeof content === "string") {
+                            text += content;
+                            break;
+                        }
                         for (const part of content) {
                             switch (part.type) {
                                 case "text": {
@@ -71,9 +80,12 @@ function chat() {
                         break;
                     }
                     case "assistant": {
-                        text += `ASSISTANT: ${content}`;
+                        text += `ASSISTANT: ${(0, ContentPart_js_1.validateContentIsString)(content, prompt)}`;
                         break;
                     }
+                    case "tool": {
+                        throw new InvalidPromptError_js_1.InvalidPromptError("Tool messages are not supported.", prompt);
+                    }
                     default: {
                         const _exhaustiveCheck = role;
                         throw new Error(`Unsupported role: ${_exhaustiveCheck}`);

package/model-provider/llamacpp/LlamaCppBakLLaVA1PromptTemplate.d.ts

@@ -1,11 +1,11 @@
 import { TextGenerationPromptTemplate } from "../../model-function/generate-text/TextGenerationPromptTemplate.js";
-import { MultiModalChatPrompt } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
-import { MultiModalInstructionPrompt } from "../../model-function/generate-text/prompt-template/InstructionPrompt.js";
+import { ChatPrompt } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
+import { InstructionPrompt } from "../../model-function/generate-text/prompt-template/InstructionPrompt.js";
 import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
 /**
  * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompt structure.
  *
  * @see https://github.com/SkunkworksAI/BakLLaVA
  */
-export declare function instruction(): TextGenerationPromptTemplate<MultiModalInstructionPrompt, LlamaCppTextGenerationPrompt>;
-export declare function chat(): TextGenerationPromptTemplate<MultiModalChatPrompt, LlamaCppTextGenerationPrompt>;
+export declare function instruction(): TextGenerationPromptTemplate<InstructionPrompt, LlamaCppTextGenerationPrompt>;
+export declare function chat(): TextGenerationPromptTemplate<ChatPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/LlamaCppBakLLaVA1PromptTemplate.js

@@ -1,4 +1,5 @@
-import { validateChatPrompt, } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
+import { validateContentIsString } from "../../model-function/generate-text/prompt-template/ContentPart.js";
+import { InvalidPromptError } from "../../model-function/generate-text/prompt-template/InvalidPromptError.js";
 // default Vicuna 1 system message
 const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
     "The assistant gives helpful, detailed, and polite answers to the user's questions.";
@@ -13,23 +14,28 @@ export function instruction() {
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             text += `USER: `;
-            // construct text and image mapping:
-            let imageCounter = 1;
             const images = {};
-            for (const content of prompt.instruction) {
-                switch (content.type) {
-                    case "text": {
-                        text += content.text;
-                        break;
-                    }
-                    case "image": {
-                        text += `[img-${imageCounter}]`;
-                        images[imageCounter.toString()] = content.base64Image;
-                        imageCounter++;
-                        break;
+            if (typeof prompt.instruction === "string") {
+                text += `${prompt.instruction}\n`;
+            }
+            else {
+                // construct text and image mapping:
+                let imageCounter = 1;
+                for (const content of prompt.instruction) {
+                    switch (content.type) {
+                        case "text": {
+                            text += content.text;
+                            break;
+                        }
+                        case "image": {
+                            text += `[img-${imageCounter}]`;
+                            images[imageCounter.toString()] = content.base64Image;
+                            imageCounter++;
+                            break;
+                        }
                     }
+                    text += `${content}\n`;
                 }
-                text += `${content}\n`;
             }
             text += `\nASSISTANT: `;
             return { text, images };
@@ -40,7 +46,6 @@ export function instruction() {
 export function chat() {
     return {
         format(prompt) {
-            validateChatPrompt(prompt);
             let text = "";
             text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
             // construct text and image mapping:
@@ -50,6 +55,10 @@ export function chat() {
                 switch (role) {
                     case "user": {
                         text += `USER: `;
+                        if (typeof content === "string") {
+                            text += content;
+                            break;
+                        }
                         for (const part of content) {
                             switch (part.type) {
                                 case "text": {
@@ -67,9 +76,12 @@ export function chat() {
                         break;
                     }
                     case "assistant": {
-                        text += `ASSISTANT: ${content}`;
+                        text += `ASSISTANT: ${validateContentIsString(content, prompt)}`;
                         break;
                     }
+                    case "tool": {
+                        throw new InvalidPromptError("Tool messages are not supported.", prompt);
+                    }
                     default: {
                         const _exhaustiveCheck = role;
                         throw new Error(`Unsupported role: ${_exhaustiveCheck}`);

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -5,11 +5,11 @@ const zod_1 = require("zod");
 const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const ZodSchema_js_1 = require("../../core/schema/ZodSchema.cjs");
+const parseJSON_js_1 = require("../../core/schema/parseJSON.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptTemplateTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptTemplateTextStreamingModel.cjs");
 const TextGenerationModel_js_1 = require("../../model-function/generate-text/TextGenerationModel.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
-const parseJSON_js_1 = require("../../core/schema/parseJSON.cjs");
 const parseEventSourceStream_js_1 = require("../../util/streaming/parseEventSourceStream.cjs");
 const LlamaCppApiConfiguration_js_1 = require("./LlamaCppApiConfiguration.cjs");
 const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
@@ -138,6 +138,9 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             responseFormat: exports.LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
+    extractTextDelta(delta) {
+        return delta.content;
+    }
     withTextPrompt() {
         return this.withPromptTemplate({
             format(prompt) {
@@ -228,7 +231,7 @@ const llamaCppTextGenerationResponseSchema = zod_1.z.object({
     tokens_predicted: zod_1.z.number(),
     truncated: zod_1.z.boolean(),
 });
-const llamaCppTextStreamingResponseSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.discriminatedUnion("stop", [
+const llamaCppTextStreamChunkSchema = (0, ZodSchema_js_1.zodSchema)(zod_1.z.discriminatedUnion("stop", [
     zod_1.z.object({
         content: zod_1.z.string(),
         stop: zod_1.z.literal(false),
@@ -237,7 +240,6 @@ const llamaCppTextStreamingResponseSchema = new ZodSchema_js_1.ZodSchema(zod_1.z
 ]));
 async function createLlamaCppFullDeltaIterableQueue(stream) {
     const queue = new AsyncQueue_js_1.AsyncQueue();
-    let content = "";
     // process the stream asynchonously (no 'await' on purpose):
     (0, parseEventSourceStream_js_1.parseEventSourceStream)({ stream })
         .then(async (events) => {
@@ -246,18 +248,9 @@ async function createLlamaCppFullDeltaIterableQueue(stream) {
                 const data = event.data;
                 const eventData = (0, parseJSON_js_1.parseJSON)({
                     text: data,
-                    schema: llamaCppTextStreamingResponseSchema,
-                });
-                content += eventData.content;
-                queue.push({
-                    type: "delta",
-                    fullDelta: {
-                        content,
-                        isComplete: eventData.stop,
-                        delta: eventData.content,
-                    },
-                    valueDelta: eventData.content,
+                    schema: llamaCppTextStreamChunkSchema,
                 });
+                queue.push({ type: "delta", deltaValue: eventData });
                 if (eventData.stop) {
                     queue.close();
                 }

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -116,7 +116,59 @@ export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends num
             totalTokens: number;
         };
     }>;
-    doStreamText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
+    doStreamText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<{
+        stop: true;
+        model: string;
+        prompt: string;
+        content: string;
+        generation_settings: {
+            stop: string[];
+            model: string;
+            stream: boolean;
+            seed: number;
+            mirostat: number;
+            frequency_penalty: number;
+            ignore_eos: boolean;
+            logit_bias: number[];
+            mirostat_eta: number;
+            mirostat_tau: number;
+            n_ctx: number;
+            n_keep: number;
+            n_predict: number;
+            n_probs: number;
+            penalize_nl: boolean;
+            presence_penalty: number;
+            repeat_last_n: number;
+            repeat_penalty: number;
+            temp: number;
+            tfs_z: number;
+            top_k: number;
+            top_p: number;
+            typical_p: number;
+        };
+        stopped_eos: boolean;
+        stopped_limit: boolean;
+        stopped_word: boolean;
+        stopping_word: string;
+        timings: {
+            predicted_ms: number;
+            predicted_n: number;
+            predicted_per_second: number | null;
+            predicted_per_token_ms: number | null;
+            prompt_ms: number | null;
+            prompt_n: number;
+            prompt_per_second: number | null;
+            prompt_per_token_ms: number | null;
+        };
+        tokens_cached: number;
+        tokens_evaluated: number;
+        tokens_predicted: number;
+        truncated: boolean;
+    } | {
+        stop: false;
+        content: string;
+    }>>>;
+    extractTextDelta(delta: unknown): string;
     withTextPrompt(): PromptTemplateTextStreamingModel<string, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     /**
      * Maps the prompt for a text version of the Llama.cpp prompt template (without image support).
@@ -340,11 +392,59 @@ declare const llamaCppTextGenerationResponseSchema: z.ZodObject<{
     truncated: boolean;
 }>;
 export type LlamaCppTextGenerationResponse = z.infer<typeof llamaCppTextGenerationResponseSchema>;
-export type LlamaCppTextGenerationDelta = {
+declare const llamaCppTextStreamChunkSchema: import("../../core/schema/ZodSchema.js").ZodSchema<{
+    stop: true;
+    model: string;
+    prompt: string;
     content: string;
-    isComplete: boolean;
-    delta: string;
-};
+    generation_settings: {
+        stop: string[];
+        model: string;
+        stream: boolean;
+        seed: number;
+        mirostat: number;
+        frequency_penalty: number;
+        ignore_eos: boolean;
+        logit_bias: number[];
+        mirostat_eta: number;
+        mirostat_tau: number;
+        n_ctx: number;
+        n_keep: number;
+        n_predict: number;
+        n_probs: number;
+        penalize_nl: boolean;
+        presence_penalty: number;
+        repeat_last_n: number;
+        repeat_penalty: number;
+        temp: number;
+        tfs_z: number;
+        top_k: number;
+        top_p: number;
+        typical_p: number;
+    };
+    stopped_eos: boolean;
+    stopped_limit: boolean;
+    stopped_word: boolean;
+    stopping_word: string;
+    timings: {
+        predicted_ms: number;
+        predicted_n: number;
+        predicted_per_second: number | null;
+        predicted_per_token_ms: number | null;
+        prompt_ms: number | null;
+        prompt_n: number;
+        prompt_per_second: number | null;
+        prompt_per_token_ms: number | null;
+    };
+    tokens_cached: number;
+    tokens_evaluated: number;
+    tokens_predicted: number;
+    truncated: boolean;
+} | {
+    stop: false;
+    content: string;
+}>;
+export type LlamaCppTextStreamChunk = (typeof llamaCppTextStreamChunkSchema)["_type"];
 export type LlamaCppTextGenerationResponseFormatType<T> = {
     stream: boolean;
     handler: ResponseHandler<T>;
@@ -413,7 +513,58 @@ export declare const LlamaCppTextGenerationResponseFormat: {
         stream: true;
         handler: ({ response }: {
             response: Response;
-        }) => Promise<AsyncIterable<Delta<string>>>;
+        }) => Promise<AsyncIterable<Delta<{
+            stop: true;
+            model: string;
+            prompt: string;
+            content: string;
+            generation_settings: {
+                stop: string[];
+                model: string;
+                stream: boolean;
+                seed: number;
+                mirostat: number;
+                frequency_penalty: number;
+                ignore_eos: boolean;
+                logit_bias: number[];
+                mirostat_eta: number;
+                mirostat_tau: number;
+                n_ctx: number;
+                n_keep: number;
+                n_predict: number;
+                n_probs: number;
+                penalize_nl: boolean;
+                presence_penalty: number;
+                repeat_last_n: number;
+                repeat_penalty: number;
+                temp: number;
+                tfs_z: number;
+                top_k: number;
+                top_p: number;
+                typical_p: number;
+            };
+            stopped_eos: boolean;
+            stopped_limit: boolean;
+            stopped_word: boolean;
+            stopping_word: string;
+            timings: {
+                predicted_ms: number;
+                predicted_n: number;
+                predicted_per_second: number | null;
+                predicted_per_token_ms: number | null;
+                prompt_ms: number | null;
+                prompt_n: number;
+                prompt_per_second: number | null;
+                prompt_per_token_ms: number | null;
+            };
+            tokens_cached: number;
+            tokens_evaluated: number;
+            tokens_predicted: number;
+            truncated: boolean;
+        } | {
+            stop: false;
+            content: string;
+        }>>>;
     };
 };
 export {};

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -1,12 +1,12 @@
 import { z } from "zod";
 import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
-import { ZodSchema } from "../../core/schema/ZodSchema.js";
+import { zodSchema } from "../../core/schema/ZodSchema.js";
+import { parseJSON } from "../../core/schema/parseJSON.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
 import { textGenerationModelProperties, } from "../../model-function/generate-text/TextGenerationModel.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
-import { parseJSON } from "../../core/schema/parseJSON.js";
 import { parseEventSourceStream } from "../../util/streaming/parseEventSourceStream.js";
 import { LlamaCppApiConfiguration } from "./LlamaCppApiConfiguration.js";
 import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
@@ -135,6 +135,9 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
             responseFormat: LlamaCppTextGenerationResponseFormat.deltaIterable,
         });
     }
+    extractTextDelta(delta) {
+        return delta.content;
+    }
     withTextPrompt() {
         return this.withPromptTemplate({
             format(prompt) {
@@ -224,7 +227,7 @@ const llamaCppTextGenerationResponseSchema = z.object({
     tokens_predicted: z.number(),
     truncated: z.boolean(),
 });
-const llamaCppTextStreamingResponseSchema = new ZodSchema(z.discriminatedUnion("stop", [
+const llamaCppTextStreamChunkSchema = zodSchema(z.discriminatedUnion("stop", [
     z.object({
         content: z.string(),
         stop: z.literal(false),
@@ -233,7 +236,6 @@ const llamaCppTextStreamingResponseSchema = new ZodSchema(z.discriminatedUnion("
 ]));
 async function createLlamaCppFullDeltaIterableQueue(stream) {
     const queue = new AsyncQueue();
-    let content = "";
     // process the stream asynchonously (no 'await' on purpose):
     parseEventSourceStream({ stream })
         .then(async (events) => {
@@ -242,18 +244,9 @@ async function createLlamaCppFullDeltaIterableQueue(stream) {
                 const data = event.data;
                 const eventData = parseJSON({
                     text: data,
-                    schema: llamaCppTextStreamingResponseSchema,
-                });
-                content += eventData.content;
-                queue.push({
-                    type: "delta",
-                    fullDelta: {
-                        content,
-                        isComplete: eventData.stop,
-                        delta: eventData.content,
-                    },
-                    valueDelta: eventData.content,
+                    schema: llamaCppTextStreamChunkSchema,
                 });
+                queue.push({ type: "delta", deltaValue: eventData });
                 if (eventData.stop) {
                     queue.close();
                 }

package/model-provider/llamacpp/LlamaCppTextGenerationModel.test.cjs

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const streamText_js_1 = require("../../model-function/generate-text/streamText.cjs");
+const StreamingTestServer_js_1 = require("../../test/StreamingTestServer.cjs");
+const arrayFromAsync_js_1 = require("../../test/arrayFromAsync.cjs");
+const LlamaCppTextGenerationModel_js_1 = require("./LlamaCppTextGenerationModel.cjs");
+describe("streamText", () => {
+    const server = new StreamingTestServer_js_1.StreamingTestServer("http://127.0.0.1:8080/completion");
+    server.setupTestEnvironment();
+    it("should return a text stream", async () => {
+        server.responseChunks = [
+            `data: {"content":"Hello","multimodal":false,"slot_id":0,"stop":false}\n\n`,
+            `data: {"content":", ","multimodal":false,"slot_id":0,"stop":false}\n\n`,
+            `data: {"content":"world!","multimodal":false,"slot_id":0,"stop":false}\n\n`,
+            `data: {"content":"","generation_settings":{"frequency_penalty":0.0,"grammar":"",` +
+                `"ignore_eos":false,"logit_bias":[],"min_p":0.05000000074505806,"mirostat":0,` +
+                `"mirostat_eta":0.10000000149011612,"mirostat_tau":5.0,"model":"models/llama-2-7b-chat.Q4_K_M.gguf",` +
+                `"n_ctx":4096,"n_keep":0,"n_predict":-1,"n_probs":0,"penalize_nl":true,"penalty_prompt_tokens":[],` +
+                `"presence_penalty":0.0,"repeat_last_n":64,"repeat_penalty":1.100000023841858,"seed":4294967295,` +
+                `"stop":[],"stream":true,"temp":0.800000011920929,"tfs_z":1.0,"top_k":40,"top_p":0.949999988079071,` +
+                `"typical_p":1.0,"use_penalty_prompt_tokens":false},"model":"models/llama-2-7b-chat.Q4_K_M.gguf",` +
+                `"prompt":"hello","slot_id":0,"stop":true,"stopped_eos":true,"stopped_limit":false,` +
+                `"stopped_word":false,"stopping_word":"","timings":{"predicted_ms":1054.704,"predicted_n":69,` +
+                `"predicted_per_second":65.421198743913,"predicted_per_token_ms":15.285565217391303,` +
+                `"prompt_ms":244.228,"prompt_n":5,"prompt_per_second":20.472673075978186,` +
+                `"prompt_per_token_ms":48.845600000000005},"tokens_cached":74,"tokens_evaluated":5,` +
+                `"tokens_predicted":69,"truncated":false}\n\n`,
+        ];
+        const stream = await (0, streamText_js_1.streamText)(new LlamaCppTextGenerationModel_js_1.LlamaCppTextGenerationModel().withTextPrompt(), "hello");
+        // note: space moved to last chunk bc of trimming
+        expect(await (0, arrayFromAsync_js_1.arrayFromAsync)(stream)).toStrictEqual([
+            "Hello",
+            ",",
+            " world!",
+        ]);
+    });
+});

package/model-provider/llamacpp/LlamaCppTextGenerationModel.test.d.ts

@@ -0,0 +1 @@
+export {};