modelfusion 0.33.1 → 0.35.0

package/model-function/generate-text/streamText.cjs

@@ -1,51 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.streamText = exports.StreamTextPromise = void 0;
+exports.streamText = void 0;
 const nanoid_1 = require("nanoid");
 const FunctionEventSource_js_1 = require("../../core/FunctionEventSource.cjs");
 const GlobalFunctionLogging_js_1 = require("../../core/GlobalFunctionLogging.cjs");
 const GlobalFunctionObservers_js_1 = require("../../core/GlobalFunctionObservers.cjs");
+const AbortError_js_1 = require("../../core/api/AbortError.cjs");
 const getFunctionCallLogger_js_1 = require("../../core/getFunctionCallLogger.cjs");
 const DurationMeasurement_js_1 = require("../../util/DurationMeasurement.cjs");
-const AbortError_js_1 = require("../../core/api/AbortError.cjs");
 const runSafe_js_1 = require("../../util/runSafe.cjs");
-const extractTextDeltas_js_1 = require("./extractTextDeltas.cjs");
-class StreamTextPromise extends Promise {
-    constructor(fullPromise) {
-        super((resolve) => {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            resolve(null); // we override the resolve function
-        });
-        Object.defineProperty(this, "fullPromise", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: fullPromise
-        });
-        Object.defineProperty(this, "outputPromise", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.outputPromise = fullPromise.then((result) => result.output);
-    }
-    asFullResponse() {
-        return this.fullPromise;
-    }
-    then(onfulfilled, onrejected) {
-        return this.outputPromise.then(onfulfilled, onrejected);
-    }
-    catch(onrejected) {
-        return this.outputPromise.catch(onrejected);
-    }
-    finally(onfinally) {
-        return this.outputPromise.finally(onfinally);
-    }
-}
-exports.StreamTextPromise = StreamTextPromise;
+const AsyncIterableResultPromise_js_1 = require("../AsyncIterableResultPromise.cjs");
 function streamText(model, prompt, options) {
-    return new StreamTextPromise(doStreamText(model, prompt, options));
+    return new AsyncIterableResultPromise_js_1.AsyncIterableResultPromise(doStreamText(model, prompt, options));
 }
 exports.streamText = streamText;
 async function doStreamText(model, prompt, options) {
@@ -87,14 +53,49 @@ async function doStreamText(model, prompt, options) {
         eventType: "started",
         ...startMetadata,
     });
-    const result = await (0, runSafe_js_1.runSafe)(async () => (0, extractTextDeltas_js_1.extractTextDeltas)({
-        deltaIterable: await model.generateDeltaStreamResponse(prompt, {
+    const result = await (0, runSafe_js_1.runSafe)(async () => {
+        const deltaIterable = await model.generateDeltaStreamResponse(prompt, {
             functionId: options?.functionId,
             settings,
             run,
-        }),
-        extractDelta: (fullDelta) => model.extractTextDelta(fullDelta),
-        onDone: (fullText, lastFullDelta) => {
+        });
+        return (async function* () {
+            let accumulatedText = "";
+            let lastFullDelta;
+            for await (const event of deltaIterable) {
+                if (event?.type === "error") {
+                    const error = event.error;
+                    const finishMetadata = {
+                        eventType: "finished",
+                        ...startMetadata,
+                        finishTimestamp: new Date(),
+                        durationInMs: durationMeasurement.durationInMs,
+                    };
+                    eventSource.notify(error instanceof AbortError_js_1.AbortError
+                        ? {
+                            ...finishMetadata,
+                            result: {
+                                status: "abort",
+                            },
+                        }
+                        : {
+                            ...finishMetadata,
+                            result: {
+                                status: "error",
+                                error,
+                            },
+                        });
+                    throw error;
+                }
+                if (event?.type === "delta") {
+                    lastFullDelta = event.fullDelta;
+                    const delta = model.extractTextDelta(lastFullDelta);
+                    if (delta != null && delta.length > 0) {
+                        accumulatedText += delta;
+                        yield delta;
+                    }
+                }
+            }
             const finishMetadata = {
                 eventType: "finished",
                 ...startMetadata,
@@ -106,33 +107,11 @@ async function doStreamText(model, prompt, options) {
                 result: {
                     status: "success",
                     response: lastFullDelta,
-                    output: fullText,
+                    output: accumulatedText,
                 },
             });
-        },
-        onError: (error) => {
-            const finishMetadata = {
-                eventType: "finished",
-                ...startMetadata,
-                finishTimestamp: new Date(),
-                durationInMs: durationMeasurement.durationInMs,
-            };
-            eventSource.notify(error instanceof AbortError_js_1.AbortError
-                ? {
-                    ...finishMetadata,
-                    result: {
-                        status: "abort",
-                    },
-                }
-                : {
-                    ...finishMetadata,
-                    result: {
-                        status: "error",
-                        error,
-                    },
-                });
-        },
-    }));
+        })();
+    });
     if (!result.ok) {
         const finishMetadata = {
             eventType: "finished",

package/model-function/generate-text/streamText.d.ts

@@ -1,23 +1,8 @@
+import { AsyncIterableResultPromise } from "../AsyncIterableResultPromise.js";
+import { DeltaEvent } from "../DeltaEvent.js";
 import { ModelFunctionOptions } from "../ModelFunctionOptions.js";
-import { ModelCallMetadata } from "../executeCall.js";
-import { DeltaEvent } from "./DeltaEvent.js";
 import { TextGenerationModel, TextGenerationModelSettings } from "./TextGenerationModel.js";
-export declare class StreamTextPromise extends Promise<AsyncIterable<string>> {
-    private fullPromise;
-    private outputPromise;
-    constructor(fullPromise: Promise<{
-        output: AsyncIterable<string>;
-        metadata: Omit<ModelCallMetadata, "durationInMs" | "finishTimestamp">;
-    }>);
-    asFullResponse(): Promise<{
-        output: AsyncIterable<string>;
-        metadata: Omit<ModelCallMetadata, "durationInMs" | "finishTimestamp">;
-    }>;
-    then<TResult1 = AsyncIterable<string>, TResult2 = never>(onfulfilled?: ((value: AsyncIterable<string>) => TResult1 | PromiseLike<TResult1>) | undefined | null, onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | undefined | null): Promise<TResult1 | TResult2>;
-    catch<TResult = never>(onrejected?: ((reason: unknown) => TResult | PromiseLike<TResult>) | undefined | null): Promise<AsyncIterable<string> | TResult>;
-    finally(onfinally?: (() => void) | undefined | null): Promise<AsyncIterable<string>>;
-}
 export declare function streamText<PROMPT, FULL_DELTA, SETTINGS extends TextGenerationModelSettings>(model: TextGenerationModel<PROMPT, unknown, FULL_DELTA, SETTINGS> & {
     generateDeltaStreamResponse: (prompt: PROMPT, options: ModelFunctionOptions<SETTINGS>) => PromiseLike<AsyncIterable<DeltaEvent<FULL_DELTA>>>;
     extractTextDelta: (fullDelta: FULL_DELTA) => string | undefined;
-}, prompt: PROMPT, options?: ModelFunctionOptions<SETTINGS>): StreamTextPromise;
+}, prompt: PROMPT, options?: ModelFunctionOptions<SETTINGS>): AsyncIterableResultPromise<string>;

package/model-function/generate-text/streamText.js

@@ -2,46 +2,13 @@ import { nanoid as createId } from "nanoid";
 import { FunctionEventSource } from "../../core/FunctionEventSource.js";
 import { getGlobalFunctionLogging } from "../../core/GlobalFunctionLogging.js";
 import { getGlobalFunctionObservers } from "../../core/GlobalFunctionObservers.js";
+import { AbortError } from "../../core/api/AbortError.js";
 import { getFunctionCallLogger } from "../../core/getFunctionCallLogger.js";
 import { startDurationMeasurement } from "../../util/DurationMeasurement.js";
-import { AbortError } from "../../core/api/AbortError.js";
 import { runSafe } from "../../util/runSafe.js";
-import { extractTextDeltas } from "./extractTextDeltas.js";
-export class StreamTextPromise extends Promise {
-    constructor(fullPromise) {
-        super((resolve) => {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            resolve(null); // we override the resolve function
-        });
-        Object.defineProperty(this, "fullPromise", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: fullPromise
-        });
-        Object.defineProperty(this, "outputPromise", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.outputPromise = fullPromise.then((result) => result.output);
-    }
-    asFullResponse() {
-        return this.fullPromise;
-    }
-    then(onfulfilled, onrejected) {
-        return this.outputPromise.then(onfulfilled, onrejected);
-    }
-    catch(onrejected) {
-        return this.outputPromise.catch(onrejected);
-    }
-    finally(onfinally) {
-        return this.outputPromise.finally(onfinally);
-    }
-}
+import { AsyncIterableResultPromise } from "../AsyncIterableResultPromise.js";
 export function streamText(model, prompt, options) {
-    return new StreamTextPromise(doStreamText(model, prompt, options));
+    return new AsyncIterableResultPromise(doStreamText(model, prompt, options));
 }
 async function doStreamText(model, prompt, options) {
     if (options?.settings != null) {
@@ -82,14 +49,49 @@ async function doStreamText(model, prompt, options) {
         eventType: "started",
         ...startMetadata,
     });
-    const result = await runSafe(async () => extractTextDeltas({
-        deltaIterable: await model.generateDeltaStreamResponse(prompt, {
+    const result = await runSafe(async () => {
+        const deltaIterable = await model.generateDeltaStreamResponse(prompt, {
             functionId: options?.functionId,
             settings,
             run,
-        }),
-        extractDelta: (fullDelta) => model.extractTextDelta(fullDelta),
-        onDone: (fullText, lastFullDelta) => {
+        });
+        return (async function* () {
+            let accumulatedText = "";
+            let lastFullDelta;
+            for await (const event of deltaIterable) {
+                if (event?.type === "error") {
+                    const error = event.error;
+                    const finishMetadata = {
+                        eventType: "finished",
+                        ...startMetadata,
+                        finishTimestamp: new Date(),
+                        durationInMs: durationMeasurement.durationInMs,
+                    };
+                    eventSource.notify(error instanceof AbortError
+                        ? {
+                            ...finishMetadata,
+                            result: {
+                                status: "abort",
+                            },
+                        }
+                        : {
+                            ...finishMetadata,
+                            result: {
+                                status: "error",
+                                error,
+                            },
+                        });
+                    throw error;
+                }
+                if (event?.type === "delta") {
+                    lastFullDelta = event.fullDelta;
+                    const delta = model.extractTextDelta(lastFullDelta);
+                    if (delta != null && delta.length > 0) {
+                        accumulatedText += delta;
+                        yield delta;
+                    }
+                }
+            }
             const finishMetadata = {
                 eventType: "finished",
                 ...startMetadata,
@@ -101,33 +103,11 @@ async function doStreamText(model, prompt, options) {
                 result: {
                     status: "success",
                     response: lastFullDelta,
-                    output: fullText,
+                    output: accumulatedText,
                 },
             });
-        },
-        onError: (error) => {
-            const finishMetadata = {
-                eventType: "finished",
-                ...startMetadata,
-                finishTimestamp: new Date(),
-                durationInMs: durationMeasurement.durationInMs,
-            };
-            eventSource.notify(error instanceof AbortError
-                ? {
-                    ...finishMetadata,
-                    result: {
-                        status: "abort",
-                    },
-                }
-                : {
-                    ...finishMetadata,
-                    result: {
-                        status: "error",
-                        error,
-                    },
-                });
-        },
-    }));
+        })();
+    });
     if (!result.ok) {
         const finishMetadata = {
             eventType: "finished",

package/model-function/index.cjs

@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./DeltaEvent.cjs"), exports);
 __exportStar(require("./Model.cjs"), exports);
 __exportStar(require("./ModelCallEvent.cjs"), exports);
 __exportStar(require("./ModelFunctionOptions.cjs"), exports);
@@ -33,11 +34,11 @@ __exportStar(require("./generate-structure/StructureFromTextGenerationModel.cjs"
 __exportStar(require("./generate-structure/StructureGenerationEvent.cjs"), exports);
 __exportStar(require("./generate-structure/StructureGenerationModel.cjs"), exports);
 __exportStar(require("./generate-structure/StructureOrTextGenerationModel.cjs"), exports);
+__exportStar(require("./generate-structure/StructureStreamingEvent.cjs"), exports);
 __exportStar(require("./generate-structure/StructureValidationError.cjs"), exports);
 __exportStar(require("./generate-structure/generateStructure.cjs"), exports);
 __exportStar(require("./generate-structure/generateStructureOrText.cjs"), exports);
-__exportStar(require("./generate-text/DeltaEvent.cjs"), exports);
-__exportStar(require("./generate-text/TextDeltaEventSource.cjs"), exports);
+__exportStar(require("./generate-structure/streamStructure.cjs"), exports);
 __exportStar(require("./generate-text/TextGenerationEvent.cjs"), exports);
 __exportStar(require("./generate-text/TextGenerationModel.cjs"), exports);
 __exportStar(require("./generate-text/TextStreamingEvent.cjs"), exports);

package/model-function/index.d.ts

@@ -1,3 +1,4 @@
+export * from "./DeltaEvent.js";
 export * from "./Model.js";
 export * from "./ModelCallEvent.js";
 export * from "./ModelFunctionOptions.js";
@@ -17,11 +18,11 @@ export * from "./generate-structure/StructureFromTextGenerationModel.js";
 export * from "./generate-structure/StructureGenerationEvent.js";
 export * from "./generate-structure/StructureGenerationModel.js";
 export * from "./generate-structure/StructureOrTextGenerationModel.js";
+export * from "./generate-structure/StructureStreamingEvent.js";
 export * from "./generate-structure/StructureValidationError.js";
 export * from "./generate-structure/generateStructure.js";
 export * from "./generate-structure/generateStructureOrText.js";
-export * from "./generate-text/DeltaEvent.js";
-export * from "./generate-text/TextDeltaEventSource.js";
+export * from "./generate-structure/streamStructure.js";
 export * from "./generate-text/TextGenerationEvent.js";
 export * from "./generate-text/TextGenerationModel.js";
 export * from "./generate-text/TextStreamingEvent.js";

package/model-function/index.js

@@ -1,3 +1,4 @@
+export * from "./DeltaEvent.js";
 export * from "./Model.js";
 export * from "./ModelCallEvent.js";
 export * from "./ModelFunctionOptions.js";
@@ -17,11 +18,11 @@ export * from "./generate-structure/StructureFromTextGenerationModel.js";
 export * from "./generate-structure/StructureGenerationEvent.js";
 export * from "./generate-structure/StructureGenerationModel.js";
 export * from "./generate-structure/StructureOrTextGenerationModel.js";
+export * from "./generate-structure/StructureStreamingEvent.js";
 export * from "./generate-structure/StructureValidationError.js";
 export * from "./generate-structure/generateStructure.js";
 export * from "./generate-structure/generateStructureOrText.js";
-export * from "./generate-text/DeltaEvent.js";
-export * from "./generate-text/TextDeltaEventSource.js";
+export * from "./generate-structure/streamStructure.js";
 export * from "./generate-text/TextGenerationEvent.js";
 export * from "./generate-text/TextGenerationModel.js";
 export * from "./generate-text/TextStreamingEvent.js";

package/model-provider/cohere/CohereTextGenerationModel.cjs

@@ -6,12 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CohereTextGenerationResponseFormat = exports.CohereTextGenerationModel = exports.COHERE_TEXT_GENERATION_MODELS = void 0;
 const secure_json_parse_1 = __importDefault(require("secure-json-parse"));
 const zod_1 = require("zod");
+const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const AsyncQueue_js_1 = require("../../event-source/AsyncQueue.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
-const AsyncQueue_js_1 = require("../../model-function/generate-text/AsyncQueue.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const PromptFormatTextGenerationModel_js_1 = require("../../prompt/PromptFormatTextGenerationModel.cjs");
-const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
-const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const CohereApiConfiguration_js_1 = require("./CohereApiConfiguration.cjs");
 const CohereError_js_1 = require("./CohereError.cjs");
 const CohereTokenizer_js_1 = require("./CohereTokenizer.cjs");

package/model-provider/cohere/CohereTextGenerationModel.d.ts

@@ -1,12 +1,12 @@
 import { z } from "zod";
-import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { ResponseHandler } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { DeltaEvent } from "../../model-function/DeltaEvent.js";
 import { ModelFunctionOptions } from "../../model-function/ModelFunctionOptions.js";
-import { DeltaEvent } from "../../model-function/generate-text/DeltaEvent.js";
 import { TextGenerationModel, TextGenerationModelSettings } from "../../model-function/generate-text/TextGenerationModel.js";
 import { PromptFormat } from "../../prompt/PromptFormat.js";
 import { PromptFormatTextGenerationModel } from "../../prompt/PromptFormatTextGenerationModel.js";
-import { ResponseHandler } from "../../core/api/postToApi.js";
 import { CohereTokenizer } from "./CohereTokenizer.js";
 export declare const COHERE_TEXT_GENERATION_MODELS: {
     command: {

package/model-provider/cohere/CohereTextGenerationModel.js

@@ -1,11 +1,11 @@
 import SecureJSON from "secure-json-parse";
 import { z } from "zod";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { AsyncQueue } from "../../event-source/AsyncQueue.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
-import { AsyncQueue } from "../../model-function/generate-text/AsyncQueue.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { PromptFormatTextGenerationModel } from "../../prompt/PromptFormatTextGenerationModel.js";
-import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
-import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { CohereApiConfiguration } from "./CohereApiConfiguration.js";
 import { failedCohereCallResponseHandler } from "./CohereError.js";
 import { CohereTokenizer } from "./CohereTokenizer.js";

package/model-provider/huggingface/HuggingFaceTextGenerationModel.cjs

@@ -56,18 +56,6 @@ class HuggingFaceTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             writable: true,
             value: undefined
         });
-        Object.defineProperty(this, "generateDeltaStreamResponse", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
-        Object.defineProperty(this, "extractTextDelta", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
     }
     get modelName() {
         return this.settings.model;

package/model-provider/huggingface/HuggingFaceTextGenerationModel.d.ts

@@ -51,8 +51,6 @@ export declare class HuggingFaceTextGenerationModel extends AbstractModel<Huggin
         generated_text: string;
     }[]>;
     extractText(response: HuggingFaceTextGenerationResponse): string;
-    generateDeltaStreamResponse: undefined;
-    extractTextDelta: undefined;
     withPromptFormat<INPUT_PROMPT>(promptFormat: PromptFormat<INPUT_PROMPT, string>): PromptFormatTextGenerationModel<INPUT_PROMPT, string, HuggingFaceTextGenerationResponse, undefined, HuggingFaceTextGenerationModelSettings, this>;
     withSettings(additionalSettings: Partial<HuggingFaceTextGenerationModelSettings>): this;
 }

package/model-provider/huggingface/HuggingFaceTextGenerationModel.js

@@ -50,18 +50,6 @@ export class HuggingFaceTextGenerationModel extends AbstractModel {
             writable: true,
             value: undefined
         });
-        Object.defineProperty(this, "generateDeltaStreamResponse", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
-        Object.defineProperty(this, "extractTextDelta", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
     }
     get modelName() {
         return this.settings.model;

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -6,12 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.LlamaCppTextGenerationResponseFormat = exports.LlamaCppTextGenerationModel = void 0;
 const secure_json_parse_1 = __importDefault(require("secure-json-parse"));
 const zod_1 = __importDefault(require("zod"));
-const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
-const AsyncQueue_js_1 = require("../../model-function/generate-text/AsyncQueue.cjs");
-const parseEventSourceReadableStream_js_1 = require("../../model-function/generate-text/parseEventSourceReadableStream.cjs");
-const PromptFormatTextGenerationModel_js_1 = require("../../prompt/PromptFormatTextGenerationModel.cjs");
 const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const AsyncQueue_js_1 = require("../../event-source/AsyncQueue.cjs");
+const parseEventSourceStream_js_1 = require("../../event-source/parseEventSourceStream.cjs");
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const PromptFormatTextGenerationModel_js_1 = require("../../prompt/PromptFormatTextGenerationModel.cjs");
 const LlamaCppApiConfiguration_js_1 = require("./LlamaCppApiConfiguration.cjs");
 const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
 const LlamaCppTokenizer_js_1 = require("./LlamaCppTokenizer.cjs");
@@ -215,14 +215,11 @@ async function createLlamaCppFullDeltaIterableQueue(stream) {
     const queue = new AsyncQueue_js_1.AsyncQueue();
     let content = "";
     // process the stream asynchonously (no 'await' on purpose):
-    (0, parseEventSourceReadableStream_js_1.parseEventSourceReadableStream)({
-        stream,
-        callback: (event) => {
-            if (event.type !== "event") {
-                return;
-            }
-            const data = event.data;
-            try {
+    (0, parseEventSourceStream_js_1.parseEventSourceStream)({ stream })
+        .then(async (events) => {
+        try {
+            for await (const event of events) {
+                const data = event.data;
                 const json = secure_json_parse_1.default.parse(data);
                 const parseResult = llamaCppTextStreamingResponseSchema.safeParse(json);
                 if (!parseResult.success) {
@@ -233,26 +230,29 @@ async function createLlamaCppFullDeltaIterableQueue(stream) {
                     queue.close();
                     return;
                 }
-                const event = parseResult.data;
-                content += event.content;
+                const eventData = parseResult.data;
+                content += eventData.content;
                 queue.push({
                     type: "delta",
                     fullDelta: {
                         content,
-                        isComplete: event.stop,
-                        delta: event.content,
+                        isComplete: eventData.stop,
+                        delta: eventData.content,
                     },
                 });
-                if (event.stop) {
+                if (eventData.stop) {
                     queue.close();
                 }
             }
-            catch (error) {
-                queue.push({ type: "error", error });
-                queue.close();
-                return;
-            }
-        },
+        }
+        catch (error) {
+            queue.push({ type: "error", error });
+            queue.close();
+        }
+    })
+        .catch((error) => {
+        queue.push({ type: "error", error });
+        queue.close();
     });
     return queue;
 }

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -1,12 +1,12 @@
 import z from "zod";
-import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { ResponseHandler } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { DeltaEvent } from "../../model-function/DeltaEvent.js";
 import { ModelFunctionOptions } from "../../model-function/ModelFunctionOptions.js";
-import { DeltaEvent } from "../../model-function/generate-text/DeltaEvent.js";
 import { TextGenerationModel, TextGenerationModelSettings } from "../../model-function/generate-text/TextGenerationModel.js";
 import { PromptFormat } from "../../prompt/PromptFormat.js";
 import { PromptFormatTextGenerationModel } from "../../prompt/PromptFormatTextGenerationModel.js";
-import { ResponseHandler } from "../../core/api/postToApi.js";
 import { LlamaCppTokenizer } from "./LlamaCppTokenizer.js";
 export interface LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE extends number | undefined> extends TextGenerationModelSettings {
     api?: ApiConfiguration;

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -1,11 +1,11 @@
 import SecureJSON from "secure-json-parse";
 import z from "zod";
-import { AbstractModel } from "../../model-function/AbstractModel.js";
-import { AsyncQueue } from "../../model-function/generate-text/AsyncQueue.js";
-import { parseEventSourceReadableStream } from "../../model-function/generate-text/parseEventSourceReadableStream.js";
-import { PromptFormatTextGenerationModel } from "../../prompt/PromptFormatTextGenerationModel.js";
 import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { AsyncQueue } from "../../event-source/AsyncQueue.js";
+import { parseEventSourceStream } from "../../event-source/parseEventSourceStream.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { PromptFormatTextGenerationModel } from "../../prompt/PromptFormatTextGenerationModel.js";
 import { LlamaCppApiConfiguration } from "./LlamaCppApiConfiguration.js";
 import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
 import { LlamaCppTokenizer } from "./LlamaCppTokenizer.js";
@@ -208,14 +208,11 @@ async function createLlamaCppFullDeltaIterableQueue(stream) {
     const queue = new AsyncQueue();
     let content = "";
     // process the stream asynchonously (no 'await' on purpose):
-    parseEventSourceReadableStream({
-        stream,
-        callback: (event) => {
-            if (event.type !== "event") {
-                return;
-            }
-            const data = event.data;
-            try {
+    parseEventSourceStream({ stream })
+        .then(async (events) => {
+        try {
+            for await (const event of events) {
+                const data = event.data;
                 const json = SecureJSON.parse(data);
                 const parseResult = llamaCppTextStreamingResponseSchema.safeParse(json);
                 if (!parseResult.success) {
@@ -226,26 +223,29 @@ async function createLlamaCppFullDeltaIterableQueue(stream) {
                     queue.close();
                     return;
                 }
-                const event = parseResult.data;
-                content += event.content;
+                const eventData = parseResult.data;
+                content += eventData.content;
                 queue.push({
                     type: "delta",
                     fullDelta: {
                         content,
-                        isComplete: event.stop,
-                        delta: event.content,
+                        isComplete: eventData.stop,
+                        delta: eventData.content,
                     },
                 });
-                if (event.stop) {
+                if (eventData.stop) {
                     queue.close();
                 }
             }
-            catch (error) {
-                queue.push({ type: "error", error });
-                queue.close();
-                return;
-            }
-        },
+        }
+        catch (error) {
+            queue.push({ type: "error", error });
+            queue.close();
+        }
+    })
+        .catch((error) => {
+        queue.push({ type: "error", error });
+        queue.close();
     });
     return queue;
 }