modelfusion 0.65.1 → 0.67.0

package/model-function/generate-text/streamText.cjs

@@ -3,8 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.streamText = void 0;
 const executeStreamCall_js_1 = require("../executeStreamCall.cjs");
 async function streamText(model, prompt, options) {
+    const shouldTrimWhitespace = model.settings.trimWhitespace ?? true;
     let accumulatedText = "";
     let lastFullDelta;
+    let isFirstDelta = true;
+    let trailingWhitespace = "";
     const fullResponse = await (0, executeStreamCall_js_1.executeStreamCall)({
         functionType: "stream-text",
         input: prompt,
@@ -13,8 +16,25 @@ async function streamText(model, prompt, options) {
         startStream: async (options) => model.doStreamText(prompt, options),
         processDelta: (delta) => {
             lastFullDelta = delta.fullDelta;
-            const textDelta = delta.valueDelta;
+            let textDelta = delta.valueDelta;
             if (textDelta != null && textDelta.length > 0) {
+                if (shouldTrimWhitespace) {
+                    if (isFirstDelta) {
+                        // remove leading whitespace:
+                        textDelta = textDelta.trimStart();
+                    }
+                    else {
+                        // restore trailing whitespace from previous chunk:
+                        textDelta = trailingWhitespace + textDelta;
+                    }
+                    // trim trailing whitespace and store it for the next chunk:
+                    const trailingWhitespaceMatch = textDelta.match(/\s+$/);
+                    trailingWhitespace = trailingWhitespaceMatch
+                        ? trailingWhitespaceMatch[0]
+                        : "";
+                    textDelta = textDelta.trimEnd();
+                }
+                isFirstDelta = false;
                 accumulatedText += textDelta;
                 return textDelta;
             }

package/model-function/generate-text/streamText.js

@@ -1,7 +1,10 @@
 import { executeStreamCall } from "../executeStreamCall.js";
 export async function streamText(model, prompt, options) {
+    const shouldTrimWhitespace = model.settings.trimWhitespace ?? true;
     let accumulatedText = "";
     let lastFullDelta;
+    let isFirstDelta = true;
+    let trailingWhitespace = "";
     const fullResponse = await executeStreamCall({
         functionType: "stream-text",
         input: prompt,
@@ -10,8 +13,25 @@ export async function streamText(model, prompt, options) {
         startStream: async (options) => model.doStreamText(prompt, options),
         processDelta: (delta) => {
             lastFullDelta = delta.fullDelta;
-            const textDelta = delta.valueDelta;
+            let textDelta = delta.valueDelta;
             if (textDelta != null && textDelta.length > 0) {
+                if (shouldTrimWhitespace) {
+                    if (isFirstDelta) {
+                        // remove leading whitespace:
+                        textDelta = textDelta.trimStart();
+                    }
+                    else {
+                        // restore trailing whitespace from previous chunk:
+                        textDelta = trailingWhitespace + textDelta;
+                    }
+                    // trim trailing whitespace and store it for the next chunk:
+                    const trailingWhitespaceMatch = textDelta.match(/\s+$/);
+                    trailingWhitespace = trailingWhitespaceMatch
+                        ? trailingWhitespaceMatch[0]
+                        : "";
+                    textDelta = textDelta.trimEnd();
+                }
+                isFirstDelta = false;
                 accumulatedText += textDelta;
                 return textDelta;
             }

package/model-provider/ollama/OllamaTextGenerationModel.cjs

@@ -72,6 +72,8 @@ class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             "system",
             "template",
             "context",
+            "format",
+            "raw",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -140,7 +142,7 @@ const ollamaTextStreamingResponseSchema = zod_1.z.discriminatedUnion("done", [
         context: zod_1.z.array(zod_1.z.number()),
     }),
 ]);
-async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration_js_1.OllamaApiConfiguration(), abortSignal, responseFormat, prompt, model, contextWindowSize, maxCompletionTokens, mirostat, mirostat_eta, mirostat_tau, num_gpu, num_gqa, num_threads, repeat_last_n, repeat_penalty, seed, stopSequences, temperature, tfs_z, top_k, top_p, system, template, context, }) {
+async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration_js_1.OllamaApiConfiguration(), abortSignal, responseFormat, prompt, model, format, contextWindowSize, maxCompletionTokens, mirostat, mirostat_eta, mirostat_tau, num_gpu, num_gqa, num_threads, repeat_last_n, repeat_penalty, seed, stopSequences, temperature, tfs_z, top_k, top_p, system, template, context, raw, }) {
     return (0, postToApi_js_1.postJsonToApi)({
         url: api.assembleUrl(`/api/generate`),
         headers: api.headers,
@@ -148,6 +150,7 @@ async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration_js
             stream: responseFormat.stream,
             model,
             prompt,
+            format,
             options: {
                 mirostat,
                 mirostat_eta,
@@ -169,6 +172,7 @@ async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration_js
             system,
             template,
             context,
+            raw,
         },
         failedResponseHandler: OllamaError_js_1.failedOllamaCallResponseHandler,
         successfulResponseHandler: responseFormat.handler,

package/model-provider/ollama/OllamaTextGenerationModel.d.ts

@@ -7,27 +7,104 @@ import { Delta } from "../../model-function/Delta.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
 import { TextGenerationModelSettings, TextStreamingModel } from "../../model-function/generate-text/TextGenerationModel.js";
 import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
+/**
+ * @see https://github.com/jmorganca/ollama/blob/main/docs/api.md#generate-a-completion
+ */
 export interface OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE extends number | undefined> extends TextGenerationModelSettings {
     api?: ApiConfiguration;
+    /**
+     * The name of the model to use. For example, 'mistral'.
+     *
+     * @see https://ollama.ai/library
+     */
     model: string;
+    /**
+     * The temperature of the model. Increasing the temperature will make the model
+     * answer more creatively. (Default: 0.8)
+     */
     temperature?: number;
     /**
      * Specify the context window size of the model that you have loaded in your
-     * Ollama server.
+     * Ollama server. (Default: 2048)
      */
     contextWindowSize?: CONTEXT_WINDOW_SIZE;
+    /**
+     * Enable Mirostat sampling for controlling perplexity.
+     * (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)
+     */
     mirostat?: number;
+    /**
+     * Influences how quickly the algorithm responds to feedback from the generated text.
+     * A lower learning rate will result in slower adjustments,
+     * while a higher learning rate will make the algorithm more responsive. (Default: 0.1)
+     */
     mirostat_eta?: number;
+    /**
+     * Controls the balance between coherence and diversity of the output.
+     * A lower value will result in more focused and coherent text. (Default: 5.0)
+     */
     mirostat_tau?: number;
+    /**
+     * The number of GQA groups in the transformer layer. Required for some models,
+     * for example it is 8 for llama2:70b
+     */
     num_gqa?: number;
+    /**
+     * The number of layers to send to the GPU(s). On macOS it defaults to 1 to
+     * enable metal support, 0 to disable.
+     */
     num_gpu?: number;
+    /**
+     * Sets the number of threads to use during computation. By default, Ollama will
+     * detect this for optimal performance. It is recommended to set this value to the
+     * number of physical CPU cores your system has (as opposed to the logical number of cores).
+     */
     num_threads?: number;
+    /**
+     * Sets how far back for the model to look back to prevent repetition.
+     * (Default: 64, 0 = disabled, -1 = num_ctx)
+     */
     repeat_last_n?: number;
+    /**
+     * Sets how strongly to penalize repetitions. A higher value (e.g., 1.5)
+     * will penalize repetitions more strongly, while a lower value (e.g., 0.9)
+     * will be more lenient. (Default: 1.1)
+     */
     repeat_penalty?: number;
+    /**
+     * Sets the random number seed to use for generation. Setting this to a
+     * specific number will make the model generate the same text for the same prompt.
+     * (Default: 0)
+     */
     seed?: number;
+    /**
+     * Tail free sampling is used to reduce the impact of less probable tokens
+     * from the output. A higher value (e.g., 2.0) will reduce the impact more,
+     * while a value of 1.0 disables this setting. (default: 1)
+     */
     tfs_z?: number;
+    /**
+     * Reduces the probability of generating nonsense. A higher value (e.g. 100)
+     * will give more diverse answers, while a lower value (e.g. 10) will be more
+     *  conservative. (Default: 40)
+     */
     top_k?: number;
+    /**
+     * Works together with top-k. A higher value (e.g., 0.95) will lead to more
+     * diverse text, while a lower value (e.g., 0.5) will generate more focused
+     * and conservative text. (Default: 0.9)
+     */
     top_p?: number;
+    /**
+     * When set to true, no formatting will be applied to the prompt and no context
+     * will be returned.
+     */
+    raw?: boolean;
+    /**
+     * The format to return a response in. Currently the only accepted value is 'json'.
+     * Leave undefined to return a string.
+     */
+    format?: "json";
     system?: string;
     template?: string;
     context?: number[];

package/model-provider/ollama/OllamaTextGenerationModel.js

@@ -69,6 +69,8 @@ export class OllamaTextGenerationModel extends AbstractModel {
             "system",
             "template",
             "context",
+            "format",
+            "raw",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -136,7 +138,7 @@ const ollamaTextStreamingResponseSchema = z.discriminatedUnion("done", [
         context: z.array(z.number()),
     }),
 ]);
-async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration(), abortSignal, responseFormat, prompt, model, contextWindowSize, maxCompletionTokens, mirostat, mirostat_eta, mirostat_tau, num_gpu, num_gqa, num_threads, repeat_last_n, repeat_penalty, seed, stopSequences, temperature, tfs_z, top_k, top_p, system, template, context, }) {
+async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration(), abortSignal, responseFormat, prompt, model, format, contextWindowSize, maxCompletionTokens, mirostat, mirostat_eta, mirostat_tau, num_gpu, num_gqa, num_threads, repeat_last_n, repeat_penalty, seed, stopSequences, temperature, tfs_z, top_k, top_p, system, template, context, raw, }) {
     return postJsonToApi({
         url: api.assembleUrl(`/api/generate`),
         headers: api.headers,
@@ -144,6 +146,7 @@ async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration(),
             stream: responseFormat.stream,
             model,
             prompt,
+            format,
             options: {
                 mirostat,
                 mirostat_eta,
@@ -165,6 +168,7 @@ async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration(),
             system,
             template,
             context,
+            raw,
         },
         failedResponseHandler: failedOllamaCallResponseHandler,
         successfulResponseHandler: responseFormat.handler,

package/model-provider/openai/OpenAICompletionModel.cjs

@@ -212,6 +212,7 @@ class OpenAICompletionModel extends AbstractModel_js_1.AbstractModel {
             "frequencyPenalty",
             "bestOf",
             "logitBias",
+            "seed",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -266,22 +267,26 @@ class OpenAICompletionModel extends AbstractModel_js_1.AbstractModel {
 exports.OpenAICompletionModel = OpenAICompletionModel;
 const OpenAICompletionResponseSchema = zod_1.z.object({
     id: zod_1.z.string(),
-    object: zod_1.z.literal("text_completion"),
-    created: zod_1.z.number(),
-    model: zod_1.z.string(),
     choices: zod_1.z.array(zod_1.z.object({
-        text: zod_1.z.string(),
+        finish_reason: zod_1.z
+            .enum(["stop", "length", "content_filter"])
+            .optional()
+            .nullable(),
         index: zod_1.z.number(),
         logprobs: zod_1.z.nullable(zod_1.z.any()),
-        finish_reason: zod_1.z.string(),
+        text: zod_1.z.string(),
     })),
+    created: zod_1.z.number(),
+    model: zod_1.z.string(),
+    system_fingerprint: zod_1.z.string().optional(),
+    object: zod_1.z.literal("text_completion"),
     usage: zod_1.z.object({
         prompt_tokens: zod_1.z.number(),
         completion_tokens: zod_1.z.number(),
         total_tokens: zod_1.z.number(),
     }),
 });
-async function callOpenAICompletionAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, responseFormat, model, prompt, suffix, maxTokens, temperature, topP, n, logprobs, echo, stop, presencePenalty, frequencyPenalty, bestOf, logitBias, user, }) {
+async function callOpenAICompletionAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, responseFormat, model, prompt, suffix, maxTokens, temperature, topP, n, logprobs, echo, stop, presencePenalty, frequencyPenalty, bestOf, logitBias, seed, user, }) {
     // empty arrays are not allowed for stop:
     if (stop != null && Array.isArray(stop) && stop.length === 0) {
         stop = undefined;
@@ -301,6 +306,7 @@ async function callOpenAICompletionAPI({ api = new OpenAIApiConfiguration_js_1.O
             logprobs,
             echo,
             stop,
+            seed,
             presence_penalty: presencePenalty,
             frequency_penalty: frequencyPenalty,
             best_of: bestOf,
@@ -332,13 +338,17 @@ exports.OpenAITextResponseFormat = {
 const textResponseStreamEventSchema = zod_1.z.object({
     choices: zod_1.z.array(zod_1.z.object({
         text: zod_1.z.string(),
-        finish_reason: zod_1.z.enum(["stop", "length"]).nullable(),
+        finish_reason: zod_1.z
+            .enum(["stop", "length", "content_filter"])
+            .optional()
+            .nullable(),
         index: zod_1.z.number(),
     })),
     created: zod_1.z.number(),
     id: zod_1.z.string(),
     model: zod_1.z.string(),
-    object: zod_1.z.string(),
+    system_fingerprint: zod_1.z.string().optional(),
+    object: zod_1.z.literal("text_completion"),
 });
 async function createOpenAITextFullDeltaIterableQueue(stream) {
     const queue = new AsyncQueue_js_1.AsyncQueue();

package/model-provider/openai/OpenAICompletionModel.d.ts

@@ -112,6 +112,7 @@ export interface OpenAICompletionCallSettings {
     frequencyPenalty?: number;
     bestOf?: number;
     logitBias?: Record<number, number>;
+    seed?: number | null;
 }
 export interface OpenAICompletionModelSettings extends TextGenerationModelSettings, Omit<OpenAICompletionCallSettings, "stop" | "maxTokens"> {
     isUserIdForwardingEnabled?: boolean;
@@ -157,11 +158,12 @@ export declare class OpenAICompletionModel extends AbstractModel<OpenAICompletio
             id: string;
             choices: {
                 text: string;
-                finish_reason: string;
                 index: number;
+                finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
                 logprobs?: any;
             }[];
             created: number;
+            system_fingerprint?: string | undefined;
         };
         text: string;
         usage: {
@@ -187,25 +189,26 @@ export declare class OpenAICompletionModel extends AbstractModel<OpenAICompletio
 }
 declare const OpenAICompletionResponseSchema: z.ZodObject<{
     id: z.ZodString;
-    object: z.ZodLiteral<"text_completion">;
-    created: z.ZodNumber;
-    model: z.ZodString;
     choices: z.ZodArray<z.ZodObject<{
-        text: z.ZodString;
+        finish_reason: z.ZodNullable<z.ZodOptional<z.ZodEnum<["stop", "length", "content_filter"]>>>;
         index: z.ZodNumber;
         logprobs: z.ZodNullable<z.ZodAny>;
-        finish_reason: z.ZodString;
+        text: z.ZodString;
     }, "strip", z.ZodTypeAny, {
         text: string;
-        finish_reason: string;
         index: number;
+        finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
         logprobs?: any;
     }, {
         text: string;
-        finish_reason: string;
         index: number;
+        finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
         logprobs?: any;
     }>, "many">;
+    created: z.ZodNumber;
+    model: z.ZodString;
+    system_fingerprint: z.ZodOptional<z.ZodString>;
+    object: z.ZodLiteral<"text_completion">;
     usage: z.ZodObject<{
         prompt_tokens: z.ZodNumber;
         completion_tokens: z.ZodNumber;
@@ -230,11 +233,12 @@ declare const OpenAICompletionResponseSchema: z.ZodObject<{
     id: string;
     choices: {
         text: string;
-        finish_reason: string;
         index: number;
+        finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
         logprobs?: any;
     }[];
     created: number;
+    system_fingerprint?: string | undefined;
 }, {
     object: "text_completion";
     usage: {
@@ -246,11 +250,12 @@ declare const OpenAICompletionResponseSchema: z.ZodObject<{
     id: string;
     choices: {
         text: string;
-        finish_reason: string;
         index: number;
+        finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
         logprobs?: any;
     }[];
     created: number;
+    system_fingerprint?: string | undefined;
 }>;
 export type OpenAICompletionResponse = z.infer<typeof OpenAICompletionResponseSchema>;
 export type OpenAITextResponseFormatType<T> = {
@@ -274,11 +279,12 @@ export declare const OpenAITextResponseFormat: {
             id: string;
             choices: {
                 text: string;
-                finish_reason: string;
                 index: number;
+                finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
                 logprobs?: any;
             }[];
             created: number;
+            system_fingerprint?: string | undefined;
         }>;
     };
     /**

package/model-provider/openai/OpenAICompletionModel.js

@@ -206,6 +206,7 @@ export class OpenAICompletionModel extends AbstractModel {
             "frequencyPenalty",
             "bestOf",
             "logitBias",
+            "seed",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -259,22 +260,26 @@ export class OpenAICompletionModel extends AbstractModel {
 }
 const OpenAICompletionResponseSchema = z.object({
     id: z.string(),
-    object: z.literal("text_completion"),
-    created: z.number(),
-    model: z.string(),
     choices: z.array(z.object({
-        text: z.string(),
+        finish_reason: z
+            .enum(["stop", "length", "content_filter"])
+            .optional()
+            .nullable(),
         index: z.number(),
         logprobs: z.nullable(z.any()),
-        finish_reason: z.string(),
+        text: z.string(),
     })),
+    created: z.number(),
+    model: z.string(),
+    system_fingerprint: z.string().optional(),
+    object: z.literal("text_completion"),
     usage: z.object({
         prompt_tokens: z.number(),
         completion_tokens: z.number(),
         total_tokens: z.number(),
     }),
 });
-async function callOpenAICompletionAPI({ api = new OpenAIApiConfiguration(), abortSignal, responseFormat, model, prompt, suffix, maxTokens, temperature, topP, n, logprobs, echo, stop, presencePenalty, frequencyPenalty, bestOf, logitBias, user, }) {
+async function callOpenAICompletionAPI({ api = new OpenAIApiConfiguration(), abortSignal, responseFormat, model, prompt, suffix, maxTokens, temperature, topP, n, logprobs, echo, stop, presencePenalty, frequencyPenalty, bestOf, logitBias, seed, user, }) {
     // empty arrays are not allowed for stop:
     if (stop != null && Array.isArray(stop) && stop.length === 0) {
         stop = undefined;
@@ -294,6 +299,7 @@ async function callOpenAICompletionAPI({ api = new OpenAIApiConfiguration(), abo
             logprobs,
             echo,
             stop,
+            seed,
             presence_penalty: presencePenalty,
             frequency_penalty: frequencyPenalty,
             best_of: bestOf,
@@ -325,13 +331,17 @@ export const OpenAITextResponseFormat = {
 const textResponseStreamEventSchema = z.object({
     choices: z.array(z.object({
         text: z.string(),
-        finish_reason: z.enum(["stop", "length"]).nullable(),
+        finish_reason: z
+            .enum(["stop", "length", "content_filter"])
+            .optional()
+            .nullable(),
         index: z.number(),
     })),
     created: z.number(),
     id: z.string(),
     model: z.string(),
-    object: z.string(),
+    system_fingerprint: z.string().optional(),
+    object: z.literal("text_completion"),
 });
 async function createOpenAITextFullDeltaIterableQueue(stream) {
     const queue = new AsyncQueue();

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -39,6 +39,8 @@ exports.OPENAI_CHAT_MODELS = {
         contextWindowSize: 8192,
         promptTokenCostInMillicents: 3,
         completionTokenCostInMillicents: 6,
+        fineTunedPromptTokenCostInMillicents: null,
+        fineTunedCompletionTokenCostInMillicents: null,
     },
     "gpt-4-1106-preview": {
         contextWindowSize: 128000,
@@ -115,7 +117,7 @@ function getOpenAIChatModelInformation(model) {
     // Extract the base model from the fine-tuned model:
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const [_, baseModel, ___, ____, _____] = model.split(":");
-    if (["gpt-3.5-turbo", "gpt-3.5-turbo-0613"].includes(baseModel)) {
+    if (["gpt-3.5-turbo", "gpt-3.5-turbo-0613", "gpt-4-0613"].includes(baseModel)) {
         const baseModelInformation = exports.OPENAI_CHAT_MODELS[baseModel];
         return {
             baseModel: baseModel,
@@ -133,11 +135,14 @@ const isOpenAIChatModel = (model) => model in exports.OPENAI_CHAT_MODELS ||
     model.startsWith("ft:gpt-3.5-turbo:");
 exports.isOpenAIChatModel = isOpenAIChatModel;
 const calculateOpenAIChatCostInMillicents = ({ model, response, }) => {
-    const modelInformation = getOpenAIChatModelInformation(model);
-    return (response.usage.prompt_tokens *
-        modelInformation.promptTokenCostInMillicents +
-        response.usage.completion_tokens *
-            modelInformation.completionTokenCostInMillicents);
+    const { promptTokenCostInMillicents, completionTokenCostInMillicents } = getOpenAIChatModelInformation(model);
+    // null: when cost is unknown, e.g. for fine-tuned models where the price is not yet known
+    if (promptTokenCostInMillicents == null ||
+        completionTokenCostInMillicents == null) {
+        return null;
+    }
+    return (response.usage.prompt_tokens * promptTokenCostInMillicents +
+        response.usage.completion_tokens * completionTokenCostInMillicents);
 };
 exports.calculateOpenAIChatCostInMillicents = calculateOpenAIChatCostInMillicents;
 /**
@@ -234,6 +239,8 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
             "presencePenalty",
             "frequencyPenalty",
             "logitBias",
+            "seed",
+            "responseFormat",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -401,13 +408,16 @@ const openAIChatResponseSchema = zod_1.z.object({
         }),
         index: zod_1.z.number(),
         logprobs: zod_1.z.nullable(zod_1.z.any()),
-        finish_reason: zod_1.z.enum([
+        finish_reason: zod_1.z
+            .enum([
             "stop",
             "length",
             "tool_calls",
             "content_filter",
             "function_call",
-        ]),
+        ])
+            .optional()
+            .nullable(),
     })),
     created: zod_1.z.number(),
     model: zod_1.z.string(),

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -27,6 +27,8 @@ export declare const OPENAI_CHAT_MODELS: {
         contextWindowSize: number;
         promptTokenCostInMillicents: number;
         completionTokenCostInMillicents: number;
+        fineTunedPromptTokenCostInMillicents: null;
+        fineTunedCompletionTokenCostInMillicents: null;
     };
     "gpt-4-1106-preview": {
         contextWindowSize: number;
@@ -92,10 +94,10 @@ export declare function getOpenAIChatModelInformation(model: OpenAIChatModelType
     baseModel: OpenAIChatBaseModelType;
     isFineTuned: boolean;
     contextWindowSize: number;
-    promptTokenCostInMillicents: number;
-    completionTokenCostInMillicents: number;
+    promptTokenCostInMillicents: number | null;
+    completionTokenCostInMillicents: number | null;
 };
-type FineTuneableOpenAIChatModelType = `gpt-3.5-turbo` | `gpt-3.5-turbo-0613`;
+type FineTuneableOpenAIChatModelType = `gpt-3.5-turbo` | `gpt-3.5-turbo-0613` | `gpt-4-0613`;
 type FineTunedOpenAIChatModelType = `ft:${FineTuneableOpenAIChatModelType}:${string}:${string}:${string}`;
 export type OpenAIChatBaseModelType = keyof typeof OPENAI_CHAT_MODELS;
 export type OpenAIChatModelType = OpenAIChatBaseModelType | FineTunedOpenAIChatModelType;
@@ -103,7 +105,7 @@ export declare const isOpenAIChatModel: (model: string) => model is OpenAIChatMo
 export declare const calculateOpenAIChatCostInMillicents: ({ model, response, }: {
     model: OpenAIChatModelType;
     response: OpenAIChatResponse;
-}) => number;
+}) => number | null;
 export interface OpenAIChatCallSettings {
     api?: ApiConfiguration;
     model: OpenAIChatModelType;
@@ -193,9 +195,9 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
+                finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
             }[];
             created: number;
             system_fingerprint?: string | undefined;
@@ -234,9 +236,9 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
+                finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
             }[];
             created: number;
             system_fingerprint?: string | undefined;
@@ -269,9 +271,9 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
+                finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
             }[];
             created: number;
             system_fingerprint?: string | undefined;
@@ -306,9 +308,9 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
+                finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
             }[];
             created: number;
             system_fingerprint?: string | undefined;
@@ -374,7 +376,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         }>;
         index: z.ZodNumber;
         logprobs: z.ZodNullable<z.ZodAny>;
-        finish_reason: z.ZodEnum<["stop", "length", "tool_calls", "content_filter", "function_call"]>;
+        finish_reason: z.ZodNullable<z.ZodOptional<z.ZodEnum<["stop", "length", "tool_calls", "content_filter", "function_call"]>>>;
     }, "strip", z.ZodTypeAny, {
         message: {
             content: string | null;
@@ -384,9 +386,9 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
+        finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
     }, {
         message: {
             content: string | null;
@@ -396,9 +398,9 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
+        finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
     }>, "many">;
     created: z.ZodNumber;
     model: z.ZodString;
@@ -435,9 +437,9 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
+        finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
     }[];
     created: number;
     system_fingerprint?: string | undefined;
@@ -459,9 +461,9 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
+        finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
     }[];
     created: number;
     system_fingerprint?: string | undefined;
@@ -495,9 +497,9 @@ export declare const OpenAIChatResponseFormat: {
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
+                finish_reason?: "length" | "stop" | "function_call" | "tool_calls" | "content_filter" | null | undefined;
             }[];
             created: number;
             system_fingerprint?: string | undefined;

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -33,6 +33,8 @@ export const OPENAI_CHAT_MODELS = {
         contextWindowSize: 8192,
         promptTokenCostInMillicents: 3,
         completionTokenCostInMillicents: 6,
+        fineTunedPromptTokenCostInMillicents: null,
+        fineTunedCompletionTokenCostInMillicents: null,
     },
     "gpt-4-1106-preview": {
         contextWindowSize: 128000,
@@ -109,7 +111,7 @@ export function getOpenAIChatModelInformation(model) {
     // Extract the base model from the fine-tuned model:
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const [_, baseModel, ___, ____, _____] = model.split(":");
-    if (["gpt-3.5-turbo", "gpt-3.5-turbo-0613"].includes(baseModel)) {
+    if (["gpt-3.5-turbo", "gpt-3.5-turbo-0613", "gpt-4-0613"].includes(baseModel)) {
         const baseModelInformation = OPENAI_CHAT_MODELS[baseModel];
         return {
             baseModel: baseModel,
@@ -125,11 +127,14 @@ export const isOpenAIChatModel = (model) => model in OPENAI_CHAT_MODELS ||
     model.startsWith("ft:gpt-3.5-turbo-0613:") ||
     model.startsWith("ft:gpt-3.5-turbo:");
 export const calculateOpenAIChatCostInMillicents = ({ model, response, }) => {
-    const modelInformation = getOpenAIChatModelInformation(model);
-    return (response.usage.prompt_tokens *
-        modelInformation.promptTokenCostInMillicents +
-        response.usage.completion_tokens *
-            modelInformation.completionTokenCostInMillicents);
+    const { promptTokenCostInMillicents, completionTokenCostInMillicents } = getOpenAIChatModelInformation(model);
+    // null: when cost is unknown, e.g. for fine-tuned models where the price is not yet known
+    if (promptTokenCostInMillicents == null ||
+        completionTokenCostInMillicents == null) {
+        return null;
+    }
+    return (response.usage.prompt_tokens * promptTokenCostInMillicents +
+        response.usage.completion_tokens * completionTokenCostInMillicents);
 };
 /**
  * Create a text generation model that calls the OpenAI chat completion API.
@@ -225,6 +230,8 @@ export class OpenAIChatModel extends AbstractModel {
             "presencePenalty",
             "frequencyPenalty",
             "logitBias",
+            "seed",
+            "responseFormat",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }
@@ -391,13 +398,16 @@ const openAIChatResponseSchema = z.object({
         }),
         index: z.number(),
         logprobs: z.nullable(z.any()),
-        finish_reason: z.enum([
+        finish_reason: z
+            .enum([
             "stop",
             "length",
             "tool_calls",
             "content_filter",
             "function_call",
-        ]),
+        ])
+            .optional()
+            .nullable(),
     })),
     created: z.number(),
     model: z.string(),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.65.1",
+  "version": "0.67.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [

package/util/streaming/parseJsonStream.cjs

@@ -8,9 +8,9 @@ function parseJsonStream({ schema, stream, process, onDone, }) {
     }
     return (async () => {
         try {
-            let unprocessedText = "";
             const reader = new ReadableStreamDefaultReader(stream);
             const utf8Decoder = new TextDecoder("utf-8");
+            let unprocessedText = "";
             // eslint-disable-next-line no-constant-condition
             while (true) {
                 const { value: chunk, done } = await reader.read();
@@ -18,8 +18,8 @@ function parseJsonStream({ schema, stream, process, onDone, }) {
                     break;
                 }
                 unprocessedText += utf8Decoder.decode(chunk, { stream: true });
-                const processableLines = unprocessedText.split(/\r\n|\n|\r/g);
-                unprocessedText = processableLines.pop() || "";
+                const processableLines = unprocessedText.split("\n");
+                unprocessedText = processableLines.pop() ?? "";
                 processableLines.forEach(processLine);
             }
             // processing remaining text:

package/util/streaming/parseJsonStream.js

@@ -5,9 +5,9 @@ export function parseJsonStream({ schema, stream, process, onDone, }) {
     }
     return (async () => {
         try {
-            let unprocessedText = "";
             const reader = new ReadableStreamDefaultReader(stream);
             const utf8Decoder = new TextDecoder("utf-8");
+            let unprocessedText = "";
             // eslint-disable-next-line no-constant-condition
             while (true) {
                 const { value: chunk, done } = await reader.read();
@@ -15,8 +15,8 @@ export function parseJsonStream({ schema, stream, process, onDone, }) {
                     break;
                 }
                 unprocessedText += utf8Decoder.decode(chunk, { stream: true });
-                const processableLines = unprocessedText.split(/\r\n|\n|\r/g);
-                unprocessedText = processableLines.pop() || "";
+                const processableLines = unprocessedText.split("\n");
+                unprocessedText = processableLines.pop() ?? "";
                 processableLines.forEach(processLine);
             }
             // processing remaining text: