modelfusion 0.22.0 → 0.23.0

package/README.md

@@ -376,6 +376,7 @@ const { chunks } = await retrieveTextChunks(
 - [Text Chunks](https://modelfusion.dev/guide/text-chunk/)
   - [Split Text](https://modelfusion.dev/guide/text-chunk/split)
 - [Utilities](https://modelfusion.dev/guide/util/)
+  - [Function Logging](https://modelfusion.dev/guide/util/logging)
   - [Function observers](https://modelfusion.dev/guide/util/observer)
   - [Retry strategies](https://modelfusion.dev/guide/util/retry)
   - [Throttling strategies](https://modelfusion.dev/guide/util/throttle)

package/core/FunctionOptions.d.ts

@@ -8,6 +8,11 @@ export type FunctionOptions = {
      * Optional function identifier that is used in events to identify the function.
      */
     functionId?: string;
+    /**
+     * Optional logging to use for the function. Logs are sent to the console.
+     * Overrides the global function logging setting.
+     */
+    logging?: FunctionLogging;
     /**
      * Optional observers that are called when the function is invoked.
      */
@@ -17,3 +22,12 @@ export type FunctionOptions = {
      */
     run?: Run;
 };
+/**
+ * The logging to use for the function. Logs are sent to the console.
+ *
+ * - `off` or undefined: No logging.
+ * - `basic-text`: Log the timestamp and the type of event as a single line of text.
+ * - `detailed-object`: Log everything except the original response as an object to the console.
+ * - `detailed-json`: Log everything except the original response as a JSON string to the console.
+ */
+export type FunctionLogging = undefined | "off" | "basic-text" | "detailed-object" | "detailed-json";

package/core/GlobalFunctionLogging.cjs

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getGlobalFunctionLogging = exports.setGlobalFunctionLogging = void 0;
+let globalFunctionLogging = undefined;
+function setGlobalFunctionLogging(logLevel) {
+    globalFunctionLogging = logLevel;
+}
+exports.setGlobalFunctionLogging = setGlobalFunctionLogging;
+function getGlobalFunctionLogging() {
+    return globalFunctionLogging;
+}
+exports.getGlobalFunctionLogging = getGlobalFunctionLogging;

package/core/GlobalFunctionLogging.d.ts

@@ -0,0 +1,3 @@
+import { FunctionLogging } from "./FunctionOptions.js";
+export declare function setGlobalFunctionLogging(logLevel: FunctionLogging): void;
+export declare function getGlobalFunctionLogging(): FunctionLogging;

package/core/GlobalFunctionLogging.js

@@ -0,0 +1,7 @@
+let globalFunctionLogging = undefined;
+export function setGlobalFunctionLogging(logLevel) {
+    globalFunctionLogging = logLevel;
+}
+export function getGlobalFunctionLogging() {
+    return globalFunctionLogging;
+}

package/core/getFunctionCallLogger.cjs

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFunctionCallLogger = void 0;
+function getFunctionCallLogger(logging) {
+    switch (logging) {
+        case "basic-text":
+            return [basicTextObserver];
+        case "detailed-object":
+            return [detailedObjectObserver];
+        case "detailed-json":
+            return [detailedJsonObserver];
+        case "off":
+        default:
+            return [];
+    }
+}
+exports.getFunctionCallLogger = getFunctionCallLogger;
+const basicTextObserver = {
+    onFunctionEvent(event) {
+        // log based on event type:
+        switch (event.eventType) {
+            case "started": {
+                console.log(`[${event.timestamp.toISOString()}] ${event.callId} - ${event.functionType} ${event.eventType}`);
+                break;
+            }
+            case "finished": {
+                console.log(`[${event.timestamp.toISOString()}] ${event.callId} - ${event.functionType} ${event.eventType} in ${event.durationInMs}ms`);
+                break;
+            }
+        }
+    },
+};
+const detailedObjectObserver = {
+    onFunctionEvent(event) {
+        // Remove the "response" property from the result (if any):
+        if (event.eventType === "finished" &&
+            event.result != null &&
+            "response" in event.result &&
+            event.result?.response != null) {
+            event = {
+                ...event,
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                ),
+            };
+        }
+        // filter all undefined properties from event for cleaner console output:
+        event = Object.fromEntries(
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        Object.entries(event).filter(([_, v]) => v !== undefined));
+        console.log(event);
+    },
+};
+const detailedJsonObserver = {
+    onFunctionEvent(event) {
+        // Remove the "response" property from the result (if any):
+        if (event.eventType === "finished" &&
+            event.result != null &&
+            "response" in event.result &&
+            event.result?.response != null) {
+            event = {
+                ...event,
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                ),
+            };
+        }
+        // filter all undefined properties from event for cleaner console output:
+        event = Object.fromEntries(
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        Object.entries(event).filter(([_, v]) => v !== undefined));
+        console.log(JSON.stringify(event));
+    },
+};

package/core/getFunctionCallLogger.d.ts

@@ -0,0 +1,3 @@
+import { FunctionOptions } from "./FunctionOptions.js";
+import { FunctionObserver } from "./FunctionObserver.js";
+export declare function getFunctionCallLogger(logging: FunctionOptions["logging"]): Array<FunctionObserver>;

package/core/getFunctionCallLogger.js

@@ -0,0 +1,70 @@
+export function getFunctionCallLogger(logging) {
+    switch (logging) {
+        case "basic-text":
+            return [basicTextObserver];
+        case "detailed-object":
+            return [detailedObjectObserver];
+        case "detailed-json":
+            return [detailedJsonObserver];
+        case "off":
+        default:
+            return [];
+    }
+}
+const basicTextObserver = {
+    onFunctionEvent(event) {
+        // log based on event type:
+        switch (event.eventType) {
+            case "started": {
+                console.log(`[${event.timestamp.toISOString()}] ${event.callId} - ${event.functionType} ${event.eventType}`);
+                break;
+            }
+            case "finished": {
+                console.log(`[${event.timestamp.toISOString()}] ${event.callId} - ${event.functionType} ${event.eventType} in ${event.durationInMs}ms`);
+                break;
+            }
+        }
+    },
+};
+const detailedObjectObserver = {
+    onFunctionEvent(event) {
+        // Remove the "response" property from the result (if any):
+        if (event.eventType === "finished" &&
+            event.result != null &&
+            "response" in event.result &&
+            event.result?.response != null) {
+            event = {
+                ...event,
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                ),
+            };
+        }
+        // filter all undefined properties from event for cleaner console output:
+        event = Object.fromEntries(
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        Object.entries(event).filter(([_, v]) => v !== undefined));
+        console.log(event);
+    },
+};
+const detailedJsonObserver = {
+    onFunctionEvent(event) {
+        // Remove the "response" property from the result (if any):
+        if (event.eventType === "finished" &&
+            event.result != null &&
+            "response" in event.result &&
+            event.result?.response != null) {
+            event = {
+                ...event,
+                result: Object.fromEntries(Object.entries(event.result).filter(([k]) => k !== "response")
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                ),
+            };
+        }
+        // filter all undefined properties from event for cleaner console output:
+        event = Object.fromEntries(
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        Object.entries(event).filter(([_, v]) => v !== undefined));
+        console.log(JSON.stringify(event));
+    },
+};

package/core/index.cjs

@@ -14,12 +14,12 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./ConsoleLogger.cjs"), exports);
 __exportStar(require("./DefaultRun.cjs"), exports);
 __exportStar(require("./FunctionEvent.cjs"), exports);
 __exportStar(require("./FunctionEventSource.cjs"), exports);
 __exportStar(require("./FunctionObserver.cjs"), exports);
 __exportStar(require("./FunctionOptions.cjs"), exports);
+__exportStar(require("./GlobalFunctionLogging.cjs"), exports);
 __exportStar(require("./GlobalFunctionObservers.cjs"), exports);
 __exportStar(require("./Run.cjs"), exports);
 __exportStar(require("./Vector.cjs"), exports);

package/core/index.d.ts

@@ -1,9 +1,9 @@
-export * from "./ConsoleLogger.js";
 export * from "./DefaultRun.js";
 export * from "./FunctionEvent.js";
 export * from "./FunctionEventSource.js";
 export * from "./FunctionObserver.js";
 export * from "./FunctionOptions.js";
+export * from "./GlobalFunctionLogging.js";
 export * from "./GlobalFunctionObservers.js";
 export * from "./Run.js";
 export * from "./Vector.js";

package/core/index.js

@@ -1,9 +1,9 @@
-export * from "./ConsoleLogger.js";
 export * from "./DefaultRun.js";
 export * from "./FunctionEvent.js";
 export * from "./FunctionEventSource.js";
 export * from "./FunctionObserver.js";
 export * from "./FunctionOptions.js";
+export * from "./GlobalFunctionLogging.js";
 export * from "./GlobalFunctionObservers.js";
 export * from "./Run.js";
 export * from "./Vector.js";

package/model-function/AbstractModel.d.ts

@@ -7,6 +7,7 @@ export declare abstract class AbstractModel<SETTINGS extends ModelSettings> impl
     abstract readonly provider: string;
     abstract readonly modelName: string | null;
     get modelInformation(): ModelInformation;
+    abstract get settingsForEvent(): Partial<SETTINGS>;
     readonly settings: SETTINGS;
     abstract withSettings(additionalSettings: Partial<SETTINGS>): this;
 }

package/model-function/Model.d.ts

@@ -6,9 +6,14 @@ export interface ModelSettings {
      */
     observers?: Array<FunctionObserver>;
 }
-export interface Model<SETTINGS> {
+export interface Model<SETTINGS extends ModelSettings> {
     modelInformation: ModelInformation;
     readonly settings: SETTINGS;
+    /**
+     * Returns settings that should be recorded in observability events.
+     * Security-related settings (e.g. API keys) should not be included here.
+     */
+    get settingsForEvent(): Partial<SETTINGS>;
     /**
      * The `withSettings` method creates a new model with the same configuration as the original model, but with the specified settings changed.
      *

package/model-function/ModelCallEvent.d.ts

@@ -9,13 +9,23 @@ import { SpeechSynthesisFinishedEvent, SpeechSynthesisStartedEvent } from "./syn
 import { TranscriptionFinishedEvent, TranscriptionStartedEvent } from "./transcribe-speech/TranscriptionEvent.js";
 export interface BaseModelCallStartedEvent extends BaseFunctionStartedEvent {
     model: ModelInformation;
-    settings: unknown;
+    /**
+     * The main input to the model call. The type depends on the call type or model.
+     */
     input: unknown;
+    /**
+     * The model settings used for the call. The type depends on the model.
+     */
+    settings: unknown;
 }
 export type BaseModelCallFinishedEventResult = {
     status: "success";
     response: unknown;
     output: unknown;
+    /**
+     * Optional usage information for the model call. The type depends on the call type.
+     */
+    usage?: unknown;
 } | {
     status: "error";
     error: unknown;
@@ -24,8 +34,17 @@ export type BaseModelCallFinishedEventResult = {
 };
 export interface BaseModelCallFinishedEvent extends BaseFunctionFinishedEvent {
     model: ModelInformation;
-    settings: unknown;
+    /**
+     * The main input to the model call. The type depends on the call type or model.
+     */
     input: unknown;
+    /**
+     * The model settings used for the call. The type depends on the model.
+     */
+    settings: unknown;
+    /**
+     * The result of the model call. Can be "success", "error", or "abort". Additional information is provided depending on the status.
+     */
     result: BaseModelCallFinishedEventResult;
 }
 export type ModelCallStartedEvent = ImageGenerationStartedEvent | JsonGenerationStartedEvent | SpeechSynthesisStartedEvent | TextEmbeddingStartedEvent | TextGenerationStartedEvent | TextStreamingStartedEvent | TranscriptionStartedEvent;

package/model-function/embed-text/embedText.d.ts

@@ -14,7 +14,7 @@ import { TextEmbeddingModel, TextEmbeddingModelSettings } from "./TextEmbeddingM
  *   ]
  * );
  */
-export declare function embedTexts<RESPONSE, SETTINGS extends TextEmbeddingModelSettings>(model: TextEmbeddingModel<RESPONSE, SETTINGS>, texts: string[], options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<TextEmbeddingModel<RESPONSE, SETTINGS>, Vector[], RESPONSE[]>;
+export declare function embedTexts<RESPONSE, SETTINGS extends TextEmbeddingModelSettings>(model: TextEmbeddingModel<RESPONSE, SETTINGS>, texts: string[], options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<Vector[], RESPONSE[]>;
 /**
  * Generate an embedding for a single text.
  *
@@ -24,4 +24,4 @@ export declare function embedTexts<RESPONSE, SETTINGS extends TextEmbeddingModel
  *   "At first, Nox didn't know what to do with the pup."
  * );
  */
-export declare function embedText<RESPONSE, SETTINGS extends TextEmbeddingModelSettings>(model: TextEmbeddingModel<RESPONSE, SETTINGS>, text: string, options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<TextEmbeddingModel<RESPONSE, SETTINGS>, Vector, RESPONSE>;
+export declare function embedText<RESPONSE, SETTINGS extends TextEmbeddingModelSettings>(model: TextEmbeddingModel<RESPONSE, SETTINGS>, text: string, options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<Vector, RESPONSE>;

package/model-function/executeCall.cjs

@@ -1,23 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ModelFunctionPromise = exports.executeCall = void 0;
+exports.executeCall = exports.ModelFunctionPromise = void 0;
 const nanoid_1 = require("nanoid");
 const FunctionEventSource_js_1 = require("../core/FunctionEventSource.cjs");
+const GlobalFunctionLogging_js_1 = require("../core/GlobalFunctionLogging.cjs");
 const GlobalFunctionObservers_js_1 = require("../core/GlobalFunctionObservers.cjs");
+const getFunctionCallLogger_js_1 = require("../core/getFunctionCallLogger.cjs");
 const DurationMeasurement_js_1 = require("../util/DurationMeasurement.cjs");
 const AbortError_js_1 = require("../util/api/AbortError.cjs");
 const runSafe_js_1 = require("../util/runSafe.cjs");
-function executeCall({ model, options, input, functionType, generateResponse, extractOutputValue, }) {
-    return new ModelFunctionPromise(doExecuteCall({
-        model,
-        options,
-        input,
-        functionType,
-        generateResponse,
-        extractOutputValue,
-    }));
-}
-exports.executeCall = executeCall;
 class ModelFunctionPromise extends Promise {
     constructor(fullPromise) {
         super((resolve) => {
@@ -52,11 +43,24 @@ class ModelFunctionPromise extends Promise {
     }
 }
 exports.ModelFunctionPromise = ModelFunctionPromise;
-async function doExecuteCall({ model, options, input, functionType, generateResponse, extractOutputValue, }) {
+function executeCall({ model, options, input, functionType, generateResponse, extractOutputValue, extractUsage, }) {
+    return new ModelFunctionPromise(doExecuteCall({
+        model,
+        options,
+        input,
+        functionType,
+        generateResponse,
+        extractOutputValue,
+        extractUsage,
+    }));
+}
+exports.executeCall = executeCall;
+async function doExecuteCall({ model, options, input, functionType, generateResponse, extractOutputValue, extractUsage, }) {
     if (options?.settings != null) {
         model = model.withSettings(options.settings);
         options = {
             functionId: options.functionId,
+            logging: options.logging,
             observers: options.observers,
             run: options.run,
         };
@@ -65,6 +69,7 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
     const settings = model.settings;
     const eventSource = new FunctionEventSource_js_1.FunctionEventSource({
         observers: [
+            ...(0, getFunctionCallLogger_js_1.getFunctionCallLogger)(options?.logging ?? (0, GlobalFunctionLogging_js_1.getGlobalFunctionLogging)()),
             ...(0, GlobalFunctionObservers_js_1.getGlobalFunctionObservers)(),
             ...(settings.observers ?? []),
             ...(run?.observers ?? []),
@@ -74,21 +79,21 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
     });
     const durationMeasurement = (0, DurationMeasurement_js_1.startDurationMeasurement)();
     const startMetadata = {
+        functionType,
         callId: `call-${(0, nanoid_1.nanoid)()}`,
         runId: run?.runId,
         sessionId: run?.sessionId,
         userId: run?.userId,
         functionId: options?.functionId,
         model: model.modelInformation,
-        functionType,
+        settings: model.settingsForEvent,
         input,
-        settings,
         timestamp: durationMeasurement.startDate,
         startTimestamp: durationMeasurement.startDate,
     };
     eventSource.notify({
-        ...startMetadata,
         eventType: "started",
+        ...startMetadata,
     });
     const result = await (0, runSafe_js_1.runSafe)(() => generateResponse({
         functionId: options?.functionId,
@@ -96,8 +101,8 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
         run,
     }));
     const finishMetadata = {
-        ...startMetadata,
         eventType: "finished",
+        ...startMetadata,
         finishTimestamp: new Date(),
         durationInMs: durationMeasurement.durationInMs,
     };
@@ -124,11 +129,13 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
     }
     const response = result.output;
     const output = extractOutputValue(response);
+    const usage = extractUsage?.(response);
     eventSource.notify({
         ...finishMetadata,
         eventType: "finished",
         result: {
             status: "success",
+            usage,
             response,
             output,
         },

package/model-function/executeCall.d.ts

@@ -1,39 +1,41 @@
 import { Model, ModelSettings } from "./Model.js";
 import { ModelCallStartedEvent } from "./ModelCallEvent.js";
 import { ModelFunctionOptions } from "./ModelFunctionOptions.js";
-export type CallMetadata<MODEL extends Model<unknown>> = {
+import { ModelInformation } from "./ModelInformation.js";
+export type ModelCallMetadata = {
     callId: string;
     runId?: string;
     sessionId?: string;
     userId?: string;
     functionId?: string;
-    model: MODEL["modelInformation"];
     startTimestamp: Date;
     finishTimestamp: Date;
     durationInMs: number;
+    model: ModelInformation;
 };
-export declare function executeCall<SETTINGS extends ModelSettings, MODEL extends Model<SETTINGS>, OUTPUT, RESPONSE>({ model, options, input, functionType, generateResponse, extractOutputValue, }: {
-    model: MODEL;
-    options?: ModelFunctionOptions<SETTINGS>;
-    input: unknown;
-    functionType: ModelCallStartedEvent["functionType"];
-    generateResponse: (options: ModelFunctionOptions<SETTINGS>) => PromiseLike<RESPONSE>;
-    extractOutputValue: (response: RESPONSE) => OUTPUT;
-}): ModelFunctionPromise<MODEL, OUTPUT, RESPONSE>;
-export declare class ModelFunctionPromise<MODEL extends Model<any>, OUTPUT, RESPONSE> extends Promise<OUTPUT> {
+export declare class ModelFunctionPromise<OUTPUT, RESPONSE> extends Promise<OUTPUT> {
     private fullPromise;
     private outputPromise;
     constructor(fullPromise: Promise<{
         output: OUTPUT;
         response: RESPONSE;
-        metadata: CallMetadata<MODEL>;
+        metadata: ModelCallMetadata;
     }>);
     asFullResponse(): Promise<{
         output: OUTPUT;
         response: RESPONSE;
-        metadata: CallMetadata<MODEL>;
+        metadata: ModelCallMetadata;
     }>;
     then<TResult1 = OUTPUT, TResult2 = never>(onfulfilled?: ((value: OUTPUT) => TResult1 | PromiseLike<TResult1>) | undefined | null, onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | undefined | null): Promise<TResult1 | TResult2>;
     catch<TResult = never>(onrejected?: ((reason: unknown) => TResult | PromiseLike<TResult>) | undefined | null): Promise<OUTPUT | TResult>;
     finally(onfinally?: (() => void) | undefined | null): Promise<OUTPUT>;
 }
+export declare function executeCall<SETTINGS extends ModelSettings, MODEL extends Model<SETTINGS>, OUTPUT, RESPONSE>({ model, options, input, functionType, generateResponse, extractOutputValue, extractUsage, }: {
+    model: MODEL;
+    options?: ModelFunctionOptions<SETTINGS>;
+    input: unknown;
+    functionType: ModelCallStartedEvent["functionType"];
+    generateResponse: (options: ModelFunctionOptions<SETTINGS>) => PromiseLike<RESPONSE>;
+    extractOutputValue: (response: RESPONSE) => OUTPUT;
+    extractUsage?: (response: RESPONSE) => unknown;
+}): ModelFunctionPromise<OUTPUT, RESPONSE>;

package/model-function/executeCall.js

@@ -1,19 +1,11 @@
 import { nanoid as createId } from "nanoid";
 import { FunctionEventSource } from "../core/FunctionEventSource.js";
+import { getGlobalFunctionLogging } from "../core/GlobalFunctionLogging.js";
 import { getGlobalFunctionObservers } from "../core/GlobalFunctionObservers.js";
+import { getFunctionCallLogger } from "../core/getFunctionCallLogger.js";
 import { startDurationMeasurement } from "../util/DurationMeasurement.js";
 import { AbortError } from "../util/api/AbortError.js";
 import { runSafe } from "../util/runSafe.js";
-export function executeCall({ model, options, input, functionType, generateResponse, extractOutputValue, }) {
-    return new ModelFunctionPromise(doExecuteCall({
-        model,
-        options,
-        input,
-        functionType,
-        generateResponse,
-        extractOutputValue,
-    }));
-}
 export class ModelFunctionPromise extends Promise {
     constructor(fullPromise) {
         super((resolve) => {
@@ -47,11 +39,23 @@ export class ModelFunctionPromise extends Promise {
         return this.outputPromise.finally(onfinally);
     }
 }
-async function doExecuteCall({ model, options, input, functionType, generateResponse, extractOutputValue, }) {
+export function executeCall({ model, options, input, functionType, generateResponse, extractOutputValue, extractUsage, }) {
+    return new ModelFunctionPromise(doExecuteCall({
+        model,
+        options,
+        input,
+        functionType,
+        generateResponse,
+        extractOutputValue,
+        extractUsage,
+    }));
+}
+async function doExecuteCall({ model, options, input, functionType, generateResponse, extractOutputValue, extractUsage, }) {
     if (options?.settings != null) {
         model = model.withSettings(options.settings);
         options = {
             functionId: options.functionId,
+            logging: options.logging,
             observers: options.observers,
             run: options.run,
         };
@@ -60,6 +64,7 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
     const settings = model.settings;
     const eventSource = new FunctionEventSource({
         observers: [
+            ...getFunctionCallLogger(options?.logging ?? getGlobalFunctionLogging()),
             ...getGlobalFunctionObservers(),
             ...(settings.observers ?? []),
             ...(run?.observers ?? []),
@@ -69,21 +74,21 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
     });
     const durationMeasurement = startDurationMeasurement();
     const startMetadata = {
+        functionType,
         callId: `call-${createId()}`,
         runId: run?.runId,
         sessionId: run?.sessionId,
         userId: run?.userId,
         functionId: options?.functionId,
         model: model.modelInformation,
-        functionType,
+        settings: model.settingsForEvent,
         input,
-        settings,
         timestamp: durationMeasurement.startDate,
         startTimestamp: durationMeasurement.startDate,
     };
     eventSource.notify({
-        ...startMetadata,
         eventType: "started",
+        ...startMetadata,
     });
     const result = await runSafe(() => generateResponse({
         functionId: options?.functionId,
@@ -91,8 +96,8 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
         run,
     }));
     const finishMetadata = {
-        ...startMetadata,
         eventType: "finished",
+        ...startMetadata,
         finishTimestamp: new Date(),
         durationInMs: durationMeasurement.durationInMs,
     };
@@ -119,11 +124,13 @@ async function doExecuteCall({ model, options, input, functionType, generateResp
     }
     const response = result.output;
     const output = extractOutputValue(response);
+    const usage = extractUsage?.(response);
     eventSource.notify({
         ...finishMetadata,
         eventType: "finished",
         result: {
             status: "success",
+            usage,
             response,
             output,
         },

package/model-function/generate-image/generateImage.d.ts

@@ -16,4 +16,4 @@ import { ImageGenerationModel, ImageGenerationModelSettings } from "./ImageGener
  *   ]
  * );
  */
-export declare function generateImage<PROMPT, RESPONSE, SETTINGS extends ImageGenerationModelSettings>(model: ImageGenerationModel<PROMPT, RESPONSE, SETTINGS>, prompt: PROMPT, options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<ImageGenerationModel<PROMPT, RESPONSE, SETTINGS>, string, RESPONSE>;
+export declare function generateImage<PROMPT, RESPONSE, SETTINGS extends ImageGenerationModelSettings>(model: ImageGenerationModel<PROMPT, RESPONSE, SETTINGS>, prompt: PROMPT, options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<string, RESPONSE>;

package/model-function/generate-json/JsonGenerationEvent.d.ts

@@ -2,6 +2,22 @@ import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelC
 export interface JsonGenerationStartedEvent extends BaseModelCallStartedEvent {
     functionType: "json-generation" | "json-or-text-generation";
 }
+export type JsonGenerationFinishedEventResult = {
+    status: "success";
+    response: unknown;
+    output: unknown;
+    usage?: {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+    };
+} | {
+    status: "error";
+    error: unknown;
+} | {
+    status: "abort";
+};
 export type JsonGenerationFinishedEvent = BaseModelCallFinishedEvent & {
     functionType: "json-generation" | "json-or-text-generation";
+    result: JsonGenerationFinishedEventResult;
 };

package/model-function/generate-json/JsonGenerationModel.d.ts

@@ -0,0 +1,13 @@
+import { ModelFunctionOptions } from "../ModelFunctionOptions.js";
+import { Model, ModelSettings } from "../Model.js";
+export interface JsonGenerationModelSettings extends ModelSettings {
+}
+export interface JsonGenerationModel<PROMPT, RESPONSE, SETTINGS extends JsonGenerationModelSettings> extends Model<SETTINGS> {
+    generateJsonResponse(prompt: PROMPT, options?: ModelFunctionOptions<SETTINGS>): PromiseLike<RESPONSE>;
+    extractJson(response: RESPONSE): unknown;
+    extractUsage?(response: RESPONSE): {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+    };
+}