modelfusion 0.126.0 → 0.128.0

package/CHANGELOG.md

@@ -1,5 +1,89 @@
 # Changelog
 
+## v0.128.0 - 2024-01-20
+
+### Changed
+
+- **breaking change**: ModelFusion uses `Uint8Array` instead of `Buffer` for better cross-platform compatibility (see also ["Goodbye, Node.js Buffer"](https://sindresorhus.com/blog/goodbye-nodejs-buffer)). This can lead to breaking changes in your code if you use `Buffer`-specific methods.
+- **breaking change**: Image content in multi-modal instruction and chat inputs (e.g. for GPT Vision) is passed in the `image` property (instead of `base64Image`) and supports both base64 strings and `Uint8Array` inputs:
+
+  ```ts
+  const image = fs.readFileSync(path.join("data", "example-image.png"), {
+    encoding: "base64",
+  });
+
+  const textStream = await streamText({
+    model: openai.ChatTextGenerator({
+      model: "gpt-4-vision-preview",
+      maxGenerationTokens: 1000,
+    }),
+
+    prompt: [
+      openai.ChatMessage.user([
+        { type: "text", text: "Describe the image in detail:\n\n" },
+        { type: "image", image, mimeType: "image/png" },
+      ]),
+    ],
+  });
+  ```
+
+- OpenAI-compatible providers with predefined API configurations have a customized provider name that shows up in the events.
+
+## v0.127.0 - 2024-01-15
+
+### Changed
+
+- **breaking change**: `streamStructure` returns an async iterable over deep partial objects. If you need to get the fully validated final result, you can use the `fullResponse: true` option and await the `structurePromise` value. Example:
+
+  ```ts
+  const { structureStream, structurePromise } = await streamStructure({
+    model: ollama
+      .ChatTextGenerator({
+        model: "openhermes2.5-mistral",
+        maxGenerationTokens: 1024,
+        temperature: 0,
+      })
+      .asStructureGenerationModel(jsonStructurePrompt.text()),
+
+    schema: zodSchema(
+      z.object({
+        characters: z.array(
+          z.object({
+            name: z.string(),
+            class: z
+              .string()
+              .describe("Character class, e.g. warrior, mage, or thief."),
+            description: z.string(),
+          })
+        ),
+      })
+    ),
+
+    prompt:
+      "Generate 3 character descriptions for a fantasy role playing game.",
+
+    fullResponse: true,
+  });
+
+  for await (const partialStructure of structureStream) {
+    console.clear();
+    console.log(partialStructure);
+  }
+
+  const structure = await structurePromise;
+
+  console.clear();
+  console.log("FINAL STRUCTURE");
+  console.log(structure);
+  ```
+
+- **breaking change**: Renamed `text` value in `streamText` with `fullResponse: true` to `textPromise`.
+
+### Fixed
+
+- Ollama streaming.
+- Ollama structure generation and streaming.
+
 ## v0.126.0 - 2024-01-15
 
 ### Changed

package/README.md

@@ -93,7 +93,7 @@ const textStream = await streamText({
   prompt: [
     openai.ChatMessage.user([
       { type: "text", text: "Describe the image in detail:" },
-      { type: "image", base64Image: image, mimeType: "image/png" },
+      { type: "image", image, mimeType: "image/png" },
     ]),
   ],
 });
@@ -184,14 +184,9 @@ const structureStream = await streamStructure({
   prompt: "Generate 3 character descriptions for a fantasy role playing game.",
 });
 
-for await (const part of structureStream) {
-  if (!part.isComplete) {
-    const unknownPartialStructure = part.value;
-    console.log("partial value", unknownPartialStructure);
-  } else {
-    const fullyTypedStructure = part.value;
-    console.log("final value", fullyTypedStructure);
-  }
+for await (const partialStructure of structureStream) {
+  console.clear();
+  console.log(partialStructure);
 }
 ```
 
@@ -580,8 +575,8 @@ modelfusion.setLogFormat("detailed-object"); // log full events
   - [Embed Value](https://modelfusion.dev/guide/function/embed)
   - [Classify Value](https://modelfusion.dev/guide/function/classify)
 - [Tools](https://modelfusion.dev/guide/tools)
-  - [Use Tool](https://modelfusion.dev/guide/tools/run-tool)
-  - [Use Tools](https://modelfusion.dev/guide/tools/run-tools)
+  - [Run Tool](https://modelfusion.dev/guide/tools/run-tool)
+  - [Run Tools](https://modelfusion.dev/guide/tools/run-tools)
   - [Agent Loop](https://modelfusion.dev/guide/tools/agent-loop)
   - [Available Tools](https://modelfusion.dev/guide/tools/available-tools/)
   - [Custom Tools](https://modelfusion.dev/guide/tools/custom-tools)

package/core/api/postToApi.cjs

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.postToApi = exports.postJsonToApi = exports.createAudioMpegResponseHandler = exports.createTextResponseHandler = exports.createJsonResponseHandler = exports.createTextErrorResponseHandler = exports.createJsonErrorResponseHandler = void 0;
+const UInt8Utils_js_1 = require("../../util/UInt8Utils.cjs");
 const parseJSON_js_1 = require("../schema/parseJSON.cjs");
 const ApiCallError_js_1 = require("./ApiCallError.cjs");
 const createJsonErrorResponseHandler = ({ errorSchema, errorToMessage, isRetryable, }) => async ({ response, url, requestBodyValues }) => {
@@ -86,8 +87,7 @@ const createAudioMpegResponseHandler = () => async ({ response, url, requestBody
             requestBodyValues,
         });
     }
-    const arrayBuffer = await response.arrayBuffer();
-    return Buffer.from(arrayBuffer);
+    return (0, UInt8Utils_js_1.toUint8Array)(await response.arrayBuffer());
 };
 exports.createAudioMpegResponseHandler = createAudioMpegResponseHandler;
 const postJsonToApi = async ({ url, headers, body, failedResponseHandler, successfulResponseHandler, abortSignal, }) => (0, exports.postToApi)({

package/core/api/postToApi.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { Schema } from "../schema/Schema.js";
 import { ApiCallError } from "./ApiCallError.js";
 export type ResponseHandler<T> = (options: {
@@ -16,7 +15,7 @@ export declare const createTextErrorResponseHandler: ({ isRetryable, }?: {
 }) => ResponseHandler<ApiCallError>;
 export declare const createJsonResponseHandler: <T>(responseSchema: Schema<T>) => ResponseHandler<T>;
 export declare const createTextResponseHandler: () => ResponseHandler<string>;
-export declare const createAudioMpegResponseHandler: () => ResponseHandler<Buffer>;
+export declare const createAudioMpegResponseHandler: () => ResponseHandler<Uint8Array>;
 export declare const postJsonToApi: <T>({ url, headers, body, failedResponseHandler, successfulResponseHandler, abortSignal, }: {
     url: string;
     headers?: Record<string, string> | undefined;
@@ -29,7 +28,7 @@ export declare const postToApi: <T>({ url, headers, body, successfulResponseHand
     url: string;
     headers?: Record<string, string> | undefined;
     body: {
-        content: string | FormData | Buffer;
+        content: string | FormData | Uint8Array;
         values: unknown;
     };
     failedResponseHandler: ResponseHandler<Error>;

package/core/api/postToApi.js

@@ -1,3 +1,4 @@
+import { toUint8Array } from "../../util/UInt8Utils.js";
 import { parseJSON, safeParseJSON } from "../schema/parseJSON.js";
 import { ApiCallError } from "./ApiCallError.js";
 export const createJsonErrorResponseHandler = ({ errorSchema, errorToMessage, isRetryable, }) => async ({ response, url, requestBodyValues }) => {
@@ -79,8 +80,7 @@ export const createAudioMpegResponseHandler = () => async ({ response, url, requ
             requestBodyValues,
         });
     }
-    const arrayBuffer = await response.arrayBuffer();
-    return Buffer.from(arrayBuffer);
+    return toUint8Array(await response.arrayBuffer());
 };
 export const postJsonToApi = async ({ url, headers, body, failedResponseHandler, successfulResponseHandler, abortSignal, }) => postToApi({
     url,

package/core/getFunctionCallLogger.cjs

@@ -61,8 +61,8 @@ const detailedObjectObserver = {
                     if (v === undefined) {
                         return [k, undefined];
                     }
-                    else if (v instanceof Buffer) {
-                        return [k, "omitted<Buffer>"];
+                    else if (v instanceof Uint8Array) {
+                        return [k, "omitted<Uint8Array>"];
                     }
                     else if (Array.isArray(v) &&
                         v.length > 20 &&

package/core/getFunctionCallLogger.js

@@ -57,8 +57,8 @@ const detailedObjectObserver = {
                     if (v === undefined) {
                         return [k, undefined];
                     }
-                    else if (v instanceof Buffer) {
-                        return [k, "omitted<Buffer>"];
+                    else if (v instanceof Uint8Array) {
+                        return [k, "omitted<Uint8Array>"];
                     }
                     else if (Array.isArray(v) &&
                         v.length > 20 &&

package/model-function/generate-image/generateImage.cjs

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateImage = void 0;
+const UInt8Utils_js_1 = require("../../util/UInt8Utils.cjs");
 const executeStandardCall_js_1 = require("../executeStandardCall.cjs");
 async function generateImage({ model, prompt, fullResponse, ...options }) {
     const callResponse = await (0, executeStandardCall_js_1.executeStandardCall)({
@@ -17,7 +18,7 @@ async function generateImage({ model, prompt, fullResponse, ...options }) {
         },
     });
     const imagesBase64 = callResponse.value;
-    const images = imagesBase64.map((imageBase64) => Buffer.from(imageBase64, "base64"));
+    const images = imagesBase64.map(UInt8Utils_js_1.base64ToUint8Array);
     return fullResponse
         ? {
             image: images[0],

package/model-function/generate-image/generateImage.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { FunctionOptions } from "../../core/FunctionOptions.js";
 import { ModelCallMetadata } from "../ModelCallMetadata.js";
 import { ImageGenerationModel, ImageGenerationModelSettings } from "./ImageGenerationModel.js";
@@ -23,21 +22,21 @@ import { ImageGenerationModel, ImageGenerationModelSettings } from "./ImageGener
  * @param {PROMPT} prompt - The prompt to be used for image generation.
  *
  * @returns {Promise} - Returns a promise that resolves to the generated image.
- * The image is a Buffer containing the image data in PNG format.
+ * The image is a Uint8Array containing the image data in PNG format.
  */
 export declare function generateImage<PROMPT>(args: {
     model: ImageGenerationModel<PROMPT, ImageGenerationModelSettings>;
     prompt: PROMPT;
     fullResponse?: false;
-} & FunctionOptions): Promise<Buffer>;
+} & FunctionOptions): Promise<Uint8Array>;
 export declare function generateImage<PROMPT>(args: {
     model: ImageGenerationModel<PROMPT, ImageGenerationModelSettings>;
     prompt: PROMPT;
     fullResponse: true;
 } & FunctionOptions): Promise<{
-    image: Buffer;
+    image: Uint8Array;
     imageBase64: string;
-    images: Buffer[];
+    images: Uint8Array[];
     imagesBase64: string[];
     rawResponse: unknown;
     metadata: ModelCallMetadata;

package/model-function/generate-image/generateImage.js

@@ -1,3 +1,4 @@
+import { base64ToUint8Array } from "../../util/UInt8Utils.js";
 import { executeStandardCall } from "../executeStandardCall.js";
 export async function generateImage({ model, prompt, fullResponse, ...options }) {
     const callResponse = await executeStandardCall({
@@ -14,7 +15,7 @@ export async function generateImage({ model, prompt, fullResponse, ...options })
         },
     });
     const imagesBase64 = callResponse.value;
-    const images = imagesBase64.map((imageBase64) => Buffer.from(imageBase64, "base64"));
+    const images = imagesBase64.map(base64ToUint8Array);
     return fullResponse
         ? {
             image: images[0],

package/model-function/generate-speech/SpeechGenerationEvent.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
 export interface SpeechGenerationStartedEvent extends BaseModelCallStartedEvent {
     functionType: "generate-speech";
@@ -7,7 +6,7 @@ export interface SpeechGenerationStartedEvent extends BaseModelCallStartedEvent
 export type SpeechGenerationFinishedEventResult = {
     status: "success";
     rawResponse: unknown;
-    value: Buffer;
+    value: Uint8Array;
 } | {
     status: "error";
     error: unknown;

package/model-function/generate-speech/SpeechGenerationModel.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { FunctionCallOptions } from "../../core/FunctionOptions.js";
 import { Delta } from "../Delta.js";
 import { Model, ModelSettings } from "../Model.js";
@@ -6,10 +5,10 @@ export interface SpeechGenerationModelSettings extends ModelSettings {
 }
 export interface SpeechGenerationModel<SETTINGS extends SpeechGenerationModelSettings = SpeechGenerationModelSettings> extends Model<SETTINGS> {
     /**
-     * Generates an mp3 audio buffer that contains the speech for the given text.
+     * Generates an mp3 audio Uint8Array that contains the speech for the given text.
      */
-    doGenerateSpeechStandard(text: string, options: FunctionCallOptions): PromiseLike<Buffer>;
+    doGenerateSpeechStandard(text: string, options: FunctionCallOptions): PromiseLike<Uint8Array>;
 }
 export interface StreamingSpeechGenerationModel<SETTINGS extends SpeechGenerationModelSettings = SpeechGenerationModelSettings> extends SpeechGenerationModel<SETTINGS> {
-    doGenerateSpeechStreamDuplex(textStream: AsyncIterable<string>, options: FunctionCallOptions): PromiseLike<AsyncIterable<Delta<Buffer>>>;
+    doGenerateSpeechStreamDuplex(textStream: AsyncIterable<string>, options: FunctionCallOptions): PromiseLike<AsyncIterable<Delta<Uint8Array>>>;
 }

package/model-function/generate-speech/generateSpeech.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { FunctionOptions } from "../../core/FunctionOptions.js";
 import { ModelCallMetadata } from "../ModelCallMetadata.js";
 import { SpeechGenerationModel, SpeechGenerationModelSettings } from "./SpeechGenerationModel.js";
@@ -17,19 +16,19 @@ import { SpeechGenerationModel, SpeechGenerationModelSettings } from "./SpeechGe
  * @param {SpeechGenerationModel<SpeechGenerationModelSettings>} model - The speech generation model.
  * @param {string} text - The text to be converted to speech.
  *
- * @returns {Promise<Buffer>} - A promise that resolves to a buffer containing the synthesized speech.
+ * @returns {Promise<Uint8Array>} - A promise that resolves to a Uint8Array containing the synthesized speech.
  */
 export declare function generateSpeech(args: {
     model: SpeechGenerationModel<SpeechGenerationModelSettings>;
     text: string;
     fullResponse?: false;
-} & FunctionOptions): Promise<Buffer>;
+} & FunctionOptions): Promise<Uint8Array>;
 export declare function generateSpeech(args: {
     model: SpeechGenerationModel<SpeechGenerationModelSettings>;
     text: string;
     fullResponse: true;
 } & FunctionOptions): Promise<{
-    speech: Buffer;
+    speech: Uint8Array;
     rawResponse: unknown;
     metadata: ModelCallMetadata;
 }>;

package/model-function/generate-speech/streamSpeech.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { FunctionOptions } from "../../core/FunctionOptions.js";
 import { ModelCallMetadata } from "../ModelCallMetadata.js";
 import { SpeechGenerationModelSettings, StreamingSpeechGenerationModel } from "./SpeechGenerationModel.js";
@@ -24,18 +23,18 @@ import { SpeechGenerationModelSettings, StreamingSpeechGenerationModel } from ".
  * @param {AsyncIterable<string> | string} text - The text to be converted to speech. Can be a string or an async iterable of strings.
  * @param {FunctionOptions} [options] - Optional function options.
  *
- * @returns {AsyncIterableResultPromise<Buffer>} An async iterable promise that contains the synthesized speech chunks.
+ * @returns {AsyncIterableResultPromise<Uint8Array>} An async iterable promise that contains the synthesized speech chunks.
  */
 export declare function streamSpeech(args: {
     model: StreamingSpeechGenerationModel<SpeechGenerationModelSettings>;
     text: AsyncIterable<string> | string;
     fullResponse?: false;
-} & FunctionOptions): Promise<AsyncIterable<Buffer>>;
+} & FunctionOptions): Promise<AsyncIterable<Uint8Array>>;
 export declare function streamSpeech(args: {
     model: StreamingSpeechGenerationModel<SpeechGenerationModelSettings>;
     text: AsyncIterable<string> | string;
     fullResponse: true;
 } & FunctionOptions): Promise<{
-    speechStream: AsyncIterable<Buffer>;
+    speechStream: AsyncIterable<Uint8Array>;
     metadata: Omit<ModelCallMetadata, "durationInMs" | "finishTimestamp">;
 }>;

package/model-function/generate-structure/StructureFromTextGenerationModel.cjs

@@ -31,7 +31,7 @@ class StructureFromTextGenerationModel {
     }
     getModelWithJsonOutput(schema) {
         if (this.template.withJsonOutput != null) {
-            return this.template.withJsonOutput?.({
+            return this.template.withJsonOutput({
                 model: this.model,
                 schema,
             });
@@ -40,7 +40,7 @@ class StructureFromTextGenerationModel {
     }
     async doGenerateStructure(schema, prompt, options) {
         const { rawResponse, text } = await (0, generateText_js_1.generateText)({
-            model: this.model,
+            model: this.getModelWithJsonOutput(schema),
             prompt: this.template.createPrompt(prompt, schema),
             fullResponse: true,
             ...options,

package/model-function/generate-structure/StructureFromTextGenerationModel.js

@@ -28,7 +28,7 @@ export class StructureFromTextGenerationModel {
     }
     getModelWithJsonOutput(schema) {
         if (this.template.withJsonOutput != null) {
-            return this.template.withJsonOutput?.({
+            return this.template.withJsonOutput({
                 model: this.model,
                 schema,
             });
@@ -37,7 +37,7 @@ export class StructureFromTextGenerationModel {
     }
     async doGenerateStructure(schema, prompt, options) {
         const { rawResponse, text } = await generateText({
-            model: this.model,
+            model: this.getModelWithJsonOutput(schema),
             prompt: this.template.createPrompt(prompt, schema),
             fullResponse: true,
             ...options,

package/model-function/generate-structure/StructureFromTextStreamingModel.cjs

@@ -11,7 +11,7 @@ class StructureFromTextStreamingModel extends StructureFromTextGenerationModel_j
     }
     async doStreamStructure(schema, prompt, options) {
         const textStream = await (0, streamText_js_1.streamText)({
-            model: this.model,
+            model: this.getModelWithJsonOutput(schema),
             prompt: this.template.createPrompt(prompt, schema),
             ...options,
         });

package/model-function/generate-structure/StructureFromTextStreamingModel.js

@@ -8,7 +8,7 @@ export class StructureFromTextStreamingModel extends StructureFromTextGeneration
     }
     async doStreamStructure(schema, prompt, options) {
         const textStream = await streamText({
-            model: this.model,
+            model: this.getModelWithJsonOutput(schema),
             prompt: this.template.createPrompt(prompt, schema),
             ...options,
         });

package/model-function/generate-structure/streamStructure.cjs

@@ -10,6 +10,12 @@ async function streamStructure({ model, schema, prompt, fullResponse, ...options
         : prompt;
     let accumulatedText = "";
     let lastStructure;
+    let resolveStructure;
+    let rejectStructure;
+    const structurePromise = new Promise((resolve, reject) => {
+        resolveStructure = resolve;
+        rejectStructure = reject;
+    });
     const callResponse = await (0, executeStreamCall_js_1.executeStreamCall)({
         functionType: "stream-structure",
         input: {
@@ -29,29 +35,25 @@ async function streamStructure({ model, schema, prompt, fullResponse, ...options
             // only send a new part into the stream when the partial structure has changed:
             if (!(0, isDeepEqualData_js_1.isDeepEqualData)(lastStructure, latestStructure)) {
                 lastStructure = latestStructure;
-                return {
-                    isComplete: false,
-                    value: lastStructure,
-                };
+                return lastStructure;
             }
             return undefined;
         },
-        processFinished: () => {
+        onDone: () => {
             // process the final result (full type validation):
             const parseResult = schema.validate(lastStructure);
-            if (!parseResult.success) {
-                reportError(parseResult.error);
-                throw parseResult.error;
+            if (parseResult.success) {
+                resolveStructure(parseResult.data);
+            }
+            else {
+                rejectStructure(parseResult.error);
             }
-            return {
-                isComplete: true,
-                value: parseResult.data,
-            };
         },
     });
     return fullResponse
         ? {
             structureStream: callResponse.value,
+            structurePromise,
             metadata: callResponse.metadata,
         }
         : callResponse.value;

package/model-function/generate-structure/streamStructure.d.ts

@@ -1,25 +1,12 @@
+import type { PartialDeep } from "type-fest";
 import { FunctionOptions } from "../../core/FunctionOptions.js";
 import { JsonSchemaProducer } from "../../core/schema/JsonSchemaProducer.js";
 import { Schema } from "../../core/schema/Schema.js";
 import { ModelCallMetadata } from "../ModelCallMetadata.js";
 import { StructureStreamingModel } from "./StructureGenerationModel.js";
-export type StructureStreamPart<STRUCTURE> = {
-    isComplete: false;
-    value: unknown;
-} | {
-    isComplete: true;
-    value: STRUCTURE;
-};
 /**
  * Generate and stream an object for a prompt and a structure definition.
  *
- * The final object is typed according to the structure definition.
- * The partial objects are of unknown type,
- * but are supposed to be partial version of the final object
- * (unless the underlying model returns invalid data).
- *
- * The structure definition is used as part of the final prompt.
- *
  * For the OpenAI chat model, this generates and parses a function call with a single function.
  *
  * @see https://modelfusion.dev/guide/function/generate-structure
@@ -44,15 +31,8 @@ export type StructureStreamPart<STRUCTURE> = {
  *   ]
  * });
  *
- * for await (const part of structureStream) {
- *   if (!part.isComplete) {
- *     const unknownPartialStructure = part.value;
- *     // use your own logic to handle partial structures, e.g. with Zod .deepPartial()
- *     // it depends on your application at which points you want to act on the partial structures
- *   } else {
- *     const fullyTypedStructure = part.value;
- *     // ...
- *   }
+ * for await (const partialStructure of structureStream) {
+ *   // ...
  * }
  *
  * @param {StructureStreamingModel<PROMPT>} structureGenerator - The model to use for streaming
@@ -60,26 +40,28 @@ export type StructureStreamPart<STRUCTURE> = {
  * @param {PROMPT | ((schema: Schema<STRUCTURE>) => PROMPT)} prompt
  * The prompt to be used.
  * You can also pass a function that takes the schema as an argument and returns the prompt.
- * @param {FunctionOptions} [options] - Optional function options
  *
  * @returns {AsyncIterableResultPromise<StructureStreamPart<STRUCTURE>>}
  * The async iterable result promise.
- * Each part of the stream is either a partial structure or the final structure.
- * It contains a isComplete flag to indicate whether the structure is complete,
- * and a value that is either the partial structure or the final structure.
+ * Each part of the stream is a partial structure.
  */
 export declare function streamStructure<STRUCTURE, PROMPT>(args: {
     model: StructureStreamingModel<PROMPT>;
     schema: Schema<STRUCTURE> & JsonSchemaProducer;
     prompt: PROMPT | ((schema: Schema<STRUCTURE>) => PROMPT);
     fullResponse?: false;
-} & FunctionOptions): Promise<AsyncIterable<StructureStreamPart<STRUCTURE>>>;
+} & FunctionOptions): Promise<AsyncIterable<PartialDeep<STRUCTURE, {
+    recurseIntoArrays: true;
+}>>>;
 export declare function streamStructure<STRUCTURE, PROMPT>(args: {
     model: StructureStreamingModel<PROMPT>;
     schema: Schema<STRUCTURE> & JsonSchemaProducer;
     prompt: PROMPT | ((schema: Schema<STRUCTURE>) => PROMPT);
     fullResponse: true;
 } & FunctionOptions): Promise<{
-    structureStream: AsyncIterable<StructureStreamPart<STRUCTURE>>;
+    structureStream: AsyncIterable<PartialDeep<STRUCTURE, {
+        recurseIntoArrays: true;
+    }>>;
+    structurePromise: PromiseLike<STRUCTURE>;
     metadata: Omit<ModelCallMetadata, "durationInMs" | "finishTimestamp">;
 }>;

package/model-function/generate-structure/streamStructure.js

@@ -7,6 +7,12 @@ export async function streamStructure({ model, schema, prompt, fullResponse, ...
         : prompt;
     let accumulatedText = "";
     let lastStructure;
+    let resolveStructure;
+    let rejectStructure;
+    const structurePromise = new Promise((resolve, reject) => {
+        resolveStructure = resolve;
+        rejectStructure = reject;
+    });
     const callResponse = await executeStreamCall({
         functionType: "stream-structure",
         input: {
@@ -26,29 +32,25 @@ export async function streamStructure({ model, schema, prompt, fullResponse, ...
             // only send a new part into the stream when the partial structure has changed:
             if (!isDeepEqualData(lastStructure, latestStructure)) {
                 lastStructure = latestStructure;
-                return {
-                    isComplete: false,
-                    value: lastStructure,
-                };
+                return lastStructure;
             }
             return undefined;
         },
-        processFinished: () => {
+        onDone: () => {
             // process the final result (full type validation):
             const parseResult = schema.validate(lastStructure);
-            if (!parseResult.success) {
-                reportError(parseResult.error);
-                throw parseResult.error;
+            if (parseResult.success) {
+                resolveStructure(parseResult.data);
+            }
+            else {
+                rejectStructure(parseResult.error);
             }
-            return {
-                isComplete: true,
-                value: parseResult.data,
-            };
         },
     });
     return fullResponse
         ? {
             structureStream: callResponse.value,
+            structurePromise,
             metadata: callResponse.metadata,
         }
         : callResponse.value;

package/model-function/generate-text/prompt-template/ContentPart.cjs

@@ -1,7 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateContentIsString = void 0;
+exports.validateContentIsString = exports.getImageAsBase64 = void 0;
+const UInt8Utils_js_1 = require("../../../util/UInt8Utils.cjs");
 const InvalidPromptError_js_1 = require("./InvalidPromptError.cjs");
+function getImageAsBase64(image) {
+    if (typeof image === "string") {
+        return image;
+    }
+    return (0, UInt8Utils_js_1.uint8ArrayToBase64)(image);
+}
+exports.getImageAsBase64 = getImageAsBase64;
 function validateContentIsString(content, prompt) {
     if (typeof content !== "string") {
         throw new InvalidPromptError_js_1.InvalidPromptError("Only text prompts are are supported by this prompt template.", prompt);

package/model-function/generate-text/prompt-template/ContentPart.d.ts

@@ -8,9 +8,9 @@ export interface TextPart {
 export interface ImagePart {
     type: "image";
     /**
-     * Base-64 encoded image.
+     * Image data. Can either be a base64-encoded string or a Uint8Array.
      */
-    base64Image: string;
+    image: string | Uint8Array;
     /**
      * Optional mime type of the image.
      */
@@ -27,4 +27,5 @@ export interface ToolResponsePart {
     id: string;
     response: unknown;
 }
+export declare function getImageAsBase64(image: string | Uint8Array): string;
 export declare function validateContentIsString(content: string | unknown, prompt: unknown): string;

package/model-function/generate-text/prompt-template/ContentPart.js

@@ -1,4 +1,11 @@
+import { uint8ArrayToBase64 } from "../../../util/UInt8Utils.js";
 import { InvalidPromptError } from "./InvalidPromptError.js";
+export function getImageAsBase64(image) {
+    if (typeof image === "string") {
+        return image;
+    }
+    return uint8ArrayToBase64(image);
+}
 export function validateContentIsString(content, prompt) {
     if (typeof content !== "string") {
         throw new InvalidPromptError("Only text prompts are are supported by this prompt template.", prompt);