modelfusion 0.34.0 → 0.35.0

package/README.md

@@ -133,7 +133,7 @@ for (const choice of response.choices) {
 console.log(`Duration: ${metadata.durationInMs}ms`);
 ```
 
-### [Generate Structure](https://modelfusion.dev/guide/function/generate-structure)
+### [Generate Structure](https://modelfusion.dev/guide/function/generate-structure#generatestructure)
 
 Generate a structure that matches a schema.
 
@@ -168,6 +168,52 @@ const sentiment = await generateStructure(
 
 Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai)
 
+### [Stream Structure](https://modelfusion.dev/guide/function/generate-structure#streamstructure)
+
+Stream a structure that matches a schema. Partial structures before the final part are untyped JSON.
+
+```ts
+const structureStream = await streamStructure(
+  new OpenAIChatModel({
+    model: "gpt-3.5-turbo",
+    temperature: 0,
+    maxCompletionTokens: 2000,
+  }),
+  new ZodStructureDefinition({
+    name: "generateCharacter" as const,
+    description: "Generate character descriptions.",
+    schema: z.object({
+      characters: z.array(
+        z.object({
+          name: z.string(),
+          class: z
+            .string()
+            .describe("Character class, e.g. warrior, mage, or thief."),
+          description: z.string(),
+        })
+      ),
+    }),
+  }),
+  [
+    OpenAIChatMessage.user(
+      "Generate 3 character descriptions for a fantasy role playing game."
+    ),
+  ]
+);
+
+for await (const part of structureStream) {
+  if (!part.isComplete) {
+    const unknownPartialStructure = part.value;
+    console.log("partial value", unknownPartialStructure);
+  } else {
+    const fullyTypedStructure = part.value;
+    console.log("final value", fullyTypedStructure);
+  }
+}
+```
+
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai)
+
 ### [Generate Structure or Text](https://modelfusion.dev/guide/function/generate-structure-or-text)
 
 Generate a structure (or text as a fallback) using a prompt and multiple schemas.

package/model-function/AsyncIterableResultPromise.cjs

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AsyncIterableResultPromise = void 0;
+class AsyncIterableResultPromise extends Promise {
+    constructor(fullPromise) {
+        super((resolve) => {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            resolve(null); // we override the resolve function
+        });
+        Object.defineProperty(this, "fullPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: fullPromise
+        });
+        Object.defineProperty(this, "outputPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.outputPromise = fullPromise.then((result) => result.output);
+    }
+    asFullResponse() {
+        return this.fullPromise;
+    }
+    then(onfulfilled, onrejected) {
+        return this.outputPromise.then(onfulfilled, onrejected);
+    }
+    catch(onrejected) {
+        return this.outputPromise.catch(onrejected);
+    }
+    finally(onfinally) {
+        return this.outputPromise.finally(onfinally);
+    }
+}
+exports.AsyncIterableResultPromise = AsyncIterableResultPromise;

package/model-function/AsyncIterableResultPromise.d.ts

@@ -0,0 +1,16 @@
+import { ModelCallMetadata } from "./executeCall.js";
+export declare class AsyncIterableResultPromise<T> extends Promise<AsyncIterable<T>> {
+    private fullPromise;
+    private outputPromise;
+    constructor(fullPromise: Promise<{
+        output: AsyncIterable<T>;
+        metadata: Omit<ModelCallMetadata, "durationInMs" | "finishTimestamp">;
+    }>);
+    asFullResponse(): Promise<{
+        output: AsyncIterable<T>;
+        metadata: Omit<ModelCallMetadata, "durationInMs" | "finishTimestamp">;
+    }>;
+    then<TResult1 = AsyncIterable<T>, TResult2 = never>(onfulfilled?: ((value: AsyncIterable<T>) => TResult1 | PromiseLike<TResult1>) | undefined | null, onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | undefined | null): Promise<TResult1 | TResult2>;
+    catch<TResult = never>(onrejected?: ((reason: unknown) => TResult | PromiseLike<TResult>) | undefined | null): Promise<AsyncIterable<T> | TResult>;
+    finally(onfinally?: (() => void) | undefined | null): Promise<AsyncIterable<T>>;
+}

package/model-function/AsyncIterableResultPromise.js

@@ -0,0 +1,33 @@
+export class AsyncIterableResultPromise extends Promise {
+    constructor(fullPromise) {
+        super((resolve) => {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            resolve(null); // we override the resolve function
+        });
+        Object.defineProperty(this, "fullPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: fullPromise
+        });
+        Object.defineProperty(this, "outputPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.outputPromise = fullPromise.then((result) => result.output);
+    }
+    asFullResponse() {
+        return this.fullPromise;
+    }
+    then(onfulfilled, onrejected) {
+        return this.outputPromise.then(onfulfilled, onrejected);
+    }
+    catch(onrejected) {
+        return this.outputPromise.catch(onrejected);
+    }
+    finally(onfinally) {
+        return this.outputPromise.finally(onfinally);
+    }
+}

package/model-function/{generate-text/DeltaEvent.d.ts → DeltaEvent.d.ts}

@@ -4,4 +4,4 @@ export type DeltaEvent<FULL_DELTA> = {
 } | {
     type: "error";
     error: unknown;
-} | undefined;
+};

package/model-function/ModelCallEvent.d.ts

@@ -4,6 +4,7 @@ import { ImageDescriptionFinishedEvent, ImageDescriptionStartedEvent } from "./d
 import { TextEmbeddingFinishedEvent, TextEmbeddingStartedEvent } from "./embed-text/TextEmbeddingEvent.js";
 import { ImageGenerationFinishedEvent, ImageGenerationStartedEvent } from "./generate-image/ImageGenerationEvent.js";
 import { StructureGenerationFinishedEvent, StructureGenerationStartedEvent } from "./generate-structure/StructureGenerationEvent.js";
+import { StructureStreamingFinishedEvent, StructureStreamingStartedEvent } from "./generate-structure/StructureStreamingEvent.js";
 import { TextGenerationFinishedEvent, TextGenerationStartedEvent } from "./generate-text/TextGenerationEvent.js";
 import { TextStreamingFinishedEvent, TextStreamingStartedEvent } from "./generate-text/TextStreamingEvent.js";
 import { SpeechSynthesisFinishedEvent, SpeechSynthesisStartedEvent } from "./synthesize-speech/SpeechSynthesisEvent.js";
@@ -48,5 +49,5 @@ export interface BaseModelCallFinishedEvent extends BaseFunctionFinishedEvent {
      */
     result: BaseModelCallFinishedEventResult;
 }
-export type ModelCallStartedEvent = ImageDescriptionStartedEvent | ImageGenerationStartedEvent | StructureGenerationStartedEvent | SpeechSynthesisStartedEvent | TextEmbeddingStartedEvent | TextGenerationStartedEvent | TextStreamingStartedEvent | TranscriptionStartedEvent;
-export type ModelCallFinishedEvent = ImageDescriptionFinishedEvent | ImageGenerationFinishedEvent | StructureGenerationFinishedEvent | SpeechSynthesisFinishedEvent | TextEmbeddingFinishedEvent | TextGenerationFinishedEvent | TextStreamingFinishedEvent | TranscriptionFinishedEvent;
+export type ModelCallStartedEvent = ImageDescriptionStartedEvent | ImageGenerationStartedEvent | StructureGenerationStartedEvent | StructureStreamingStartedEvent | SpeechSynthesisStartedEvent | TextEmbeddingStartedEvent | TextGenerationStartedEvent | TextStreamingStartedEvent | TranscriptionStartedEvent;
+export type ModelCallFinishedEvent = ImageDescriptionFinishedEvent | ImageGenerationFinishedEvent | StructureGenerationFinishedEvent | StructureStreamingFinishedEvent | SpeechSynthesisFinishedEvent | TextEmbeddingFinishedEvent | TextGenerationFinishedEvent | TextStreamingFinishedEvent | TranscriptionFinishedEvent;

package/model-function/generate-structure/StructureFromTextGenerationModel.d.ts

@@ -6,7 +6,7 @@ export type StructureFromTextPromptFormat<PROMPT> = {
     createPrompt: (prompt: PROMPT, structure: StructureDefinition<string, unknown>) => string;
     extractStructure: (response: string) => unknown;
 };
-export declare class StructureFromTextGenerationModel<PROMPT, MODEL extends TextGenerationModel<string, any, any, TextGenerationModelSettings>> implements StructureGenerationModel<PROMPT, string, MODEL["settings"]> {
+export declare class StructureFromTextGenerationModel<PROMPT, MODEL extends TextGenerationModel<string, any, any, TextGenerationModelSettings>> implements StructureGenerationModel<PROMPT, string, undefined, MODEL["settings"]> {
     private readonly model;
     private readonly format;
     constructor({ model, format, }: {

package/model-function/generate-structure/StructureGenerationModel.d.ts

@@ -1,9 +1,10 @@
 import { StructureDefinition } from "../../core/structure/StructureDefinition.js";
+import { DeltaEvent } from "../DeltaEvent.js";
 import { Model, ModelSettings } from "../Model.js";
 import { ModelFunctionOptions } from "../ModelFunctionOptions.js";
 export interface StructureGenerationModelSettings extends ModelSettings {
 }
-export interface StructureGenerationModel<PROMPT, RESPONSE, SETTINGS extends StructureGenerationModelSettings> extends Model<SETTINGS> {
+export interface StructureGenerationModel<PROMPT, RESPONSE, FULL_DELTA, SETTINGS extends StructureGenerationModelSettings> extends Model<SETTINGS> {
     generateStructureResponse(structure: StructureDefinition<string, unknown>, prompt: PROMPT, options?: ModelFunctionOptions<SETTINGS>): PromiseLike<RESPONSE>;
     extractStructure(response: RESPONSE): unknown;
     extractUsage?(response: RESPONSE): {
@@ -11,4 +12,12 @@ export interface StructureGenerationModel<PROMPT, RESPONSE, SETTINGS extends Str
         completionTokens: number;
         totalTokens: number;
     };
+    /**
+     * Optional. Implement for streaming support.
+     */
+    readonly generateStructureStreamResponse?: (structureDefinition: StructureDefinition<string, unknown>, prompt: PROMPT, options?: ModelFunctionOptions<SETTINGS>) => PromiseLike<AsyncIterable<DeltaEvent<FULL_DELTA>>>;
+    /**
+     * Optional. Implement for streaming support.
+     */
+    readonly extractPartialStructure?: (fullDelta: FULL_DELTA) => unknown | undefined;
 }

package/model-function/generate-structure/StructureStreamingEvent.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-structure/StructureStreamingEvent.d.ts

@@ -0,0 +1,7 @@
+import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
+export interface StructureStreamingStartedEvent extends BaseModelCallStartedEvent {
+    functionType: "structure-streaming";
+}
+export interface StructureStreamingFinishedEvent extends BaseModelCallFinishedEvent {
+    functionType: "structure-streaming";
+}

package/model-function/generate-structure/StructureStreamingEvent.js

@@ -0,0 +1 @@
+export {};

package/model-function/generate-structure/fixJson.cjs

@@ -0,0 +1,215 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fixJson = void 0;
+// Implemented as a scanner with additional fixing
+// that performs a single linear time scan pass over the partial JSON:
+function fixJson(input) {
+    const stack = ["BEFORE_VALUE"];
+    let lastValidIndex = -1;
+    let literalStart = null;
+    for (let i = 0; i < input.length; i++) {
+        const char = input[i];
+        const currentState = stack[stack.length - 1];
+        switch (currentState) {
+            case "BEFORE_VALUE": {
+                switch (char) {
+                    case '"': {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_STRING");
+                        break;
+                    }
+                    case "f":
+                    case "t":
+                    case "n": {
+                        lastValidIndex = i;
+                        literalStart = i;
+                        stack.pop();
+                        stack.push("INSIDE_LITERAL");
+                        break;
+                    }
+                    case "-": {
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "{": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_OBJECT");
+                        break;
+                    }
+                    case "[": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_ARRAY");
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                switch (char) {
+                    case '"': {
+                        stack.push("INSIDE_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT_KEY": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        stack.push("AFTER_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "AFTER_OBJECT_KEY": {
+                switch (char) {
+                    case ":": {
+                        stack.pop();
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "\\": {
+                        stack.push("INSIDE_STRING_ESCAPE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                switch (char) {
+                    case ",": {
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING_ESCAPE": {
+                stack.pop();
+                lastValidIndex = i;
+                break;
+            }
+            case "INSIDE_NUMBER": {
+                switch (char) {
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "e":
+                    case "E":
+                    case "-":
+                    case ".": {
+                        break;
+                    }
+                    case ",": {
+                        stack.pop();
+                        if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                            stack.push("BEFORE_VALUE");
+                        }
+                        break;
+                    }
+                    default: {
+                        stack.pop();
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, i);
+                if (!"false".startsWith(partialLiteral) &&
+                    !"true".startsWith(partialLiteral) &&
+                    !"null".startsWith(partialLiteral)) {
+                    stack.pop();
+                    if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                        stack.push("BEFORE_VALUE");
+                    }
+                }
+                else {
+                    lastValidIndex = i;
+                }
+                break;
+            }
+        }
+    }
+    let result = input.slice(0, lastValidIndex + 1);
+    for (let i = stack.length - 1; i >= 0; i--) {
+        const state = stack[i];
+        switch (state) {
+            case "INSIDE_STRING": {
+                result += '"';
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                result += "}";
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                result += "]";
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, input.length);
+                if ("true".startsWith(partialLiteral)) {
+                    result += "true".slice(partialLiteral.length);
+                }
+                else if ("false".startsWith(partialLiteral)) {
+                    result += "false".slice(partialLiteral.length);
+                }
+                else if ("null".startsWith(partialLiteral)) {
+                    result += "null".slice(partialLiteral.length);
+                }
+            }
+        }
+    }
+    return result;
+}
+exports.fixJson = fixJson;

package/model-function/generate-structure/fixJson.d.ts

@@ -0,0 +1 @@
+export declare function fixJson(input: string): string;

package/model-function/generate-structure/fixJson.js

@@ -0,0 +1,211 @@
+// Implemented as a scanner with additional fixing
+// that performs a single linear time scan pass over the partial JSON:
+export function fixJson(input) {
+    const stack = ["BEFORE_VALUE"];
+    let lastValidIndex = -1;
+    let literalStart = null;
+    for (let i = 0; i < input.length; i++) {
+        const char = input[i];
+        const currentState = stack[stack.length - 1];
+        switch (currentState) {
+            case "BEFORE_VALUE": {
+                switch (char) {
+                    case '"': {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_STRING");
+                        break;
+                    }
+                    case "f":
+                    case "t":
+                    case "n": {
+                        lastValidIndex = i;
+                        literalStart = i;
+                        stack.pop();
+                        stack.push("INSIDE_LITERAL");
+                        break;
+                    }
+                    case "-": {
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "{": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_OBJECT");
+                        break;
+                    }
+                    case "[": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_ARRAY");
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                switch (char) {
+                    case '"': {
+                        stack.push("INSIDE_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT_KEY": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        stack.push("AFTER_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "AFTER_OBJECT_KEY": {
+                switch (char) {
+                    case ":": {
+                        stack.pop();
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "\\": {
+                        stack.push("INSIDE_STRING_ESCAPE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                switch (char) {
+                    case ",": {
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING_ESCAPE": {
+                stack.pop();
+                lastValidIndex = i;
+                break;
+            }
+            case "INSIDE_NUMBER": {
+                switch (char) {
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "e":
+                    case "E":
+                    case "-":
+                    case ".": {
+                        break;
+                    }
+                    case ",": {
+                        stack.pop();
+                        if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                            stack.push("BEFORE_VALUE");
+                        }
+                        break;
+                    }
+                    default: {
+                        stack.pop();
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, i);
+                if (!"false".startsWith(partialLiteral) &&
+                    !"true".startsWith(partialLiteral) &&
+                    !"null".startsWith(partialLiteral)) {
+                    stack.pop();
+                    if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                        stack.push("BEFORE_VALUE");
+                    }
+                }
+                else {
+                    lastValidIndex = i;
+                }
+                break;
+            }
+        }
+    }
+    let result = input.slice(0, lastValidIndex + 1);
+    for (let i = stack.length - 1; i >= 0; i--) {
+        const state = stack[i];
+        switch (state) {
+            case "INSIDE_STRING": {
+                result += '"';
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                result += "}";
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                result += "]";
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, input.length);
+                if ("true".startsWith(partialLiteral)) {
+                    result += "true".slice(partialLiteral.length);
+                }
+                else if ("false".startsWith(partialLiteral)) {
+                    result += "false".slice(partialLiteral.length);
+                }
+                else if ("null".startsWith(partialLiteral)) {
+                    result += "null".slice(partialLiteral.length);
+                }
+            }
+        }
+    }
+    return result;
+}