modelfusion 0.33.1 → 0.35.0

package/model-function/generate-structure/StructureStreamingEvent.d.ts

@@ -0,0 +1,7 @@
+import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
+export interface StructureStreamingStartedEvent extends BaseModelCallStartedEvent {
+    functionType: "structure-streaming";
+}
+export interface StructureStreamingFinishedEvent extends BaseModelCallFinishedEvent {
+    functionType: "structure-streaming";
+}

package/model-function/generate-structure/StructureStreamingEvent.js

@@ -0,0 +1 @@
+export {};

package/model-function/generate-structure/fixJson.cjs

@@ -0,0 +1,215 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fixJson = void 0;
+// Implemented as a scanner with additional fixing
+// that performs a single linear time scan pass over the partial JSON:
+function fixJson(input) {
+    const stack = ["BEFORE_VALUE"];
+    let lastValidIndex = -1;
+    let literalStart = null;
+    for (let i = 0; i < input.length; i++) {
+        const char = input[i];
+        const currentState = stack[stack.length - 1];
+        switch (currentState) {
+            case "BEFORE_VALUE": {
+                switch (char) {
+                    case '"': {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_STRING");
+                        break;
+                    }
+                    case "f":
+                    case "t":
+                    case "n": {
+                        lastValidIndex = i;
+                        literalStart = i;
+                        stack.pop();
+                        stack.push("INSIDE_LITERAL");
+                        break;
+                    }
+                    case "-": {
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "{": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_OBJECT");
+                        break;
+                    }
+                    case "[": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_ARRAY");
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                switch (char) {
+                    case '"': {
+                        stack.push("INSIDE_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT_KEY": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        stack.push("AFTER_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "AFTER_OBJECT_KEY": {
+                switch (char) {
+                    case ":": {
+                        stack.pop();
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "\\": {
+                        stack.push("INSIDE_STRING_ESCAPE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                switch (char) {
+                    case ",": {
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING_ESCAPE": {
+                stack.pop();
+                lastValidIndex = i;
+                break;
+            }
+            case "INSIDE_NUMBER": {
+                switch (char) {
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "e":
+                    case "E":
+                    case "-":
+                    case ".": {
+                        break;
+                    }
+                    case ",": {
+                        stack.pop();
+                        if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                            stack.push("BEFORE_VALUE");
+                        }
+                        break;
+                    }
+                    default: {
+                        stack.pop();
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, i);
+                if (!"false".startsWith(partialLiteral) &&
+                    !"true".startsWith(partialLiteral) &&
+                    !"null".startsWith(partialLiteral)) {
+                    stack.pop();
+                    if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                        stack.push("BEFORE_VALUE");
+                    }
+                }
+                else {
+                    lastValidIndex = i;
+                }
+                break;
+            }
+        }
+    }
+    let result = input.slice(0, lastValidIndex + 1);
+    for (let i = stack.length - 1; i >= 0; i--) {
+        const state = stack[i];
+        switch (state) {
+            case "INSIDE_STRING": {
+                result += '"';
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                result += "}";
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                result += "]";
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, input.length);
+                if ("true".startsWith(partialLiteral)) {
+                    result += "true".slice(partialLiteral.length);
+                }
+                else if ("false".startsWith(partialLiteral)) {
+                    result += "false".slice(partialLiteral.length);
+                }
+                else if ("null".startsWith(partialLiteral)) {
+                    result += "null".slice(partialLiteral.length);
+                }
+            }
+        }
+    }
+    return result;
+}
+exports.fixJson = fixJson;

package/model-function/generate-structure/fixJson.d.ts

@@ -0,0 +1 @@
+export declare function fixJson(input: string): string;

package/model-function/generate-structure/fixJson.js

@@ -0,0 +1,211 @@
+// Implemented as a scanner with additional fixing
+// that performs a single linear time scan pass over the partial JSON:
+export function fixJson(input) {
+    const stack = ["BEFORE_VALUE"];
+    let lastValidIndex = -1;
+    let literalStart = null;
+    for (let i = 0; i < input.length; i++) {
+        const char = input[i];
+        const currentState = stack[stack.length - 1];
+        switch (currentState) {
+            case "BEFORE_VALUE": {
+                switch (char) {
+                    case '"': {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_STRING");
+                        break;
+                    }
+                    case "f":
+                    case "t":
+                    case "n": {
+                        lastValidIndex = i;
+                        literalStart = i;
+                        stack.pop();
+                        stack.push("INSIDE_LITERAL");
+                        break;
+                    }
+                    case "-": {
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_NUMBER");
+                        break;
+                    }
+                    case "{": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_OBJECT");
+                        break;
+                    }
+                    case "[": {
+                        lastValidIndex = i;
+                        stack.pop();
+                        stack.push("INSIDE_ARRAY");
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                switch (char) {
+                    case '"': {
+                        stack.push("INSIDE_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_OBJECT_KEY": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        stack.push("AFTER_OBJECT_KEY");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "AFTER_OBJECT_KEY": {
+                switch (char) {
+                    case ":": {
+                        stack.pop();
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING": {
+                switch (char) {
+                    case '"': {
+                        stack.pop();
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "\\": {
+                        stack.push("INSIDE_STRING_ESCAPE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                switch (char) {
+                    case ",": {
+                        stack.push("BEFORE_VALUE");
+                        break;
+                    }
+                    default: {
+                        lastValidIndex = i;
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_STRING_ESCAPE": {
+                stack.pop();
+                lastValidIndex = i;
+                break;
+            }
+            case "INSIDE_NUMBER": {
+                switch (char) {
+                    case "0":
+                    case "1":
+                    case "2":
+                    case "3":
+                    case "4":
+                    case "5":
+                    case "6":
+                    case "7":
+                    case "8":
+                    case "9": {
+                        lastValidIndex = i;
+                        break;
+                    }
+                    case "e":
+                    case "E":
+                    case "-":
+                    case ".": {
+                        break;
+                    }
+                    case ",": {
+                        stack.pop();
+                        if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                            stack.push("BEFORE_VALUE");
+                        }
+                        break;
+                    }
+                    default: {
+                        stack.pop();
+                        break;
+                    }
+                }
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, i);
+                if (!"false".startsWith(partialLiteral) &&
+                    !"true".startsWith(partialLiteral) &&
+                    !"null".startsWith(partialLiteral)) {
+                    stack.pop();
+                    if (stack[stack.length - 1] === "INSIDE_ARRAY") {
+                        stack.push("BEFORE_VALUE");
+                    }
+                }
+                else {
+                    lastValidIndex = i;
+                }
+                break;
+            }
+        }
+    }
+    let result = input.slice(0, lastValidIndex + 1);
+    for (let i = stack.length - 1; i >= 0; i--) {
+        const state = stack[i];
+        switch (state) {
+            case "INSIDE_STRING": {
+                result += '"';
+                break;
+            }
+            case "INSIDE_OBJECT": {
+                result += "}";
+                break;
+            }
+            case "INSIDE_ARRAY": {
+                result += "]";
+                break;
+            }
+            case "INSIDE_LITERAL": {
+                const partialLiteral = input.substring(literalStart, input.length);
+                if ("true".startsWith(partialLiteral)) {
+                    result += "true".slice(partialLiteral.length);
+                }
+                else if ("false".startsWith(partialLiteral)) {
+                    result += "false".slice(partialLiteral.length);
+                }
+                else if ("null".startsWith(partialLiteral)) {
+                    result += "null".slice(partialLiteral.length);
+                }
+            }
+        }
+    }
+    return result;
+}

package/model-function/generate-structure/fixJson.test.cjs

@@ -0,0 +1,130 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const fixJson_1 = require("./fixJson");
+(0, vitest_1.describe)("fixJson", () => {
+    (0, vitest_1.test)("should handle empty input", () => {
+        (0, vitest_1.expect)((0, fixJson_1.fixJson)("")).toBe("");
+    });
+    (0, vitest_1.describe)("literals", () => {
+        (0, vitest_1.test)("should handle incomplete null", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("nul")).toBe("null");
+        });
+        (0, vitest_1.test)("should handle incomplete true", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("t")).toBe("true");
+        });
+        (0, vitest_1.test)("should handle incomplete false", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("fals")).toBe("false");
+        });
+    });
+    (0, vitest_1.describe)("number", () => {
+        (0, vitest_1.test)("should handle incomplete numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("12.")).toBe("12");
+        });
+        (0, vitest_1.test)("should handle numbers with dot", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("12.2")).toBe("12.2");
+        });
+        (0, vitest_1.test)("should handle negative numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("-12")).toBe("-12");
+        });
+        (0, vitest_1.test)("should handle incomplete negative numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("-")).toBe("");
+        });
+        (0, vitest_1.test)("should handle e-notation numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("2.5e")).toBe("2.5");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("2.5e-")).toBe("2.5");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("2.5e3")).toBe("2.5e3");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("-2.5e3")).toBe("-2.5e3");
+        });
+        (0, vitest_1.test)("should handle uppercase e-notation numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("2.5E")).toBe("2.5");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("2.5E-")).toBe("2.5");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("2.5E3")).toBe("2.5E3");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("-2.5E3")).toBe("-2.5E3");
+        });
+        (0, vitest_1.test)("should handle incomplete numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("12.e")).toBe("12");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("12.34e")).toBe("12.34");
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("5e")).toBe("5");
+        });
+    });
+    (0, vitest_1.describe)("string", () => {
+        (0, vitest_1.test)("should handle incomplete strings", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('"abc')).toBe('"abc"');
+        });
+        (0, vitest_1.test)("should handle escape sequences", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('"value with \\"quoted\\" text and \\\\ escape')).toBe('"value with \\"quoted\\" text and \\\\ escape"');
+        });
+        (0, vitest_1.test)("should handle incomplete escape sequences", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('"value with \\')).toBe('"value with "');
+        });
+        (0, vitest_1.test)("should handle unicode characters", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('"value with unicode \u003C"')).toBe('"value with unicode \u003C"');
+        });
+    });
+    (0, vitest_1.describe)("array", () => {
+        (0, vitest_1.test)("should handle incomplete array", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("[")).toBe("[]");
+        });
+        (0, vitest_1.test)("should handle trailing comma", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("[1, ")).toBe("[1]");
+        });
+        (0, vitest_1.test)("should handle closing array", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('["a": [], "b": 123')).toBe('["a": [], "b": 123]');
+        });
+    });
+    (0, vitest_1.describe)("object", () => {
+        (0, vitest_1.test)("should handle keys without values", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"key":')).toBe("{}");
+        });
+        (0, vitest_1.test)("should handle closing brace in object", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"a": {"b": 1}')).toBe('{"a": {"b": 1}}');
+        });
+        (0, vitest_1.test)("should handle partial keys (first key)", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"ke')).toBe("{}");
+        });
+        (0, vitest_1.test)("should handle partial keys (second key)", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"k1": 1, "k2')).toBe('{"k1": 1}');
+        });
+        (0, vitest_1.test)("should handle partial keys with colon (second key)", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"k1": 1, "k2":')).toBe('{"k1": 1}');
+        });
+        (0, vitest_1.test)("should handle trailing whitespaces", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"key": "value"  ')).toBe('{"key": "value"}');
+        });
+    });
+    (0, vitest_1.describe)("nesting", () => {
+        (0, vitest_1.test)("should handle nested arrays with numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("[1, [2, 3, [")).toBe("[1, [2, 3, []]]");
+        });
+        (0, vitest_1.test)("should handle nested arrays with literals", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)("[false, [true, [")).toBe("[false, [true, []]]");
+        });
+        (0, vitest_1.test)("should handle nested objects", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"key": {"subKey":')).toBe('{"key": {}}');
+        });
+        (0, vitest_1.test)("should handle nested objects with numbers", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"key": 123, "key2": {"subKey":')).toBe('{"key": 123, "key2": {}}');
+        });
+        (0, vitest_1.test)("should handle nested objects with literals", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"key": null, "key2": {"subKey":')).toBe('{"key": null, "key2": {}}');
+        });
+        (0, vitest_1.test)("should handle arrays within objects", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"key": [1, 2, {')).toBe('{"key": [1, 2, {}]}');
+        });
+        (0, vitest_1.test)("should handle objects within arrays", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('[1, 2, {"key": "value",')).toBe('[1, 2, {"key": "value"}]');
+        });
+        (0, vitest_1.test)("should handle nested arrays and objects", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"a": {"b": ["c", {"d": "e",')).toBe('{"a": {"b": ["c", {"d": "e"}]}}');
+        });
+        (0, vitest_1.test)("should handle deeply nested structures", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"a": {"b": {"c": {"d":')).toBe('{"a": {"b": {"c": {}}}}');
+        });
+        (0, vitest_1.test)("should handle potential nested arrays or objects", () => {
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"a": 1, "b": [')).toBe('{"a": 1, "b": []}');
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"a": 1, "b": {')).toBe('{"a": 1, "b": {}}');
+            (0, vitest_1.expect)((0, fixJson_1.fixJson)('{"a": 1, "b": "')).toBe('{"a": 1, "b": ""}');
+        });
+    });
+});

package/model-function/generate-structure/fixJson.test.d.ts

@@ -0,0 +1 @@
+export {};