@llumiverse/core 0.22.0 → 0.22.2

package/src/json.ts

@@ -1,181 +1,16 @@
 import { JSONValue } from "@llumiverse/common";
+import { jsonrepair } from 'jsonrepair';
 
 function extractJsonFromText(text: string): string {
     const start = text.indexOf("{");
     const end = text.lastIndexOf("}");
-    text = text.substring(start, end + 1);
-    return text.replace(/\\n/g, "");
+    return text.substring(start, end + 1);
 }
 
 export function extractAndParseJSON(text: string): JSONValue {
     return parseJSON(extractJsonFromText(text));
 }
 
-const RX_DQUOTE = /^"([^"\\]|\\.)*"/us;
-const RX_SQUOTE = /^'([^'\\]|\\.)*'/us;
-const RX_NUMBER = /^-?\d+(\.\d+)?/;
-const RX_BOOLEAN = /^true|false/;
-const RX_NULL = /^null/;
-const RX_KEY = /^[$_a-zA-Z][$_a-zA-Z0-9]*/;
-const RX_PUNCTUATION = /^\s*([\[\]{}:,])\s*/;
-
-function fixText(value: string) {
-    return value.replaceAll('\n', '\\n').replaceAll('\r', '\\r');
-}
-
-function decodeSingleQuotedString(value: string) {
-    return JSON.parse('"' + value.slice(1, -1).replaceAll(/(?<!\\)"/g, '\\"') + '"');
-}
-
-export class JsonParser {
-    pos: number = 0;
-
-    constructor(public text: string) { }
-
-    skip(n: number) {
-        this.text = this.text.substring(n);
-        this.pos += n;
-    }
-
-    tryReadPunctuation() {
-        const m = RX_PUNCTUATION.exec(this.text);
-        if (m) {
-            this.skip(m[0].length);
-            return m[1];
-        }
-    }
-
-    readKey() {
-        const first = this.text.charCodeAt(0);
-        if (first === 34) { // "
-            const m = RX_DQUOTE.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return JSON.parse(m[0]);
-            }
-        } else if (first === 39) { // '
-            const m = RX_SQUOTE.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return decodeSingleQuotedString(m[0]);
-            }
-        } else {
-            const m = RX_KEY.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return m[0];
-            }
-        }
-        throw new Error('Expected a key at position ' + this.pos + ' but found ' + this.text);
-    }
-
-    readScalar() {
-        const first = this.text.charCodeAt(0);
-        if (first === 34) { // "
-            const m = RX_DQUOTE.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return JSON.parse(fixText(m[0]));
-            }
-        } else if (first === 39) { // '
-            const m = RX_SQUOTE.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return decodeSingleQuotedString(fixText(m[0]));
-            }
-        } else {
-            let m = RX_NUMBER.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return parseFloat(m[0]);
-            }
-            m = RX_BOOLEAN.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return m[0] === 'true';
-            }
-            m = RX_NULL.exec(this.text);
-            if (m) {
-                this.skip(m[0].length);
-                return null;
-            }
-        }
-        throw new Error('Expected a value at position ' + this.pos + ' but found ' + this.text);
-    }
-
-    readObject() {
-        let key: string | undefined;
-        const obj: any = {};
-        while (true) {
-            if (!key) { // read key
-                const p = this.tryReadPunctuation();
-                if (p === '}') {
-                    return obj;
-                } else if (p === ',') {
-                    continue;
-                } else if (p) {
-                    throw new Error('Expected a key at position ' + this.pos + ' but found ' + this.text);
-                }
-                key = this.readKey();
-                if (!key) {
-                    throw new Error('Expected a key at position ' + this.pos + ' but found ' + this.text);
-                }
-                if (this.tryReadPunctuation() !== ':') {
-                    throw new Error('Expected a colon at position ' + this.pos + ' but found ' + this.text);
-                };
-            } else { // read value
-                const value = this.readValue();
-                if (value === undefined) {
-                    throw new Error('Expected a value at position ' + this.pos + ' but found ' + this.text);
-                }
-                obj[key] = value;
-                key = undefined;
-            }
-        }
-    }
-
-    readArray() {
-        const ar: any[] = [];
-        while (true) {
-            const p = this.tryReadPunctuation();
-            if (p === ',') {
-                continue;
-            } else if (p === ']') {
-                return ar;
-            } else if (p === '[') {
-                ar.push(this.readArray());
-            } else if (p === '{') {
-                ar.push(this.readObject());
-            } else if (!p) {
-                ar.push(this.readScalar());
-            } else {
-                throw new Error('Expected a value at position ' + this.pos + ' but found ' + this.text);
-            }
-        }
-    }
-
-    readValue() {
-        const p = this.tryReadPunctuation();
-        if (p === '{') {
-            return this.readObject();
-        } else if (p === '[') {
-            return this.readArray();
-        } else if (!p) {
-            return this.readScalar();
-        }
-    }
-
-    static parse(text: string) {
-        const parser = new JsonParser(text);
-        const r = parser.readValue();
-        if (r === undefined) {
-            throw new Error('Not a valid JSON');
-        }
-        return r;
-    }
-}
-
-
 export function parseJSON(text: string): JSONValue {
     text = text.trim();
     try {
@@ -183,7 +18,7 @@ export function parseJSON(text: string): JSONValue {
     } catch (err: any) {
         // use a relaxed parser
         try {
-            return JsonParser.parse(text);
+            return JSON.parse(jsonrepair(text));
         } catch (err2: any) { // throw the original error
             throw err;
         }

package/src/stream.ts

@@ -1,22 +1,30 @@
 
 export async function readStreamAsBase64(stream: ReadableStream): Promise<string> {
-    return (await _readStreamAsBuffer(stream)).toString('base64');
+    const uint8Array = await readStreamAsUint8Array(stream);
+    return Buffer.from(uint8Array).toString('base64');
 }
 
 export async function readStreamAsString(stream: ReadableStream): Promise<string> {
-    return (await _readStreamAsBuffer(stream)).toString();
+    const uint8Array = await readStreamAsUint8Array(stream);
+    return Buffer.from(uint8Array).toString();
 }
 
 export async function readStreamAsUint8Array(stream: ReadableStream): Promise<Uint8Array> {
-    // We return a Uint8Array for strict type checking, even though the buffer extends Uint8Array.
-    const buffer = await _readStreamAsBuffer(stream);
-    return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
-}
-
-async function _readStreamAsBuffer(stream: ReadableStream): Promise<Buffer> {
-    const out: Buffer[] = [];
+    const chunks: Uint8Array[] = [];
+    let totalLength = 0;
+    
     for await (const chunk of stream) {
-        out.push(Buffer.from(chunk));
+        const uint8Chunk = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);
+        chunks.push(uint8Chunk);
+        totalLength += uint8Chunk.length;
+    }
+    
+    const combined = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        combined.set(chunk, offset);
+        offset += chunk.length;
     }
-    return Buffer.concat(out);
+    
+    return combined;
 }

package/src/validation.ts

@@ -1,8 +1,8 @@
+import { CompletionResult, ResultValidationError } from "@llumiverse/common";
 import { Ajv } from 'ajv';
 import addFormats from 'ajv-formats';
 import { extractAndParseJSON } from "./json.js";
 import { resolveField } from './resolver.js';
-import { ResultValidationError } from "@llumiverse/common";
 
 
 const ajv = new Ajv({
@@ -28,17 +28,40 @@ export class ValidationError extends Error implements ResultValidationError {
     }
 }
 
-export function validateResult(data: any, schema: Object) {
-    let json;
+function parseCompletionAsJson(data: CompletionResult[]) {
+    let lastError: ValidationError | undefined;
+    for (const part of data) {
+        if (part.type === "text") {
+            const text = part.value.trim();
+            try {
+                return extractAndParseJSON(text);
+            } catch (error: any) {
+                lastError = new ValidationError("json_error", error.message);
+            }
+        }
+    }
+    if (!lastError) {
+        lastError = new ValidationError("json_error", "No JSON compatible response found in completion result");
+    }
+    throw lastError;
+}
+
 
-    if (typeof data === "string") {
-        try {
-            json = extractAndParseJSON(data);
-        } catch (error: any) {
-            throw new ValidationError("json_error", error.message)
+export function validateResult(data: CompletionResult[], schema: Object): CompletionResult[] {
+    let json;
+    if (Array.isArray(data)) {
+        const jsonResults = data.filter(r => r.type === "json");
+        if (jsonResults.length > 0) {
+            json = jsonResults[0].value;
+        } else {
+            try {
+                json = parseCompletionAsJson(data);
+            } catch (error: any) {
+                throw new ValidationError("json_error", error.message)
+            }
         }
     } else {
-        json = data;
+        throw new Error("Data to validate must be an array")
     }
 
     const validate = ajv.compile(schema);
@@ -71,5 +94,5 @@ export function validateResult(data: any, schema: Object) {
         }
     }
 
-    return json;
+    return [{ type: "json", value: json }];
 }