@elaraai/east 0.0.1-beta.0

package/dist/src/serialization/beast2-stream.js

@@ -0,0 +1,665 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+import {} from "../types.js";
+import { isVariant, variant } from "../containers/variant.js";
+import { BufferWriter, StreamBufferReader } from "./binary-utils.js";
+import { encodeBeast2ValueToBufferFor, MAGIC_BYTES } from "./beast2.js";
+import { EastTypeValueType, toEastTypeValue } from "../type_of_type.js";
+import { equalFor } from "../comparison.js";
+import { printFor } from "./east.js";
+import { ref } from "../containers/ref.js";
+const printTypeValue = printFor(EastTypeValueType);
+const isTypeValueEqual = equalFor(EastTypeValueType);
+// Chunk size for streaming (64KB - good balance of memory vs throughput)
+const CHUNK_SIZE = 64 * 1024;
+/**
+ * Determines if a type can be encoded synchronously without needing to yield.
+ * Sync types are small and bounded - they can't cause excessive buffer growth.
+ */
+function isSyncType(type) {
+    switch (type.type) {
+        case "Never":
+        case "Null":
+        case "Boolean":
+        case "Integer":
+        case "Float":
+        case "DateTime":
+            return true;
+        case "String":
+        case "Blob":
+        case "Ref":
+        case "Array":
+        case "Set":
+        case "Dict":
+            // These can be large
+            return false;
+        case "Struct":
+            // Sync only if all fields are sync
+            return type.value.every(({ type }) => isSyncType(type));
+        case "Variant":
+            // Sync only if all cases are sync
+            return type.value.every(({ type }) => isSyncType(type));
+        case "Recursive":
+            return false;
+        case "Function":
+            return true; // Will throw, but no need to yield
+        default:
+            return false;
+    }
+}
+function encodeBeast2ValueToStreamFor(type, typeCtx = []) {
+    // Convert EastType to EastTypeValue if necessary
+    if (!isVariant(type)) {
+        type = toEastTypeValue(type);
+    }
+    // For fully sync types, use the sync encoder directly
+    if (isSyncType(type)) {
+        const syncEncoder = encodeBeast2ValueToBufferFor(type);
+        return function* (value, writer, ctx) {
+            syncEncoder(value, writer, ctx);
+        };
+    }
+    // Async types - need generator logic
+    if (type.type === "String") {
+        return function* (value, writer, _ctx) {
+            writer.writeStringUtf8Varint(value);
+            if (writer.size >= CHUNK_SIZE) {
+                yield writer.pop();
+            }
+        };
+    }
+    else if (type.type === "Blob") {
+        return function* (value, writer, _ctx) {
+            // Write length prefix
+            writer.writeVarint(value.length);
+            // Check if we should chunk the blob
+            if (value.length > CHUNK_SIZE) {
+                // Large blob - flush buffer first, then send blob in chunks
+                if (writer.size > 0) {
+                    yield writer.pop();
+                }
+                let offset = 0;
+                while (offset < value.length) {
+                    const chunkSize = Math.min(CHUNK_SIZE, value.length - offset);
+                    const chunk = value.subarray(offset, offset + chunkSize);
+                    writer.writeBytes(chunk);
+                    yield writer.pop();
+                    offset += chunkSize;
+                }
+            }
+            else {
+                // Small blob - write it all
+                writer.writeBytes(value);
+                if (writer.size >= CHUNK_SIZE) {
+                    yield writer.pop();
+                }
+            }
+        };
+    }
+    else if (type.type === "Ref") {
+        // Branch on whether value type is sync
+        if (isSyncType(type.value)) {
+            const syncValueEncoder = encodeBeast2ValueToBufferFor(type.value);
+            return function* (value, writer, ctx = { refs: new Map() }) {
+                // Check for backreference
+                if (ctx.refs.has(value)) {
+                    const offset = ctx.refs.get(value);
+                    writer.writeVarint(writer.currentOffset - offset);
+                    return;
+                }
+                // Write inline marker and register
+                writer.writeVarint(0);
+                ctx.refs.set(value, writer.currentOffset);
+                syncValueEncoder(value.value, writer, ctx);
+            };
+        }
+        else {
+            // Use late binding for async value type
+            let valueGen;
+            const ret = function* (value, writer, ctx = { refs: new Map() }) {
+                // Check for backreference
+                if (ctx.refs.has(value)) {
+                    const offset = ctx.refs.get(value);
+                    writer.writeVarint(writer.currentOffset - offset);
+                    return;
+                }
+                // Write inline marker and register
+                writer.writeVarint(0);
+                ctx.refs.set(value, writer.currentOffset);
+                yield* valueGen(value.value, writer, ctx);
+            };
+            typeCtx.push(ret);
+            valueGen = encodeBeast2ValueToStreamFor(type.value, typeCtx);
+            typeCtx.pop();
+            return ret;
+        }
+    }
+    else if (type.type === "Array") {
+        // Branch on whether value type is sync
+        if (isSyncType(type.value)) {
+            const syncValueEncoder = encodeBeast2ValueToBufferFor(type.value);
+            return function* (value, writer, ctx = { refs: new Map() }) {
+                // Check for backreference
+                if (ctx.refs.has(value)) {
+                    const offset = ctx.refs.get(value);
+                    writer.writeVarint(writer.currentOffset - offset);
+                    return;
+                }
+                // Write inline marker and register
+                writer.writeVarint(0);
+                ctx.refs.set(value, writer.currentOffset);
+                // Encode length
+                writer.writeVarint(value.length);
+                for (const item of value) {
+                    syncValueEncoder(item, writer, ctx);
+                    if (writer.size >= CHUNK_SIZE) {
+                        yield writer.pop();
+                    }
+                }
+            };
+        }
+        else {
+            // Use late binding for async value type
+            let valueGen;
+            const ret = function* (value, writer, ctx = { refs: new Map() }) {
+                // Check for backreference
+                if (ctx.refs.has(value)) {
+                    const offset = ctx.refs.get(value);
+                    writer.writeVarint(writer.currentOffset - offset);
+                    return;
+                }
+                // Write inline marker and register
+                writer.writeVarint(0);
+                ctx.refs.set(value, writer.currentOffset);
+                // Encode length
+                writer.writeVarint(value.length);
+                for (const item of value) {
+                    yield* valueGen(item, writer, ctx);
+                    if (writer.size >= CHUNK_SIZE) {
+                        yield writer.pop();
+                    }
+                }
+            };
+            typeCtx.push(ret);
+            valueGen = encodeBeast2ValueToStreamFor(type.value, typeCtx);
+            typeCtx.pop();
+            return ret;
+        }
+    }
+    else if (type.type === "Set") {
+        // Keys are assumed small - always use sync
+        const syncKeyEncoder = encodeBeast2ValueToBufferFor(type.value);
+        return function* (value, writer, ctx = { refs: new Map() }) {
+            // Check for backreference
+            if (ctx.refs.has(value)) {
+                const offset = ctx.refs.get(value);
+                writer.writeVarint(writer.currentOffset - offset);
+                return;
+            }
+            // Write inline marker and register
+            writer.writeVarint(0);
+            ctx.refs.set(value, writer.currentOffset);
+            // Encode length
+            writer.writeVarint(value.size);
+            for (const key of value) {
+                syncKeyEncoder(key, writer, ctx);
+                if (writer.size >= CHUNK_SIZE) {
+                    yield writer.pop();
+                }
+            }
+        };
+    }
+    else if (type.type === "Dict") {
+        // Keys are assumed small - always use sync. Check value type.
+        const syncKeyEncoder = encodeBeast2ValueToBufferFor(type.value.key);
+        if (isSyncType(type.value.value)) {
+            const syncValueEncoder = encodeBeast2ValueToBufferFor(type.value.value);
+            return function* (value, writer, ctx = { refs: new Map() }) {
+                // Check for backreference
+                if (ctx.refs.has(value)) {
+                    const offset = ctx.refs.get(value);
+                    writer.writeVarint(writer.currentOffset - offset);
+                    return;
+                }
+                // Write inline marker and register
+                writer.writeVarint(0);
+                ctx.refs.set(value, writer.currentOffset);
+                // Encode length
+                writer.writeVarint(value.size);
+                for (const [k, v] of value) {
+                    syncKeyEncoder(k, writer, ctx);
+                    syncValueEncoder(v, writer, ctx);
+                    if (writer.size >= CHUNK_SIZE) {
+                        yield writer.pop();
+                    }
+                }
+            };
+        }
+        else {
+            // Use late binding for async value type
+            let valueGen;
+            const ret = function* (value, writer, ctx = { refs: new Map() }) {
+                // Check for backreference
+                if (ctx.refs.has(value)) {
+                    const offset = ctx.refs.get(value);
+                    writer.writeVarint(writer.currentOffset - offset);
+                    return;
+                }
+                // Write inline marker and register
+                writer.writeVarint(0);
+                ctx.refs.set(value, writer.currentOffset);
+                // Encode length
+                writer.writeVarint(value.size);
+                for (const [k, v] of value) {
+                    syncKeyEncoder(k, writer, ctx);
+                    yield* valueGen(v, writer, ctx);
+                    if (writer.size >= CHUNK_SIZE) {
+                        yield writer.pop();
+                    }
+                }
+            };
+            typeCtx.push(ret);
+            valueGen = encodeBeast2ValueToStreamFor(type.value.value, typeCtx);
+            typeCtx.pop();
+            return ret;
+        }
+    }
+    else if (type.type === "Struct") {
+        // Annotate each field with sync flag and appropriate encoder
+        const fieldInfo = [];
+        const ret = function* (value, writer, _ctx) {
+            // Struct is immutable - no backreference checking needed
+            // Encode fields directly
+            for (const { name, isSync, encoder } of fieldInfo) {
+                if (isSync) {
+                    encoder(value[name], writer);
+                }
+                else {
+                    yield* encoder(value[name], writer);
+                }
+            }
+        };
+        typeCtx.push(ret);
+        for (const { name, type: fieldType } of type.value)
+            fieldInfo.push({
+                name,
+                isSync: isSyncType(fieldType),
+                encoder: isSyncType(fieldType)
+                    ? encodeBeast2ValueToBufferFor(fieldType)
+                    : encodeBeast2ValueToStreamFor(fieldType, typeCtx),
+            });
+        typeCtx.pop();
+        return ret;
+    }
+    else if (type.type === "Variant") {
+        // Annotate each case with sync flag and appropriate encoder
+        const caseInfo = {};
+        const ret = function* (value, writer, _ctx) {
+            // Variant is immutable - no backreference checking needed
+            // Encode tag and value directly
+            const tag = value.type;
+            const { tagIndex, isSync, encoder } = caseInfo[tag]; // Assume valid input
+            writer.writeVarint(tagIndex);
+            if (isSync) {
+                encoder(value.value, writer);
+            }
+            else {
+                yield* encoder(value.value, writer);
+            }
+        };
+        typeCtx.push(ret);
+        let i = 0;
+        for (const { name, type: caseType } of type.value) {
+            caseInfo[name] = {
+                tagIndex: i,
+                isSync: isSyncType(caseType),
+                encoder: isSyncType(caseType)
+                    ? encodeBeast2ValueToBufferFor(caseType)
+                    : encodeBeast2ValueToStreamFor(caseType, typeCtx),
+            };
+            i += 1;
+        }
+        typeCtx.pop();
+        return ret;
+    }
+    else if (type.type === "Recursive") {
+        // Type stack lookup: index from the end
+        const targetEncoder = typeCtx[typeCtx.length - Number(type.value)];
+        if (!targetEncoder) {
+            throw new Error(`Recursive type depth ${type.value} exceeds type context stack size ${typeCtx.length}`);
+        }
+        return targetEncoder;
+    }
+    else {
+        // Should never reach here - all types handled above
+        throw new Error(`Unhandled type: ${type.type}`);
+    }
+}
+const typeEncoder = encodeBeast2ValueToBufferFor(EastTypeValueType);
+export function encodeBeast2ToStreamFor(type) {
+    // Convert EastType to EastTypeValue if necessary
+    if (!isVariant(type)) {
+        type = toEastTypeValue(type);
+    }
+    const valueGen = encodeBeast2ValueToStreamFor(type);
+    return (value) => {
+        let writer;
+        let generator;
+        let ctx;
+        return new ReadableStream({
+            start(controller) {
+                // Initialize writer and write header
+                writer = new BufferWriter();
+                writer.writeBytes(MAGIC_BYTES);
+                // Write type schema
+                typeEncoder(type, writer, { refs: new Map() });
+                // Check if header exceeded chunk size (rare)
+                if (writer.size >= CHUNK_SIZE) {
+                    controller.enqueue(writer.pop());
+                }
+                // Create context for value encoding
+                ctx = { refs: new Map() };
+                // Create value generator
+                generator = valueGen(value, writer, ctx);
+            },
+            pull(controller) {
+                while (true) {
+                    const { done, value: chunk } = generator.next();
+                    if (done) {
+                        // Flush any remaining data
+                        if (writer.size > 0) {
+                            controller.enqueue(writer.pop());
+                        }
+                        controller.close();
+                        return;
+                    }
+                    if (chunk !== undefined) {
+                        controller.enqueue(chunk);
+                        return; // Backpressure - wait for next pull()
+                    }
+                    // chunk is void from yield* delegation - keep looping
+                }
+            },
+        });
+    };
+}
+/**
+ * Create a streaming decoder function for a given Beast v2 type.
+ * Factory pattern: analyzes type once, returns async decoder.
+ */
+function decodeBeast2ValueFromStreamFor(type, typeCtx = []) {
+    if (type.type === "Never") {
+        return (_reader, _ctx) => { throw new Error(`Cannot decode Never type`); };
+    }
+    else if (type.type === "Null") {
+        return (_reader, _ctx) => null;
+    }
+    else if (type.type === "Boolean") {
+        return async (reader, _ctx) => {
+            const byte = await reader.readByte();
+            return byte !== 0;
+        };
+    }
+    else if (type.type === "Integer") {
+        return (reader, _ctx) => reader.readZigzag();
+    }
+    else if (type.type === "Float") {
+        return (reader, _ctx) => reader.readFloat64LE();
+    }
+    else if (type.type === "String") {
+        return (reader, _ctx) => reader.readStringUtf8Varint();
+    }
+    else if (type.type === "DateTime") {
+        return (reader, _ctx) => {
+            const millis = reader.readZigzag();
+            if (millis instanceof Promise) {
+                return millis.then(m => new Date(Number(m)));
+            }
+            return new Date(Number(millis));
+        };
+    }
+    else if (type.type === "Blob") {
+        return async (reader, _ctx) => {
+            const length = await reader.readVarint();
+            const blob = new Uint8Array(length);
+            await reader.readBytes(blob);
+            return blob;
+        };
+    }
+    else if (type.type === "Ref") {
+        // Use late binding for element decoder
+        let elemDecoder;
+        const ret = async (reader, ctx = { refs: new Map() }) => {
+            const startOffset = reader.position;
+            const refOrZero = await reader.readVarint();
+            // Check if this is a backreference
+            if (refOrZero > 0) {
+                const targetOffset = startOffset - refOrZero;
+                if (!ctx.refs.has(targetOffset)) {
+                    throw new Error(`Undefined backreference at offset ${startOffset}, target ${targetOffset}`);
+                }
+                return ctx.refs.get(targetOffset);
+            }
+            // Inline ref
+            const result = ref(undefined);
+            ctx.refs.set(startOffset, result);
+            result.value = await elemDecoder(reader, ctx);
+            return result;
+        };
+        typeCtx.push(ret);
+        elemDecoder = decodeBeast2ValueFromStreamFor(type.value, typeCtx);
+        typeCtx.pop();
+        return ret;
+    }
+    else if (type.type === "Array") {
+        // Use late binding for element decoder
+        let elemDecoder;
+        const ret = async (reader, ctx = { refs: new Map() }) => {
+            const startOffset = reader.position;
+            const refOrZero = await reader.readVarint();
+            // Check if this is a backreference
+            if (refOrZero > 0) {
+                const targetOffset = startOffset - refOrZero;
+                if (!ctx.refs.has(targetOffset)) {
+                    throw new Error(`Undefined backreference at offset ${startOffset}, target ${targetOffset}`);
+                }
+                return ctx.refs.get(targetOffset);
+            }
+            // Inline array
+            const result = [];
+            ctx.refs.set(startOffset, result);
+            const length = await reader.readVarint();
+            for (let i = 0; i < length; i++) {
+                result.push(await elemDecoder(reader, ctx));
+            }
+            return result;
+        };
+        typeCtx.push(ret);
+        elemDecoder = decodeBeast2ValueFromStreamFor(type.value, typeCtx);
+        typeCtx.pop();
+        return ret;
+    }
+    else if (type.type === "Set") {
+        // Keys are standalone - no need for late binding
+        const keyDecoder = decodeBeast2ValueFromStreamFor(type.value, []);
+        return async (reader, ctx = { refs: new Map() }) => {
+            const startOffset = reader.position;
+            const refOrZero = await reader.readVarint();
+            // Check if this is a backreference
+            if (refOrZero > 0) {
+                const targetOffset = startOffset - refOrZero;
+                if (!ctx.refs.has(targetOffset)) {
+                    throw new Error(`Undefined backreference at offset ${startOffset}, target ${targetOffset}`);
+                }
+                return ctx.refs.get(targetOffset);
+            }
+            // Inline set
+            const result = new Set();
+            ctx.refs.set(startOffset, result);
+            const length = await reader.readVarint();
+            for (let i = 0; i < length; i++) {
+                result.add(await keyDecoder(reader));
+            }
+            return result;
+        };
+    }
+    else if (type.type === "Dict") {
+        // Keys are standalone, values use late binding
+        const keyDecoder = decodeBeast2ValueFromStreamFor(type.value.key, []);
+        let valueDecoder;
+        const ret = async (reader, ctx = { refs: new Map() }) => {
+            const startOffset = reader.position;
+            const refOrZero = await reader.readVarint();
+            // Check if this is a backreference
+            if (refOrZero > 0) {
+                const targetOffset = startOffset - refOrZero;
+                if (!ctx.refs.has(targetOffset)) {
+                    throw new Error(`Undefined backreference at offset ${startOffset}, target ${targetOffset}`);
+                }
+                return ctx.refs.get(targetOffset);
+            }
+            // Inline dict
+            const result = new Map();
+            ctx.refs.set(startOffset, result);
+            const length = await reader.readVarint();
+            for (let i = 0; i < length; i++) {
+                const key = await keyDecoder(reader);
+                const value = await valueDecoder(reader, ctx);
+                result.set(key, value);
+            }
+            return result;
+        };
+        typeCtx.push(ret);
+        valueDecoder = decodeBeast2ValueFromStreamFor(type.value.value, typeCtx);
+        typeCtx.pop();
+        return ret;
+    }
+    else if (type.type === "Struct") {
+        // Struct is immutable - fields use typeCtx for recursive types
+        let fieldDecoders = [];
+        const ret = async (reader, _ctx) => {
+            // Struct is immutable - no backreference checking needed
+            const result = {};
+            for (const [key, decoder] of fieldDecoders) {
+                result[key] = await decoder(reader);
+            }
+            return result;
+        };
+        typeCtx.push(ret);
+        for (const { name, type: fieldType } of type.value) {
+            fieldDecoders.push([name, decodeBeast2ValueFromStreamFor(fieldType, typeCtx)]);
+        }
+        ;
+        typeCtx.pop();
+        return ret;
+    }
+    else if (type.type === "Variant") {
+        // Variant is immutable - cases use typeCtx for recursive types
+        let caseDecoders = [];
+        const ret = async (reader, _ctx) => {
+            // Variant is immutable - no backreference checking needed
+            const tagIndex = await reader.readVarint();
+            if (tagIndex >= caseDecoders.length) {
+                throw new Error(`Invalid variant tag ${tagIndex}`);
+            }
+            const [caseName, caseDecoder] = caseDecoders[tagIndex];
+            const value = await caseDecoder(reader);
+            return variant(caseName, value);
+        };
+        typeCtx.push(ret);
+        for (const { name, type: caseType } of type.value) {
+            caseDecoders.push([name, decodeBeast2ValueFromStreamFor(caseType, typeCtx)]);
+        }
+        typeCtx.pop();
+        return ret;
+    }
+    else if (type.type === "Recursive") {
+        // Type stack lookup: index from the end
+        const targetDecoder = typeCtx[typeCtx.length - Number(type.value)];
+        if (!targetDecoder) {
+            throw new Error(`Recursive type depth ${type.value} exceeds type context stack size ${typeCtx.length}`);
+        }
+        return targetDecoder;
+    }
+    else if (type.type === "Function") {
+        return (_reader, _ctx) => { throw new Error(`Functions cannot be deserialized`); };
+    }
+    else {
+        throw new Error(`Unhandled type ${type.type}`);
+    }
+}
+/**
+ * Internal helper to decode a value from a stream.
+ * Uses the streaming decoder factory with context threading.
+ */
+async function decodeBeast2ValueFromStream(type, reader) {
+    const decoder = decodeBeast2ValueFromStreamFor(type);
+    const ctx = { refs: new Map() };
+    return await decoder(reader, ctx);
+}
+const streamingTypeDecode = decodeBeast2ValueFromStreamFor(EastTypeValueType);
+/**
+ * Decode a Beast v2 format stream to an East value.
+ * Reads the type schema from the stream and returns both type and value.
+ *
+ * @param stream The input stream to decode
+ * @returns Promise resolving to { type, value }
+ */
+export async function decodeBeast2FromStream(stream) {
+    const reader = new StreamBufferReader(stream);
+    try {
+        // Verify magic bytes
+        for (let i = 0; i < MAGIC_BYTES.length; i++) {
+            const byte = await reader.readByte();
+            if (byte !== MAGIC_BYTES[i]) {
+                throw new Error(`Invalid Beast v2 magic bytes at offset ${i}: expected 0x${MAGIC_BYTES[i].toString(16)}, got 0x${byte.toString(16)}`);
+            }
+        }
+        // Decode type schema directly from stream
+        const type = await streamingTypeDecode(reader, { refs: new Map() });
+        // Decode value
+        const value = await decodeBeast2ValueFromStream(type, reader);
+        return { type, value };
+    }
+    finally {
+        await reader.release();
+    }
+}
+/**
+ * Create a streaming decoder for a specific Beast v2 type.
+ * Factory pattern: the type is verified once, returns async decoder.
+ *
+ * @param type The expected type to decode
+ * @returns A function that takes a stream and returns a Promise of the decoded value
+ */
+export function decodeBeast2FromStreamFor(type) {
+    const valueDecoder = decodeBeast2ValueFromStreamFor(type);
+    return async (stream) => {
+        const reader = new StreamBufferReader(stream);
+        try {
+            // Verify magic bytes
+            for (let i = 0; i < MAGIC_BYTES.length; i++) {
+                const byte = await reader.readByte();
+                if (byte !== MAGIC_BYTES[i]) {
+                    throw new Error(`Invalid Beast v2 magic bytes at offset ${i}: expected 0x${MAGIC_BYTES[i].toString(16)}, got 0x${byte.toString(16)}`);
+                }
+            }
+            // Decode type schema directly from stream
+            const decodedType = await streamingTypeDecode(reader, { refs: new Map() });
+            // Verify type matches expected type
+            if (!isTypeValueEqual(decodedType, type)) {
+                throw new Error(`Type mismatch: expected ${printTypeValue(type)}, got ${printTypeValue(decodedType)}`);
+            }
+            // Decode value with context
+            const ctx = { refs: new Map() };
+            const value = await valueDecoder(reader, ctx);
+            return value;
+        }
+        finally {
+            await reader.release();
+        }
+    };
+}
+//# sourceMappingURL=beast2-stream.js.map