@nnilky/structo 1.0.1 → 1.0.3

package/README.md

@@ -1 +1,131 @@
 # Structo
+
+Define binary objects in zod style schemas
+
+```ts
+import * as st from "@nnilky/structo";
+
+type Vec3 = st.InferOutput<typeof Vec3>
+const Vec3 = st.object({
+    x: st.f64(),
+    y: st.f64(),
+    z: st.f64(),
+});
+
+type Entity = st.InferOutput<typeof Entity>
+const Entity = st.object({
+    id: st.u64(),
+    position: Vec3,
+});
+
+```
+
+- Lightweight, base size is <1KB and each datatype is a few hundred bytes
+- Fast, from [benchmarks](./benchmark) only 2-
+- Supports Web/Node.js compatible
+- Easily implemented
+
+Each serializer is completely seperate from the base library, meaning you only pay for what you use.
+
+## Implementing your own Serializer
+
+Implementing your own serializer is incredibly simple, heres the `f64` serializer for example
+
+```ts
+export function f64(endian: "little" | "big" = "little"): st.Serializer<number> {
+    return {
+        size: 8,
+        write: (ctx, value) => {
+            ctx.alloc(8);
+            ctx.view.setFloat64(ctx.offset, value, endian === "little");
+            ctx.offset += 8;
+        },
+        read: (ctx) => {
+            const value = ctx.view.getFloat64(ctx.offset, endian === "little");
+            ctx.offset += 8;
+            return value;
+        },
+    };
+}
+```
+
+The `alloc()` in the write function is a utility provided to you to ensure you have enough space ahead of you in the buffer to write to.
+
+When reading and writing, you must increment the `ctx.offset` so the next value can be read from. However,
+
+The `size` attribute is optional, but if included can be used to do allocations in advance.
+
+> Important: You must modify the ctx object, do not spread over it since this will break the reference
+
+### Container Example
+
+Here is the list serializer
+
+```ts
+export function list<T>(options: {
+    type: st.Serializer<T>;
+    length: st.Serializer<number>;
+}): st.Serializer<T[]> {
+    const lengthType = options.length;
+    const valueType = options.type;
+
+    return {
+        write: (ctx, value) => {
+            lengthType.write(ctx, value.length);
+            for (const v of value) {
+                lengthType.write(ctx, v);
+            }
+        },
+        read: (ctx) => {
+            const size = lengthType.read(ctx);
+
+            const arr = []
+            for (let i = 0; i < size; i++) {
+                arr.push(valueType.read(ctx))
+            }
+            return arr;
+        },
+    };
+}
+```
+
+## Transforms
+
+```ts
+const Bits = st.pipe(
+    st.u32()
+    st.transform(v => v * 8)
+)
+```
+
+Transforms are utility that let you modify a read/written value. They allow you to processing declaratively.
+
+### Writing your Own
+
+Here is the `transform` transformation function, a `transform` just a function that takes a type and returns a serializer.
+
+```ts
+export function transform<T>(callback: (value: T) => T) {
+    return (type: st.Serializer<T>): st.Serializer<T> => ({
+        size: type.size,
+        read: (ctx) => {
+            const value = type.read(ctx);
+            return callback(value);
+        },
+        write: (ctx, value) => {
+            let outValue = callback(value);
+            type.write(ctx, outValue);
+        },
+    });
+}
+```
+
+## FaQ
+
+- Why do I have to do `import * as st from "@nnilky/structo"`?
+  - By using an `* as` import, bundles can erase the names as runtime without having to worry about runtime effects
+  - additionally, by putting it under a namespace it reduces con
+- Will data streaming be implemented?
+  - Sadly this library isn't built for data streaming, the exposed API for serializers need the full array to be accessible
+  - If you want data streaming, your going to have to fork this and re-implement the standard types
+- What's the difference`

package/dist/datatypes/containers/array.d.ts

@@ -0,0 +1,2 @@
+import type { Serializer } from "../../types";
+export declare function array<T>(size: number, type: Serializer<T>): Serializer<T[]>;

package/dist/datatypes/containers/array.js

@@ -0,0 +1,22 @@
+export function array(size, type) {
+    const { read: readType, write: writeType, size: typeSize } = type;
+    return {
+        size: type.size ? size * type.size : undefined,
+        write: (ctx, value) => {
+            if (value.length !== size)
+                throw new Error("Invalid Size");
+            if (typeSize)
+                ctx.alloc(size * typeSize);
+            for (let i = 0; i < size; i++) {
+                writeType(ctx, value[i]);
+            }
+        },
+        read: (ctx) => {
+            const arr = new Array(size);
+            for (let i = 0; i < size; i++) {
+                arr[i] = readType(ctx);
+            }
+            return arr;
+        },
+    };
+}

package/dist/datatypes/containers/array.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/datatypes/containers/array.test.js

@@ -0,0 +1,22 @@
+import { describe, it } from "bun:test";
+import { expectEncode, expectError } from "../utils.test";
+import * as st from "../../index";
+describe("st.array", () => {
+    it("encode correctly", () => {
+        const spec = st.array(4, st.u16());
+        expectEncode(spec, [1, 2, 3, 4]);
+    });
+    it("works on empty lists", () => {
+        const spec = st.array(0, st.u16());
+        expectEncode(spec, []);
+    });
+    it("throws error on invalid size", () => {
+        const spec = st.array(2, st.u16());
+        expectError(() => st.write(spec, [1]));
+    });
+    it("throws error on invalid value", () => {
+        const spec = st.array(2, st.u16());
+        expectError(() => st.write(spec, [1, -1]));
+        expectError(() => st.write(spec, [2 ** 16 + 1, 0]));
+    });
+});

package/dist/datatypes/containers/fastObject.d.ts

@@ -0,0 +1,19 @@
+import type { InferInput, Serializer } from "../../types";
+type InferObject<T> = T extends Record<string, Serializer<any>> ? {
+    [Key in keyof T]: InferInput<T[Key]>;
+} : never;
+/**
+ * `fastObject` is equivelent to object but it uses eval to improve performance
+ *
+ * This means that is should be avoided in scenarios where CSP is required (so can't be the default), but should be fine for all other cases
+ *
+ * ```ts
+ * st.fastObject({
+ *     name: st.string(st.u32()),
+ *     age: st.u8(),
+ *     createdAt: st.f64(),
+ * })
+ * ```
+ */
+export declare function fastObject<T extends Record<string, Serializer<any>>>(definition: T): Serializer<InferObject<T>>;
+export {};

package/dist/datatypes/containers/fastObject.js

@@ -0,0 +1,59 @@
+const definitionSymbol = Symbol();
+/**
+ * `fastObject` is equivelent to object but it uses eval to improve performance
+ *
+ * This means that is should be avoided in scenarios where CSP is required (so can't be the default), but should be fine for all other cases
+ *
+ * ```ts
+ * st.fastObject({
+ *     name: st.string(st.u32()),
+ *     age: st.u8(),
+ *     createdAt: st.f64(),
+ * })
+ * ```
+ */
+export function fastObject(definition) {
+    let serializers = [];
+    let writeBody = "";
+    let readBody = "";
+    function generateSerializers(serializer, keys) {
+        if (definitionSymbol in serializer) {
+            const definition = serializer[definitionSymbol];
+            readBody += `${keys[keys.length - 1]}: {`;
+            for (const key of Object.keys(definition)) {
+                generateSerializers(definition[key], [...keys, key]);
+            }
+            readBody += `},`;
+        }
+        else {
+            let path = "";
+            for (const key of keys) {
+                if (key.match(/[a-zA-Z_][a-zA-Z0-9_]*/)) {
+                    path += `.${key}`;
+                }
+                else {
+                    path += `[${JSON.stringify(key)}]`;
+                }
+            }
+            let name = `s${Object.keys(serializers).length}`;
+            serializers.push([name, serializer]);
+            writeBody += `${name}(c, v${path});`;
+            readBody += `${keys[keys.length - 1]}: ${name}(c),`;
+        }
+    }
+    for (const key of Object.keys(definition)) {
+        generateSerializers(definition[key], [key]);
+    }
+    const writeFactory = new Function(...serializers.map((v) => v[0]), `return (c, v) => {${writeBody}}`);
+    const readFactory = new Function(...serializers.map((v) => v[0]), `return (c, v) => ({${readBody}})`);
+    const write = writeFactory(...serializers.map((v) => v[1].write));
+    const read = readFactory(...serializers.map((v) => v[1].read));
+    let size = Object.values(definition).reduce((total, v) => total + v.size, 0);
+    return {
+        //@ts-ignore
+        [definitionSymbol]: definition,
+        size: isNaN(size) ? undefined : size,
+        write,
+        read,
+    };
+}

package/dist/datatypes/containers/list.d.ts

@@ -0,0 +1,5 @@
+import type { Serializer } from "../../types";
+export declare function list<T>(options: {
+    type: Serializer<T>;
+    length: Serializer<number>;
+}): Serializer<T[]>;

package/dist/datatypes/containers/list.js

@@ -0,0 +1,22 @@
+export function list(options) {
+    const { read: readLength, write: writeLength } = options.length;
+    const { read: readType, write: writeType, size: sizeType } = options.type;
+    return {
+        write: (ctx, value) => {
+            writeLength(ctx, value.length);
+            if (sizeType)
+                ctx.alloc(value.length * sizeType);
+            for (const v of value) {
+                writeType(ctx, v);
+            }
+        },
+        read: (ctx) => {
+            const size = readLength(ctx);
+            const arr = new Array(size);
+            for (let i = 0; i < size; i++) {
+                arr[i] = readType(ctx);
+            }
+            return arr;
+        },
+    };
+}

package/dist/datatypes/containers/list.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/datatypes/containers/list.test.js

@@ -0,0 +1,31 @@
+import { describe, it } from "bun:test";
+import { expectEncode, expectEncodeSnapshot, expectError } from "../utils.test";
+import * as st from "../../index";
+describe("st.list", () => {
+    it("encode correctly", () => {
+        const spec = st.list({ type: st.u16(), length: st.u8() });
+        expectEncode(spec, [1, 2, 3, 4]);
+    });
+    it("works on empty lists", () => {
+        const spec = st.list({ type: st.u16(), length: st.u8() });
+        expectEncode(spec, []);
+    });
+    it("throws error on too large array", () => {
+        const spec = st.list({ type: st.u16(), length: st.u8() });
+        expectError(() => {
+            st.write(spec, Array.from({ length: 10000 }, () => 0));
+        });
+    });
+    it("throws error on invalid value", () => {
+        const spec = st.list({ type: st.u16(), length: st.u8() });
+        expectError(() => {
+            st.write(spec, [-1]);
+        });
+    });
+    it(`matches snapshots`, () => {
+        expectEncodeSnapshot(st.list({
+            type: st.string(st.u16()),
+            length: st.s32(),
+        }), ["foo", "bar", "baz"]);
+    });
+});

package/dist/datatypes/containers/object.d.ts

@@ -0,0 +1,20 @@
+import type { InferInput, InferOutput, Serializer } from "../../types";
+type InferObjectInfer<T> = T extends Record<string, Serializer<any>> ? {
+    [Key in keyof T]: InferInput<T[Key]>;
+} : never;
+type InferObjectOutput<T> = T extends Record<string, Serializer<any>> ? {
+    [Key in keyof T]: InferOutput<T[Key]>;
+} : never;
+/**
+ * `object` is equivelent to a C struct, values are stored in the order defined
+ *
+ * ```ts
+ * st.object({
+ *     name: st.string(st.u32()),
+ *     age: st.u8(),
+ *     createdAt: st.f64(),
+ * })
+ * ```
+ */
+export declare function object<T extends Record<string, Serializer<any>>>(definition: T): Serializer<InferObjectInfer<T>, InferObjectOutput<T>>;
+export {};

package/dist/datatypes/containers/object.js

@@ -0,0 +1,35 @@
+/**
+ * `object` is equivelent to a C struct, values are stored in the order defined
+ *
+ * ```ts
+ * st.object({
+ *     name: st.string(st.u32()),
+ *     age: st.u8(),
+ *     createdAt: st.f64(),
+ * })
+ * ```
+ */
+export function object(definition) {
+    const entires = Object.entries(definition);
+    // Use the fact: number + undefined = NaN
+    // Check for NaN afterwards
+    let computedSize = Object.values(definition).reduce((total, v) => total + v.size, 0);
+    const size = isNaN(computedSize) ? undefined : computedSize;
+    return {
+        size,
+        write: (ctx, value) => {
+            if (size)
+                ctx.alloc(size);
+            for (let i = 0; i < entires.length; i++) {
+                entires[i][1].write(ctx, value[entires[i][0]]);
+            }
+        },
+        read: (ctx) => {
+            const output = new Array(entires.length);
+            for (let i = 0; i < entires.length; i++) {
+                output[i] = [entires[i][0], entires[i][1].read(ctx)];
+            }
+            return Object.fromEntries(output);
+        },
+    };
+}

package/dist/datatypes/containers/object.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/datatypes/containers/object.test.js

@@ -0,0 +1,33 @@
+//@ts-ignore TODO
+import { describe, it, expect } from "bun:test";
+import { bytes, expectEncode, expectEncodeSnapshot } from "../utils.test";
+import * as st from "../../index";
+describe("st.object", () => {
+    const test = st.object({
+        a: st.u8(),
+        b: st.u8(),
+    });
+    it("encodes correctly", () => {
+        st.write(test, { b: 2, a: 1 });
+    });
+    it("encode in correct order", () => {
+        const data = st.write(test, { b: 2, a: 1 });
+        const arr = new Uint8Array(data);
+        expect(arr[0]).toBe(1);
+        expect(arr[1]).toBe(2);
+    });
+    it("encodes empty", () => {
+        expectEncode(st.object({}), {});
+    });
+    it(`matches snapshots`, () => {
+        expectEncodeSnapshot(st.object({
+            number: st.u32(),
+            puppy: st.string(st.s32()),
+            buffer: st.buffer(4),
+        }), {
+            number: 1,
+            puppy: "woof woof bark bark",
+            buffer: bytes([19, 87, 19, 83]),
+        });
+    });
+});

package/dist/datatypes/containers/sizedbuffer.d.ts

@@ -0,0 +1,2 @@
+import type { Serializer } from "../../types";
+export declare function sizedBuffer(length: Serializer<number>): Serializer<ArrayBuffer>;

package/dist/datatypes/containers/sizedbuffer.js

@@ -0,0 +1,17 @@
+export function sizedBuffer(length) {
+    return {
+        write: (ctx, value) => {
+            length.write(ctx, value.byteLength);
+            const bytes = new Uint8Array(value);
+            ctx.alloc(value.byteLength);
+            new Uint8Array(ctx.view.buffer).set(bytes, ctx.offset);
+            ctx.offset += value.byteLength;
+        },
+        read: (ctx) => {
+            const size = length.read(ctx);
+            const slice = ctx.buffer.slice(ctx.offset, ctx.offset + size);
+            ctx.offset += size;
+            return slice;
+        },
+    };
+}

package/dist/datatypes/containers/sizedbuffer.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/datatypes/containers/sizedbuffer.test.js

@@ -0,0 +1,27 @@
+import { describe, it } from "bun:test";
+import { bytes, expectEncode, expectError } from "../utils.test";
+import * as st from "../../index";
+describe("st.buffer", () => {
+    it("encode correctly", () => {
+        const buffer = st.sizedBuffer(st.u8());
+        expectEncode(buffer, bytes([1, 2]));
+    });
+    it("encodes empty correctly", () => {
+        const spec = st.sizedBuffer(st.u8());
+        expectEncode(spec, bytes([]));
+    });
+    it("holds large data", () => {
+        const size = 1024 * 1024 * 8; // 8MB
+        const spec = st.sizedBuffer(st.u32());
+        const data = new Uint8Array(size);
+        data.set([3, 4], 1000);
+        data.set([3, 4], 2000);
+        expectEncode(spec, data.buffer);
+    });
+    it("errors on invalid length", () => {
+        const spec = st.sizedBuffer(st.u8());
+        expectError(() => {
+            st.write(spec, bytes(Array.from({ length: 256 }, () => 0)));
+        });
+    });
+});

package/dist/datatypes/index.d.ts

@@ -0,0 +1,16 @@
+export { u64Bigint, s64Bigint } from "./numbers/bigints";
+export { f32, f64 } from "./numbers/floats";
+export { s8, s16, s32, s64 } from "./numbers/sints";
+export { u8, u16, u32, u64 } from "./numbers/uints";
+export { array } from "./containers/array";
+export { fastObject } from "./containers/fastObject";
+export { object } from "./containers/object";
+export { list } from "./containers/list";
+export { sizedBuffer } from "./containers/sizedbuffer";
+export { string } from "./values/string";
+export { buffer } from "./values/buffer";
+export { byteLiteral } from "./values/byteliteral";
+export { createRememberedValue } from "./utilities/remember";
+export { type Transform as Pipeline, pipe } from "./transforms/pipe";
+export { positionOffset } from "./transforms/readOffset";
+export { transform } from "./transforms/transform";

package/dist/datatypes/index.js

@@ -0,0 +1,16 @@
+export { u64Bigint, s64Bigint } from "./numbers/bigints";
+export { f32, f64 } from "./numbers/floats";
+export { s8, s16, s32, s64 } from "./numbers/sints";
+export { u8, u16, u32, u64 } from "./numbers/uints";
+export { array } from "./containers/array";
+export { fastObject } from "./containers/fastObject";
+export { object } from "./containers/object";
+export { list } from "./containers/list";
+export { sizedBuffer } from "./containers/sizedbuffer";
+export { string } from "./values/string";
+export { buffer } from "./values/buffer";
+export { byteLiteral } from "./values/byteliteral";
+export { createRememberedValue } from "./utilities/remember";
+export { pipe } from "./transforms/pipe";
+export { positionOffset } from "./transforms/readOffset";
+export { transform } from "./transforms/transform";

package/dist/datatypes/lazy.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/datatypes/lazy.js

@@ -0,0 +1 @@
+export {};

package/dist/datatypes/numbers/bigints.d.ts

@@ -0,0 +1,3 @@
+import type { Serializer } from "../../types";
+export declare function u64Bigint(endian?: "little" | "big"): Serializer<number | bigint, bigint>;
+export declare function s64Bigint(endian?: "little" | "big"): Serializer<number | bigint, bigint>;

package/dist/datatypes/numbers/bigints.js

@@ -0,0 +1,37 @@
+const checkValue = (value, start, end) => {
+    if (value < start || value >= end) {
+        throw new Error("Out of Range");
+    }
+};
+export function u64Bigint(endian = "little") {
+    return {
+        size: 8,
+        write: (ctx, value) => {
+            checkValue(value, 0n, 2n ** 64n);
+            ctx.alloc(8);
+            ctx.view.setBigUint64(ctx.offset, BigInt(value), endian === "little");
+            ctx.offset += 8;
+        },
+        read: (ctx) => {
+            const value = ctx.view.getBigUint64(ctx.offset, endian === "little");
+            ctx.offset += 8;
+            return value;
+        },
+    };
+}
+export function s64Bigint(endian = "little") {
+    return {
+        size: 8,
+        write: (ctx, value) => {
+            checkValue(value, -(2n ** 63n), 2n ** 63n);
+            ctx.alloc(8);
+            ctx.view.setBigInt64(ctx.offset, BigInt(value), endian === "little");
+            ctx.offset += 8;
+        },
+        read: (ctx) => {
+            const value = ctx.view.getBigInt64(ctx.offset, endian === "little");
+            ctx.offset += 8;
+            return value;
+        },
+    };
+}

package/dist/datatypes/numbers/bigints.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/datatypes/numbers/bigints.test.js

@@ -0,0 +1,61 @@
+import { describe, it, expect } from "bun:test";
+import { expectEncode, expectEncodeSnapshot, expectError, randbigint } from "../utils.test";
+import * as st from "../../index";
+function test_bigint(options) {
+    const { name, range: [start, end], serializer, size, } = options;
+    describe(name, () => {
+        it("works in bounds", () => {
+            expectEncode(serializer, start);
+            for (let i = 0; i < 100; i++) {
+                expectEncode(serializer, randbigint(start, end));
+            }
+            expectEncode(serializer, end);
+        });
+        it("outside bounds", () => {
+            expectError(() => st.write(serializer, start - 1n));
+            expectError(() => st.write(serializer, end + 1n));
+        });
+        it(`is right size`, () => {
+            const expectValueSize = (value) => {
+                const data = st.write(serializer, value);
+                expect(data.byteLength).toBe(size);
+            };
+            for (let i = 0; i < 100; i++) {
+                expectValueSize(randbigint(start, end));
+            }
+            expectValueSize(start);
+            expectValueSize(end);
+        });
+        it(`matches snapshots`, () => {
+            expectEncodeSnapshot(serializer, 0n);
+            expectEncodeSnapshot(serializer, start);
+            expectEncodeSnapshot(serializer, start + 10n);
+            expectEncodeSnapshot(serializer, end - 10n);
+            expectEncodeSnapshot(serializer, end);
+        });
+    });
+}
+test_bigint({
+    name: "st.u64(little)",
+    serializer: st.u64Bigint("little"),
+    range: [0n, 18446744073709551615n],
+    size: 8,
+});
+test_bigint({
+    name: "st.u64(big)",
+    serializer: st.u64Bigint("big"),
+    range: [0n, 18446744073709551615n],
+    size: 8,
+});
+test_bigint({
+    name: "st.s64(little)",
+    serializer: st.s64Bigint("little"),
+    range: [-9223372036854775808n, 9223372036854775807n],
+    size: 8,
+});
+test_bigint({
+    name: "st.s64(big)",
+    serializer: st.s64Bigint("big"),
+    range: [-9223372036854775808n, 9223372036854775807n],
+    size: 8,
+});

package/dist/datatypes/numbers/floats.d.ts

@@ -0,0 +1,3 @@
+import type { Serializer } from "../../types";
+export declare function f32(endian?: "little" | "big"): Serializer<number>;
+export declare function f64(endian?: "little" | "big"): Serializer<number>;

package/dist/datatypes/numbers/floats.js

@@ -0,0 +1,30 @@
+export function f32(endian = "little") {
+    return {
+        size: 4,
+        write: (ctx, value) => {
+            ctx.alloc(4);
+            ctx.view.setFloat32(ctx.offset, value, endian === "little");
+            ctx.offset += 4;
+        },
+        read: (ctx) => {
+            const value = ctx.view.getFloat32(ctx.offset, endian === "little");
+            ctx.offset += 4;
+            return value;
+        },
+    };
+}
+export function f64(endian = "little") {
+    return {
+        size: 8,
+        write: (ctx, value) => {
+            ctx.alloc(8);
+            ctx.view.setFloat64(ctx.offset, value, endian === "little");
+            ctx.offset += 8;
+        },
+        read: (ctx) => {
+            const value = ctx.view.getFloat64(ctx.offset, endian === "little");
+            ctx.offset += 8;
+            return value;
+        },
+    };
+}

package/dist/datatypes/numbers/floats.test.d.ts

@@ -0,0 +1 @@
+export {};