@based/schema 5.0.0-alpha.20 → 5.0.0-alpha.21

package/dist/def/DEFAULT_MAP.d.ts

@@ -0,0 +1,3 @@
+import { TypeIndex } from './types.js';
+export declare const DEFAULT_MAP: Record<TypeIndex, any>;
+//# sourceMappingURL=DEFAULT_MAP.d.ts.map

package/dist/def/DEFAULT_MAP.js

@@ -0,0 +1,29 @@
+import { TYPE_INDEX_MAP } from './types.js';
+// TODO update defaults
+export const DEFAULT_MAP = {
+    [TYPE_INDEX_MAP.alias]: '',
+    [TYPE_INDEX_MAP.binary]: undefined,
+    [TYPE_INDEX_MAP.boolean]: false,
+    [TYPE_INDEX_MAP.cardinality]: 0,
+    [TYPE_INDEX_MAP.created]: 0,
+    [TYPE_INDEX_MAP.updated]: 0,
+    [TYPE_INDEX_MAP.number]: 0,
+    [TYPE_INDEX_MAP.timestamp]: 0,
+    [TYPE_INDEX_MAP.enum]: 0,
+    [TYPE_INDEX_MAP.id]: 0,
+    [TYPE_INDEX_MAP.int16]: 0,
+    [TYPE_INDEX_MAP.int32]: 0,
+    [TYPE_INDEX_MAP.int8]: 0,
+    [TYPE_INDEX_MAP.uint8]: 0,
+    [TYPE_INDEX_MAP.uint16]: 0,
+    [TYPE_INDEX_MAP.uint32]: 0,
+    [TYPE_INDEX_MAP.json]: undefined,
+    [TYPE_INDEX_MAP.microbuffer]: undefined,
+    [TYPE_INDEX_MAP.reference]: undefined,
+    [TYPE_INDEX_MAP.references]: [],
+    [TYPE_INDEX_MAP.string]: '',
+    [TYPE_INDEX_MAP.aliases]: [],
+    [TYPE_INDEX_MAP.text]: '',
+    [TYPE_INDEX_MAP.vector]: undefined, // maybe not can set a vec with 0
+};
+//# sourceMappingURL=DEFAULT_MAP.js.map

package/dist/def/createEmptyDef.d.ts

@@ -2,6 +2,8 @@ import { SchemaLocales, SchemaObject, StrictSchemaType } from '../types.js';
 export declare const createEmptyDef: (typeName: string, type: StrictSchemaType | SchemaObject, locales: Partial<SchemaLocales>) => {
     cnt: number;
     blockCapacity: number;
+    insertOnly: boolean;
+    partial: boolean;
     checksum: number;
     type: string;
     props: {};

package/dist/def/createEmptyDef.js

@@ -3,6 +3,8 @@ export const createEmptyDef = (typeName, type, locales) => {
     return {
         cnt: 0,
         blockCapacity: 0,
+        insertOnly: false,
+        partial: false,
         checksum: hashObjectIgnoreKeyOrder(type),
         type: typeName,
         props: {},

package/dist/def/defaultMap.js

@@ -23,5 +23,6 @@ export const DEFAULT_MAP = {
     [TYPE_INDEX_MAP.aliases]: [],
     [TYPE_INDEX_MAP.text]: {},
     [TYPE_INDEX_MAP.vector]: undefined, // maybe not can set a vec with 0
+    [TYPE_INDEX_MAP.colvec]: undefined, // maybe not can set a vec with 0
 };
 //# sourceMappingURL=defaultMap.js.map

package/dist/def/getPropLen.d.ts

@@ -0,0 +1,3 @@
+import { SchemaProp } from '../types.js';
+export declare function getPropLen(schemaProp: SchemaProp): any;
+//# sourceMappingURL=getPropLen.d.ts.map

package/dist/def/getPropLen.js

@@ -0,0 +1,23 @@
+import { getPropType } from '../parse/utils.js';
+import { isPropType } from '../types.js';
+import { SIZE_MAP } from './types.js';
+export function getPropLen(schemaProp) {
+    let len = SIZE_MAP[getPropType(schemaProp)];
+    if (isPropType('string', schemaProp) ||
+        isPropType('alias', schemaProp) ||
+        isPropType('cardinality', schemaProp)) {
+        if (typeof schemaProp === 'object') {
+            if (schemaProp.maxBytes < 61) {
+                len = schemaProp.maxBytes + 1;
+            }
+            else if ('max' in schemaProp && schemaProp.max < 31) {
+                len = schemaProp.max * 2 + 1;
+            }
+        }
+    }
+    else if (isPropType('vector', schemaProp)) {
+        len = 4 * schemaProp.size;
+    }
+    return len;
+}
+//# sourceMappingURL=getPropLen.js.map

package/dist/def/makePacked.d.ts

@@ -0,0 +1,3 @@
+import { SchemaTypeDef, PropDef } from './types.js';
+export declare function makePacked(result: Partial<SchemaTypeDef>, typeName: string, vals: PropDef[], len: number): void;
+//# sourceMappingURL=makePacked.d.ts.map

package/dist/def/makePacked.js

@@ -0,0 +1,50 @@
+export function makePacked(result, typeName, vals, len) {
+    const encoder = new TextEncoder();
+    result.buf = new Uint8Array(len);
+    result.buf[0] = result.idUint8[0];
+    result.buf[1] = result.idUint8[1];
+    const fieldNames = [];
+    const tNameBuf = encoder.encode(typeName);
+    fieldNames.push(tNameBuf);
+    let fieldNameLen = tNameBuf.byteLength + 1;
+    let i = 2;
+    if (result.mainLen) {
+        result.buf[i] = 0;
+        for (const f of vals) {
+            if (!f.separate) {
+                i++;
+                result.buf[i] = f.typeIndex;
+                const name = encoder.encode(f.path.join('.'));
+                fieldNames.push(name);
+                fieldNameLen += name.byteLength + 1;
+            }
+        }
+        i++;
+        result.buf[i] = 0;
+    }
+    for (const f of vals) {
+        if (f.separate) {
+            i++;
+            result.buf[i] = f.prop;
+            i++;
+            result.buf[i] = f.typeIndex;
+            const name = encoder.encode(f.path.join('.'));
+            fieldNames.push(name);
+            fieldNameLen += name.byteLength + 1;
+        }
+    }
+    result.propNames = new Uint8Array(fieldNameLen);
+    let lastWritten = 0;
+    for (const f of fieldNames) {
+        result.propNames[lastWritten] = f.byteLength;
+        result.propNames.set(f, lastWritten + 1);
+        lastWritten += f.byteLength + 1;
+    }
+    let bufLen = result.buf.length;
+    result.packed = new Uint8Array(2 + bufLen + result.propNames.length);
+    result.packed[0] = bufLen;
+    result.packed[1] = bufLen >>>= 8;
+    result.packed.set(result.buf, 2);
+    result.packed.set(result.propNames, result.buf.length + 2);
+}
+//# sourceMappingURL=makePacked.js.map

package/dist/def/readFromPacked.d.ts

@@ -0,0 +1,3 @@
+import { SchemaTypeDef } from './types.js';
+export declare const readFromPacked: (packed: Uint8Array) => SchemaTypeDef;
+//# sourceMappingURL=readFromPacked.d.ts.map

package/dist/def/readFromPacked.js

@@ -0,0 +1,140 @@
+import { REVERSE_SIZE_MAP } from './types.js';
+import { DEFAULT_MAP } from './defaultMap.js';
+import { VALIDATION_MAP } from './validation.js';
+export const readFromPacked = (packed) => {
+    const size = (packed[0] | (packed[1] << 8)) >>> 0;
+    const props = [];
+    const b = packed.subarray(2, 2 + size);
+    let collectMain = false;
+    const mainProps = [];
+    const typeId = b.subarray(0, 2);
+    const typeIdNr = (typeId[0] | (typeId[1] << 8)) >>> 0;
+    for (let i = 2; i < b.length; i++) {
+        const prop = b[i];
+        if (collectMain) {
+            if (prop === 0) {
+                collectMain = false;
+            }
+            else {
+                mainProps.push({
+                    prop: 0,
+                    typeIndex: b[i],
+                });
+            }
+        }
+        else {
+            if (prop == 0) {
+                collectMain = true;
+            }
+            else {
+                props.push({ prop, typeIndex: b[i + 1] });
+                i++;
+            }
+        }
+    }
+    const decoder = new TextDecoder();
+    const fields = [];
+    const f = packed.subarray(2 + size, packed.length);
+    for (let i = 0; i < f.length; i++) {
+        const size = f[i];
+        fields.push(decoder.decode(f.subarray(i + 1, i + 1 + size)));
+        i += size;
+    }
+    for (let i = 0; i < mainProps.length; i++) {
+        mainProps[i].path = fields[i + 1];
+    }
+    for (let i = 0; i < props.length; i++) {
+        props[i].path = fields[i + 1 + mainProps.length];
+    }
+    // Fixed len strings not supported
+    // Refs also not supported
+    // Text not supported yet
+    // Ref not supported yet
+    //   compression: 1 (0)
+    const result = {
+        cnt: 0,
+        checksum: 0,
+        total: 0,
+        type: fields[0],
+        lastId: 0,
+        blockCapacity: 0,
+        mainLen: mainProps.length,
+        buf: b,
+        propNames: f,
+        packed,
+        props: {},
+        reverseProps: {}, // in a bit
+        id: typeIdNr,
+        idUint8: typeId,
+        separate: [],
+        main: {},
+        tree: {},
+        // not nessecary...
+        hasSeperateSort: false,
+        seperateSort: {
+            size: 0,
+            buffer: new Uint8Array([]),
+            bufferTmp: new Uint8Array([]),
+            props: [],
+        },
+        hasSeperateTextSort: false,
+        seperateTextSort: {
+            localeToIndex: new Map(),
+            localeStringToIndex: new Map(),
+            noUndefined: new Uint8Array([]),
+            size: 0,
+            buffer: new Uint8Array([]),
+            bufferTmp: new Uint8Array([]),
+            props: [],
+        },
+        mainEmpty: new Uint8Array([]),
+        mainEmptyAllZeroes: true,
+        // need this...
+        locales: {},
+        localeSize: 0,
+    };
+    let s = 0;
+    for (const p of mainProps) {
+        const len = REVERSE_SIZE_MAP[p.typeIndex];
+        const prop = {
+            prop: p.prop,
+            separate: false,
+            __isPropDef: true,
+            validation: VALIDATION_MAP[p.typeIndex],
+            start: s,
+            default: DEFAULT_MAP[p.typeIndex], // tmp
+            typeIndex: p.typeIndex,
+            path: p.path.split('.'),
+            len,
+        };
+        result.props[p.path] = prop;
+        result.main[prop.start] = prop;
+        s += len;
+    }
+    for (const p of props) {
+        const prop = {
+            prop: p.prop,
+            separate: true,
+            __isPropDef: true,
+            validation: VALIDATION_MAP[p.typeIndex],
+            start: 0,
+            typeIndex: p.typeIndex,
+            default: DEFAULT_MAP[p.typeIndex], // tmp
+            path: p.path.split('.'),
+            len: 0,
+            compression: 1,
+        };
+        result.props[p.path] = prop;
+        result.reverseProps[prop.prop] = prop;
+    }
+    // make this into a typeDef
+    //   return {
+    //     type: fields[0],
+    //     fields,
+    //     typeId,
+    //     props,
+    //     mainProps,
+    //   }
+    return result;
+};
+//# sourceMappingURL=readFromPacked.js.map

package/dist/def/selvaBuffer.js

@@ -1,4 +1,4 @@
-import { ALIAS, ALIASES, BINARY, EMPTY_MICRO_BUFFER, CARDINALITY, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, WEAK_REFERENCE, WEAK_REFERENCES, JSON, } from './types.js';
+import { ALIAS, ALIASES, BINARY, EMPTY_MICRO_BUFFER, CARDINALITY, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, WEAK_REFERENCE, WEAK_REFERENCES, JSON, COLVEC, } from './types.js';
 const selvaFieldType = {
     NULL: 0,
     MICRO_BUFFER: 1,
@@ -26,6 +26,7 @@ selvaTypeMap[WEAK_REFERENCE] = selvaFieldType.WEAK_REFERENCE;
 selvaTypeMap[WEAK_REFERENCES] = selvaFieldType.WEAK_REFERENCES;
 selvaTypeMap[ALIAS] = selvaFieldType.ALIAS;
 selvaTypeMap[ALIASES] = selvaFieldType.ALIASES;
+selvaTypeMap[COLVEC] = selvaFieldType.COLVEC;
 const EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT = 0x01;
 const EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80;
 function blockCapacity(blockCapacity) {
@@ -53,6 +54,14 @@ const propDefBuffer = (schema, prop, isEdge) => {
         view.setUint16(1, prop.len, true);
         return [...buf];
     }
+    else if (prop.len && (type === COLVEC)) {
+        const buf = new Uint8Array(5);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        view.setUint16(1, prop.len, true);
+        view.setUint16(3, 4, true); // TODO Other types than f32
+        return [...buf];
+    }
     else if (type === REFERENCE || type === REFERENCES) {
         const buf = new Uint8Array(9);
         const view = new DataView(buf.buffer);

package/dist/def/timestamp.d.ts

@@ -0,0 +1,2 @@
+export declare const convertToTimestamp: (value: string | Date | number) => number;
+//# sourceMappingURL=timestamp.d.ts.map

package/dist/def/timestamp.js

@@ -0,0 +1,67 @@
+const timeToNumber = (ex) => {
+    if (ex === 's') {
+        return 1000;
+    }
+    if (ex === 'm') {
+        return 1000 * 60;
+    }
+    if (ex === 'h') {
+        return 1000 * 60 * 60;
+    }
+    if (ex === 'd') {
+        return 1000 * 60 * 60 * 24;
+    }
+    if (ex === 'y') {
+        return 31556952000;
+    }
+    return 1;
+};
+export const convertToTimestamp = (value) => {
+    if (value instanceof Date) {
+        return value.valueOf();
+    }
+    if (typeof value === 'string') {
+        if (value === 'now') {
+            return Date.now();
+        }
+        const y = value.replace(/([+-])/g, ' $1 ');
+        const arr = y.split(/ +/);
+        let newValue = 0;
+        let now;
+        let op = 1;
+        for (const seg of arr) {
+            if (seg === '-') {
+                op = -1;
+            }
+            else if (seg === '+') {
+                op = 1;
+            }
+            else {
+                var v = 0;
+                if (seg === 'now') {
+                    if (!now) {
+                        now = Date.now();
+                    }
+                    v = now;
+                }
+                else if (/[smhdy]$/.test(seg)) {
+                    const ex = seg[seg.length - 1];
+                    const number = parseInt(seg, 10);
+                    v = number * timeToNumber(ex);
+                }
+                else if (seg) {
+                    v = new Date(seg).valueOf();
+                }
+                if (op === -1) {
+                    newValue -= v;
+                }
+                else {
+                    newValue += v;
+                }
+            }
+        }
+        return newValue;
+    }
+    return value;
+};
+//# sourceMappingURL=timestamp.js.map

package/dist/def/typeDef.d.ts

@@ -1,7 +1,6 @@
 import { SchemaObject, StrictSchemaType, SchemaLocales } from '../index.js';
 import { SchemaTypeDef, SchemaTypesParsed } from './types.js';
 import { StrictSchema } from '../types.js';
-export declare const DEFAULT_BLOCK_CAPACITY = 100000;
 export declare const updateTypeDefs: (schema: StrictSchema) => {
     schemaTypesParsed: {
         [key: string]: SchemaTypeDef;

package/dist/def/typeDef.js

@@ -1,6 +1,6 @@
 import { isPropType, getPropType, } from '../index.js';
 import { setByPath } from '@saulx/utils';
-import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, NUMBER, } from './types.js';
+import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, NUMBER, BLOCK_CAPACITY_MAX, BLOCK_CAPACITY_DEFAULT, BLOCK_CAPACITY_MIN, } from './types.js';
 import { DEFAULT_MAP } from './defaultMap.js';
 import { makeSeparateTextSort } from './makeSeparateTextSort.js';
 import { makeSeparateSort } from './makeSeparateSort.js';
@@ -9,39 +9,56 @@ import { addEdges } from './addEdges.js';
 import { createEmptyDef } from './createEmptyDef.js';
 import { fillEmptyMain, isZeroes } from './fillEmptyMain.js';
 import { defaultValidation, VALIDATION_MAP } from './validation.js';
-export const DEFAULT_BLOCK_CAPACITY = 100_000;
 export const updateTypeDefs = (schema) => {
     const schemaTypesParsed = {};
     const schemaTypesParsedById = {};
-    for (const field in schemaTypesParsed) {
-        if (field in schema.types) {
+    for (const typeName in schemaTypesParsed) {
+        if (typeName in schema.types) {
             continue;
         }
-        const id = schemaTypesParsed[field].id;
-        delete schemaTypesParsed[field];
+        const id = schemaTypesParsed[typeName].id;
+        delete schemaTypesParsed[typeName];
         delete schemaTypesParsedById[id];
     }
-    for (const field in schema.types) {
-        const type = schema.types[field];
+    for (const typeName in schema.types) {
+        const type = schema.types[typeName];
         if (!type.id) {
             throw new Error('NEED ID ON TYPE');
         }
-        const def = createSchemaTypeDef(field, type, schemaTypesParsed, schema.locales ?? {
+        const def = createSchemaTypeDef(typeName, type, schemaTypesParsed, schema.locales ?? {
             en: {},
         });
-        def.blockCapacity = field === '_root' ? 2147483647 : DEFAULT_BLOCK_CAPACITY;
-        schemaTypesParsed[field] = def;
+        schemaTypesParsed[typeName] = def;
         schemaTypesParsedById[type.id] = def;
     }
     return { schemaTypesParsed, schemaTypesParsedById };
 };
 export const createSchemaTypeDef = (typeName, type, parsed, locales, result = createEmptyDef(typeName, type, locales), path = [], top = true) => {
-    if (result.id == 0 && top) {
-        if ('id' in type) {
-            result.id = type.id;
+    if (top) {
+        if (result.id == 0) {
+            if ('id' in type) {
+                result.id = type.id;
+            }
+            else {
+                throw new Error(`Invalid schema type id ${result.type}`);
+            }
         }
-        else {
-            throw new Error(`Invalid schema type id ${result.type}`);
+        if (result.blockCapacity == 0) {
+            if ('blockCapacity' in type) {
+                if (typeof type.blockCapacity !== 'number' || type.blockCapacity < BLOCK_CAPACITY_MIN || type.blockCapacity > BLOCK_CAPACITY_MAX) {
+                    throw new Error('Invalid blockCapacity');
+                }
+                result.blockCapacity = type.blockCapacity;
+            }
+            else {
+                result.blockCapacity = typeName === '_root' ? BLOCK_CAPACITY_MAX : BLOCK_CAPACITY_DEFAULT;
+            }
+        }
+        if (result.insertOnly == false && 'insertOnly' in type) {
+            result.insertOnly = !!type.insertOnly;
+        }
+        if (result.partial == false && 'partial' in type) {
+            result.partial = !!type.partial;
         }
     }
     result.locales = locales;
@@ -75,6 +92,11 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = cr
             else if (isPropType('text', schemaProp)) {
                 result.separateSortText++;
             }
+            else if (isPropType('colvec', schemaProp)) {
+                if (!result.insertOnly) {
+                    throw new Error('colvec requires insertOnly');
+                }
+            }
             const isseparate = isSeparate(schemaProp, len);
             const typeIndex = TYPE_INDEX_MAP[propType];
             const prop = {
@@ -110,12 +132,18 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = cr
                 }
             }
             else if (isPropType('references', schemaProp)) {
+                if (result.partial) {
+                    throw new Error('references is not supported with partial');
+                }
                 prop.inversePropName = schemaProp.items.prop;
                 prop.inverseTypeName = schemaProp.items.ref;
                 prop.dependent = schemaProp.items.dependent;
                 addEdges(prop, schemaProp.items);
             }
             else if (isPropType('reference', schemaProp)) {
+                if (result.partial) {
+                    throw new Error('reference is not supported with partial');
+                }
                 prop.inversePropName = schemaProp.prop;
                 prop.inverseTypeName = schemaProp.ref;
                 prop.dependent = schemaProp.dependent;

package/dist/def/types.d.ts

@@ -26,6 +26,7 @@ export declare const ID = 26;
 export declare const VECTOR = 27;
 export declare const JSON = 28;
 export declare const OBJECT = 29;
+export declare const COLVEC = 30;
 export declare const TYPE_INDEX_MAP: {
     alias: number;
     aliases: number;
@@ -50,6 +51,7 @@ export declare const TYPE_INDEX_MAP: {
     cardinality: number;
     json: number;
     object: number;
+    colvec: number;
 };
 export declare const enum numberTypes {
     number = 4,
@@ -126,6 +128,9 @@ export type SchemaSortUndefinedHandler = {
     bufferTmp: Uint8Array;
     props: PropDef[];
 };
+export declare const BLOCK_CAPACITY_MIN = 1025;
+export declare const BLOCK_CAPACITY_MAX = 2147483647;
+export declare const BLOCK_CAPACITY_DEFAULT = 100000;
 export type SchemaTypeDef = {
     cnt: number;
     checksum: number;
@@ -133,6 +138,8 @@ export type SchemaTypeDef = {
     lastId: number;
     blockCapacity: number;
     mainLen: number;
+    insertOnly: boolean;
+    partial: boolean;
     buf: Uint8Array;
     propNames: Uint8Array;
     props: {

package/dist/def/types.js

@@ -25,6 +25,7 @@ export const ID = 26;
 export const VECTOR = 27;
 export const JSON = 28;
 export const OBJECT = 29;
+export const COLVEC = 30;
 export const TYPE_INDEX_MAP = {
     alias: ALIAS,
     aliases: ALIASES,
@@ -49,6 +50,7 @@ export const TYPE_INDEX_MAP = {
     cardinality: CARDINALITY,
     json: JSON,
     object: OBJECT,
+    colvec: COLVEC,
 };
 const numberTypeValues = [
     NUMBER,
@@ -63,6 +65,9 @@ const numberTypeValues = [
 export function isNumberType(type) {
     return numberTypeValues.includes(type);
 }
+export const BLOCK_CAPACITY_MIN = 1025;
+export const BLOCK_CAPACITY_MAX = 2147483647;
+export const BLOCK_CAPACITY_DEFAULT = 100_000;
 export const SIZE_MAP = {
     timestamp: 8, // 64bit
     // double-precision 64-bit binary format IEEE 754 value
@@ -88,6 +93,7 @@ export const SIZE_MAP = {
     vector: 0, // separate
     json: 0,
     object: 0,
+    colvec: 0, // separate
 };
 const reverseMap = {};
 for (const k in TYPE_INDEX_MAP) {

package/dist/def/utils.js

@@ -3,7 +3,7 @@ import { isPropType } from '../types.js';
 import { getPropType } from '../parse/utils.js';
 import { convertToTimestamp } from '@saulx/utils';
 export function isSeparate(schemaProp, len) {
-    return len === 0 || isPropType('vector', schemaProp);
+    return len === 0 || isPropType('vector', schemaProp) || isPropType('colvec', schemaProp);
 }
 export const propIsSigned = (prop) => {
     const t = prop.typeIndex;
@@ -43,6 +43,9 @@ export function getPropLen(schemaProp) {
     else if (isPropType('vector', schemaProp)) {
         len = 4 * schemaProp.size;
     }
+    else if (isPropType('colvec', schemaProp)) {
+        len = schemaProp.size;
+    }
     return len;
 }
 export const parseMinMaxStep = (val) => {

package/dist/def/validation.js

@@ -10,6 +10,7 @@ export const VALIDATION_MAP = {
         return true;
     },
     [TYPE_INDEX_MAP.binary]: (value) => {
+        console.log('DERP ', value, typeof value);
         if (value instanceof Uint8Array) {
             return true;
         }

package/dist/lang.d.ts

@@ -147,7 +147,7 @@ declare const langCodes: {
     readonly ka: 145;
     readonly cnr: 146;
 };
-export declare const langCodesMap: Map<string, 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146>;
+export declare const langCodesMap: Map<string, 0 | 2 | 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146>;
 export declare const inverseLangMap: Map<any, any>;
 export type LangName = keyof typeof langCodes;
 export type LangCode = (typeof langCodes)[LangName];

package/dist/mermaid.d.ts

@@ -0,0 +1,3 @@
+import { StrictSchema } from './types.js';
+export declare const mermaid: (schema: StrictSchema) => string;
+//# sourceMappingURL=mermaid.d.ts.map

package/dist/mermaid.js

@@ -0,0 +1,24 @@
+import { getPropType } from './parse/utils.js';
+import { isPropType } from './types.js';
+export const mermaid = (schema) => {
+    let mermaid = 'classDiagram';
+    if (schema.types) {
+        for (const type in schema.types) {
+            for (const key in schema.types[type].props) {
+                const prop = schema.types[type].props[key];
+                const propType = getPropType(prop);
+                if (isPropType('reference', prop)) {
+                    mermaid += `\n${type} --> ${prop.ref} : ${key}`;
+                }
+                else if (isPropType('references', prop)) {
+                    mermaid += `\n${type} --> ${prop.items.ref} : ${key}[]`;
+                }
+                else {
+                    mermaid += `\n${type} : ${propType} ${key}`;
+                }
+            }
+        }
+    }
+    return mermaid;
+};
+//# sourceMappingURL=mermaid.js.map

package/dist/parse/index.js

@@ -65,7 +65,6 @@ export class SchemaParser {
         for (const locale in locales) {
             const opts = locales[locale];
             if (opts === true) {
-                console.log(locale, opts);
                 continue;
             }
             expectObject(opts);

package/dist/parse/props.js

@@ -157,6 +157,15 @@ p.vector = propParser({
         return isDefault(val, prop, ctx);
     },
 }, 0);
+p.colvec = propParser({
+    size(val) {
+        expectNumber(val);
+    },
+}, {
+    default(val, prop, ctx) {
+        return isDefault(val, prop, ctx);
+    },
+}, 0);
 p.enum = propParser({
     enum(items) {
         if (!Array.isArray(items)) {

package/dist/serialize.d.ts

@@ -1,5 +1,14 @@
 import { StrictSchema } from './types.js';
-export declare const serialize: (schema: any, noCompression?: boolean) => Uint8Array;
-export declare const deSerializeInner: (buf: Uint8Array, obj: any, start: number) => number;
+type Opts = {
+    readOnly?: boolean;
+    stripMetaInformation?: boolean;
+};
+export declare const serialize: (schema: any, opts?: Opts) => Uint8Array;
+export declare const deSerializeKey: (buf: Uint8Array, keySize: number, i: number) => {
+    size: number;
+    value: string;
+};
+export declare const deSerializeInner: (buf: Uint8Array, obj: any, start: number, fromArray: boolean) => number;
 export declare const deSerialize: (buf: Uint8Array) => StrictSchema;
+export {};
 //# sourceMappingURL=serialize.d.ts.map

package/dist/serialize.js

@@ -1,108 +1,335 @@
-import * as deflate from 'fflate';
+// import * as deflate from 'fflate'
+import { stringFormats } from './types.js';
 import { REVERSE_TYPE_INDEX_MAP, TYPE_INDEX_MAP } from './def/types.js';
+import { readDoubleLE, readUint16, readUint24, readUint32, writeDoubleLE, writeUint16, writeUint24, writeUint32, } from '@saulx/utils';
 const ENCODER = new TextEncoder();
+const UINT8 = 245;
+const FALSE = 246;
+const TRUE = 247;
+const FUNCTION = 248;
+const STRING = 249;
+const ARRAY = 250;
+const BINARY = 251;
+const UINT32 = 252;
+const FLOAT64 = 253;
+const SCHEMA_PROP = 254;
+const OBJECT = 255;
+// Key Address encoding types
+const KEY_ADDRESS_1_BYTE = 0;
+const KEY_ADDRESS_2_BYTES = 1;
+const KEY_ADDRESS_3_BYTES = 2;
+// Key types
+const PROPS = 3;
+const TYPES = 4;
+const READONLY = 5;
+const FORMAT = 6;
+const REQUIRED = 7;
+const REF = 8;
+const PROP = 9;
+const KEY_OPTS = PROP;
+const ensureCapacity = (required) => {
+    if (schemaBuffer.len + required > schemaBuffer.buf.length) {
+        const newBuf = new Uint8Array(Math.max(schemaBuffer.buf.length * 2, schemaBuffer.len + required));
+        newBuf.set(schemaBuffer.buf);
+        schemaBuffer.buf = newBuf;
+    }
+};
 let schemaBuffer;
+const handleSingleValue = (ops, val, obj, prev, fromObject, key, isTypes) => {
+    const type = typeof val;
+    // typed Array - single PROP
+    if (val instanceof Uint8Array) {
+        ensureCapacity(1 + 2 + val.byteLength);
+        schemaBuffer.buf[schemaBuffer.len] = BINARY;
+        schemaBuffer.len += 1;
+        schemaBuffer.buf[schemaBuffer.len] = val.byteLength;
+        schemaBuffer.len += 1;
+        schemaBuffer.buf[schemaBuffer.len] = val.byteLength >>> 8;
+        schemaBuffer.len += 1;
+        schemaBuffer.buf.set(val, schemaBuffer.len);
+        schemaBuffer.len += val.byteLength;
+    }
+    else if (type === 'function') {
+        const str = val.toString();
+        // Pessimistically assume 4 bytes per char for UTF-8 to be safe.
+        ensureCapacity(1 + 2 + str.length * 4);
+        schemaBuffer.buf[schemaBuffer.len] = FUNCTION;
+        schemaBuffer.len += 1;
+        const sizeIndex = schemaBuffer.len;
+        schemaBuffer.len += 2;
+        // encodeInto is much faster as it avoids intermediate allocation.
+        const r = ENCODER.encodeInto(str, schemaBuffer.buf.subarray(schemaBuffer.len));
+        schemaBuffer.len += r.written;
+        schemaBuffer.buf[sizeIndex] = r.written;
+        schemaBuffer.buf[sizeIndex + 1] = r.written >>> 8;
+    }
+    else if (type === 'object') {
+        // fromObject
+        if (val === null) {
+        }
+        else {
+            if (!fromObject && key === 'props' && obj.type === 'object') {
+                walk(ops, val, obj, prev, true, schemaBuffer, false);
+            }
+            else {
+                walk(ops, val, obj, prev, fromObject, schemaBuffer, isTypes);
+            }
+        }
+    }
+    else if (type === 'boolean') {
+        ensureCapacity(1);
+        schemaBuffer.buf[schemaBuffer.len] = val ? TRUE : FALSE;
+        schemaBuffer.len += 1;
+    }
+    else if (type === 'string') {
+        // Pessimistically assume 4 bytes per char for UTF-8 to be safe.
+        ensureCapacity(1 + 2 + val.length * 4);
+        schemaBuffer.buf[schemaBuffer.len] = STRING;
+        schemaBuffer.len += 1;
+        const sizeIndex = schemaBuffer.len;
+        schemaBuffer.len += 2;
+        // encodeInto is much faster as it avoids intermediate allocation.
+        const r = ENCODER.encodeInto(val, schemaBuffer.buf.subarray(schemaBuffer.len));
+        schemaBuffer.len += r.written;
+        schemaBuffer.buf[sizeIndex] = r.written;
+        schemaBuffer.buf[sizeIndex + 1] = r.written >>> 8;
+    }
+    else if (type === 'number') {
+        const isInt = val % 1 === 0;
+        if (val < 256 && val > 0 && isInt) {
+            ensureCapacity(2);
+            schemaBuffer.buf[schemaBuffer.len] = UINT8;
+            schemaBuffer.len += 1;
+            schemaBuffer.buf[schemaBuffer.len] = val;
+            schemaBuffer.len += 1;
+        }
+        else if ((val < 4294967295 || val > 0) && isInt) {
+            ensureCapacity(5);
+            schemaBuffer.buf[schemaBuffer.len] = UINT32;
+            schemaBuffer.len += 1;
+            writeUint32(schemaBuffer.buf, val, schemaBuffer.len);
+            schemaBuffer.len += 4;
+        }
+        else {
+            ensureCapacity(9);
+            schemaBuffer.buf[schemaBuffer.len] = FLOAT64;
+            schemaBuffer.len += 1;
+            writeDoubleLE(schemaBuffer.buf, val, schemaBuffer.len);
+            schemaBuffer.len += 8;
+        }
+    }
+};
+const encodeKey = (key, schemaBuffer) => {
+    let address = schemaBuffer.dictMap[key];
+    // if len == 1 never from address
+    if (!address) {
+        // pessimistically assume 4 bytes per char for UTF-8 to be safe.
+        ensureCapacity(1 + key.length * 4);
+        address = schemaBuffer.len;
+        schemaBuffer.len += 1;
+        const r = ENCODER.encodeInto(key, schemaBuffer.buf.subarray(schemaBuffer.len));
+        schemaBuffer.buf[address] = r.written + KEY_OPTS;
+        schemaBuffer.len += r.written;
+        schemaBuffer.dictMap[key] = address;
+    }
+    else {
+        ensureCapacity(4);
+        if (address > 65025) {
+            schemaBuffer.buf[schemaBuffer.len] = KEY_ADDRESS_3_BYTES;
+            schemaBuffer.len += 1;
+            writeUint24(schemaBuffer.buf, address, schemaBuffer.len);
+            schemaBuffer.len += 3;
+        }
+        else if (address > 255) {
+            schemaBuffer.buf[schemaBuffer.len] = KEY_ADDRESS_2_BYTES;
+            schemaBuffer.len += 1;
+            writeUint16(schemaBuffer.buf, address, schemaBuffer.len);
+            schemaBuffer.len += 2;
+        }
+        else {
+            schemaBuffer.buf[schemaBuffer.len] = KEY_ADDRESS_1_BYTE;
+            schemaBuffer.len += 1;
+            schemaBuffer.buf[schemaBuffer.len] = address;
+            schemaBuffer.len += 1;
+        }
+    }
+};
 // 3 level
 // 0 for queries (min)
 // 1 for modify
 // 2 fulls schema
-const walk = (obj, prev, prev2, fromObject, schemaBuffer) => {
+const walk = (opts, obj, prev, prev2, fromObject, schemaBuffer, isTypes) => {
     let start = schemaBuffer.len;
-    // HANDLE ENUM
-    const isSchemaProp = 'type' in obj && (prev2?.type === 'object' || fromObject === false);
+    const isArray = Array.isArray(obj);
+    const isFromObj = prev2?.type === 'object' || fromObject === false;
+    const isSchemaProp = 'type' in obj && isFromObj;
+    ensureCapacity(1 + 4); // Type byte + size
     if (isSchemaProp) {
-        schemaBuffer.buf[schemaBuffer.len++] = 254;
+        schemaBuffer.buf[schemaBuffer.len++] = SCHEMA_PROP;
         const typeIndex = TYPE_INDEX_MAP[obj.type];
         schemaBuffer.buf[schemaBuffer.len++] = typeIndex;
     }
     else {
-        schemaBuffer.buf[schemaBuffer.len++] = 255;
+        schemaBuffer.buf[schemaBuffer.len++] = isArray ? ARRAY : OBJECT;
     }
     let sizeIndex = schemaBuffer.len;
     schemaBuffer.len += 2;
-    for (const key in obj) {
-        if (key === 'type' && isSchemaProp) {
-            continue;
-        }
-        else {
-            let address = schemaBuffer.dictMap[key];
-            // if len == 1 never from address
-            if (!address) {
-                address = schemaBuffer.len;
+    if (isArray) {
+        const len = obj.length;
+        ensureCapacity(2 * len + 2);
+        writeUint16(schemaBuffer.buf, len, schemaBuffer.len);
+        schemaBuffer.len += 2;
+        for (let j = 0; j < len; j++) {
+            if (len < 256) {
+                schemaBuffer.buf[schemaBuffer.len] = j;
                 schemaBuffer.len += 1;
-                const r = ENCODER.encodeInto(key, schemaBuffer.buf.subarray(schemaBuffer.len));
-                schemaBuffer.buf[address] = r.written;
-                schemaBuffer.len += r.written;
-                schemaBuffer.dictMap[key] = address;
             }
             else {
-                schemaBuffer.buf[schemaBuffer.len] = 0;
-                schemaBuffer.len += 1;
-                schemaBuffer.buf[schemaBuffer.len] = address;
-                schemaBuffer.buf[schemaBuffer.len + 1] = address >>> 8;
+                writeUint16(schemaBuffer.buf, j, schemaBuffer.len);
                 schemaBuffer.len += 2;
             }
-            const val = obj[key];
-            const type = typeof val;
-            // typed Array
-            if (Array.isArray(val)) {
-                // derp
+            handleSingleValue(opts, obj[j], obj, prev, fromObject, j);
+        }
+    }
+    else {
+        for (const key in obj) {
+            if (opts.readOnly &&
+                isFromObj &&
+                (key === 'validation' || key === 'default')) {
+                if (key === 'validation' && typeof obj[key] === 'function') {
+                    continue;
+                }
+                else if (key === 'default') {
+                    continue;
+                }
+            }
+            else if (isFromObj &&
+                (opts.stripMetaInformation || opts.readOnly) &&
+                (key === 'title' ||
+                    key === 'description' ||
+                    key === 'format' ||
+                    key === 'display') &&
+                typeof obj[key] === 'string') {
+                continue;
+            }
+            else if (key === 'type' && isSchemaProp) {
+                continue;
             }
-            else if (type === 'function') {
-                // derp
+            else if (key === 'required' && obj[key] === true) {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = REQUIRED;
+                schemaBuffer.len += 1;
+                continue;
+            }
+            // Add this later
+            else if (key == 'ref' && isFromObj && typeof obj.ref === 'string') {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = REF;
+                schemaBuffer.len += 1;
+                encodeKey(obj[key], schemaBuffer);
+                continue;
             }
-            else if (type === 'object') {
-                // fromObject
-                if (val === null) {
+            else if (key === 'prop' && isFromObj && typeof obj.prop === 'string') {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = PROP;
+                schemaBuffer.len += 1;
+                encodeKey(obj[key], schemaBuffer);
+                continue;
+            }
+            else if (key === 'readOnly' && obj[key] === true) {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = READONLY;
+                schemaBuffer.len += 1;
+                continue;
+            }
+            else if (key === 'format' && isFromObj) {
+                ensureCapacity(2);
+                schemaBuffer.buf[schemaBuffer.len] = FORMAT;
+                schemaBuffer.len += 1;
+                schemaBuffer.buf[schemaBuffer.len] = stringFormats.indexOf(obj.format);
+                schemaBuffer.len += 1;
+                continue;
+            }
+            else {
+                let isTypes = false;
+                if (key === 'types') {
+                    //  undefined undefined false
+                    if (!prev && !prev2 && !fromObject) {
+                        isTypes = true;
+                    }
+                    ensureCapacity(1);
+                    schemaBuffer.buf[schemaBuffer.len] = TYPES;
+                    schemaBuffer.len += 1;
+                }
+                else if (key === 'props') {
+                    ensureCapacity(1);
+                    schemaBuffer.buf[schemaBuffer.len] = PROPS;
+                    schemaBuffer.len += 1;
                 }
                 else {
-                    if (!fromObject && key === 'props' && obj.type === 'object') {
-                        walk(val, obj, prev, true, schemaBuffer);
-                    }
-                    else {
-                        walk(val, obj, prev, fromObject, schemaBuffer);
-                    }
+                    encodeKey(key, schemaBuffer);
                 }
-            }
-            else if (type === 'string') {
-                // derp
-            }
-            else if (type === 'number') {
-                // do stuff
+                handleSingleValue(opts, obj[key], obj, prev, fromObject, key, isTypes);
             }
         }
     }
-    const size = schemaBuffer.len - start;
+    let size = schemaBuffer.len - start;
     schemaBuffer.buf[sizeIndex] = size;
     schemaBuffer.buf[sizeIndex + 1] = size >>> 8;
 };
-export const serialize = (schema, 
-// schema: StrictSchema,
-noCompression = false) => {
+export const serialize = (schema, opts = {}) => {
     if (!schemaBuffer) {
-        // 1mb buffer add check if its large enough else increase
         schemaBuffer = {
-            buf: new Uint8Array(1e6),
+            buf: new Uint8Array(10e3), // 10kb default
             len: 0,
             dictMap: {},
         };
     }
-    schemaBuffer.dictMap = {};
     schemaBuffer.len = 0;
-    const isDeflate = noCompression ? 0 : 1;
-    walk(schema, undefined, undefined, false, schemaBuffer);
+    schemaBuffer.dictMap = {};
+    // defalte not supported in unpacking yet
+    const isDeflate = 0; // opts.deflate ? 1 : 0
+    walk(opts, schema, undefined, undefined, false, schemaBuffer, false);
     const packed = new Uint8Array(schemaBuffer.buf.subarray(0, schemaBuffer.len));
-    if (isDeflate) {
-        return deflate.deflateSync(packed);
+    // if (isDeflate) {
+    //   // add extra byte! see if nessecary
+    //   return deflate.deflateSync(packed)
+    // } else {
+    return packed;
+    // }
+};
+const decoder = new TextDecoder();
+export const deSerializeKey = (buf, keySize, i) => {
+    let size = 0;
+    let value;
+    if (keySize === KEY_ADDRESS_3_BYTES) {
+        const dictAddress = readUint24(buf, i);
+        size += 3;
+        const actualKeySize = buf[dictAddress] - KEY_OPTS;
+        value = decoder.decode(buf.subarray(dictAddress + 1, actualKeySize + dictAddress + 1));
+    }
+    else if (keySize === KEY_ADDRESS_2_BYTES) {
+        const dictAddress = readUint16(buf, i);
+        size += 2;
+        const actualKeySize = buf[dictAddress] - KEY_OPTS;
+        value = decoder.decode(buf.subarray(dictAddress + 1, actualKeySize + dictAddress + 1));
+    }
+    else if (keySize === KEY_ADDRESS_1_BYTE) {
+        const dictAddress = buf[i];
+        size += 1;
+        const actualKeySize = buf[dictAddress] - KEY_OPTS;
+        value = decoder.decode(buf.subarray(dictAddress + 1, actualKeySize + dictAddress + 1));
     }
     else {
-        return packed;
+        const actualKeySize = keySize - KEY_OPTS;
+        value = decoder.decode(buf.subarray(i, actualKeySize + i));
+        size += actualKeySize;
     }
+    return { size, value };
 };
-const decoder = new TextDecoder();
-export const deSerializeInner = (buf, obj, start) => {
+export const deSerializeInner = (buf, obj, start, fromArray) => {
     let i = start;
-    const isSchemaProp = buf[i] === 254;
+    const isSchemaProp = buf[i] === SCHEMA_PROP;
     i += 1;
     if (isSchemaProp) {
         const type = buf[i];
@@ -110,33 +337,137 @@ export const deSerializeInner = (buf, obj, start) => {
         obj.type = parsedType;
         i += 1;
     }
-    const size = buf[i] | ((buf[i + 1] << 8) >>> 0);
+    const size = readUint16(buf, i);
     i += 2;
     const end = size + start;
+    if (fromArray) {
+        i += 2;
+    }
     while (i < end) {
-        let keySize = buf[i];
-        i += 1;
         let key;
-        if (keySize === 0) {
-            const dictAddress = buf[i] | ((buf[i + 1] << 8) >>> 0);
+        if (fromArray) {
+            if (obj.length < 256) {
+                key = buf[i];
+                i += 1;
+            }
+            else {
+                key = readUint16(buf, i);
+                i += 2;
+            }
+        }
+        else {
+            let keySize = buf[i];
+            i += 1;
+            // format!
+            if (keySize === REQUIRED) {
+                obj.required = true;
+                continue;
+            }
+            else if (keySize === FORMAT) {
+                obj.format = stringFormats[buf[i]];
+                i += 1;
+                continue;
+            }
+            else if (keySize === READONLY) {
+                obj.readOnly = true;
+                continue;
+            }
+            else if (keySize === TYPES) {
+                key = 'types';
+            }
+            else if (keySize === PROPS) {
+                key = 'props';
+            }
+            else if (keySize === REF) {
+                const valueKeySize = buf[i];
+                i += 1;
+                const { size, value } = deSerializeKey(buf, valueKeySize, i);
+                i += size;
+                obj.ref = value;
+                continue;
+            }
+            else if (keySize === PROP) {
+                const valueKeySize = buf[i];
+                i += 1;
+                const { size, value } = deSerializeKey(buf, valueKeySize, i);
+                i += size;
+                obj.prop = value;
+                continue;
+            }
+            else {
+                const { size, value } = deSerializeKey(buf, keySize, i);
+                i += size;
+                key = value;
+            }
+        }
+        if (buf[i] === UINT8) {
+            i += 1;
+            obj[key] = buf[i];
+            i += 1;
+        }
+        else if (buf[i] === FALSE) {
+            i += 1;
+            obj[key] = false;
+        }
+        else if (buf[i] === TRUE) {
+            i += 1;
+            obj[key] = true;
+        }
+        else if (buf[i] === FUNCTION) {
+            i += 1;
+            const size = readUint16(buf, i);
             i += 2;
-            keySize = buf[dictAddress];
-            key = decoder.decode(buf.subarray(dictAddress + 1, keySize + dictAddress + 1));
+            const fn = `return (${decoder.decode(buf.subarray(i, i + size))})(payload, prop)`;
+            obj[key] = new Function('payload', 'prop', fn);
+            i += size;
+        }
+        else if (buf[i] === STRING) {
+            i += 1;
+            const size = readUint16(buf, i);
+            i += 2;
+            obj[key] = decoder.decode(buf.subarray(i, i + size));
+            i += size;
+        }
+        else if (buf[i] === BINARY) {
+            i += 1;
+            const size = readUint16(buf, i);
+            i += 2;
+            obj[key] = buf.subarray(i, size + i);
+            i += size;
+        }
+        else if (buf[i] === UINT32) {
+            obj[key] = readUint32(buf, i + 1);
+            i += 5;
+        }
+        else if (buf[i] === FLOAT64) {
+            obj[key] = readDoubleLE(buf, i + 1);
+            i += 9;
+        }
+        else if (buf[i] === OBJECT || buf[i] === SCHEMA_PROP) {
+            const nest = (obj[key] = {});
+            const fieldSize = deSerializeInner(buf, nest, i, false);
+            i += fieldSize;
+        }
+        else if (buf[i] === ARRAY) {
+            const len = readUint16(buf, i + 3);
+            const nest = (obj[key] = new Array(len));
+            const fieldSize = deSerializeInner(buf, nest, i, true);
+            i += fieldSize;
         }
         else {
-            key = decoder.decode(buf.subarray(i, keySize + i));
-            i += keySize;
+            console.warn('Invalid value type', buf[i], 'skip');
+            // Invalid value type
+            i += 1;
+            const size = buf[i] | ((buf[i + 1] << 8) >>> 0);
+            i += size;
         }
-        const nest = (obj[key] = {});
-        const fieldSize = deSerializeInner(buf, nest, i);
-        i += fieldSize;
     }
     return i - start;
 };
 export const deSerialize = (buf) => {
     // if first byte is deflate
     const schema = {};
-    deSerializeInner(buf, schema, 0);
+    deSerializeInner(buf, schema, 0, false);
     return schema;
 };
 //# sourceMappingURL=serialize.js.map

package/dist/types.d.ts

@@ -96,6 +96,11 @@ export type SchemaVector = Prop<{
     default?: Float32Array;
     size: number;
 }>;
+export type SchemaColvec = Prop<{
+    type: 'colvec';
+    default?: Float32Array;
+    size: number;
+}>;
 export type SchemaTimestamp = Prop<{
     type: 'timestamp';
     default?: number | Date | string;
@@ -150,7 +155,7 @@ export type SchemaSet<ItemsType extends SetItems = SetItems> = Prop<{
     } ? ItemsType['default'][] : undefined;
     items: ItemsType & NeverInItems;
 }>;
-type NonRefSchemaProps<isStrict = false> = SchemaTimestamp | SchemaBoolean | SchemaNumber | SchemaString | SchemaAlias | SchemaText | SchemaEnum | SchemaJson | SchemaBinary | SchemaCardinality | SchemaVector | (isStrict extends true ? SchemaSet<SetItems<true>> : SchemaPropShorthand | SchemaSet);
+type NonRefSchemaProps<isStrict = false> = SchemaTimestamp | SchemaBoolean | SchemaNumber | SchemaString | SchemaAlias | SchemaText | SchemaEnum | SchemaJson | SchemaBinary | SchemaCardinality | SchemaVector | SchemaColvec | (isStrict extends true ? SchemaSet<SetItems<true>> : SchemaPropShorthand | SchemaSet);
 export type SchemaProp<isStrict = false> = SchemaReferencesWithQuery | SchemaReferenceWithQuery | NonRefSchemaProps<isStrict> | SchemaReferences | SchemaReference | SchemaObject | SchemaBinary;
 export type SchemaPropOneWay<isStrict = false> = SchemaReferencesOneWay | SchemaReferenceOneWay | SchemaObjectOneWay | NonRefSchemaProps<isStrict>;
 export type SchemaAnyProp = SchemaPropOneWay | SchemaProp;
@@ -165,6 +170,9 @@ type GenericSchemaType<isStrict = false> = {
         delete?: SchemaHook;
     };
     id?: number;
+    blockCapacity?: number;
+    insertOnly?: boolean;
+    partial?: boolean;
     props: SchemaProps<isStrict>;
 };
 export type StrictSchemaType = GenericSchemaType<true>;
@@ -204,6 +212,7 @@ export type SchemaPropTypeMap = {
     binary: SchemaBinary;
     cardinality: SchemaCardinality;
     vector: SchemaVector;
+    colvec: SchemaColvec;
 } & Record<NumberType, SchemaNumber>;
 export type SchemaPropTypes = keyof SchemaPropTypeMap;
 export declare const isPropType: <T extends SchemaPropTypes>(type: T, prop: SchemaProp) => prop is SchemaPropTypeMap[T];

package/dist/validation/validation.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=validation.d.ts.map

package/dist/validation/validation.js

@@ -0,0 +1,6 @@
+export {};
+// use typeIndex here
+// end export them per type
+// can also add validate on the prop def
+// this way we can actually write custom ones
+//# sourceMappingURL=validation.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/schema",
-  "version": "5.0.0-alpha.20",
+  "version": "5.0.0-alpha.21",
   "license": "MIT",
   "files": [
     "dist",
@@ -29,7 +29,6 @@
     "typescript": "^5.6.3"
   },
   "dependencies": {
-    "fflate": "0.8.1",
     "@saulx/utils": "^6.7.0",
     "picocolors": "^1.1.0"
   }