@based/schema 5.0.0-alpha.20 → 5.0.0-alpha.22

package/dist/def/createEmptyDef.d.ts

@@ -2,13 +2,15 @@ import { SchemaLocales, SchemaObject, StrictSchemaType } from '../types.js';
 export declare const createEmptyDef: (typeName: string, type: StrictSchemaType | SchemaObject, locales: Partial<SchemaLocales>) => {
     cnt: number;
     blockCapacity: number;
+    insertOnly: boolean;
+    partial: boolean;
     checksum: number;
     type: string;
     props: {};
     reverseProps: {};
-    idUint8: Uint8Array;
+    idUint8: Uint8Array<ArrayBuffer>;
     id: number;
-    mainEmpty: Uint8Array;
+    mainEmpty: Uint8Array<ArrayBuffer>;
     mainLen: number;
     separate: any[];
     tree: {};
@@ -23,16 +25,16 @@ export declare const createEmptyDef: (typeName: string, type: StrictSchemaType |
     seperateSort: {
         size: number;
         props: any[];
-        buffer: Uint8Array;
-        bufferTmp: Uint8Array;
+        buffer: Uint8Array<ArrayBuffer>;
+        bufferTmp: Uint8Array<ArrayBuffer>;
     };
     hasSeperateTextSort: boolean;
     seperateTextSort: {
         size: number;
         props: any[];
-        buffer: Uint8Array;
-        noUndefined: Uint8Array;
-        bufferTmp: Uint8Array;
+        buffer: Uint8Array<ArrayBuffer>;
+        noUndefined: Uint8Array<ArrayBuffer>;
+        bufferTmp: Uint8Array<ArrayBuffer>;
         localeStringToIndex: Map<any, any>;
         localeToIndex: Map<any, any>;
     };

package/dist/def/createEmptyDef.js

@@ -3,6 +3,8 @@ export const createEmptyDef = (typeName, type, locales) => {
     return {
         cnt: 0,
         blockCapacity: 0,
+        insertOnly: false,
+        partial: false,
         checksum: hashObjectIgnoreKeyOrder(type),
         type: typeName,
         props: {},

package/dist/def/defaultMap.js

@@ -23,5 +23,6 @@ export const DEFAULT_MAP = {
     [TYPE_INDEX_MAP.aliases]: [],
     [TYPE_INDEX_MAP.text]: {},
     [TYPE_INDEX_MAP.vector]: undefined, // maybe not can set a vec with 0
+    [TYPE_INDEX_MAP.colvec]: undefined, // maybe not can set a vec with 0
 };
 //# sourceMappingURL=defaultMap.js.map

package/dist/def/fillEmptyMain.d.ts

@@ -1,5 +1,5 @@
 import { PropDef } from './types.js';
 export declare const ENCODER: TextEncoder;
-export declare const fillEmptyMain: (vals: PropDef[], mainLen: number) => Uint8Array;
+export declare const fillEmptyMain: (vals: PropDef[], mainLen: number) => Uint8Array<ArrayBuffer>;
 export declare const isZeroes: (buf: Uint8Array) => boolean;
 //# sourceMappingURL=fillEmptyMain.d.ts.map

package/dist/def/selvaBuffer.js

@@ -1,4 +1,4 @@
-import { ALIAS, ALIASES, BINARY, EMPTY_MICRO_BUFFER, CARDINALITY, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, WEAK_REFERENCE, WEAK_REFERENCES, JSON, } from './types.js';
+import { ALIAS, ALIASES, BINARY, EMPTY_MICRO_BUFFER, CARDINALITY, MICRO_BUFFER, REFERENCE, REFERENCES, STRING, TEXT, VECTOR, WEAK_REFERENCE, WEAK_REFERENCES, JSON, COLVEC, } from './types.js';
 const selvaFieldType = {
     NULL: 0,
     MICRO_BUFFER: 1,
@@ -26,6 +26,7 @@ selvaTypeMap[WEAK_REFERENCE] = selvaFieldType.WEAK_REFERENCE;
 selvaTypeMap[WEAK_REFERENCES] = selvaFieldType.WEAK_REFERENCES;
 selvaTypeMap[ALIAS] = selvaFieldType.ALIAS;
 selvaTypeMap[ALIASES] = selvaFieldType.ALIASES;
+selvaTypeMap[COLVEC] = selvaFieldType.COLVEC;
 const EDGE_FIELD_CONSTRAINT_FLAG_DEPENDENT = 0x01;
 const EDGE_FIELD_CONSTRAINT_FLAG_SKIP_DUMP = 0x80;
 function blockCapacity(blockCapacity) {
@@ -53,6 +54,14 @@ const propDefBuffer = (schema, prop, isEdge) => {
         view.setUint16(1, prop.len, true);
         return [...buf];
     }
+    else if (prop.len && (type === COLVEC)) {
+        const buf = new Uint8Array(5);
+        const view = new DataView(buf.buffer);
+        buf[0] = selvaType;
+        view.setUint16(1, prop.len, true);
+        view.setUint16(3, 4, true); // TODO Other types than f32
+        return [...buf];
+    }
     else if (type === REFERENCE || type === REFERENCES) {
         const buf = new Uint8Array(9);
         const view = new DataView(buf.buffer);

package/dist/def/typeDef.d.ts

@@ -1,7 +1,6 @@
 import { SchemaObject, StrictSchemaType, SchemaLocales } from '../index.js';
 import { SchemaTypeDef, SchemaTypesParsed } from './types.js';
 import { StrictSchema } from '../types.js';
-export declare const DEFAULT_BLOCK_CAPACITY = 100000;
 export declare const updateTypeDefs: (schema: StrictSchema) => {
     schemaTypesParsed: {
         [key: string]: SchemaTypeDef;

package/dist/def/typeDef.js

@@ -1,6 +1,6 @@
 import { isPropType, getPropType, } from '../index.js';
 import { setByPath } from '@saulx/utils';
-import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, NUMBER, } from './types.js';
+import { TYPE_INDEX_MAP, REFERENCES, REFERENCE, NUMBER, BLOCK_CAPACITY_MAX, BLOCK_CAPACITY_DEFAULT, BLOCK_CAPACITY_MIN, } from './types.js';
 import { DEFAULT_MAP } from './defaultMap.js';
 import { makeSeparateTextSort } from './makeSeparateTextSort.js';
 import { makeSeparateSort } from './makeSeparateSort.js';
@@ -9,39 +9,59 @@ import { addEdges } from './addEdges.js';
 import { createEmptyDef } from './createEmptyDef.js';
 import { fillEmptyMain, isZeroes } from './fillEmptyMain.js';
 import { defaultValidation, VALIDATION_MAP } from './validation.js';
-export const DEFAULT_BLOCK_CAPACITY = 100_000;
 export const updateTypeDefs = (schema) => {
     const schemaTypesParsed = {};
     const schemaTypesParsedById = {};
-    for (const field in schemaTypesParsed) {
-        if (field in schema.types) {
+    for (const typeName in schemaTypesParsed) {
+        if (typeName in schema.types) {
             continue;
         }
-        const id = schemaTypesParsed[field].id;
-        delete schemaTypesParsed[field];
+        const id = schemaTypesParsed[typeName].id;
+        delete schemaTypesParsed[typeName];
         delete schemaTypesParsedById[id];
     }
-    for (const field in schema.types) {
-        const type = schema.types[field];
+    for (const typeName in schema.types) {
+        const type = schema.types[typeName];
         if (!type.id) {
             throw new Error('NEED ID ON TYPE');
         }
-        const def = createSchemaTypeDef(field, type, schemaTypesParsed, schema.locales ?? {
+        const def = createSchemaTypeDef(typeName, type, schemaTypesParsed, schema.locales ?? {
             en: {},
         });
-        def.blockCapacity = field === '_root' ? 2147483647 : DEFAULT_BLOCK_CAPACITY;
-        schemaTypesParsed[field] = def;
+        schemaTypesParsed[typeName] = def;
         schemaTypesParsedById[type.id] = def;
     }
     return { schemaTypesParsed, schemaTypesParsedById };
 };
 export const createSchemaTypeDef = (typeName, type, parsed, locales, result = createEmptyDef(typeName, type, locales), path = [], top = true) => {
-    if (result.id == 0 && top) {
-        if ('id' in type) {
-            result.id = type.id;
+    if (top) {
+        if (result.id == 0) {
+            if ('id' in type) {
+                result.id = type.id;
+            }
+            else {
+                throw new Error(`Invalid schema type id ${result.type}`);
+            }
         }
-        else {
-            throw new Error(`Invalid schema type id ${result.type}`);
+        if (result.blockCapacity == 0) {
+            if ('blockCapacity' in type) {
+                if (typeof type.blockCapacity !== 'number' ||
+                    type.blockCapacity < BLOCK_CAPACITY_MIN ||
+                    type.blockCapacity > BLOCK_CAPACITY_MAX) {
+                    throw new Error('Invalid blockCapacity');
+                }
+                result.blockCapacity = type.blockCapacity;
+            }
+            else {
+                result.blockCapacity =
+                    typeName === '_root' ? BLOCK_CAPACITY_MAX : BLOCK_CAPACITY_DEFAULT;
+            }
+        }
+        if (result.insertOnly == false && 'insertOnly' in type) {
+            result.insertOnly = !!type.insertOnly;
+        }
+        if (result.partial == false && 'partial' in type) {
+            result.partial = !!type.partial;
         }
     }
     result.locales = locales;
@@ -75,6 +95,11 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = cr
             else if (isPropType('text', schemaProp)) {
                 result.separateSortText++;
             }
+            else if (isPropType('colvec', schemaProp)) {
+                if (!result.insertOnly) {
+                    throw new Error('colvec requires insertOnly');
+                }
+            }
             const isseparate = isSeparate(schemaProp, len);
             const typeIndex = TYPE_INDEX_MAP[propType];
             const prop = {
@@ -110,12 +135,18 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = cr
                 }
             }
             else if (isPropType('references', schemaProp)) {
+                if (result.partial) {
+                    throw new Error('references is not supported with partial');
+                }
                 prop.inversePropName = schemaProp.items.prop;
                 prop.inverseTypeName = schemaProp.items.ref;
                 prop.dependent = schemaProp.items.dependent;
                 addEdges(prop, schemaProp.items);
             }
             else if (isPropType('reference', schemaProp)) {
+                if (result.partial) {
+                    throw new Error('reference is not supported with partial');
+                }
                 prop.inversePropName = schemaProp.prop;
                 prop.inverseTypeName = schemaProp.ref;
                 prop.dependent = schemaProp.dependent;
@@ -169,6 +200,13 @@ export const createSchemaTypeDef = (typeName, type, parsed, locales, result = cr
             if (f.separate) {
                 len += 2;
                 setByPath(result.tree, f.path, f);
+                if (f.default !== undefined) {
+                    result.hasSeperateDefaults = true;
+                    if (!result.seperateDefaults) {
+                        // result.seperateDefaults = []
+                    }
+                    // result.seperateDefaults.push(f)
+                }
             }
             else {
                 if (!result.mainLen) {

package/dist/def/types.d.ts

@@ -26,6 +26,7 @@ export declare const ID = 26;
 export declare const VECTOR = 27;
 export declare const JSON = 28;
 export declare const OBJECT = 29;
+export declare const COLVEC = 30;
 export declare const TYPE_INDEX_MAP: {
     alias: number;
     aliases: number;
@@ -50,6 +51,7 @@ export declare const TYPE_INDEX_MAP: {
     cardinality: number;
     json: number;
     object: number;
+    colvec: number;
 };
 export declare const enum numberTypes {
     number = 4,
@@ -126,6 +128,9 @@ export type SchemaSortUndefinedHandler = {
     bufferTmp: Uint8Array;
     props: PropDef[];
 };
+export declare const BLOCK_CAPACITY_MIN = 1025;
+export declare const BLOCK_CAPACITY_MAX = 2147483647;
+export declare const BLOCK_CAPACITY_DEFAULT = 100000;
 export type SchemaTypeDef = {
     cnt: number;
     checksum: number;
@@ -133,6 +138,8 @@ export type SchemaTypeDef = {
     lastId: number;
     blockCapacity: number;
     mainLen: number;
+    insertOnly: boolean;
+    partial: boolean;
     buf: Uint8Array;
     propNames: Uint8Array;
     props: {
@@ -160,6 +167,11 @@ export type SchemaTypeDef = {
         localeStringToIndex: Map<string, Uint8Array>;
         localeToIndex: Map<LangCode, number>;
     };
+    hasSeperateDefaults: boolean;
+    seperateDefaults?: {
+        props: Map<number, PropDef>;
+        bufferTmp: Uint8Array;
+    };
     createTs?: PropDef[];
     updateTs?: PropDef[];
     locales: Partial<SchemaLocales>;

package/dist/def/types.js

@@ -25,6 +25,7 @@ export const ID = 26;
 export const VECTOR = 27;
 export const JSON = 28;
 export const OBJECT = 29;
+export const COLVEC = 30;
 export const TYPE_INDEX_MAP = {
     alias: ALIAS,
     aliases: ALIASES,
@@ -49,6 +50,7 @@ export const TYPE_INDEX_MAP = {
     cardinality: CARDINALITY,
     json: JSON,
     object: OBJECT,
+    colvec: COLVEC,
 };
 const numberTypeValues = [
     NUMBER,
@@ -63,6 +65,9 @@ const numberTypeValues = [
 export function isNumberType(type) {
     return numberTypeValues.includes(type);
 }
+export const BLOCK_CAPACITY_MIN = 1025;
+export const BLOCK_CAPACITY_MAX = 2147483647;
+export const BLOCK_CAPACITY_DEFAULT = 100_000;
 export const SIZE_MAP = {
     timestamp: 8, // 64bit
     // double-precision 64-bit binary format IEEE 754 value
@@ -88,6 +93,7 @@ export const SIZE_MAP = {
     vector: 0, // separate
     json: 0,
     object: 0,
+    colvec: 0, // separate
 };
 const reverseMap = {};
 for (const k in TYPE_INDEX_MAP) {

package/dist/def/utils.js

@@ -3,7 +3,7 @@ import { isPropType } from '../types.js';
 import { getPropType } from '../parse/utils.js';
 import { convertToTimestamp } from '@saulx/utils';
 export function isSeparate(schemaProp, len) {
-    return len === 0 || isPropType('vector', schemaProp);
+    return len === 0 || isPropType('vector', schemaProp) || isPropType('colvec', schemaProp);
 }
 export const propIsSigned = (prop) => {
     const t = prop.typeIndex;
@@ -43,6 +43,9 @@ export function getPropLen(schemaProp) {
     else if (isPropType('vector', schemaProp)) {
         len = 4 * schemaProp.size;
     }
+    else if (isPropType('colvec', schemaProp)) {
+        len = schemaProp.size;
+    }
     return len;
 }
 export const parseMinMaxStep = (val) => {

package/dist/parse/index.js

@@ -7,6 +7,7 @@ import { deepCopy } from '@saulx/utils';
 export { getPropType };
 export class SchemaParser {
     constructor(schema) {
+        // uint8Array is not working
         this.schema = deepCopy(schema);
     }
     isItems;
@@ -65,7 +66,6 @@ export class SchemaParser {
         for (const locale in locales) {
             const opts = locales[locale];
             if (opts === true) {
-                console.log(locale, opts);
                 continue;
             }
             expectObject(opts);

package/dist/parse/props.js

@@ -128,7 +128,7 @@ export const isDefault = (val, prop, ctx) => {
         enum: prop.enum,
         validation,
         default: DEFAULT_MAP[typeIndex],
-        step: parseMinMaxStep(prop.step ?? typeIndex === NUMBER ? 0 : 1),
+        step: parseMinMaxStep((prop.step ?? typeIndex === NUMBER) ? 0 : 1),
         max: parseMinMaxStep(prop.max),
         min: parseMinMaxStep(prop.min),
     };
@@ -157,6 +157,15 @@ p.vector = propParser({
         return isDefault(val, prop, ctx);
     },
 }, 0);
+p.colvec = propParser({
+    size(val) {
+        expectNumber(val);
+    },
+}, {
+    default(val, prop, ctx) {
+        return isDefault(val, prop, ctx);
+    },
+}, 0);
 p.enum = propParser({
     enum(items) {
         if (!Array.isArray(items)) {

package/dist/serialize.d.ts

@@ -1,5 +1,14 @@
 import { StrictSchema } from './types.js';
-export declare const serialize: (schema: any, noCompression?: boolean) => Uint8Array;
-export declare const deSerializeInner: (buf: Uint8Array, obj: any, start: number) => number;
+type Opts = {
+    readOnly?: boolean;
+    stripMetaInformation?: boolean;
+};
+export declare const serialize: (schema: any, opts?: Opts) => Uint8Array;
+export declare const deSerializeKey: (buf: Uint8Array, keySize: number, i: number) => {
+    size: number;
+    value: string;
+};
+export declare const deSerializeInner: (buf: Uint8Array, obj: any, start: number, fromArray: boolean) => number;
 export declare const deSerialize: (buf: Uint8Array) => StrictSchema;
+export {};
 //# sourceMappingURL=serialize.d.ts.map

package/dist/serialize.js

@@ -1,142 +1,490 @@
-import * as deflate from 'fflate';
-import { REVERSE_TYPE_INDEX_MAP, TYPE_INDEX_MAP } from './def/types.js';
+// import * as deflate from 'fflate'
+import { stringFormats } from './types.js';
+import { ENUM, REVERSE_TYPE_INDEX_MAP, TYPE_INDEX_MAP } from './def/types.js';
+import { readDoubleLE, readUint16, readUint24, readUint32, writeDoubleLE, writeUint16, writeUint24, writeUint32, } from '@saulx/utils';
 const ENCODER = new TextEncoder();
+const UINT8 = 245;
+const FALSE = 246;
+const TRUE = 247;
+const FUNCTION = 248;
+const STRING = 249;
+const ARRAY = 250;
+const BINARY = 251;
+const UINT32 = 252;
+const FLOAT64 = 253;
+const SCHEMA_PROP = 254;
+const OBJECT = 255;
+// Key Address encoding types
+const KEY_ADDRESS_1_BYTE = 0;
+const KEY_ADDRESS_2_BYTES = 1;
+const KEY_ADDRESS_3_BYTES = 2;
+// Key types
+const PROPS = 3;
+const TYPES = 4;
+const READONLY = 5;
+const FORMAT = 6;
+const REQUIRED = 7;
+const REF = 8;
+const PROP = 9;
+const KEY_OPTS = PROP;
+const ensureCapacity = (required) => {
+    if (schemaBuffer.len + required > schemaBuffer.buf.length) {
+        const newBuf = new Uint8Array(Math.max(schemaBuffer.buf.length * 2, schemaBuffer.len + required));
+        newBuf.set(schemaBuffer.buf);
+        schemaBuffer.buf = newBuf;
+    }
+};
 let schemaBuffer;
+const handleSingleValue = (ops, val, obj, prev, fromObject, key) => {
+    const type = typeof val;
+    // typed Array - single PROP
+    if (val instanceof Uint8Array) {
+        ensureCapacity(1 + 2 + val.byteLength);
+        schemaBuffer.buf[schemaBuffer.len] = BINARY;
+        schemaBuffer.len += 1;
+        schemaBuffer.buf[schemaBuffer.len] = val.byteLength;
+        schemaBuffer.len += 1;
+        schemaBuffer.buf[schemaBuffer.len] = val.byteLength >>> 8;
+        schemaBuffer.len += 1;
+        schemaBuffer.buf.set(val, schemaBuffer.len);
+        schemaBuffer.len += val.byteLength;
+    }
+    else if (type === 'function') {
+        const str = val.toString();
+        // Pessimistically assume 4 bytes per char for UTF-8 to be safe.
+        ensureCapacity(1 + 2 + str.length * 4);
+        schemaBuffer.buf[schemaBuffer.len] = FUNCTION;
+        schemaBuffer.len += 1;
+        const sizeIndex = schemaBuffer.len;
+        schemaBuffer.len += 2;
+        // encodeInto is much faster as it avoids intermediate allocation.
+        const r = ENCODER.encodeInto(str, schemaBuffer.buf.subarray(schemaBuffer.len));
+        schemaBuffer.len += r.written;
+        schemaBuffer.buf[sizeIndex] = r.written;
+        schemaBuffer.buf[sizeIndex + 1] = r.written >>> 8;
+    }
+    else if (type === 'object') {
+        // fromObject
+        if (val === null) {
+        }
+        else {
+            if (!fromObject && key === 'props' && obj.type === 'object') {
+                walk(ops, val, obj, prev, true, schemaBuffer);
+            }
+            else {
+                walk(ops, val, obj, prev, fromObject, schemaBuffer);
+            }
+        }
+    }
+    else if (type === 'boolean') {
+        ensureCapacity(1);
+        schemaBuffer.buf[schemaBuffer.len] = val ? TRUE : FALSE;
+        schemaBuffer.len += 1;
+    }
+    else if (type === 'string') {
+        // Pessimistically assume 4 bytes per char for UTF-8 to be safe.
+        ensureCapacity(1 + 2 + val.length * 4);
+        schemaBuffer.buf[schemaBuffer.len] = STRING;
+        schemaBuffer.len += 1;
+        const sizeIndex = schemaBuffer.len;
+        schemaBuffer.len += 2;
+        // encodeInto is much faster as it avoids intermediate allocation.
+        const r = ENCODER.encodeInto(val, schemaBuffer.buf.subarray(schemaBuffer.len));
+        schemaBuffer.len += r.written;
+        schemaBuffer.buf[sizeIndex] = r.written;
+        schemaBuffer.buf[sizeIndex + 1] = r.written >>> 8;
+    }
+    else if (type === 'number') {
+        const isInt = val % 1 === 0;
+        if (val < 256 && val > 0 && isInt) {
+            ensureCapacity(2);
+            schemaBuffer.buf[schemaBuffer.len] = UINT8;
+            schemaBuffer.len += 1;
+            schemaBuffer.buf[schemaBuffer.len] = val;
+            schemaBuffer.len += 1;
+        }
+        else if ((val < 4294967295 || val > 0) && isInt) {
+            ensureCapacity(5);
+            schemaBuffer.buf[schemaBuffer.len] = UINT32;
+            schemaBuffer.len += 1;
+            writeUint32(schemaBuffer.buf, val, schemaBuffer.len);
+            schemaBuffer.len += 4;
+        }
+        else {
+            ensureCapacity(9);
+            schemaBuffer.buf[schemaBuffer.len] = FLOAT64;
+            schemaBuffer.len += 1;
+            writeDoubleLE(schemaBuffer.buf, val, schemaBuffer.len);
+            schemaBuffer.len += 8;
+        }
+    }
+};
+const encodeKey = (key, schemaBuffer) => {
+    let address = schemaBuffer.dictMap[key];
+    // if len == 1 never from address
+    if (!address) {
+        // pessimistically assume 4 bytes per char for UTF-8 to be safe.
+        ensureCapacity(1 + key.length * 4);
+        address = schemaBuffer.len;
+        schemaBuffer.len += 1;
+        const r = ENCODER.encodeInto(key, schemaBuffer.buf.subarray(schemaBuffer.len));
+        schemaBuffer.buf[address] = r.written + KEY_OPTS;
+        schemaBuffer.len += r.written;
+        schemaBuffer.dictMap[key] = address;
+    }
+    else {
+        ensureCapacity(4);
+        if (address > 65025) {
+            schemaBuffer.buf[schemaBuffer.len] = KEY_ADDRESS_3_BYTES;
+            schemaBuffer.len += 1;
+            writeUint24(schemaBuffer.buf, address, schemaBuffer.len);
+            schemaBuffer.len += 3;
+        }
+        else if (address > 255) {
+            schemaBuffer.buf[schemaBuffer.len] = KEY_ADDRESS_2_BYTES;
+            schemaBuffer.len += 1;
+            writeUint16(schemaBuffer.buf, address, schemaBuffer.len);
+            schemaBuffer.len += 2;
+        }
+        else {
+            schemaBuffer.buf[schemaBuffer.len] = KEY_ADDRESS_1_BYTE;
+            schemaBuffer.len += 1;
+            schemaBuffer.buf[schemaBuffer.len] = address;
+            schemaBuffer.len += 1;
+        }
+    }
+};
 // 3 level
 // 0 for queries (min)
 // 1 for modify
 // 2 fulls schema
-const walk = (obj, prev, prev2, fromObject, schemaBuffer) => {
+const walk = (opts, obj, prev, prev2, fromObject, schemaBuffer) => {
     let start = schemaBuffer.len;
-    // HANDLE ENUM
-    const isSchemaProp = 'type' in obj && (prev2?.type === 'object' || fromObject === false);
+    const isArray = Array.isArray(obj);
+    const isFromObj = prev2?.type === 'object' || fromObject === false;
+    const isSchemaProp = ('enum' in obj || ('type' in obj && TYPE_INDEX_MAP[obj.type])) && isFromObj;
+    ensureCapacity(1 + 5); // Type byte + size
     if (isSchemaProp) {
-        schemaBuffer.buf[schemaBuffer.len++] = 254;
-        const typeIndex = TYPE_INDEX_MAP[obj.type];
+        schemaBuffer.buf[schemaBuffer.len++] = SCHEMA_PROP;
+        const typeIndex = TYPE_INDEX_MAP['enum' in obj ? 'enum' : obj.type];
         schemaBuffer.buf[schemaBuffer.len++] = typeIndex;
     }
     else {
-        schemaBuffer.buf[schemaBuffer.len++] = 255;
+        schemaBuffer.buf[schemaBuffer.len++] = isArray ? ARRAY : OBJECT;
     }
     let sizeIndex = schemaBuffer.len;
-    schemaBuffer.len += 2;
-    for (const key in obj) {
-        if (key === 'type' && isSchemaProp) {
-            continue;
-        }
-        else {
-            let address = schemaBuffer.dictMap[key];
-            // if len == 1 never from address
-            if (!address) {
-                address = schemaBuffer.len;
+    schemaBuffer.len += 5;
+    if (isArray) {
+        const len = obj.length;
+        ensureCapacity(2 * len + 2);
+        writeUint16(schemaBuffer.buf, len, schemaBuffer.len);
+        schemaBuffer.len += 2;
+        for (let j = 0; j < len; j++) {
+            if (len < 256) {
+                schemaBuffer.buf[schemaBuffer.len] = j;
                 schemaBuffer.len += 1;
-                const r = ENCODER.encodeInto(key, schemaBuffer.buf.subarray(schemaBuffer.len));
-                schemaBuffer.buf[address] = r.written;
-                schemaBuffer.len += r.written;
-                schemaBuffer.dictMap[key] = address;
             }
             else {
-                schemaBuffer.buf[schemaBuffer.len] = 0;
-                schemaBuffer.len += 1;
-                schemaBuffer.buf[schemaBuffer.len] = address;
-                schemaBuffer.buf[schemaBuffer.len + 1] = address >>> 8;
+                writeUint16(schemaBuffer.buf, j, schemaBuffer.len);
                 schemaBuffer.len += 2;
             }
-            const val = obj[key];
-            const type = typeof val;
-            // typed Array
-            if (Array.isArray(val)) {
-                // derp
+            handleSingleValue(opts, obj[j], obj, prev, fromObject, j);
+        }
+    }
+    else {
+        for (const key in obj) {
+            if (opts.readOnly &&
+                isFromObj &&
+                (key === 'validation' || key === 'default')) {
+                if (key === 'validation' && typeof obj[key] === 'function') {
+                    continue;
+                }
+                else if (key === 'default') {
+                    continue;
+                }
             }
-            else if (type === 'function') {
-                // derp
+            else if (isFromObj &&
+                (opts.stripMetaInformation || opts.readOnly) &&
+                (key === 'title' ||
+                    key === 'description' ||
+                    key === 'format' ||
+                    key === 'display') &&
+                typeof obj[key] === 'string') {
+                continue;
             }
-            else if (type === 'object') {
-                // fromObject
-                if (val === null) {
+            else if (key === 'type' && isSchemaProp) {
+                continue;
+            }
+            else if (key === 'required' && obj[key] === true) {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = REQUIRED;
+                schemaBuffer.len += 1;
+                continue;
+            }
+            // Add this later
+            else if (key == 'ref' && isFromObj && typeof obj.ref === 'string') {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = REF;
+                schemaBuffer.len += 1;
+                encodeKey(obj[key], schemaBuffer);
+                continue;
+            }
+            else if (key === 'prop' && isFromObj && typeof obj.prop === 'string') {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = PROP;
+                schemaBuffer.len += 1;
+                encodeKey(obj[key], schemaBuffer);
+                continue;
+            }
+            else if (key === 'readOnly' && obj[key] === true) {
+                ensureCapacity(1);
+                schemaBuffer.buf[schemaBuffer.len] = READONLY;
+                schemaBuffer.len += 1;
+                continue;
+            }
+            else if (key === 'format' && isFromObj) {
+                ensureCapacity(2);
+                schemaBuffer.buf[schemaBuffer.len] = FORMAT;
+                schemaBuffer.len += 1;
+                schemaBuffer.buf[schemaBuffer.len] = stringFormats.indexOf(obj.format);
+                schemaBuffer.len += 1;
+                continue;
+            }
+            else {
+                if (key === 'types') {
+                    ensureCapacity(1);
+                    schemaBuffer.buf[schemaBuffer.len] = TYPES;
+                    schemaBuffer.len += 1;
+                }
+                else if (key === 'props') {
+                    ensureCapacity(1);
+                    schemaBuffer.buf[schemaBuffer.len] = PROPS;
+                    schemaBuffer.len += 1;
                 }
                 else {
-                    if (!fromObject && key === 'props' && obj.type === 'object') {
-                        walk(val, obj, prev, true, schemaBuffer);
-                    }
-                    else {
-                        walk(val, obj, prev, fromObject, schemaBuffer);
-                    }
+                    encodeKey(key, schemaBuffer);
                 }
-            }
-            else if (type === 'string') {
-                // derp
-            }
-            else if (type === 'number') {
-                // do stuff
+                // important to handle the size here...
+                handleSingleValue(opts, obj[key], obj, prev, fromObject, key);
             }
         }
     }
-    const size = schemaBuffer.len - start;
-    schemaBuffer.buf[sizeIndex] = size;
-    schemaBuffer.buf[sizeIndex + 1] = size >>> 8;
+    let size = schemaBuffer.len - start;
+    // 3
+    // if (size < 252) {
+    //   console.log('FLAP>', size)
+    //   schemaBuffer.buf[sizeIndex] = size + 3
+    //   schemaBuffer.buf.set(
+    //     schemaBuffer.buf.subarray(sizeIndex + 4, sizeIndex + size),
+    //     sizeIndex + 1,
+    //   )
+    //   schemaBuffer.len -= 4
+    // } else {
+    schemaBuffer.buf[sizeIndex] = 0; // means 4
+    writeUint32(schemaBuffer.buf, size, sizeIndex + 1);
+    // }
 };
-export const serialize = (schema, 
-// schema: StrictSchema,
-noCompression = false) => {
+export const serialize = (schema, opts = {}) => {
     if (!schemaBuffer) {
-        // 1mb buffer add check if its large enough else increase
         schemaBuffer = {
-            buf: new Uint8Array(1e6),
+            buf: new Uint8Array(10e3), // 10kb default
             len: 0,
             dictMap: {},
         };
     }
-    schemaBuffer.dictMap = {};
     schemaBuffer.len = 0;
-    const isDeflate = noCompression ? 0 : 1;
-    walk(schema, undefined, undefined, false, schemaBuffer);
+    schemaBuffer.dictMap = {};
+    // defalte not supported in unpacking yet
+    const isDeflate = 0; // opts.deflate ? 1 : 0
+    walk(opts, schema, undefined, undefined, false, schemaBuffer);
     const packed = new Uint8Array(schemaBuffer.buf.subarray(0, schemaBuffer.len));
-    if (isDeflate) {
-        return deflate.deflateSync(packed);
+    // if (isDeflate) {
+    //   // add extra byte! see if nessecary
+    //   return deflate.deflateSync(packed)
+    // } else {
+    return packed;
+    // }
+};
+const decoder = new TextDecoder();
+export const deSerializeKey = (buf, keySize, i) => {
+    let size = 0;
+    let value;
+    if (keySize === KEY_ADDRESS_3_BYTES) {
+        const dictAddress = readUint24(buf, i);
+        size += 3;
+        const actualKeySize = buf[dictAddress] - KEY_OPTS;
+        value = decoder.decode(buf.subarray(dictAddress + 1, actualKeySize + dictAddress + 1));
+    }
+    else if (keySize === KEY_ADDRESS_2_BYTES) {
+        const dictAddress = readUint16(buf, i);
+        size += 2;
+        const actualKeySize = buf[dictAddress] - KEY_OPTS;
+        value = decoder.decode(buf.subarray(dictAddress + 1, actualKeySize + dictAddress + 1));
+    }
+    else if (keySize === KEY_ADDRESS_1_BYTE) {
+        const dictAddress = buf[i];
+        size += 1;
+        const actualKeySize = buf[dictAddress] - KEY_OPTS;
+        value = decoder.decode(buf.subarray(dictAddress + 1, actualKeySize + dictAddress + 1));
     }
     else {
-        return packed;
+        const actualKeySize = keySize - KEY_OPTS;
+        value = decoder.decode(buf.subarray(i, actualKeySize + i));
+        size += actualKeySize;
     }
+    return { size, value };
 };
-const decoder = new TextDecoder();
-export const deSerializeInner = (buf, obj, start) => {
+export const deSerializeInner = (buf, obj, start, fromArray) => {
     let i = start;
-    const isSchemaProp = buf[i] === 254;
+    const isSchemaProp = buf[i] === SCHEMA_PROP;
     i += 1;
     if (isSchemaProp) {
         const type = buf[i];
         const parsedType = REVERSE_TYPE_INDEX_MAP[type];
-        obj.type = parsedType;
+        if (type !== ENUM) {
+            obj.type = parsedType;
+        }
+        i += 1;
+    }
+    let size;
+    if (buf[i] === 0) {
+        size = readUint32(buf, i + 1);
+        i += 5;
+    }
+    else {
+        size = buf[i] - 3;
+        console.log('yo', size);
         i += 1;
     }
-    const size = buf[i] | ((buf[i + 1] << 8) >>> 0);
-    i += 2;
     const end = size + start;
+    if (fromArray) {
+        i += 2;
+    }
     while (i < end) {
-        let keySize = buf[i];
-        i += 1;
         let key;
-        if (keySize === 0) {
-            const dictAddress = buf[i] | ((buf[i + 1] << 8) >>> 0);
+        if (fromArray) {
+            if (obj.length < 256) {
+                key = buf[i];
+                i += 1;
+            }
+            else {
+                key = readUint16(buf, i);
+                i += 2;
+            }
+        }
+        else {
+            let keySize = buf[i];
+            i += 1;
+            // format!
+            if (keySize === REQUIRED) {
+                obj.required = true;
+                continue;
+            }
+            else if (keySize === FORMAT) {
+                obj.format = stringFormats[buf[i]];
+                i += 1;
+                continue;
+            }
+            else if (keySize === READONLY) {
+                obj.readOnly = true;
+                continue;
+            }
+            else if (keySize === TYPES) {
+                key = 'types';
+            }
+            else if (keySize === PROPS) {
+                key = 'props';
+            }
+            else if (keySize === REF) {
+                const valueKeySize = buf[i];
+                i += 1;
+                const { size, value } = deSerializeKey(buf, valueKeySize, i);
+                i += size;
+                obj.ref = value;
+                continue;
+            }
+            else if (keySize === PROP) {
+                const valueKeySize = buf[i];
+                i += 1;
+                const { size, value } = deSerializeKey(buf, valueKeySize, i);
+                i += size;
+                obj.prop = value;
+                continue;
+            }
+            else {
+                const { size, value } = deSerializeKey(buf, keySize, i);
+                i += size;
+                key = value;
+            }
+        }
+        if (buf[i] === UINT8) {
+            i += 1;
+            obj[key] = buf[i];
+            i += 1;
+        }
+        else if (buf[i] === FALSE) {
+            i += 1;
+            obj[key] = false;
+        }
+        else if (buf[i] === TRUE) {
+            i += 1;
+            obj[key] = true;
+        }
+        else if (buf[i] === FUNCTION) {
+            i += 1;
+            const size = readUint16(buf, i);
             i += 2;
-            keySize = buf[dictAddress];
-            key = decoder.decode(buf.subarray(dictAddress + 1, keySize + dictAddress + 1));
+            const fn = `return (${decoder.decode(buf.subarray(i, i + size))})(payload, prop)`;
+            obj[key] = new Function('payload', 'prop', fn);
+            i += size;
+        }
+        else if (buf[i] === STRING) {
+            i += 1;
+            const size = readUint16(buf, i);
+            i += 2;
+            obj[key] = decoder.decode(buf.subarray(i, i + size));
+            i += size;
+        }
+        else if (buf[i] === BINARY) {
+            i += 1;
+            const size = readUint16(buf, i);
+            i += 2;
+            obj[key] = buf.subarray(i, size + i);
+            i += size;
+        }
+        else if (buf[i] === UINT32) {
+            obj[key] = readUint32(buf, i + 1);
+            i += 5;
+        }
+        else if (buf[i] === FLOAT64) {
+            obj[key] = readDoubleLE(buf, i + 1);
+            i += 9;
+        }
+        else if (buf[i] === OBJECT || buf[i] === SCHEMA_PROP) {
+            const nest = (obj[key] = {});
+            const fieldSize = deSerializeInner(buf, nest, i, false);
+            i += fieldSize;
+        }
+        else if (buf[i] === ARRAY) {
+            const len = readUint16(buf, i + 3);
+            const nest = (obj[key] = new Array(len));
+            const fieldSize = deSerializeInner(buf, nest, i, true);
+            i += fieldSize;
         }
         else {
-            key = decoder.decode(buf.subarray(i, keySize + i));
-            i += keySize;
+            console.warn('Invalid value type', buf[i], 'skip');
+            // Invalid value type
+            i += 1;
+            const size = buf[i] | ((buf[i + 1] << 8) >>> 0);
+            i += size;
         }
-        const nest = (obj[key] = {});
-        const fieldSize = deSerializeInner(buf, nest, i);
-        i += fieldSize;
     }
     return i - start;
 };
 export const deSerialize = (buf) => {
     // if first byte is deflate
     const schema = {};
-    deSerializeInner(buf, schema, 0);
+    deSerializeInner(buf, schema, 0, false);
     return schema;
 };
 //# sourceMappingURL=serialize.js.map

package/dist/types.d.ts

@@ -96,6 +96,11 @@ export type SchemaVector = Prop<{
     default?: Float32Array;
     size: number;
 }>;
+export type SchemaColvec = Prop<{
+    type: 'colvec';
+    default?: Float32Array;
+    size: number;
+}>;
 export type SchemaTimestamp = Prop<{
     type: 'timestamp';
     default?: number | Date | string;
@@ -150,7 +155,7 @@ export type SchemaSet<ItemsType extends SetItems = SetItems> = Prop<{
     } ? ItemsType['default'][] : undefined;
     items: ItemsType & NeverInItems;
 }>;
-type NonRefSchemaProps<isStrict = false> = SchemaTimestamp | SchemaBoolean | SchemaNumber | SchemaString | SchemaAlias | SchemaText | SchemaEnum | SchemaJson | SchemaBinary | SchemaCardinality | SchemaVector | (isStrict extends true ? SchemaSet<SetItems<true>> : SchemaPropShorthand | SchemaSet);
+type NonRefSchemaProps<isStrict = false> = SchemaTimestamp | SchemaBoolean | SchemaNumber | SchemaString | SchemaAlias | SchemaText | SchemaEnum | SchemaJson | SchemaBinary | SchemaCardinality | SchemaVector | SchemaColvec | (isStrict extends true ? SchemaSet<SetItems<true>> : SchemaPropShorthand | SchemaSet);
 export type SchemaProp<isStrict = false> = SchemaReferencesWithQuery | SchemaReferenceWithQuery | NonRefSchemaProps<isStrict> | SchemaReferences | SchemaReference | SchemaObject | SchemaBinary;
 export type SchemaPropOneWay<isStrict = false> = SchemaReferencesOneWay | SchemaReferenceOneWay | SchemaObjectOneWay | NonRefSchemaProps<isStrict>;
 export type SchemaAnyProp = SchemaPropOneWay | SchemaProp;
@@ -165,6 +170,9 @@ type GenericSchemaType<isStrict = false> = {
         delete?: SchemaHook;
     };
     id?: number;
+    blockCapacity?: number;
+    insertOnly?: boolean;
+    partial?: boolean;
     props: SchemaProps<isStrict>;
 };
 export type StrictSchemaType = GenericSchemaType<true>;
@@ -204,6 +212,7 @@ export type SchemaPropTypeMap = {
     binary: SchemaBinary;
     cardinality: SchemaCardinality;
     vector: SchemaVector;
+    colvec: SchemaColvec;
 } & Record<NumberType, SchemaNumber>;
 export type SchemaPropTypes = keyof SchemaPropTypeMap;
 export declare const isPropType: <T extends SchemaPropTypes>(type: T, prop: SchemaProp) => prop is SchemaPropTypeMap[T];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@based/schema",
-  "version": "5.0.0-alpha.20",
+  "version": "5.0.0-alpha.22",
   "license": "MIT",
   "files": [
     "dist",
@@ -29,8 +29,7 @@
     "typescript": "^5.6.3"
   },
   "dependencies": {
-    "fflate": "0.8.1",
-    "@saulx/utils": "^6.7.0",
+    "@saulx/utils": "^6.7.2",
     "picocolors": "^1.1.0"
   }
 }